Merge tag 'release-0.60.0'
Slider 0.60.0 incubating
diff --git a/README.md b/README.md
index 480502d..a25b83a 100644
--- a/README.md
+++ b/README.md
@@ -22,9 +22,9 @@
monitor them and make them larger or smaller as desired -even while
the cluster is running.
-Clusters can be stopped, "frozen" and restarted, "thawed" later; the distribution
+Clusters can be stopped and restarted later; the distribution
of the deployed application across the YARN cluster is persisted -enabling
-a best-effort placement close to the previous locations on a cluster thaw.
+a best-effort placement close to the previous locations on a cluster start.
Applications which remember the previous placement of data (such as HBase)
can exhibit fast start-up times from this feature.
@@ -91,3 +91,31 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
+
+# Export Control
+
+This distribution includes cryptographic software. The country in which you
+currently reside may have restrictions on the import, possession, use, and/or
+re-export to another country, of encryption software. BEFORE using any
+encryption software, please check your country's laws, regulations and
+policies concerning the import, possession, or use, and re-export of encryption
+software, to see if this is permitted. See <http://www.wassenaar.org/> for more
+information.
+
+The U.S. Government Department of Commerce, Bureau of Industry and Security
+(BIS), has classified this software as Export Commodity Control Number (ECCN)
+5D002.C.1, which includes information security software using or performing
+cryptographic functions with asymmetric algorithms. The form and manner of this
+Apache Software Foundation distribution makes it eligible for export under the
+License Exception ENC Technology Software Unrestricted (TSU) exception (see the
+BIS Export Administration Regulations, Section 740.13) for both object code and
+source code.
+
+The following provides more details on the included cryptographic software:
+
+Apache Slider uses the built-in java cryptography libraries. See Oracle's
+information regarding Java cryptographic export regulations for more details:
+http://www.oracle.com/us/products/export/export-regulations-345813.html
+
+Apache Slider uses the SSL libraries from the Jetty project distributed by the
+Eclipse Foundation (http://eclipse.org/jetty).
diff --git a/app-packages/accumulo/README.md b/app-packages/accumulo/README.md
new file mode 100644
index 0000000..537d769
--- /dev/null
+++ b/app-packages/accumulo/README.md
@@ -0,0 +1,113 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# How to create a Slider package for Accumulo?
+
+ mvn clean package -DskipTests -Paccumulo-app-package-maven
+
+OR
+
+ mvn clean package -DskipTests -Paccumulo-app-package -Dpkg.version=1.6.1 \
+ -Dpkg.name=accumulo-1.6.1-bin.tar.gz -Dpkg.src=/local/path/to/tarball
+
+App package can be found in
+
+ app-packages/accumulo/target/slider-accumulo-app-package-*.zip
+
+
+
+In the first case, the version number of the app package will match the
+slider version, and in the second case it will match the `pkg.version`
+(intended to be the accumulo version).
+
+Verify the content using
+
+ zip -Tv slider-accumulo-app-package*.zip
+
+`appConfig-default.json` and `resources-default.json` are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
+
+In the maven packaging case, the version of Accumulo used for the app package
+can be adjusted by adding a flag such as
+
+ -Daccumulo.version=1.5.1
+
+**Note that the LICENSE.txt and NOTICE.txt that are bundled with the app
+package are designed for Accumulo 1.6.0 only and may need to be modified to be
+applicable for other versions of the app package.
+
+Note also that the sample `appConfig-default.json` provided only works with Accumulo 1.6.
+For Accumulo 1.5 the instance.volumes property must be replaced with
+instance.dfs.dir (and it cannot use the provided variable `${DEFAULT_DATA_DIR}`
+which is an HDFS URI).
+
+A less descriptive file name can be specified with
+`-Dapp.package.name=accumulo_160` which would create a file `accumulo_160.zip`
+
+# Building Native Libraries
+
+Accumulo works better with its native libraries, and these must be built
+manually for Accumulo releases 1.6.0 and greater. They should be built on a
+machine Accumulo will be deployed on, or an equivalent. The procedure below
+illustrates the steps for extracting and rebuilding the Accumulo app package
+with native libraries, in the case of Accumulo version 1.6.0. You will need a
+C++ compiler/toolchain installed to build this library, and `JAVA_HOME` must be
+set.
+
+ unzip ${app.package.name}.zip package/files/accumulo*gz
+ cd package/files/
+ gunzip accumulo-1.6.0-bin.tar.gz
+ tar xvf accumulo-1.6.0-bin.tar
+ accumulo-1.6.0/bin/build_native_library.sh
+ tar uvf accumulo-1.6.0-bin.tar accumulo-1.6.0
+ rm -rf accumulo-1.6.0
+ gzip accumulo-1.6.0-bin.tar
+ cd ../../
+ zip ${app.package.name}.zip -r package
+ rm -rf package
+
+# Export Control
+
+This distribution includes cryptographic software. The country in which you
+currently reside may have restrictions on the import, possession, use, and/or
+re-export to another country, of encryption software. BEFORE using any
+encryption software, please check your country's laws, regulations and
+policies concerning the import, possession, or use, and re-export of encryption
+software, to see if this is permitted. See [http://www.wassenaar.org/](http://www.wassenaar.org/) for more
+information.
+
+The U.S. Government Department of Commerce, Bureau of Industry and Security
+(BIS), has classified this software as Export Commodity Control Number (ECCN)
+5D002.C.1, which includes information security software using or performing
+cryptographic functions with asymmetric algorithms. The form and manner of this
+Apache Software Foundation distribution makes it eligible for export under the
+License Exception ENC Technology Software Unrestricted (TSU) exception (see the
+BIS Export Administration Regulations, Section 740.13) for both object code and
+source code.
+
+The following provides more details on the included cryptographic software:
+
+Apache Slider uses the built-in java cryptography libraries. See Oracle's
+information regarding Java cryptographic export regulations for more details:
+[http://www.oracle.com/us/products/export/export-regulations-345813.html](http://www.oracle.com/us/products/export/export-regulations-345813.html)
+
+Apache Slider uses the SSL libraries from the Jetty project distributed by the
+Eclipse Foundation [http://eclipse.org/jetty](http://eclipse.org/jetty).
+
+See also the Apache Accumulo export control notice in the README:
+[http://accumulo.apache.org/downloads](http://accumulo.apache.org/downloads)
diff --git a/app-packages/accumulo/README.txt b/app-packages/accumulo/README.txt
deleted file mode 100644
index 8e8fac2..0000000
--- a/app-packages/accumulo/README.txt
+++ /dev/null
@@ -1,47 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-How to create a Slider package for Accumulo?
-
- mvn clean package -DskipTests -Paccumulo-app-package
-
-App package can be found in
- app-packages/accumulo/target/apache-slider-accumulo-${accumulo.version}-app-package-${slider.version}.zip
-
-Verify the content using
- zip -Tv apache-slider-accumulo-*.zip
-
-While appConfig.json and resources.json are not required for the package they
-work well as the default configuration for Slider apps. So it is advisable that
-when you create an application package for Slider, include sample/default
-resources.json and appConfig.json for a minimal Yarn cluster.
-
-The version of Accumulo used for the app package can be adjusted by adding a
-flag such as
- -Daccumulo.version=1.5.1
-
-**Note that the LICENSE.txt and NOTICE.txt that are bundled with the app
-package are designed for Accumulo 1.6.0 only and may need to be modified to be
-applicable for other versions of the app package.
-
-Note also that the sample appConfig.json provided only works with Accumulo 1.6,
-while for Accumulo 1.5 the instance.volumes property must be replaced with
-instance.dfs.dir (and it cannot use the provided variable ${DEFAULT_DATA_DIR}
-which is an HDFS URI).
-
-A less descriptive file name can be specified with
--Dapp.package.name=accumulo_160 which would create a file accumulo_160.zip.
diff --git a/app-packages/accumulo/appConfig-default.json b/app-packages/accumulo/appConfig-default.json
new file mode 100644
index 0000000..9d11bae
--- /dev/null
+++ b/app-packages/accumulo/appConfig-default.json
@@ -0,0 +1,69 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
+ "java_home": "${app.java.home}",
+
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
+ "site.global.app_user": "${app.user}",
+ "site.global.user_group": "${app.user.group}",
+
+ "site.accumulo-env.java_home": "${JAVA_HOME}",
+ "site.accumulo-env.tserver_heapsize": "256m",
+ "site.accumulo-env.master_heapsize": "128m",
+ "site.accumulo-env.monitor_heapsize": "64m",
+ "site.accumulo-env.gc_heapsize": "64m",
+ "site.accumulo-env.other_heapsize": "128m",
+ "site.accumulo-env.hadoop_prefix": "${hadoop.dir}",
+ "site.accumulo-env.hadoop_conf_dir": "/etc/hadoop/conf",
+ "site.accumulo-env.zookeeper_home": "${zk.dir}",
+
+ "site.client.instance.name": "${USER}-${CLUSTER_NAME}",
+
+ "site.global.accumulo_root_password": "NOT_USED",
+ "site.global.ssl_cert_dir": "ssl",
+ "site.global.monitor_protocol": "http",
+
+ "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
+ "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
+ "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
+
+ "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.instance.rpc.ssl.enabled": "false",
+ "site.accumulo-site.instance.rpc.ssl.clientAuth": "false",
+ "site.accumulo-site.general.kerberos.keytab": "${accumulo.keytab}",
+ "site.accumulo-site.general.kerberos.principal": "${accumulo.principal}",
+
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
+ "site.accumulo-site.tserver.memory.maps.max": "80M",
+ "site.accumulo-site.tserver.cache.data.size": "7M",
+ "site.accumulo-site.tserver.cache.index.size": "20M",
+ "site.accumulo-site.tserver.sort.buffer.size": "50M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
+
+ "site.accumulo-site.trace.user": "root",
+
+ "site.accumulo-site.master.port.client": "0",
+ "site.accumulo-site.trace.port.client": "0",
+ "site.accumulo-site.tserver.port.client": "0",
+ "site.accumulo-site.gc.port.client": "0",
+ "site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
+ "site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
+
+ "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n${@//site/accumulo-env/hadoop_conf_dir},\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n${hadoop.dir}/.*.jar,\n${hadoop.dir}/lib/.*.jar,\n${hdfs.dir}/.*.jar,\n${mapred.dir}/.*.jar,\n${yarn.dir}/.*.jar,"
+ },
+ "credentials": {
+ "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root.initial.password", "instance.secret", "trace.token.property.password"]
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M",
+ "slider.am.keytab.local.path": "${accumulo.headless.keytab}",
+ "slider.keytab.principal.name": "${accumulo.headless.principal}"
+ }
+ }
+}
diff --git a/app-packages/accumulo/appConfig-secured-default.json b/app-packages/accumulo/appConfig-secured-default.json
new file mode 100644
index 0000000..b493ccc
--- /dev/null
+++ b/app-packages/accumulo/appConfig-secured-default.json
@@ -0,0 +1,70 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
+ "java_home": "${app.java.home}",
+
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
+ "site.global.app_user": "${USER}",
+ "site.global.user_group": "${USER}",
+
+ "site.accumulo-env.java_home": "${JAVA_HOME}",
+ "site.accumulo-env.tserver_heapsize": "256m",
+ "site.accumulo-env.master_heapsize": "128m",
+ "site.accumulo-env.monitor_heapsize": "64m",
+ "site.accumulo-env.gc_heapsize": "64m",
+ "site.accumulo-env.other_heapsize": "128m",
+ "site.accumulo-env.hadoop_prefix": "${hadoop.dir}",
+ "site.accumulo-env.hadoop_conf_dir": "/etc/hadoop/conf",
+ "site.accumulo-env.zookeeper_home": "${zk.dir}",
+
+ "site.client.instance.name": "${USER}-${CLUSTER_NAME}",
+
+ "site.global.accumulo_root_password": "NOT_USED",
+ "site.global.ssl_cert_dir": "ssl",
+ "site.global.monitor_protocol": "http",
+
+ "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
+ "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
+ "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
+
+ "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.instance.rpc.ssl.enabled": "false",
+ "site.accumulo-site.instance.rpc.ssl.clientAuth": "false",
+ "site.accumulo-site.general.kerberos.keytab": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.ACCUMULO.service.keytab",
+ "site.accumulo-site.general.kerberos.principal": "${USER_NAME}/_HOST@EXAMPLE.COM",
+
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
+ "site.accumulo-site.tserver.memory.maps.max": "80M",
+ "site.accumulo-site.tserver.cache.data.size": "7M",
+ "site.accumulo-site.tserver.cache.index.size": "20M",
+ "site.accumulo-site.tserver.sort.buffer.size": "50M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
+
+ "site.accumulo-site.trace.user": "root",
+
+ "site.accumulo-site.master.port.client": "0",
+ "site.accumulo-site.trace.port.client": "0",
+ "site.accumulo-site.tserver.port.client": "0",
+ "site.accumulo-site.gc.port.client": "0",
+ "site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
+ "site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
+
+ "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n${@//site/accumulo-env/hadoop_conf_dir},\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n${hadoop.dir}/.*.jar,\n${hadoop.dir}/lib/.*.jar,\n${hdfs.dir}/.*.jar,\n${mapred.dir}/.*.jar,\n${yarn.dir}/.*.jar,"
+ },
+ "credentials": {
+ "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root.initial.password", "instance.secret", "trace.token.property.password"]
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M",
+ "slider.hdfs.keytab.dir": ".slider/keytabs/accumulo",
+ "slider.am.login.keytab.name": "${USER_NAME}.headless.keytab",
+ "slider.keytab.principal.name": "${USER_NAME}"
+ }
+ }
+}
diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json
deleted file mode 100644
index 6b7033e..0000000
--- a/app-packages/accumulo/appConfig.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": "${app.package.name}.zip",
- "config_types": "accumulo-site",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/accumulo-${accumulo.version}-bin.tar.gz",
- "site.global.app_user": "yarn",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.global.tserver_heapsize": "128m",
- "site.global.master_heapsize": "128m",
- "site.global.monitor_heapsize": "64m",
- "site.global.gc_heapsize": "64m",
- "site.global.other_heapsize": "128m",
- "site.global.hadoop_prefix": "/usr/lib/hadoop",
- "site.global.hadoop_conf_dir": "/etc/hadoop/conf",
- "site.global.zookeeper_home": "/usr/lib/zookeeper",
- "site.global.accumulo_instance_name": "instancename",
- "site.global.accumulo_root_password": "secret",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.global.monitor_protocol": "http",
- "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
- "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
- "site.accumulo-site.instance.secret": "DEFAULT",
- "site.accumulo-site.tserver.memory.maps.max": "80M",
- "site.accumulo-site.tserver.cache.data.size": "7M",
- "site.accumulo-site.tserver.cache.index.size": "20M",
- "site.accumulo-site.trace.token.property.password": "secret",
- "site.accumulo-site.trace.user": "root",
- "site.accumulo-site.tserver.sort.buffer.size": "50M",
- "site.accumulo-site.tserver.walog.max.size": "100M",
- "site.accumulo-site.master.port.client": "0",
- "site.accumulo-site.trace.port.client": "0",
- "site.accumulo-site.tserver.port.client": "0",
- "site.accumulo-site.gc.port.client": "0",
- "site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
- "site.accumulo-site.monitor.port.log4j": "0",
- "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
- },
- "components": {
- "ACCUMULO_MASTER": {
- },
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- },
- "ACCUMULO_TSERVER": {
- },
- "ACCUMULO_MONITOR": {
- },
- "ACCUMULO_GC": {
- },
- "ACCUMULO_TRACER": {
- }
- }
-}
diff --git a/app-packages/accumulo/configuration/global.xml b/app-packages/accumulo/configuration/accumulo-env.xml
similarity index 63%
rename from app-packages/accumulo/configuration/global.xml
rename to app-packages/accumulo/configuration/accumulo-env.xml
index 5d39dca..65b6804 100644
--- a/app-packages/accumulo/configuration/global.xml
+++ b/app-packages/accumulo/configuration/accumulo-env.xml
@@ -22,18 +22,8 @@
<configuration>
<property>
- <name>app_log_dir</name>
- <value>/var/log/accumulo</value>
- <description>Log Directories for Accumulo.</description>
- </property>
- <property>
- <name>app_pid_dir</name>
- <value>/var/run/accumulo</value>
- <description>Pid Directories for Accumulo.</description>
- </property>
- <property>
<name>tserver_heapsize</name>
- <value>128m</value>
+ <value>256m</value>
<description>TServer heap size.</description>
</property>
<property>
@@ -57,21 +47,6 @@
<description>Other Heap Size</description>
</property>
<property>
- <name>accumulo_hdfs_root_dir</name>
- <value>/apps/accumulo/data</value>
- <description>Accumulo Relative Path to HDFS.</description>
- </property>
- <property>
- <name>accumulo_conf_dir</name>
- <value>/etc/accumulo</value>
- <description>Config Directory for Accumulo.</description>
- </property>
- <property>
- <name>app_user</name>
- <value>yarn</value>
- <description>Accumulo User Name.</description>
- </property>
- <property>
<name>hadoop_prefix</name>
<value>/usr/lib/hadoop</value>
<description>Hadoop directory.</description>
@@ -91,4 +66,24 @@
<value>accumulo-instance</value>
<description>Accumulo Instance Name.</description>
</property>
+ <property>
+ <name>content</name>
+ <description>This is the template for a client accumulo-env.sh file</description>
+ <value>
+#! /usr/bin/env bash
+export HADOOP_PREFIX=${@//site/accumulo-env/hadoop_prefix}
+export HADOOP_CONF_DIR=${@//site/accumulo-env/hadoop_conf_dir}
+export JAVA_HOME=${@//site/accumulo-env/java_home}
+export ZOOKEEPER_HOME=${@//site/accumulo-env/zookeeper_home}
+export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
+export ACCUMULO_TSERVER_OPTS="-Xmx${@//site/accumulo-env/tserver_heapsize} -Xms${@//site/accumulo-env/tserver_heapsize}"
+export ACCUMULO_MASTER_OPTS="-Xmx${@//site/accumulo-env/master_heapsize} -Xms${@//site/accumulo-env/master_heapsize}"
+export ACCUMULO_MONITOR_OPTS="-Xmx${@//site/accumulo-env/monitor_heapsize} -Xms${@//site/accumulo-env/monitor_heapsize}"
+export ACCUMULO_GC_OPTS="-Xmx${@//site/accumulo-env/gc_heapsize} -Xms${@//site/accumulo-env/gc_heapsize}"
+export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true"
+export ACCUMULO_OTHER_OPTS="-Xmx${@//site/accumulo-env/other_heapsize} -Xms${@//site/accumulo-env/other_heapsize}"
+# what do when the JVM runs out of heap memory
+export ACCUMULO_KILL_CMD='kill -9 %p'
+ </value>
+ </property>
</configuration>
diff --git a/app-packages/accumulo/configuration/accumulo-site.xml b/app-packages/accumulo/configuration/accumulo-site.xml
index 269cc2b..3001c45 100644
--- a/app-packages/accumulo/configuration/accumulo-site.xml
+++ b/app-packages/accumulo/configuration/accumulo-site.xml
@@ -28,17 +28,6 @@
</property>
<property>
- <name>instance.secret</name>
- <value>DEFAULT</value>
- <description>A secret unique to a given instance that all servers
- must know in order to communicate with one another.
- Change it before initialization. To
- change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd],
- and then update this file.
- </description>
- </property>
-
- <property>
<name>tserver.memory.maps.max</name>
<value>80M</value>
</property>
@@ -54,12 +43,6 @@
</property>
<property>
- <name>trace.token.property.password</name>
- <!-- change this to the root user's password, and/or change the user below -->
- <value>secret</value>
- </property>
-
- <property>
<name>trace.user</name>
<value>root</value>
</property>
@@ -71,7 +54,7 @@
<property>
<name>tserver.walog.max.size</name>
- <value>100M</value>
+ <value>40M</value>
</property>
<property>
diff --git a/app-packages/accumulo/configuration/client.xml b/app-packages/accumulo/configuration/client.xml
new file mode 100644
index 0000000..481b7d1
--- /dev/null
+++ b/app-packages/accumulo/configuration/client.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>instance.name</name>
+ <value>accumulo-instance</value>
+ <description>Accumulo Instance Name.</description>
+ </property>
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>${@//site/accumulo-site/instance.zookeeper.host}</value>
+ <description>Zookeeper hosts.</description>
+ </property>
+ <property>
+ <name>instance.rpc.ssl.enabled</name>
+ <value>${@//site/accumulo-site/instance.rpc.ssl.enabled}</value>
+ <description>SSL enabled.</description>
+ </property>
+ <property>
+ <name>instance.rpc.ssl.clientAuth</name>
+ <value>${@//site/accumulo-site/instance.rpc.ssl.clientAuth}</value>
+ <description>SSL client auth enabled.</description>
+ </property>
+ <property>
+ <name>general.security.credential.provider.paths</name>
+ <value>${@//site/accumulo-site/general.security.credential.provider.paths}</value>
+ <description>Client credential provider containing cert passwords.</description>
+ </property>
+</configuration>
diff --git a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE b/app-packages/accumulo/getconf.sh
old mode 100644
new mode 100755
similarity index 63%
copy from app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
copy to app-packages/accumulo/getconf.sh
index dd934d5..7d6a1ac
--- a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
+++ b/app-packages/accumulo/getconf.sh
@@ -13,4 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-Replace with the actual storm package.
+CLUSTER=$1
+
+slider registry --getconf accumulo-site --name $CLUSTER --format xml --dest accumulo-site.xml
+slider registry --getconf client --name $CLUSTER --format properties --dest client.conf
+slider registry --getconf accumulo-env --name $CLUSTER --format json --dest accumulo-env.json
+python -c "import json; file = open('accumulo-env.json'); content = json.load(file); file.close(); print content['content']" > accumulo-env.sh
diff --git a/app-packages/accumulo/metainfo.xml b/app-packages/accumulo/metainfo.xml
index b1aa9de..d3fb263 100644
--- a/app-packages/accumulo/metainfo.xml
+++ b/app-packages/accumulo/metainfo.xml
@@ -40,7 +40,7 @@
</value>
</export>
<export>
- <name>app.jmx</name>
+ <name>org.apache.slider.jmx</name>
<value>
${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}/xml
</value>
@@ -69,6 +69,14 @@
<command>ACCUMULO_TRACER-START</command>
<requires>ACCUMULO_MASTER-STARTED</requires>
</commandOrder>
+ <commandOrder>
+ <command>ACCUMULO_GC-START</command>
+ <requires>ACCUMULO_TSERVER-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>ACCUMULO_TRACER-START</command>
+ <requires>ACCUMULO_TSERVER-STARTED</requires>
+ </commandOrder>
</commandOrders>
<components>
<component>
@@ -85,7 +93,7 @@
<name>ACCUMULO_MONITOR</name>
<category>MASTER</category>
<publishConfig>true</publishConfig>
- <appExports>QuickLinks-app.jmx,QuickLinks-org.apache.slider.monitor</appExports>
+ <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor</appExports>
<commandScript>
<script>scripts/accumulo_monitor.py</script>
<scriptType>PYTHON</scriptType>
@@ -144,5 +152,23 @@
</osSpecific>
</osSpecifics>
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>accumulo-site.xml</fileName>
+ <dictionaryName>accumulo-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>accumulo-env.sh</fileName>
+ <dictionaryName>accumulo-env</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>properties</type>
+ <fileName>client.conf</fileName>
+ <dictionaryName>client</dictionaryName>
+ </configFile>
+ </configFiles>
+
</application>
</metainfo>
diff --git a/app-packages/accumulo/package/files/accumulo-metrics.xml b/app-packages/accumulo/package/files/accumulo-metrics.xml
index 60f9f8d..3b97809 100644
--- a/app-packages/accumulo/package/files/accumulo-metrics.xml
+++ b/app-packages/accumulo/package/files/accumulo-metrics.xml
@@ -33,10 +33,6 @@
<enabled type="boolean">false</enabled>
<logging type="boolean">false</logging>
</master>
- <logger>
- <enabled type="boolean">false</enabled>
- <logging type="boolean">false</logging>
- </logger>
<tserver>
<enabled type="boolean">false</enabled>
<logging type="boolean">false</logging>
@@ -57,4 +53,8 @@
<enabled type="boolean">false</enabled>
<logging type="boolean">false</logging>
</thrift>
+ <replication>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </replication>
</config>
diff --git a/app-packages/accumulo/package/files/log4j.properties b/app-packages/accumulo/package/files/log4j.properties
index a4bcb2e..f3eaddc 100644
--- a/app-packages/accumulo/package/files/log4j.properties
+++ b/app-packages/accumulo/package/files/log4j.properties
@@ -20,8 +20,9 @@
# hide Jetty junk
log4j.logger.org.mortbay.log=WARN,A1
-# hide "Got brand-new compresssor" messages
+# hide "Got brand-new compressor" messages
log4j.logger.org.apache.hadoop.io.compress=WARN,A1
+log4j.logger.org.apache.accumulo.core.file.rfile.bcfile.Compression=WARN,A1
# hide junk from TestRandomDeletes
log4j.logger.org.apache.accumulo.test.TestRandomDeletes=WARN,A1
diff --git a/app-packages/accumulo/package/scripts/accumulo_client.py b/app-packages/accumulo/package/scripts/accumulo_client.py
index 45d07dd..f50addf 100644
--- a/app-packages/accumulo/package/scripts/accumulo_client.py
+++ b/app-packages/accumulo/package/scripts/accumulo_client.py
@@ -36,7 +36,7 @@
setup_conf_dir(name='client')
def status(self, env):
- raise ClientComponentHasNoStatus()
+ pass
if __name__ == "__main__":
diff --git a/app-packages/accumulo/package/scripts/accumulo_configuration.py b/app-packages/accumulo/package/scripts/accumulo_configuration.py
index 8299c36..fb4410e 100644
--- a/app-packages/accumulo/package/scripts/accumulo_configuration.py
+++ b/app-packages/accumulo/package/scripts/accumulo_configuration.py
@@ -20,8 +20,7 @@
from resource_management import *
-def setup_conf_dir(name=None, # 'master' or 'tserver' or 'monitor' or 'gc' or 'tracer' or 'client'
- extra_params=None):
+def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or 'tracer' or 'client'
import params
# create the conf directory
@@ -31,6 +30,51 @@
recursive = True
)
+ ssl_params = False
+ if params.ssl_enabled or (params.monitor_security_enabled and
+ name == 'monitor'):
+ import os
+
+ ssl_params = True
+ if os.path.exists(params.keystore_path) or os.path.exists(params.truststore_path):
+ if os.path.exists(params.keystore_path) and os.path.exists(params.truststore_path):
+ # assume keystores were already set up properly
+ pass
+ else:
+ self.fail_with_error("something went wrong when certs were created")
+
+ Directory( format("{params.conf_dir}/ssl"),
+ owner = params.accumulo_user,
+ group = params.user_group,
+ recursive = True
+ )
+ if not os.path.exists(params.truststore_path):
+ Execute( format("{hadoop_prefix}/bin/hadoop fs -get {params.ssl_cert_dir}/truststore.jks "
+ "{params.truststore_path}"),
+ user=params.accumulo_user)
+ File( params.truststore_path,
+ mode=0600,
+ group=params.user_group,
+ owner=params.accumulo_user,
+ replace=False)
+ if not os.path.exists(params.keystore_path):
+ Execute( format("{hadoop_prefix}/bin/hadoop fs -get {params.ssl_cert_dir}/{params.hostname}.jks "
+ "{params.keystore_path}"),
+ user=params.accumulo_user)
+ File( params.keystore_path,
+ mode=0600,
+ group=params.user_group,
+ owner=params.accumulo_user,
+ replace=False)
+
+ jarname = "SliderAccumuloUtils.jar"
+ File(format("{params.accumulo_root}/lib/{jarname}"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.accumulo_user,
+ content=StaticFile(jarname)
+ )
+
if name != "client":
# create pid dir
Directory( params.pid_dir,
@@ -47,12 +91,16 @@
)
configs = {}
- if extra_params == None:
- configs = params.config['configurations']['accumulo-site']
- else:
+ if ssl_params:
configs.update(params.config['configurations']['accumulo-site'])
- for k in extra_params:
- configs[k] = extra_params[k]
+ if (params.monitor_security_enabled and name == 'monitor'):
+ configs[params.monitor_keystore_property] = params.keystore_path
+ configs[params.monitor_truststore_property] = params.truststore_path
+ if params.ssl_enabled:
+ configs[params.ssl_keystore_file_property] = params.keystore_path
+ configs[params.ssl_truststore_file_property] = params.truststore_path
+ else:
+ configs = params.config['configurations']['accumulo-site']
# create a site file for server processes
XmlConfig( "accumulo-site.xml",
@@ -66,7 +114,6 @@
# create a minimal site file for client processes
client_configurations = {}
client_configurations['instance.zookeeper.host'] = params.config['configurations']['accumulo-site']['instance.zookeeper.host']
- client_configurations['instance.dfs.dir'] = params.config['configurations']['accumulo-site']['instance.dfs.dir']
client_configurations['instance.volumes'] = params.config['configurations']['accumulo-site']['instance.volumes']
client_configurations['general.classpaths'] = params.config['configurations']['accumulo-site']['general.classpaths']
XmlConfig( "accumulo-site.xml",
@@ -79,6 +126,13 @@
# create env file
accumulo_TemplateConfig( 'accumulo-env.sh')
+ # create client.conf file
+ PropertiesFile(format("{params.conf_dir}/client.conf"),
+ properties = params.config['configurations']['client'],
+ owner = params.accumulo_user,
+ group = params.user_group
+ )
+
# create host files
accumulo_StaticFile( 'masters')
accumulo_StaticFile( 'slaves')
diff --git a/app-packages/accumulo/package/scripts/accumulo_script.py b/app-packages/accumulo/package/scripts/accumulo_script.py
index 5e2ceba..6227261 100644
--- a/app-packages/accumulo/package/scripts/accumulo_script.py
+++ b/app-packages/accumulo/package/scripts/accumulo_script.py
@@ -22,10 +22,8 @@
from resource_management.core.environment import Environment
from accumulo_configuration import setup_conf_dir
-from accumulo_configuration import accumulo_StaticFile
from accumulo_service import accumulo_service
-
class AccumuloScript(Script):
def __init__(self, component):
self.component = component
@@ -37,44 +35,7 @@
import params
env.set_params(params)
- if params.monitor_security_enabled and self.component == 'monitor':
- import os
- import random
- import string
-
- basedir = Environment.get_instance().config.basedir
- keystore_file = os.path.join(basedir, "files", "keystore.jks")
- truststore_file = os.path.join(basedir, "files", "cacerts.jks")
- cert_file = os.path.join(basedir, "files", "server.cer")
-
- if os.path.exists(keystore_file) or os.path.exists(truststore_file) or os.path.exists(cert_file):
- self.fail_with_error("trying to create monitor certs but they already existed")
-
- goodchars = string.lowercase + string.uppercase + string.digits + '#%+,-./:=?@^_'
- keypass = ''.join(random.choice(goodchars) for x in range(20))
- storepass = ''.join(random.choice(goodchars) for x in range(20))
-
- https_params = {}
- https_params[params.keystore_property] = params.keystore_path
- https_params[params.truststore_property] = params.truststore_path
- https_params[params.keystore_password_property] = keypass
- https_params[params.truststore_password_property] = storepass
-
- setup_conf_dir(name=self.component, extra_params=https_params)
-
- Execute( format("{java64_home}/bin/keytool -genkey -alias \"default\" -keyalg RSA -keypass {keypass} -storepass {storepass} -keystore {keystore_file} -dname \"CN=Unknown, OU=Unknown, O=Unknown, L=Unknown, ST=Unknown, C=Unknown\""),
- user=params.accumulo_user)
- Execute( format("{java64_home}/bin/keytool -export -alias \"default\" -storepass {storepass} -file {cert_file} -keystore {keystore_file}"),
- user=params.accumulo_user)
- Execute( format("echo \"yes\" | {java64_home}/bin/keytool -import -v -trustcacerts -alias \"default\" -file {cert_file} -keystore {truststore_file} -keypass {keypass} -storepass {storepass}"),
- user=params.accumulo_user)
-
- accumulo_StaticFile("keystore.jks")
- accumulo_StaticFile("cacerts.jks")
-
- else:
- setup_conf_dir(name=self.component)
-
+ setup_conf_dir(name=self.component)
def start(self, env):
import params
@@ -82,9 +43,17 @@
self.configure(env) # for security
if self.component == 'master':
- Execute( format("{daemon_script} init --instance-name {accumulo_instance_name} --password {accumulo_root_password} --clear-instance-name"),
- not_if=format("hadoop fs -stat {accumulo_hdfs_root_dir}"),
+ try:
+ Execute( format("{daemon_script} init --instance-name {accumulo_instance_name} --password {accumulo_root_password} --clear-instance-name >{log_dir}/accumulo-{accumulo_user}-init.out 2>{log_dir}/accumulo-{accumulo_user}-init.err"),
+ not_if=format("{hadoop_prefix}/bin/hadoop fs -stat {accumulo_hdfs_root_dir}"),
user=params.accumulo_user)
+ except Exception, e:
+ try:
+ Execute( format("{hadoop_prefix}/bin/hadoop fs -rm -R {accumulo_hdfs_root_dir}"),
+ user=params.accumulo_user)
+ except:
+ pass
+ raise e
accumulo_service( self.component,
action = 'start'
diff --git a/app-packages/accumulo/package/scripts/accumulo_service.py b/app-packages/accumulo/package/scripts/accumulo_service.py
index 562ef5d..ca21cc8 100644
--- a/app-packages/accumulo/package/scripts/accumulo_service.py
+++ b/app-packages/accumulo/package/scripts/accumulo_service.py
@@ -30,7 +30,7 @@
pid_exists = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
if action == 'start':
- daemon_cmd = format("{daemon_script} {role} > {log_dir}/accumulo-{accumulo_user}-{role}.out 2>{log_dir}/accumulo-{accumulo_user}-{role}.err & echo $! > {pid_file}")
+ daemon_cmd = format("{daemon_script} {role} --address {params.hostname} > {log_dir}/accumulo-{accumulo_user}-{role}.out 2>{log_dir}/accumulo-{accumulo_user}-{role}.err & echo $! > {pid_file}")
Execute ( daemon_cmd,
not_if=pid_exists,
user=params.accumulo_user
diff --git a/app-packages/accumulo/package/scripts/params.py b/app-packages/accumulo/package/scripts/params.py
index 3eaa1ab..9e6e8dd 100644
--- a/app-packages/accumulo/package/scripts/params.py
+++ b/app-packages/accumulo/package/scripts/params.py
@@ -23,6 +23,7 @@
# server configurations
config = Script.get_config()
+hostname = config["public_hostname"]
# user and status
accumulo_user = status_params.accumulo_user
@@ -31,43 +32,51 @@
# accumulo env
java64_home = config['hostLevelParams']['java_home']
-hadoop_prefix = config['configurations']['global']['hadoop_prefix']
-hadoop_conf_dir = config['configurations']['global']['hadoop_conf_dir']
-zookeeper_home = config['configurations']['global']['zookeeper_home']
-master_heapsize = config['configurations']['global']['master_heapsize']
-tserver_heapsize = config['configurations']['global']['tserver_heapsize']
-monitor_heapsize = config['configurations']['global']['monitor_heapsize']
-gc_heapsize = config['configurations']['global']['gc_heapsize']
-other_heapsize = config['configurations']['global']['other_heapsize']
+hadoop_prefix = config['configurations']['accumulo-env']['hadoop_prefix']
+hadoop_conf_dir = config['configurations']['accumulo-env']['hadoop_conf_dir']
+zookeeper_home = config['configurations']['accumulo-env']['zookeeper_home']
+zookeeper_host = config['configurations']['accumulo-site']['instance.zookeeper.host']
+master_heapsize = config['configurations']['accumulo-env']['master_heapsize']
+tserver_heapsize = config['configurations']['accumulo-env']['tserver_heapsize']
+monitor_heapsize = config['configurations']['accumulo-env']['monitor_heapsize']
+gc_heapsize = config['configurations']['accumulo-env']['gc_heapsize']
+other_heapsize = config['configurations']['accumulo-env']['other_heapsize']
+env_sh_template = config['configurations']['accumulo-env']['content']
# accumulo local directory structure
accumulo_root = config['configurations']['global']['app_root']
-conf_dir = None
-if ('accumulo_conf_dir' in config['configurations']['global']):
- conf_dir = config['configurations']['global']['accumulo_conf_dir']
-else:
- conf_dir = format("{accumulo_root}/conf")
+conf_dir = format("{accumulo_root}/conf")
log_dir = config['configurations']['global']['app_log_dir']
daemon_script = format("{accumulo_root}/bin/accumulo")
# accumulo monitor certificate properties
monitor_security_enabled = config['configurations']['global']['monitor_protocol'] == "https"
-keystore_path = format("{accumulo_root}/conf/keystore.jks")
-truststore_path = format("{accumulo_root}/conf/cacerts.jks")
-cert_path = format("{accumulo_root}/conf/server.cer")
-keystore_property = "monitor.ssl.keyStore"
-keystore_password_property = "monitor.ssl.keyStorePassword"
-truststore_property = "monitor.ssl.trustStore"
-truststore_password_property = "monitor.ssl.trustStorePassword"
+monitor_keystore_property = "monitor.ssl.keyStore"
+monitor_truststore_property = "monitor.ssl.trustStore"
+
+# accumulo ssl properties
+ssl_enabled = False
+if 'instance.rpc.ssl.enabled' in config['configurations']['accumulo-site']:
+ ssl_enabled = config['configurations']['accumulo-site']['instance.rpc.ssl.enabled']
+clientauth_enabled = False
+if 'instance.rpc.ssl.clientAuth' in config['configurations']['accumulo-site']:
+ clientauth_enabled = config['configurations']['accumulo-site']['instance.rpc.ssl.clientAuth']
+ssl_cert_dir = config['configurations']['global']['ssl_cert_dir']
+keystore_path = format("{conf_dir}/ssl/keystore.jks")
+truststore_path = format("{conf_dir}/ssl/truststore.jks")
+ssl_keystore_file_property = "rpc.javax.net.ssl.keyStore"
+ssl_truststore_file_property = "rpc.javax.net.ssl.trustStore"
+credential_provider = config['configurations']['accumulo-site']["general.security.credential.provider.paths"]
+#credential_provider = credential_provider.replace("${HOST}", hostname) # if enabled, must propagate to configuration
+if ssl_keystore_file_property in config['configurations']['accumulo-site']:
+ keystore_path = config['configurations']['accumulo-site'][ssl_keystore_file_property]
+if ssl_truststore_file_property in config['configurations']['accumulo-site']:
+ truststore_path = config['configurations']['accumulo-site'][ssl_truststore_file_property]
# accumulo initialization parameters
-accumulo_instance_name = config['configurations']['global']['accumulo_instance_name']
+accumulo_instance_name = config['configurations']['client']['instance.name']
accumulo_root_password = config['configurations']['global']['accumulo_root_password']
-accumulo_hdfs_root_dir = None
-if ('instance.dfs.dir' in config['configurations']['accumulo-site']):
- accumulo_hdfs_root_dir = config['configurations']['accumulo-site']['instance.dfs.dir']
-else:
- accumulo_hdfs_root_dir = config['configurations']['accumulo-site']['instance.volumes'].split(",")[0]
+accumulo_hdfs_root_dir = config['configurations']['accumulo-site']['instance.volumes'].split(",")[0]
#log4j.properties
if (('accumulo-log4j' in config['configurations']) and ('content' in config['configurations']['accumulo-log4j'])):
diff --git a/app-packages/accumulo/package/templates/accumulo-env.sh.j2 b/app-packages/accumulo/package/templates/accumulo-env.sh.j2
index 7ffec53..9e365af 100755
--- a/app-packages/accumulo/package/templates/accumulo-env.sh.j2
+++ b/app-packages/accumulo/package/templates/accumulo-env.sh.j2
@@ -36,7 +36,7 @@
export ACCUMULO_MASTER_OPTS="-Xmx{{master_heapsize}} -Xms{{master_heapsize}}"
export ACCUMULO_MONITOR_OPTS="-Xmx{{monitor_heapsize}} -Xms{{monitor_heapsize}}"
export ACCUMULO_GC_OPTS="-Xmx{{gc_heapsize}} -Xms{{gc_heapsize}}"
-export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75"
+export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true"
export ACCUMULO_OTHER_OPTS="-Xmx{{other_heapsize}} -Xms{{other_heapsize}}"
# what do when the JVM runs out of heap memory
export ACCUMULO_KILL_CMD='kill -9 %p'
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index fe71c70..469ca85 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -30,11 +30,43 @@
<properties>
<work.dir>package-tmp</work.dir>
- <app.package.name>apache-slider-accumulo-${accumulo.version}-app-package-${project.version}</app.package.name>
+ <app.package.name>${project.artifactId}-${pkg.version}</app.package.name>
+ <pkg.src>${project.build.directory}/${work.dir}</pkg.src>
+ <pkg.version>${project.version}</pkg.version>
+ <pkg.name>accumulo-${accumulo.version}-bin.tar.gz</pkg.name>
+ <!-- the following properties are used for testing -->
+ <slider.bin.dir>../../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
+ <test.app.pkg.dir>${project.build.directory}</test.app.pkg.dir>
+ <test.app.resources.dir>${project.build.directory}/test-config</test.app.resources.dir>
+ <!-- these properties are used in the default and the test appConfigs -->
+ <hadoop.dir>/usr/lib/hadoop</hadoop.dir>
+ <hdfs.dir>/usr/lib/hadoop-hdfs</hdfs.dir>
+ <yarn.dir>/usr/lib/hadoop-yarn</yarn.dir>
+ <mapred.dir>/usr/lib/hadoop-mapred</mapred.dir>
+ <zk.dir>/usr/lib/zookeeper</zk.dir>
+ <app.java.home>${java.home}</app.java.home>
+ <app.user>yarn</app.user>
+ <app.user.group>hadoop</app.user.group>
+ <!-- these are for accumulo processes -->
+ <accumulo.keytab></accumulo.keytab>
+ <accumulo.principal></accumulo.principal>
+ <!-- these are for the AM -->
+ <accumulo.headless.keytab>${accumulo.keytab}</accumulo.headless.keytab>
+ <accumulo.headless.principal>${accumulo.principal}</accumulo.headless.principal>
</properties>
<profiles>
<profile>
+ <id>hdp</id>
+ <properties>
+ <hadoop.dir>/usr/hdp/current/hadoop-client</hadoop.dir>
+ <hdfs.dir>/usr/hdp/current/hadoop-hdfs-client</hdfs.dir>
+ <yarn.dir>/usr/hdp/current/hadoop-yarn-client</yarn.dir>
+ <mapred.dir>/usr/hdp/current/hadoop-mapreduce-client</mapred.dir>
+ <zk.dir>/usr/hdp/current/zookeeper-client</zk.dir>
+ </properties>
+ </profile>
+ <profile>
<id>accumulo-app-package</id>
<build>
<plugins>
@@ -59,6 +91,78 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>${maven-antrun-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${work.dir}/${pkg.name}" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>accumulo-app-package-maven</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo</artifactId>
+ <version>${accumulo.version}</version>
+ <classifier>bin</classifier>
+ <type>tar.gz</type>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-fate</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-gc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-master</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-minicluster</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/assembly/accumulo.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ <finalName>${project.artifactId}-${pkg.version}</finalName>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-app-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
<executions>
@@ -77,7 +181,13 @@
</execution>
</executions>
</plugin>
-
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>accumulo-funtest</id>
+ <build>
+ <plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
@@ -97,11 +207,11 @@
<java.awt.headless>true</java.awt.headless>
<!-- this property must be supplied-->
<slider.conf.dir>${slider.conf.dir}</slider.conf.dir>
- <slider.bin.dir>../../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
- <test.app.pkg.dir>target</test.app.pkg.dir>
+ <slider.bin.dir>${slider.bin.dir}</slider.bin.dir>
+ <test.app.pkg.dir>${test.app.pkg.dir}</test.app.pkg.dir>
<test.app.pkg.file>${app.package.name}.zip</test.app.pkg.file>
- <test.app.resource>target/test-config/resources.json</test.app.resource>
- <test.app.template>target/${app.package.name}/appConfig.json</test.app.template>
+ <test.app.pkg.name>ACCUMULO</test.app.pkg.name>
+ <test.app.resources.dir>${test.app.resources.dir}</test.app.resources.dir>
</systemPropertyVariables>
</configuration>
</plugin>
@@ -116,7 +226,15 @@
<resource>
<directory>src/test/resources</directory>
<filtering>true</filtering>
- <targetPath>${project.build.directory}/test-config</targetPath>
+ <targetPath>${test.app.resources.dir}</targetPath>
+ </resource>
+ <resource>
+ <directory>.</directory>
+ <filtering>true</filtering>
+ <targetPath>${test.app.resources.dir}</targetPath>
+ <includes>
+ <include>appConfig-default.json</include>
+ </includes>
</resource>
</resources>
@@ -142,19 +260,22 @@
<dependencies>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo</artifactId>
+ <artifactId>accumulo-server-base</artifactId>
<version>${accumulo.version}</version>
- <classifier>bin</classifier>
- <type>tar.gz</type>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/app-packages/accumulo/resources.json b/app-packages/accumulo/resources-default.json
similarity index 88%
rename from app-packages/accumulo/resources.json
rename to app-packages/accumulo/resources-default.json
index f876901..f0923f2 100644
--- a/app-packages/accumulo/resources.json
+++ b/app-packages/accumulo/resources-default.json
@@ -3,6 +3,8 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": ""
},
"components": {
"ACCUMULO_MASTER": {
@@ -15,7 +17,7 @@
"ACCUMULO_TSERVER": {
"yarn.role.priority": "2",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"ACCUMULO_MONITOR": {
"yarn.role.priority": "3",
diff --git a/app-packages/accumulo/src/assembly/accumulo.xml b/app-packages/accumulo/src/assembly/accumulo.xml
index a8f9578..7be1942 100644
--- a/app-packages/accumulo/src/assembly/accumulo.xml
+++ b/app-packages/accumulo/src/assembly/accumulo.xml
@@ -24,13 +24,18 @@
<id>accumulo_v${accumulo.version}</id>
<formats>
<format>zip</format>
- <format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<files>
<file>
- <source>appConfig.json</source>
+ <source>appConfig-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>appConfig-secured-default.json</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
<fileMode>0755</fileMode>
@@ -41,6 +46,19 @@
<filtered>true</filtered>
<fileMode>0755</fileMode>
</file>
+ <file>
+ <source>${project.build.directory}/${work.dir}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${project.build.directory}/slider-accumulo-app-package-${project.version}.jar</source>
+ <outputDirectory>package/files</outputDirectory>
+ <destName>SliderAccumuloUtils.jar</destName>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
</files>
<fileSets>
@@ -51,22 +69,12 @@
<exclude>pom.xml</exclude>
<exclude>src/**</exclude>
<exclude>target/**</exclude>
- <exclude>appConfig.json</exclude>
+ <exclude>appConfig-default.json</exclude>
+ <exclude>appConfig-secured-default.json</exclude>
<exclude>metainfo.xml</exclude>
</excludes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
-
- <fileSet>
- <directory>${project.build.directory}/${work.dir}</directory>
- <outputDirectory>package/files</outputDirectory>
- <includes>
- <include>accumulo-${accumulo.version}-bin.tar.gz</include>
- </includes>
- <fileMode>0755</fileMode>
- <directoryMode>0755</directoryMode>
- </fileSet>
-
</fileSets>
</assembly>
diff --git a/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java b/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
new file mode 100644
index 0000000..0f50838
--- /dev/null
+++ b/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/CustomAuthenticator.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.accumulo;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.server.security.handler.Authenticator;
+import org.apache.accumulo.server.security.handler.Authorizor;
+import org.apache.accumulo.server.security.handler.PermissionHandler;
+import org.apache.accumulo.server.security.handler.ZKAuthenticator;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Set;
+
+public final class CustomAuthenticator implements Authenticator {
+ public static final String ROOT_INITIAL_PASSWORD_PROPERTY =
+ "root.initial.password";
+ private static ZKAuthenticator zkAuthenticator = null;
+
+ public CustomAuthenticator() {
+ zkAuthenticator = new ZKAuthenticator();
+ }
+
+ @Override
+ public void initialize(String instanceId, boolean initialize) {
+ zkAuthenticator.initialize(instanceId, initialize);
+ }
+
+ @Override
+ public void initializeSecurity(TCredentials credentials, String principal,
+ byte[] token) throws AccumuloSecurityException {
+ String pass = null;
+ SiteConfiguration siteconf = SiteConfiguration.getInstance(
+ DefaultConfiguration.getInstance());
+ String jksFile = siteconf.get(
+ Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS);
+
+ if (jksFile == null) {
+ throw new RuntimeException(
+ Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS +
+ " not specified in accumulo-site.xml");
+ }
+ try {
+ pass = new String(ProviderUtil.getPassword(jksFile,
+ ROOT_INITIAL_PASSWORD_PROPERTY));
+ } catch (IOException ioe) {
+ throw new RuntimeException("Can't get key " +
+ ROOT_INITIAL_PASSWORD_PROPERTY + " from " + jksFile, ioe);
+ }
+ zkAuthenticator.initializeSecurity(credentials, principal,
+ pass.getBytes(Charset.forName("UTF-8")));
+ }
+
+ @Override
+ public Set<String> listUsers() {
+ return zkAuthenticator.listUsers();
+ }
+
+ @Override
+ public void createUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ zkAuthenticator.createUser(principal, token);
+ }
+
+ @Override
+ public void dropUser(String user) throws AccumuloSecurityException {
+ zkAuthenticator.dropUser(user);
+ }
+
+ @Override
+ public void changePassword(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ zkAuthenticator.changePassword(principal, token);
+ }
+
+ @Override
+ public boolean userExists(String user) {
+ return zkAuthenticator.userExists(user);
+ }
+
+ @Override
+ public boolean validSecurityHandlers(Authorizor auth, PermissionHandler pm) {
+ return true;
+ }
+
+ @Override
+ public boolean authenticateUser(String principal, AuthenticationToken token) throws AccumuloSecurityException {
+ return zkAuthenticator.authenticateUser(principal, token);
+ }
+
+ @Override
+ public Set<Class<? extends AuthenticationToken>> getSupportedTokenTypes() {
+ return zkAuthenticator.getSupportedTokenTypes();
+ }
+
+ @Override
+ public boolean validTokenClass(String tokenClass) {
+ return zkAuthenticator.validTokenClass(tokenClass);
+ }
+}
diff --git a/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/ProviderUtil.java b/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/ProviderUtil.java
new file mode 100644
index 0000000..ee5a781
--- /dev/null
+++ b/app-packages/accumulo/src/main/java/org/apache/slider/accumulo/ProviderUtil.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.accumulo;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+public class ProviderUtil {
+ public static char[] getPassword(String credentialProvider, String alias)
+ throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ credentialProvider);
+ return conf.getPassword(alias);
+ }
+}
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloAgentCommandTestBase.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloAgentCommandTestBase.groovy
index 50ecfcd..b619f7e 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloAgentCommandTestBase.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloAgentCommandTestBase.groovy
@@ -25,17 +25,19 @@
@Slf4j
abstract class AccumuloAgentCommandTestBase extends AgentCommandTestBase {
protected static final int ACCUMULO_LAUNCH_WAIT_TIME
- protected static final int ACCUMULO_GO_LIVE_TIME = 60000
+ protected static final int ACCUMULO_GO_LIVE_TIME
- // parameters must match those found in the default appConfig.json
- protected static final String INSTANCE_NAME = "instancename"
protected static final String USER = "root"
- protected static final String PASSWORD = "secret"
+ protected static final String PASSWORD = "secret_password"
+ protected static final String INSTANCE_SECRET = "other_secret_password"
static {
ACCUMULO_LAUNCH_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
KEY_ACCUMULO_LAUNCH_TIME,
1000 * DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS)
+ ACCUMULO_GO_LIVE_TIME = getTimeOptionMillis(SLIDER_CONFIG,
+ KEY_ACCUMULO_GO_LIVE_TIME,
+ 1000 * DEFAULT_ACCUMULO_LIVE_TIME_SECONDS)
}
abstract public String getClusterName();
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index bcb952b..bb9abba 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -17,18 +17,102 @@
package org.apache.slider.funtest.accumulo
import groovy.util.logging.Slf4j
+import org.apache.accumulo.core.conf.Property
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.security.ProviderUtils
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.security.alias.CredentialProvider
+import org.apache.hadoop.security.alias.CredentialProviderFactory
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.slider.accumulo.CustomAuthenticator
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.core.persist.ConfTreeSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfiguration
-import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.core.registry.retrieve.RegistryRetriever
import org.apache.slider.funtest.framework.SliderShell
-import org.apache.slider.server.services.curator.CuratorServiceInstance
+
+import org.junit.Before
import org.junit.Test
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath
+
@Slf4j
class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
+ protected static final String PROVIDER_PROPERTY = "site.accumulo-site." +
+ Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS
+ protected static final String KEY_PASS = "keypass"
+ protected static final String TRUST_PASS = "trustpass"
+ protected ConfTree tree
+
+ protected String getAppResource() {
+ return sysprop("test.app.resources.dir") + "/resources.json"
+ }
+
+ protected String getAppTemplate() {
+ String appTemplateFile = templateName()
+ Configuration conf = new Configuration()
+ FileSystem fs = FileSystem.getLocal(conf)
+ InputStream stream = new FileInputStream(sysprop("test.app.resources.dir") + "/appConfig-default.json")
+ assert stream!=null, "Couldn't pull appConfig.json from app pkg"
+ ConfTreeSerDeser c = new ConfTreeSerDeser()
+ ConfTree t = c.fromStream(stream)
+ t = modifyTemplate(t)
+ c.save(fs, new Path(appTemplateFile), t, true)
+ return appTemplateFile
+ }
+
+ protected String templateName() {
+ return sysprop("test.app.resources.dir") + "/appConfig.json"
+ }
+
+ protected ConfTree modifyTemplate(ConfTree original) {
+ return original
+ }
+
+ @Before
+ public void createKeyStore() {
+ ConfTreeSerDeser c = new ConfTreeSerDeser()
+ tree = c.fromFile(new File(APP_TEMPLATE))
+ assume tree.credentials.size() > 0, "No credentials requested, " +
+ "skipping creation of credentials"
+ SliderClient.replaceTokens(tree, UserGroupInformation.getCurrentUser()
+ .getShortUserName(), getClusterName())
+ String jks = tree.global.get(PROVIDER_PROPERTY)
+ def keys = tree.credentials.get(jks)
+ assert keys!=null, "jks specified in $PROVIDER_PROPERTY wasn't requested " +
+ "in credentials"
+ Path jksPath = ProviderUtils.unnestUri(new URI(jks))
+ if (clusterFS.exists(jksPath)) {
+ clusterFS.delete(jksPath, false)
+ }
+ Configuration conf = loadSliderConf()
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, jks)
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0)
+ provider.createCredentialEntry(
+ CustomAuthenticator.ROOT_INITIAL_PASSWORD_PROPERTY, PASSWORD.toCharArray())
+ provider.createCredentialEntry(Property.INSTANCE_SECRET.toString(),
+ INSTANCE_SECRET.toCharArray())
+ provider.createCredentialEntry(Property.TRACE_TOKEN_PROPERTY_PREFIX
+ .toString() + "password", PASSWORD.toCharArray())
+ provider.createCredentialEntry(Property.RPC_SSL_KEYSTORE_PASSWORD
+ .toString(), KEY_PASS.toCharArray())
+ provider.createCredentialEntry(Property.RPC_SSL_TRUSTSTORE_PASSWORD
+ .toString(), TRUST_PASS.toCharArray())
+ provider.createCredentialEntry(Property.MONITOR_SSL_KEYSTOREPASS
+ .toString(), KEY_PASS.toCharArray())
+ provider.createCredentialEntry(Property.MONITOR_SSL_TRUSTSTOREPASS
+ .toString(), TRUST_PASS.toCharArray())
+ provider.flush()
+ assert clusterFS.exists(jksPath), "jks $jks not created"
+ log.info("Created credential provider $jks for test")
+ }
@Override
public String getClusterName() {
@@ -46,7 +130,6 @@
SliderShell shell = slider(EXIT_SUCCESS,
[
ACTION_CREATE, getClusterName(),
- ARG_IMAGE, agentTarballPath.toString(),
ARG_TEMPLATE, APP_TEMPLATE,
ARG_RESOURCES, APP_RESOURCE
])
@@ -85,25 +168,39 @@
}
public static String getMonitorUrl(SliderClient sliderClient, String clusterName) {
- CuratorServiceInstance<ServiceInstanceData> instance =
- sliderClient.getRegistry().queryForInstance(SliderKeys.APP_TYPE, clusterName)
- ServiceInstanceData serviceInstanceData = instance.payload
- RegistryRetriever retriever = new RegistryRetriever(serviceInstanceData)
- PublishedConfiguration configuration = retriever.retrieveConfiguration(
- retriever.getConfigurations(true), "quicklinks", true)
+ int tries = 5
+ Exception caught;
+ while (true) {
+ try {
+ String path = servicePath(currentUser(),
+ SliderKeys.APP_TYPE,
+ clusterName);
+ ServiceRecord instance = sliderClient.resolve(path)
+ RegistryRetriever retriever = new RegistryRetriever(instance)
+ PublishedConfiguration configuration = retriever.retrieveConfiguration(
+ retriever.getConfigurations(true), "quicklinks", true)
- // must match name set in metainfo.xml
- String monitorUrl = configuration.entries.get("org.apache.slider.monitor")
-
- assertNotNull monitorUrl
- return monitorUrl
+ // must match name set in metainfo.xml
+ String monitorUrl = configuration.entries.get("org.apache.slider.monitor")
+ assertNotNull monitorUrl
+ return monitorUrl
+ } catch (Exception e) {
+ caught = e;
+ log.info("Got exception trying to read quicklinks")
+ if (tries-- == 0) {
+ break
+ }
+ sleep(20000)
+ }
+ }
+ throw caught;
}
public static void checkMonitorPage(String monitorUrl) {
String monitor = fetchWebPageWithoutError(monitorUrl);
- assume monitor != null, "Monitor page null"
- assume monitor.length() > 100, "Monitor page too short"
- assume monitor.contains("Accumulo Overview"), "Monitor page didn't contain expected text"
+ assert monitor != null, "Monitor page null"
+ assert monitor.length() > 100, "Monitor page too short"
+ assert monitor.contains("Accumulo Overview"), "Monitor page didn't contain expected text"
}
/**
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLIT.groovy
index 6f68e13..12f89e0 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLIT.groovy
@@ -20,19 +20,21 @@
import groovy.util.logging.Slf4j
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
-
-import javax.net.ssl.KeyManager
-import javax.net.ssl.SSLContext
-import javax.net.ssl.TrustManager
-import javax.net.ssl.X509TrustManager
-import java.security.SecureRandom
-import java.security.cert.CertificateException
-import java.security.cert.X509Certificate
+import org.apache.slider.core.conf.ConfTree
@Slf4j
-class AccumuloMonitorSSLIT extends AccumuloBasicIT {
- AccumuloMonitorSSLIT() {
- APP_TEMPLATE = "target/test-config/appConfig_monitor_ssl.json"
+class AccumuloMonitorSSLIT extends AccumuloSSLTestBase {
+ protected String templateName() {
+ return sysprop("test.app.resources.dir") + "/appConfig_monitor_ssl.json"
+ }
+
+ protected ConfTree modifyTemplate(ConfTree confTree) {
+ confTree.global.put("site.global.monitor_protocol", "https")
+ String jks = confTree.global.get(PROVIDER_PROPERTY)
+ def keys = confTree.credentials.get(jks)
+ keys.add("monitor.ssl.keyStorePassword")
+ keys.add("monitor.ssl.trustStorePassword")
+ return confTree
}
@Override
@@ -49,25 +51,6 @@
public void clusterLoadOperations(ClusterDescription cd, SliderClient sliderClient) {
String monitorUrl = getMonitorUrl(sliderClient, getClusterName())
assert monitorUrl.startsWith("https://"), "Monitor URL didn't have expected protocol"
-
- SSLContext ctx = SSLContext.getInstance("SSL");
- TrustManager[] t = new TrustManager[1];
- t[0] = new DefaultTrustManager();
- ctx.init(new KeyManager[0], t, new SecureRandom());
- SSLContext.setDefault(ctx);
checkMonitorPage(monitorUrl)
}
-
- private static class DefaultTrustManager implements X509TrustManager {
- @Override
- public void checkClientTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {}
-
- @Override
- public void checkServerTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {}
-
- @Override
- public X509Certificate[] getAcceptedIssuers() {
- return null;
- }
- }
-}
\ No newline at end of file
+}
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
index cdbbcce..b4118d2 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
@@ -25,6 +25,7 @@
import org.apache.accumulo.core.client.security.tokens.PasswordToken
import org.apache.accumulo.test.TestIngest
import org.apache.accumulo.test.VerifyIngest
+import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXmlConfKeys
@@ -48,10 +49,12 @@
@Override
public void clusterLoadOperations(ClusterDescription cd, SliderClient sliderClient) {
try {
- String zookeepers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+ String zookeepers = SLIDER_CONFIG.get(
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM,
FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
- ZooKeeperInstance instance = new ZooKeeperInstance(INSTANCE_NAME, zookeepers)
+ ZooKeeperInstance instance = new ZooKeeperInstance(
+ tree.global.get("site.client.instance.name"), zookeepers)
Connector connector = instance.getConnector(USER, new PasswordToken(PASSWORD))
ingest(connector, 200000, 1, 50, 0);
@@ -77,7 +80,7 @@
TestIngest.ingest(connector, opts, new BatchWriterOpts());
}
- private static void verify(Connector connector, int rows, int cols, int width, int offset) throws Exception {
+ public static void verify(Connector connector, int rows, int cols, int width, int offset) throws Exception {
ScannerOpts scannerOpts = new ScannerOpts();
VerifyIngest.Opts opts = new VerifyIngest.Opts();
opts.rows = rows;
@@ -88,7 +91,7 @@
VerifyIngest.verifyIngest(connector, opts, scannerOpts);
}
- static void interleaveTest(final Connector connector) throws Exception {
+ public static void interleaveTest(final Connector connector) throws Exception {
final int ROWS = 200000;
final AtomicBoolean fail = new AtomicBoolean(false);
final int CHUNKSIZE = ROWS / 10;
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
new file mode 100644
index 0000000..0464cec
--- /dev/null
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.funtest.accumulo
+
+import org.apache.accumulo.core.client.ClientConfiguration
+import org.apache.accumulo.core.client.Connector
+import org.apache.accumulo.core.client.ZooKeeperInstance
+import org.apache.accumulo.core.client.security.tokens.PasswordToken
+import org.apache.hadoop.registry.client.api.RegistryConstants
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.funtest.framework.FuntestProperties
+
+import static org.apache.slider.funtest.accumulo.AccumuloReadWriteIT.ingest
+import static org.apache.slider.funtest.accumulo.AccumuloReadWriteIT.interleaveTest
+import static org.apache.slider.funtest.accumulo.AccumuloReadWriteIT.verify
+
+class AccumuloReadWriteSSLIT extends AccumuloSSLTestBase {
+ @Override
+ public String getClusterName() {
+ return "test_read_write_ssl";
+ }
+
+ @Override
+ public String getDescription() {
+ return "Test reading and writing to Accumulo cluster SSL $clusterName"
+ }
+
+ public ZooKeeperInstance getInstance() {
+ String zookeepers = SLIDER_CONFIG.get(
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM,
+ FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
+ ClientConfiguration conf = new ClientConfiguration()
+ .withInstance(tree.global.get("site.client.instance.name"))
+ .withZkHosts(zookeepers)
+ .withSsl(true)
+ .withKeystore(clientKeyStoreFile.toString(), KEY_PASS, null)
+ .withTruststore(trustStoreFile.toString(), TRUST_PASS, null)
+ return new ZooKeeperInstance(conf)
+ }
+
+ @Override
+ public void clusterLoadOperations(ClusterDescription cd, SliderClient sliderClient) {
+ try {
+ ZooKeeperInstance instance = getInstance()
+ Connector connector = instance.getConnector(USER, new PasswordToken(PASSWORD))
+
+ ingest(connector, 200000, 1, 50, 0);
+ verify(connector, 200000, 1, 50, 0);
+
+ ingest(connector, 2, 1, 500000, 0);
+ verify(connector, 2, 1, 500000, 0);
+
+ interleaveTest(connector);
+ } catch (Exception e) {
+ fail("Got exception connecting/reading/writing "+e)
+ }
+ }
+}
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
new file mode 100644
index 0000000..d3165dc
--- /dev/null
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.funtest.accumulo
+
+import groovy.json.JsonSlurper
+import org.apache.accumulo.core.conf.Property
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.funtest.framework.AgentUploads
+import org.junit.Before
+import org.junit.BeforeClass
+
+import javax.net.ssl.KeyManager
+import javax.net.ssl.SSLContext
+import javax.net.ssl.TrustManager
+import javax.net.ssl.X509TrustManager
+import java.security.SecureRandom
+import java.security.cert.CertificateException
+import java.security.cert.X509Certificate
+
+class AccumuloSSLTestBase extends AccumuloBasicIT {
+ protected static final File trustStoreFile = new File(TEST_APP_PKG_DIR, "truststore.jks")
+ protected static final File clientKeyStoreFile = new File(TEST_APP_PKG_DIR, "keystore.jks")
+
+ protected String templateName() {
+ return sysprop("test.app.resources.dir") + "/appConfig_ssl.json"
+ }
+
+ protected ConfTree modifyTemplate(ConfTree confTree) {
+ confTree.global.put("site.accumulo-site.instance.rpc.ssl.enabled", "true")
+ confTree.global.put("site.accumulo-site.instance.rpc.ssl.clientAuth", "true")
+ String jks = confTree.global.get(PROVIDER_PROPERTY)
+ def keys = confTree.credentials.get(jks)
+ keys.add("rpc.javax.net.ssl.keyStorePassword")
+ keys.add("rpc.javax.net.ssl.trustStorePassword")
+ return confTree
+ }
+
+ @Override
+ public String getClusterName() {
+ return "test_ssl";
+ }
+
+ @Override
+ public String getDescription() {
+ return "Test enable SSL $clusterName"
+ }
+
+ @BeforeClass
+ public static void initHttps() {
+ SSLContext ctx = SSLContext.getInstance("SSL");
+ TrustManager[] t = new TrustManager[1];
+ t[0] = new DefaultTrustManager();
+ ctx.init(new KeyManager[0], t, new SecureRandom());
+ SSLContext.setDefault(ctx);
+ }
+
+ @Before
+ public void createCerts() {
+ Path certDir = new Path(clusterFS.homeDirectory,
+ tree.global.get("site.global.ssl_cert_dir"))
+ if (clusterFS.exists(certDir)) {
+ clusterFS.delete(certDir, true)
+ }
+ clusterFS.mkdirs(certDir)
+
+ Configuration conf = loadSliderConf()
+ String provider = tree.global.get(PROVIDER_PROPERTY)
+ provider = provider.replace("hdfs/user",
+ conf.get("fs.defaultFS").replace("://", "@") + "/user")
+ System.out.println("provider after "+provider)
+ File rootKeyStoreFile = new File(TEST_APP_PKG_DIR, "root.jks")
+
+ if (!rootKeyStoreFile.exists() && !trustStoreFile.exists()) {
+ CertUtil.createRootKeyPair(rootKeyStoreFile.toString(),
+ Property.INSTANCE_SECRET.toString(), trustStoreFile.toString(),
+ Property.RPC_SSL_TRUSTSTORE_PASSWORD.toString(), provider);
+ }
+
+ AgentUploads agentUploads = new AgentUploads(SLIDER_CONFIG)
+ agentUploads.uploader.copyIfOutOfDate(trustStoreFile, new Path(certDir,
+ "truststore.jks"), false)
+
+ for (node in getNodeList(conf)) {
+ File keyStoreFile = new File(TEST_APP_PKG_DIR, node + ".jks")
+ if (!keyStoreFile.exists()) {
+ CertUtil.createServerKeyPair(keyStoreFile.toString(),
+ Property.RPC_SSL_KEYSTORE_PASSWORD.toString(),
+ rootKeyStoreFile.toString(), Property.INSTANCE_SECRET.toString(),
+ provider, node);
+ }
+ agentUploads.uploader.copyIfOutOfDate(keyStoreFile, new Path(certDir,
+ node + ".jks"), false)
+ }
+
+ if (!clientKeyStoreFile.exists()) {
+ CertUtil.createServerKeyPair(clientKeyStoreFile.toString(),
+ Property.RPC_SSL_KEYSTORE_PASSWORD.toString(),
+ rootKeyStoreFile.toString(), Property.INSTANCE_SECRET.toString(),
+ provider, InetAddress.getLocalHost().getHostName());
+ }
+ }
+
+ def getNodeList(Configuration conf) {
+ String address
+ if (YarnConfiguration.useHttps(conf)) {
+ address = "https://" + conf.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS);
+ } else {
+ address = "http://" + conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS);
+ }
+ address = address.replace("0.0.0.0", conf.get(YarnConfiguration.RM_ADDRESS)
+ .split(":")[0])
+ address = address + "/ws/v1/cluster/nodes"
+ def slurper = new JsonSlurper()
+ def result = slurper.parse(new URL(address))
+ def hosts = []
+ for (host in result.nodes.node) {
+ hosts.add(host.nodeHostName)
+ }
+ return hosts.unique()
+ }
+
+ private static class DefaultTrustManager implements X509TrustManager {
+ @Override
+ public void checkClientTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {}
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {}
+
+ @Override
+ public X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ }
+}
diff --git a/app-packages/accumulo/src/test/java/org/apache/slider/funtest/accumulo/CertUtil.java b/app-packages/accumulo/src/test/java/org/apache/slider/funtest/accumulo/CertUtil.java
new file mode 100644
index 0000000..8bac58f
--- /dev/null
+++ b/app-packages/accumulo/src/test/java/org/apache/slider/funtest/accumulo/CertUtil.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.funtest.accumulo;
+
+
+import org.apache.slider.accumulo.ProviderUtil;
+import sun.security.x509.AlgorithmId;
+import sun.security.x509.CertificateAlgorithmId;
+import sun.security.x509.CertificateIssuerName;
+import sun.security.x509.CertificateSerialNumber;
+import sun.security.x509.CertificateSubjectName;
+import sun.security.x509.CertificateValidity;
+import sun.security.x509.CertificateVersion;
+import sun.security.x509.CertificateX509Key;
+import sun.security.x509.X500Name;
+import sun.security.x509.X509CertImpl;
+import sun.security.x509.X509CertInfo;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.SecureRandom;
+import java.security.SignatureException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.Date;
+import java.util.Enumeration;
+
+public class CertUtil {
+
+ public static void createRootKeyPair(String keyStoreFile,
+ String keyStorePasswordProperty, String trustStoreFile,
+ String trustStorePasswordProperty, String credentialProvider)
+ throws Exception {
+ char[] keyStorePassword = ProviderUtil.getPassword(credentialProvider,
+ keyStorePasswordProperty);
+ char[] trustStorePassword = ProviderUtil.getPassword(credentialProvider,
+ trustStorePasswordProperty);
+
+ createSelfSignedCert(keyStoreFile, "root", keyStorePassword);
+ createPublicCert(trustStoreFile, "root", keyStoreFile, keyStorePassword,
+ trustStorePassword);
+ }
+
+ public static void createServerKeyPair(String keyStoreFile,
+ String keyStorePasswordProperty, String rootKeyStoreFile,
+ String rootKeyStorePasswordProperty, String credentialProvider,
+ String hostname)
+ throws Exception {
+ char[] keyStorePassword = ProviderUtil.getPassword(credentialProvider,
+ keyStorePasswordProperty);
+ char[] rootKeyStorePassword = ProviderUtil.getPassword(credentialProvider,
+ rootKeyStorePasswordProperty);
+
+ createSignedCert(keyStoreFile, "server", hostname, keyStorePassword,
+ rootKeyStoreFile, rootKeyStorePassword);
+ }
+
+
+ private static final String keystoreType = "JKS";
+ private static final int keysize = 2048;
+ private static final String encryptionAlgorithm = "RSA";
+ private static final String signingAlgorithm = "SHA256WITHRSA";
+ private static final String issuerDirString = ",O=Apache Slider";
+
+ public static void createPublicCert(String targetKeystoreFile, String keyName,
+ String rootKeystorePath, char[] rootKeystorePassword,
+ char[] truststorePassword) throws KeyStoreException,
+ IOException, CertificateException, NoSuchAlgorithmException {
+ KeyStore signerKeystore = KeyStore.getInstance(keystoreType);
+ char[] signerPasswordArray = rootKeystorePassword;
+ FileInputStream rootKeystoreInputStream = null;
+ try{
+ rootKeystoreInputStream = new FileInputStream(rootKeystorePath);
+ signerKeystore.load(rootKeystoreInputStream, signerPasswordArray);
+ } finally {
+ if(rootKeystoreInputStream != null) {
+ rootKeystoreInputStream.close();
+ }
+ }
+ Certificate rootCert = findCert(signerKeystore);
+
+ KeyStore keystore = KeyStore.getInstance(keystoreType);
+ keystore.load(null, null);
+ keystore.setCertificateEntry(keyName + "Cert", rootCert);
+ FileOutputStream targetKeystoreOutputStream = null;
+ try{
+ targetKeystoreOutputStream = new FileOutputStream(targetKeystoreFile);
+ keystore.store(targetKeystoreOutputStream, truststorePassword);
+ } finally {
+ if(targetKeystoreOutputStream != null) {
+ targetKeystoreOutputStream.close();
+ }
+ }
+ }
+
+ public static void createSignedCert(String targetKeystoreFile,
+ String keyName, String hostname, char[] keystorePassword,
+ String signerKeystorePath, char[] signerKeystorePassword)
+ throws Exception {
+ KeyStore signerKeystore = KeyStore.getInstance(keystoreType);
+ char[] signerPasswordArray = signerKeystorePassword;
+ FileInputStream signerKeystoreInputStream = null;
+ try{
+ signerKeystoreInputStream = new FileInputStream(signerKeystorePath);
+ signerKeystore.load(signerKeystoreInputStream, signerPasswordArray);
+ } finally {
+ if (signerKeystoreInputStream != null) {
+ signerKeystoreInputStream.close();
+ }
+ }
+ Certificate signerCert = findCert(signerKeystore);
+ PrivateKey signerKey = findPrivateKey(signerKeystore, signerPasswordArray);
+
+ KeyPair kp = generateKeyPair();
+ Certificate cert = generateCert(hostname, kp, false,
+ signerCert.getPublicKey(), signerKey);
+
+ char[] password = keystorePassword;
+ KeyStore keystore = KeyStore.getInstance(keystoreType);
+ keystore.load(null, null);
+ keystore.setCertificateEntry(keyName + "Cert", cert);
+ keystore.setKeyEntry(keyName + "Key", kp.getPrivate(), password, new Certificate[] {cert, signerCert});
+ FileOutputStream targetKeystoreOutputStream = null;
+ try{
+ targetKeystoreOutputStream = new FileOutputStream(targetKeystoreFile);
+ keystore.store(targetKeystoreOutputStream, password);
+ } finally {
+ if (targetKeystoreOutputStream != null){
+ targetKeystoreOutputStream.close();
+ }
+ }
+ }
+
+ public static void createSelfSignedCert(String targetKeystoreFileName,
+ String keyName, char[] keystorePassword)
+ throws IOException, NoSuchAlgorithmException, CertificateException,
+ NoSuchProviderException, InvalidKeyException, SignatureException,
+ KeyStoreException {
+ File targetKeystoreFile = new File(targetKeystoreFileName);
+ if (targetKeystoreFile.exists()) {
+ throw new IOException("File exists: "+targetKeystoreFile);
+ }
+
+ KeyPair kp = generateKeyPair();
+
+ Certificate cert = generateCert(null, kp, true,
+ kp.getPublic(), kp.getPrivate());
+
+ char[] password = keystorePassword;
+ KeyStore keystore = KeyStore.getInstance(keystoreType);
+ keystore.load(null, null);
+ keystore.setCertificateEntry(keyName + "Cert", cert);
+ keystore.setKeyEntry(keyName + "Key", kp.getPrivate(), password, new Certificate[] {cert});
+ FileOutputStream targetKeystoreOutputStream = null;
+ try{
+ targetKeystoreOutputStream = new FileOutputStream(targetKeystoreFile);
+ keystore.store(targetKeystoreOutputStream, password);
+ } finally {
+ if (targetKeystoreOutputStream != null) {
+ targetKeystoreOutputStream.close();
+ }
+ }
+ }
+
+ private static KeyPair generateKeyPair() throws NoSuchAlgorithmException {
+ KeyPairGenerator gen = KeyPairGenerator.getInstance(encryptionAlgorithm);
+ gen.initialize(keysize);
+ return gen.generateKeyPair();
+ }
+
+ private static X509Certificate generateCert(
+ String hostname, KeyPair kp, boolean isCertAuthority,
+ PublicKey signerPublicKey, PrivateKey signerPrivateKey)
+ throws IOException, CertificateException, NoSuchProviderException,
+ NoSuchAlgorithmException, InvalidKeyException, SignatureException {
+ X500Name issuer = new X500Name("CN=root" + issuerDirString);
+ X500Name subject;
+ if (hostname == null) {
+ subject = issuer;
+ } else {
+ subject = new X500Name("CN=" + hostname + issuerDirString);
+ }
+
+ X509CertInfo info = new X509CertInfo();
+ Date from = new Date();
+ Date to = new Date(from.getTime() + 365 * 86400000l);
+ CertificateValidity interval = new CertificateValidity(from, to);
+ BigInteger sn = new BigInteger(64, new SecureRandom());
+
+ info.set(X509CertInfo.VALIDITY, interval);
+ info.set(X509CertInfo.SERIAL_NUMBER, new CertificateSerialNumber(sn));
+ info.set(X509CertInfo.SUBJECT, new CertificateSubjectName(subject));
+ info.set(X509CertInfo.ISSUER, new CertificateIssuerName(issuer));
+ info.set(X509CertInfo.KEY, new CertificateX509Key(kp.getPublic()));
+ info.set(X509CertInfo.VERSION, new CertificateVersion(CertificateVersion.V3));
+ AlgorithmId algo = new AlgorithmId(AlgorithmId.md5WithRSAEncryption_oid);
+ info.set(X509CertInfo.ALGORITHM_ID, new CertificateAlgorithmId(algo));
+
+ // Sign the cert to identify the algorithm that's used.
+ X509CertImpl cert = new X509CertImpl(info);
+ cert.sign(signerPrivateKey, signingAlgorithm);
+
+ // Update the algorithm, and resign.
+ algo = (AlgorithmId)cert.get(X509CertImpl.SIG_ALG);
+ info.set(CertificateAlgorithmId.NAME + "." + CertificateAlgorithmId.ALGORITHM, algo);
+ cert = new X509CertImpl(info);
+ cert.sign(signerPrivateKey, signingAlgorithm);
+ return cert;
+ }
+
+ private static Certificate findCert(KeyStore keyStore) throws KeyStoreException {
+ Enumeration<String> aliases = keyStore.aliases();
+ Certificate cert = null;
+ while (aliases.hasMoreElements()) {
+ String alias = aliases.nextElement();
+ if (keyStore.isCertificateEntry(alias)) {
+ // assume only one cert
+ cert = keyStore.getCertificate(alias);
+ break;
+ }
+ }
+ if (cert == null) {
+ throw new KeyStoreException("Could not find cert in keystore");
+ }
+ return cert;
+ }
+
+ private static PrivateKey findPrivateKey(KeyStore keyStore, char[] keystorePassword)
+ throws KeyStoreException, UnrecoverableKeyException, NoSuchAlgorithmException {
+ Enumeration<String> aliases = keyStore.aliases();
+ PrivateKey key = null;
+ while (aliases.hasMoreElements()) {
+ String alias = aliases.nextElement();
+ if (keyStore.isKeyEntry(alias)) {
+ // assume only one key
+ key = (PrivateKey) keyStore.getKey(alias, keystorePassword);
+ break;
+ }
+ }
+ if (key == null) {
+ throw new KeyStoreException("Could not find private key in keystore");
+ }
+ return key;
+ }
+
+}
diff --git a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
deleted file mode 100644
index 8b63d06..0000000
--- a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "agent.conf": "agent.ini",
- "application.def": "${app.package.name}.zip",
- "config_types": "accumulo-site",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/accumulo-${accumulo.version}-bin.tar.gz",
- "site.global.app_user": "yarn",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.global.tserver_heapsize": "128m",
- "site.global.master_heapsize": "128m",
- "site.global.monitor_heapsize": "64m",
- "site.global.gc_heapsize": "64m",
- "site.global.other_heapsize": "128m",
- "site.global.hadoop_prefix": "/usr/lib/hadoop",
- "site.global.hadoop_conf_dir": "/etc/hadoop/conf",
- "site.global.zookeeper_home": "/usr/lib/zookeeper",
- "site.global.accumulo_instance_name": "instancename",
- "site.global.accumulo_root_password": "secret",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.global.monitor_protocol": "https",
- "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
- "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
- "site.accumulo-site.instance.secret": "DEFAULT",
- "site.accumulo-site.tserver.memory.maps.max": "80M",
- "site.accumulo-site.tserver.cache.data.size": "7M",
- "site.accumulo-site.tserver.cache.index.size": "20M",
- "site.accumulo-site.trace.token.property.password": "secret",
- "site.accumulo-site.trace.user": "root",
- "site.accumulo-site.tserver.sort.buffer.size": "50M",
- "site.accumulo-site.tserver.walog.max.size": "100M",
- "site.accumulo-site.master.port.client": "0",
- "site.accumulo-site.trace.port.client": "0",
- "site.accumulo-site.tserver.port.client": "0",
- "site.accumulo-site.gc.port.client": "0",
- "site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
- "site.accumulo-site.monitor.port.log4j": "0",
- "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
- },
- "components": {
- "ACCUMULO_MASTER": {
- },
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- },
- "ACCUMULO_TSERVER": {
- },
- "ACCUMULO_MONITOR": {
- },
- "ACCUMULO_GC": {
- },
- "ACCUMULO_TRACER": {
- }
- }
-}
diff --git a/app-packages/accumulo/src/test/resources/resources.json b/app-packages/accumulo/src/test/resources/resources.json
index 0d536aa..1c5dd97 100644
--- a/app-packages/accumulo/src/test/resources/resources.json
+++ b/app-packages/accumulo/src/test/resources/resources.json
@@ -3,12 +3,14 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": ""
},
"components": {
"ACCUMULO_MASTER": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "64"
},
"slider-appmaster": {
},
@@ -20,7 +22,7 @@
"ACCUMULO_MONITOR": {
"yarn.role.priority": "3",
"yarn.component.instances": "1",
- "yarn.memory": "128"
+ "yarn.memory": "64"
},
"ACCUMULO_GC": {
"yarn.role.priority": "4",
diff --git a/app-packages/accumulo/resources.json b/app-packages/accumulo/src/test/resources/resources_with_client.json
similarity index 78%
copy from app-packages/accumulo/resources.json
copy to app-packages/accumulo/src/test/resources/resources_with_client.json
index f876901..297a232 100644
--- a/app-packages/accumulo/resources.json
+++ b/app-packages/accumulo/src/test/resources/resources_with_client.json
@@ -14,7 +14,7 @@
},
"ACCUMULO_TSERVER": {
"yarn.role.priority": "2",
- "yarn.component.instances": "1",
+ "yarn.component.instances": "2",
"yarn.memory": "256"
},
"ACCUMULO_MONITOR": {
@@ -30,7 +30,12 @@
"ACCUMULO_TRACER": {
"yarn.role.priority": "5",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "128"
+ },
+ "ACCUMULO_CLIENT": {
+ "yarn.role.priority": "6",
+ "yarn.component.instances": "0",
+ "yarn.memory": "128"
}
}
}
diff --git a/app-packages/app-pkg-template/README.txt b/app-packages/app-pkg-template/README.txt
index 00dfdbc..266f34f 100644
--- a/app-packages/app-pkg-template/README.txt
+++ b/app-packages/app-pkg-template/README.txt
@@ -28,7 +28,6 @@
Verify the content using
zip -Tv myapp-1.0.0.zip
-While appConfig.json and resources.json are not required for the package they work
-well as the default configuration for Slider apps. So its advisable that when you
-create an application package for Slider, include sample/default resources.json and
-appConfig.json for a one-node Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
diff --git a/app-packages/app-pkg-template/appConfig.json b/app-packages/app-pkg-template/appConfig-default.json
similarity index 73%
rename from app-packages/app-pkg-template/appConfig.json
rename to app-packages/app-pkg-template/appConfig-default.json
index a6f61f9..cc65503 100644
--- a/app-packages/app-pkg-template/appConfig.json
+++ b/app-packages/app-pkg-template/appConfig-default.json
@@ -3,10 +3,9 @@
"metadata": {
},
"global": {
- "application.def": "package/myapp-1.0.0.zip",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "application.def": "myapp-1.0.0.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
- "site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/myapp-1.0.0",
"site.global.listen_port": "${MYAPP_COMPONENT.ALLOCATED_PORT}"
diff --git a/app-packages/app-pkg-template/metainfo.xml b/app-packages/app-pkg-template/metainfo.xml
index c6e1485..50c0fbd 100644
--- a/app-packages/app-pkg-template/metainfo.xml
+++ b/app-packages/app-pkg-template/metainfo.xml
@@ -28,12 +28,12 @@
<component>
<name>MYAPP_COMPONENT</name>
<category>MASTER</category>
- <exports>
- <export>
+ <componentExports>
+ <componentExport>
<name>host_port</name>
<value>${THIS_HOST}:${site.global.listen_port}</value>
- </export>
- </exports>
+ </componentExport>
+ </componentExports>
<commandScript>
<script>scripts/myapp_component.py</script>
<scriptType>PYTHON</scriptType>
diff --git a/app-packages/app-pkg-template/resources.json b/app-packages/app-pkg-template/resources-default.json
similarity index 100%
rename from app-packages/app-pkg-template/resources.json
rename to app-packages/app-pkg-template/resources-default.json
diff --git a/app-packages/command-logger/application-pkg/pom.xml b/app-packages/command-logger/application-pkg/pom.xml
index 71e4d82..2c1fd46 100644
--- a/app-packages/command-logger/application-pkg/pom.xml
+++ b/app-packages/command-logger/application-pkg/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -54,20 +54,8 @@
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
</plugins>
+
<extensions>
<extension>
<groupId>org.apache.maven.wagon</groupId>
@@ -75,4 +63,28 @@
</extension>
</extensions>
</build>
+
+ <profiles>
+ <profile>
+ <id>rat</id>
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
diff --git a/app-packages/command-logger/slider-pkg/appConfig.json b/app-packages/command-logger/slider-pkg/appConfig.json
index 1d92c59..d4082a8 100644
--- a/app-packages/command-logger/slider-pkg/appConfig.json
+++ b/app-packages/command-logger/slider-pkg/appConfig.json
@@ -3,18 +3,14 @@
"metadata": {
},
"global": {
- "application.def": "apache-slider-command-logger.zip",
- "config_types": "cl-site",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/command-logger.tar",
- "site.global.app_user": "yarn",
+ "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.application_id": "CommandLogger",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log",
- "site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
+
+ "site.cl-site.logfile.location": "${AGENT_WORK_ROOT}/app/install/command-logger-app/operations.log",
+ "site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p",
+ "site.cl-site.pattern.for.test.to.verify": "verify this pattern"
},
"components": {
"COMMAND_LOGGER": {
diff --git a/app-packages/command-logger/slider-pkg/metainfo.xml b/app-packages/command-logger/slider-pkg/metainfo.xml
index e17413d..5de2c37 100644
--- a/app-packages/command-logger/slider-pkg/metainfo.xml
+++ b/app-packages/command-logger/slider-pkg/metainfo.xml
@@ -24,10 +24,12 @@
log file. When stopped it renames the file.
</comment>
<version>0.1.0</version>
+ <exportedConfigs>cl-site</exportedConfigs>
<components>
<component>
<name>COMMAND_LOGGER</name>
<category>MASTER</category>
+ <publishConfig>true</publishConfig>
<commandScript>
<script>scripts/cl.py</script>
<scriptType>PYTHON</scriptType>
@@ -42,11 +44,19 @@
<packages>
<package>
<type>tarball</type>
- <name>files/command_log.tar</name>
+ <name>files/command-logger.tar</name>
</package>
</packages>
</osSpecific>
</osSpecifics>
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>cl-site.xml</fileName>
+ <dictionaryName>cl-site</dictionaryName>
+ </configFile>
+ </configFiles>
+
</application>
</metainfo>
diff --git a/app-packages/command-logger/slider-pkg/package/scripts/cl.py b/app-packages/command-logger/slider-pkg/package/scripts/cl.py
index 6b18faa..b15bbfd 100644
--- a/app-packages/command-logger/slider-pkg/package/scripts/cl.py
+++ b/app-packages/command-logger/slider-pkg/package/scripts/cl.py
@@ -81,7 +81,6 @@
file_location = params.file_location
TemplateConfig( file_location,
- owner = params.app_user,
template_tag = None
)
diff --git a/app-packages/command-logger/slider-pkg/package/scripts/params.py b/app-packages/command-logger/slider-pkg/package/scripts/params.py
index 3d388ae..b135539 100644
--- a/app-packages/command-logger/slider-pkg/package/scripts/params.py
+++ b/app-packages/command-logger/slider-pkg/package/scripts/params.py
@@ -25,7 +25,6 @@
container_id = config['hostLevelParams']['container_id']
application_id = config['configurations']['global']['application_id']
-app_user = config['configurations']['global']['app_user']
datetime_format = config['configurations']['cl-site']['datetime.format']
file_location = config['configurations']['cl-site']['logfile.location']
diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml
index bd46cbb..f7514dc 100644
--- a/app-packages/command-logger/slider-pkg/pom.xml
+++ b/app-packages/command-logger/slider-pkg/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -93,20 +93,6 @@
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
</plugins>
</build>
@@ -118,5 +104,34 @@
<type>tar</type>
</dependency>
</dependencies>
+
+ <profiles>
+ <profile>
+ <id>apache-release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt
new file mode 100644
index 0000000..6389fb2
--- /dev/null
+++ b/app-packages/hbase-win/README.txt
@@ -0,0 +1,38 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+Create Slider App Package for HBase on Windows
+
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
+
+
+To create the app package you will need the HBase tarball and invoke mvn command
+with appropriate parameters.
+
+Command:
+mvn clean package -Phbase-app-package-win -Dpkg.version=<version>
+ -Dpkg.name=<file name of app zip file> -Dpkg.src=<folder location where the pkg is available>
+
+Example:
+mvn clean package -Phbase-app-package-win -Dpkg.version=0.98.5-hadoop2
+ -Dpkg.name=hbase-0.98.5-hadoop2-bin.zip
+ -Dpkg.src=/Users/user1/Downloads
+
+App package can be found in
+ app-packages/hbase/target/slider-hbase-app-win-package-${pkg.version}.zip
diff --git a/app-packages/hbase-win/appConfig-default.json b/app-packages/hbase-win/appConfig-default.json
new file mode 100644
index 0000000..04cb9a9
--- /dev/null
+++ b/app-packages/hbase-win/appConfig-default.json
@@ -0,0 +1,38 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/HBASE/slider-hbase-app-win-package-${pkg.version}.zip",
+ "create.default.zookeeper.node": "true",
+ "java_home": "C:\\java",
+
+ "site.global.app_user": "hadoop",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
+ "site.global.hbase_instance_name": "instancename",
+ "site.global.user_group": "hadoop",
+ "site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
+ "site.global.java_library_path": "c:\\hdp\\hadoop\\bin",
+ "site.global.hbase_rest_port": "17000",
+ "site.global.hbase_thrift_port": "9090",
+ "site.global.hbase_thrift2_port": "9091",
+
+ "site.hbase-env.hbase_master_heapsize": "1024m",
+ "site.hbase-env.hbase_regionserver_heapsize": "1024m",
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
+ "site.hbase-site.hbase.superuser": "hadoop",
+ "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+ "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+ "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
+ "site.hbase-site.zookeeper.znode.parent": "${DEFAULT_ZK_PATH}",
+ "site.hbase-site.hbase.regionserver.info.port": "0",
+ "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
+ "site.hbase-site.hbase.regionserver.port": "0",
+ "site.hbase-site.hbase.master.port": "0"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
diff --git a/app-packages/hbase-win/configuration/hbase-env.xml b/app-packages/hbase-win/configuration/hbase-env.xml
new file mode 100644
index 0000000..fa5686f
--- /dev/null
+++ b/app-packages/hbase-win/configuration/hbase-env.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>hbase_regionserver_heapsize</name>
+ <value>1024</value>
+ <description>HBase RegionServer Heap Size.</description>
+ </property>
+ <property>
+ <name>hbase_regionserver_xmn_max</name>
+ <value>512</value>
+ <description>HBase RegionServer maximum value for minimum heap size.</description>
+ </property>
+ <property>
+ <name>hbase_regionserver_xmn_ratio</name>
+ <value>0.2</value>
+ <description>HBase RegionServer minimum heap size is calculated as a percentage of max heap size.</description>
+ </property>
+ <property>
+ <name>hbase_master_heapsize</name>
+ <value>1024</value>
+ <description>HBase Master Heap Size</description>
+ </property>
+
+ <!-- hbase-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for start command</description>
+ <value>
+ -Xmx{{heap_size}} "-XX:+UseConcMarkSweepGC" "-XX:CMSInitiatingOccupancyFraction=70" "-Djava.net.preferIPv4Stack=true" "-XX:+ForceTimeHighResolution" "-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" -Xloggc:"{{log_dir}}\hbase-{{role_user}}.gc" -Dhbase.log.dir="{{log_dir}}" -Dhbase.log.file="hbase-{{role_user}}.log" -Dhbase.home.dir="{{hbase_root}}" -Dhbase.id.str="{{hbase_instance_name}}" -XX:OnOutOfMemoryError="taskkill /F /PID p" -Dhbase.root.logger="INFO,DRFA" -Djava.library.path="{{java_library_path}}" -Dhbase.security.logger="INFO,DRFAS" -classpath "{{conf_dir}};{{hbase_root}}\lib\*;{{hbase_additional_cp}}"
+ </value>
+ </property>
+
+</configuration>
diff --git a/app-packages/hbase-win/configuration/hbase-log4j.xml b/app-packages/hbase-win/configuration/hbase-log4j.xml
new file mode 100644
index 0000000..d488c4e
--- /dev/null
+++ b/app-packages/hbase-win/configuration/hbase-log4j.xml
@@ -0,0 +1,143 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+ <property>
+ <name>content</name>
+ <description>Custom log4j.properties</description>
+ <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.security.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+# Rolling File Appender properties
+hbase.log.maxfilesize=256MB
+hbase.log.maxbackupindex=20
+
+# Rolling File Appender
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file}
+
+log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize}
+log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex}
+
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+#
+# Security audit appender
+#
+hbase.security.log.file=SecurityAuth.audit
+hbase.security.log.maxfilesize=256MB
+hbase.security.log.maxbackupindex=20
+log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
+log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file}
+log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize}
+log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex}
+log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+# Enable this to get detailed connection error/retry logging.
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE
+
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of .META. messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
+
+ </value>
+ </property>
+
+</configuration>
diff --git a/app-packages/hbase-win/configuration/hbase-policy.xml b/app-packages/hbase-win/configuration/hbase-policy.xml
new file mode 100644
index 0000000..e45f23c
--- /dev/null
+++ b/app-packages/hbase-win/configuration/hbase-policy.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HRegionInterface protocol implementations (ie.
+ clients talking to HRegionServers)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterInterface protocol implementation (ie.
+ clients talking to HMaster for admin operations).
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterRegionInterface protocol implementations
+ (for HRegionServers communicating with HMaster)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+</configuration>
diff --git a/app-packages/hbase-win/configuration/hbase-site.xml b/app-packages/hbase-win/configuration/hbase-site.xml
new file mode 100644
index 0000000..a9711d3
--- /dev/null
+++ b/app-packages/hbase-win/configuration/hbase-site.xml
@@ -0,0 +1,370 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>hbase.rootdir</name>
+ <value>hdfs://localhost:8020/apps/hbase/data</value>
+ <description>The directory shared by region servers and into
+ which HBase persists. The URL should be 'fully-qualified'
+ to include the filesystem scheme. For example, to specify the
+ HDFS directory '/hbase' where the HDFS instance's namenode is
+ running at namenode.example.org on port 9000, set this value to:
+ hdfs://namenode.example.org:9000/hbase. By default HBase writes
+ into /tmp. Change this configuration else all data will be lost
+ on machine restart.
+ </description>
+ </property>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ <description>The mode the cluster will be in. Possible values are
+ false for standalone mode and true for distributed mode. If
+ false, startup will run all HBase and ZooKeeper daemons together
+ in the one JVM.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.port</name>
+ <value>60000</value>
+ <description>The port the HBase Master should bind to.</description>
+ </property>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>/hadoop/hbase</value>
+ <description>Temporary directory on the local filesystem.
+ Change this setting to point to a location more permanent
+ than '/tmp' (The '/tmp' directory is often cleared on
+ machine restart).
+ </description>
+ </property>
+ <property>
+ <name>hbase.local.dir</name>
+ <value>${hbase.tmp.dir}/local</value>
+ <description>Directory on the local filesystem to be used as a local storage
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.info.bindAddress</name>
+ <value>0.0.0.0</value>
+ <description>The bind address for the HBase Master web UI
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.info.port</name>
+ <value>60010</value>
+ <description>The port for the HBase Master web UI.</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.info.port</name>
+ <value>60030</value>
+ <description>The port for the HBase RegionServer web UI.</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.global.memstore.upperLimit</name>
+ <value>0.4</value>
+ <description>Maximum size of all memstores in a region server before new
+ updates are blocked and flushes are forced. Defaults to 40% of heap
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>60</value>
+ <description>Count of RPC Listener instances spun up on RegionServers.
+ Same property is used by the Master for count of master handlers.
+ Default is 10.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.majorcompaction</name>
+ <value>86400000</value>
+ <description>The time (in milliseconds) between 'major' compactions of all
+ HStoreFiles in a region. Default: 1 day.
+ Set to 0 to disable automated major compactions.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.global.memstore.lowerLimit</name>
+ <value>0.38</value>
+ <description>When memstores are being forced to flush to make room in
+ memory, keep flushing until we hit this mark. Defaults to 35% of heap.
+ This value equal to hbase.regionserver.global.memstore.upperLimit causes
+ the minimum possible flushing to occur when updates are blocked due to
+ memstore limiting.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.block.multiplier</name>
+ <value>2</value>
+ <description>Block updates if memstore has hbase.hregion.memstore.block.multiplier
+ time hbase.hregion.flush.size bytes. Useful preventing
+ runaway memstore during spikes in update traffic. Without an
+ upper-bound, memstore fills such that when it flushes the
+ resultant flush files take a long time to compact or split, or
+ worse, we OOME
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.flush.size</name>
+ <value>134217728</value>
+ <description>
+ Memstore will be flushed to disk if size of the memstore
+ exceeds this number of bytes. Value is checked by a thread that runs
+ every hbase.server.thread.wakefrequency.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.mslab.enabled</name>
+ <value>true</value>
+ <description>
+ Enables the MemStore-Local Allocation Buffer,
+ a feature which works to prevent heap fragmentation under
+ heavy write loads. This can reduce the frequency of stop-the-world
+ GC pauses on large heaps.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.max.filesize</name>
+ <value>10737418240</value>
+ <description>
+ Maximum HStoreFile size. If any one of a column families' HStoreFiles has
+ grown to exceed this value, the hosting HRegion is split in two.
+ Default: 1G.
+ </description>
+ </property>
+ <property>
+ <name>hbase.client.scanner.caching</name>
+ <value>100</value>
+ <description>Number of rows that will be fetched when calling next
+ on a scanner if it is not served from (local, client) memory. Higher
+ caching values will enable faster scanners but will eat up more memory
+ and some calls of next may take longer and longer times when the cache is empty.
+ Do not set this value such that the time between invocations is greater
+ than the scanner timeout; i.e. hbase.regionserver.lease.period
+ </description>
+ </property>
+ <property>
+ <name>zookeeper.session.timeout</name>
+ <value>30000</value>
+ <description>ZooKeeper session timeout.
+ HBase passes this to the zk quorum as suggested maximum time for a
+ session (This setting becomes zookeeper's 'maxSessionTimeout'). See
+ http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
+ "The client sends a requested timeout, the server responds with the
+ timeout that it can give the client. " In milliseconds.
+ </description>
+ </property>
+ <property>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value>10485760</value>
+ <description>Specifies the combined maximum allowed size of a KeyValue
+ instance. This is to set an upper boundary for a single entry saved in a
+ storage file. Since they cannot be split it helps avoiding that a region
+ cannot be split any further because the data is too large. It seems wise
+ to set this to a fraction of the maximum region size. Setting it to zero
+ or less disables the check.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hstore.compactionThreshold</name>
+ <value>3</value>
+ <description>
+ If more than this number of HStoreFiles in any one HStore
+ (one HStoreFile is written per flush of memstore) then a compaction
+ is run to rewrite all HStoreFiles files as one. Larger numbers
+ put off compaction but when it runs, it takes longer to complete.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hstore.flush.retries.number</name>
+ <value>120</value>
+ <description>
+ The number of times the region flush operation will be retried.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.hstore.blockingStoreFiles</name>
+ <value>10</value>
+ <description>
+ If more than this number of StoreFiles in any one Store
+ (one StoreFile is written per flush of MemStore) then updates are
+ blocked for this HRegion until a compaction is completed, or
+ until hbase.hstore.blockingWaitTime has been exceeded.
+ </description>
+ </property>
+ <property>
+ <name>hfile.block.cache.size</name>
+ <value>0.40</value>
+ <description>
+ Percentage of maximum heap (-Xmx setting) to allocate to block cache
+ used by HFile/StoreFile. Default of 0.25 means allocate 25%.
+ Set to 0 to disable but it's not recommended.
+ </description>
+ </property>
+
+ <!-- The following properties configure authentication information for
+ HBase processes when using Kerberos security. There are no default
+ values, included here for documentation purposes -->
+ <property>
+ <name>hbase.master.keytab.file</name>
+ <value>/etc/security/keytabs/hbase.service.keytab</value>
+ <description>Full path to the kerberos keytab file to use for logging in
+ the configured HMaster server principal.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.kerberos.principal</name>
+ <value>hbase/_HOST@EXAMPLE.COM</value>
+ <description>Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name
+ that should be used to run the HMaster process. The principal name should
+ be in the form: user/hostname@DOMAIN. If "_HOST" is used as the hostname
+ portion, it will be replaced with the actual hostname of the running
+ instance.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.keytab.file</name>
+ <value>/etc/security/keytabs/hbase.service.keytab</value>
+ <description>Full path to the kerberos keytab file to use for logging in
+ the configured HRegionServer server principal.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.kerberos.principal</name>
+ <value>hbase/_HOST@EXAMPLE.COM</value>
+ <description>Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name
+ that should be used to run the HRegionServer process. The principal name
+ should be in the form: user/hostname@DOMAIN. If "_HOST" is used as the
+ hostname portion, it will be replaced with the actual hostname of the
+ running instance. An entry for this principal must exist in the file
+ specified in hbase.regionserver.keytab.file
+ </description>
+ </property>
+
+ <!-- Additional configuration specific to HBase security -->
+ <property>
+ <name>hbase.superuser</name>
+ <value>hbase</value>
+ <description>List of users or groups (comma-separated), who are allowed
+ full privileges, regardless of stored ACLs, across the cluster.
+ Only used when HBase security is enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.security.authentication</name>
+ <value>simple</value>
+ <description> Controls whether or not secure authentication is enabled for HBase. Possible values are 'simple'
+ (no authentication), and 'kerberos'.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.security.authorization</name>
+ <value>false</value>
+ <description>Enables HBase authorization. Set the value of this property to false to disable HBase authorization.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.coprocessor.region.classes</name>
+ <value></value>
+ <description>A comma-separated list of Coprocessors that are loaded by
+ default on all tables. For any override coprocessor method, these classes
+ will be called in order. After implementing your own Coprocessor, just put
+ it in HBase's classpath and add the fully qualified class name here.
+ A coprocessor can also be loaded on demand by setting HTableDescriptor.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.coprocessor.master.classes</name>
+ <value></value>
+ <description>A comma-separated list of
+ org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are
+ loaded by default on the active HMaster process. For any implemented
+ coprocessor methods, the listed classes will be called in order. After
+ implementing your own MasterObserver, just put it in HBase's classpath
+ and add the fully qualified class name here.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>2181</value>
+ <description>Property from ZooKeeper's config zoo.cfg.
+ The port at which the clients will connect.
+ </description>
+ </property>
+
+ <!--
+ The following three properties are used together to create the list of
+ host:peer_port:leader_port quorum servers for ZooKeeper.
+ -->
+ <property>
+ <name>hbase.zookeeper.quorum</name>
+ <value>localhost</value>
+ <description>Comma separated list of servers in the ZooKeeper Quorum.
+ For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
+ By default this is set to localhost for local and pseudo-distributed modes
+ of operation. For a fully-distributed setup, this should be set to a full
+ list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
+ this is the list of servers which we will start/stop ZooKeeper on.
+ </description>
+ </property>
+ <!-- End of properties used to generate ZooKeeper host:port quorum list. -->
+
+ <property>
+ <name>hbase.zookeeper.useMulti</name>
+ <value>true</value>
+ <description>Instructs HBase to make use of ZooKeeper's multi-update functionality.
+ This allows certain ZooKeeper operations to complete more quickly and prevents some issues
+ with rare Replication failure scenarios (see the release note of HBASE-2611 for an example).·
+ IMPORTANT: only set this to true if all ZooKeeper servers in the cluster are on version 3.4+
+ and will not be downgraded. ZooKeeper versions before 3.4 do not support multi-update and will
+ not fail gracefully if multi-update is invoked (see ZOOKEEPER-1495).
+ </description>
+ </property>
+ <property>
+ <name>zookeeper.znode.parent</name>
+ <value>/hbase-unsecure</value>
+ <description>Root ZNode for HBase in ZooKeeper. All of HBase's ZooKeeper
+ files that are configured with a relative path will go under this node.
+ By default, all of HBase's ZooKeeper file path are configured with a
+ relative path, so they will all go under this directory unless changed.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.defaults.for.version.skip</name>
+ <value>true</value>
+ <description>Disables version verification.</description>
+ </property>
+
+ <property>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ <description>Path to domain socket.</description>
+ </property>
+
+</configuration>
diff --git a/app-packages/hbase-win/jmx_metrics.json b/app-packages/hbase-win/jmx_metrics.json
new file mode 100644
index 0000000..ac0640e
--- /dev/null
+++ b/app-packages/hbase-win/jmx_metrics.json
@@ -0,0 +1,56 @@
+{
+ "Component": {
+ "HBASE_MASTER": {
+ "MetricAverageLoad": {
+ "metric": "Hadoop:service=HBase,name=Master,sub=Server.averageLoad",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "DeadRegionServers": {
+ "metric": "Hadoop:service=HBase,name=Master,sub=Server.numDeadRegionServers",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ClusterId": {
+ "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.clusterId",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "IsActiveMaster": {
+ "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.isActiveMaster",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "MasterActiveTime": {
+ "metric": "Hadoop:service=HBase,name=Master,sub=Server.masterActiveTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "MasterStartTime": {
+ "metric": "Hadoop:service=HBase,name=Master,sub=Server.masterStartTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "RegionServers": {
+ "metric": "Hadoop:service=HBase,name=Master,sub=Server.numRegionServers",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ServerName": {
+ "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.serverName",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ZookeeperQuorum": {
+ "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.zookeeperQuorum",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ClusterRequests": {
+ "metric": "Hadoop:service=HBase,name=Master,sub=Server.clusterRequests",
+ "pointInTime": true,
+ "temporal": false
+ }
+ }
+ }
+}
diff --git a/app-packages/hbase-win/metainfo.xml b/app-packages/hbase-win/metainfo.xml
new file mode 100644
index 0000000..da6121d
--- /dev/null
+++ b/app-packages/hbase-win/metainfo.xml
@@ -0,0 +1,170 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <application>
+ <name>HBASE</name>
+ <comment>
+ Apache HBase is the Hadoop database, a distributed, scalable, big data store.
+ Requirements:
+ 1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner.
+ 2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance.
+ </comment>
+ <version>${pkg.version}</version>
+ <type>YARN-APP</type>
+ <minHadoopVersion>2.1.0</minHadoopVersion>
+ <exportedConfigs>hbase-site</exportedConfigs>
+ <exportGroups>
+ <exportGroup>
+ <name>QuickLinks</name>
+ <exports>
+ <export>
+ <name>org.apache.slider.jmx</name>
+ <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
+ </export>
+ <export>
+ <name>org.apache.slider.monitor</name>
+ <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
+ </export>
+ <export>
+ <name>org.apache.slider.hbase.rest</name>
+ <value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value>
+ </export>
+ <export>
+ <name>org.apache.slider.hbase.thrift2</name>
+ <value>http://${HBASE_THRIFT2_HOST}:${site.global.hbase_thrift2_port}</value>
+ </export>
+ <export>
+ <name>org.apache.slider.hbase.thrift</name>
+ <value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
+ <commandOrders>
+ <commandOrder>
+ <command>HBASE_REGIONSERVER-START</command>
+ <requires>HBASE_MASTER-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>HBASE_MASTER-START</command>
+ <requires>HBASE_REST-INSTALLED</requires>
+ </commandOrder>
+ </commandOrders>
+ <components>
+ <component>
+ <name>HBASE_MASTER</name>
+ <category>MASTER</category>
+ <minInstanceCount>1</minInstanceCount>
+ <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor</appExports>
+ <componentExports>
+ <componentExport>
+ <name>org.apache.slider.jmx</name>
+ <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
+ </componentExport>
+ <componentExport>
+ <name>org.apache.slider.monitor</name>
+ <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
+ </componentExport>
+ </componentExports>
+ <commandScript>
+ <script>scripts/hbase_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>HBASE_REGIONSERVER</name>
+ <category>SLAVE</category>
+ <minInstanceCount>0</minInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_regionserver.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>HBASE_REST</name>
+ <category>MASTER</category>
+ <appExports>QuickLinks-org.apache.slider.hbase.rest</appExports>
+ <commandScript>
+ <script>scripts/hbase_rest.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>HBASE_THRIFT</name>
+ <category>MASTER</category>
+ <appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports>
+ <commandScript>
+ <script>scripts/hbase_thrift.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>HBASE_THRIFT2</name>
+ <category>MASTER</category>
+ <minInstanceCount>0</minInstanceCount>
+ <appExports>QuickLinks-org.apache.slider.hbase.thrift2</appExports>
+ <commandScript>
+ <script>scripts/hbase_thrift2.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>zip</type>
+ <name>files/${pkg.name}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>hbase-site.xml</fileName>
+ <dictionaryName>hbase-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>hbase-env.sh</fileName>
+ <dictionaryName>hbase-env</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>hbase-log4j.properties</fileName>
+ <dictionaryName>hbase-log4j</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>hbase-policy.xml</fileName>
+ <dictionaryName>hbase-policy</dictionaryName>
+ </configFile>
+ </configFiles>
+
+ </application>
+</metainfo>
diff --git a/app-packages/hbase-win/package/scripts/__init__.py b/app-packages/hbase-win/package/scripts/__init__.py
new file mode 100644
index 0000000..5561e10
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/__init__.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
diff --git a/app-packages/hbase-win/package/scripts/functions.py b/app-packages/hbase-win/package/scripts/functions.py
new file mode 100644
index 0000000..e6e7fb9
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/functions.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import math
+import datetime
+
+from resource_management.core.shell import checked_call
+
+def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max):
+ """
+ @param heapsize_str: str (e.g '1000m')
+ @param xmn_percent: float (e.g 0.2)
+ @param xmn_max: integer (e.g 512)
+ """
+ heapsize = int(re.search('\d+',heapsize_str).group(0))
+ heapsize_unit = re.search('\D+',heapsize_str).group(0)
+ xmn_val = int(math.floor(heapsize*xmn_percent))
+ xmn_val -= xmn_val % 8
+
+ result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val
+ return str(result_xmn_val) + heapsize_unit
diff --git a/app-packages/hbase-win/package/scripts/hbase.py b/app-packages/hbase-win/package/scripts/hbase.py
new file mode 100644
index 0000000..0962149
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/hbase.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+
+from resource_management import *
+import sys
+import shutil
+
+def hbase(name=None # 'master' or 'regionserver'
+ ):
+ import params
+
+ Directory( params.conf_dir,
+ recursive = True,
+ content = params.input_conf_files_dir
+ )
+
+ Directory (params.tmp_dir,
+ recursive = True
+ )
+
+ Directory (os.path.join(params.local_dir, "jars"),
+ recursive = True
+ )
+
+ XmlConfig( "hbase-site.xml",
+ conf_dir = params.conf_dir,
+ configurations = params.config['configurations']['hbase-site'],
+ owner = params.hbase_user,
+ group = params.user_group
+ )
+
+
+ if (params.log4j_props != None):
+ File(format("{params.conf_dir}/log4j.properties"),
+ group=params.user_group,
+ owner=params.hbase_user,
+ content=params.log4j_props
+ )
+ elif (os.path.exists(format("{conf_dir}/log4j.properties"))):
+ File(format("{params.conf_dir}/log4j.properties"),
+ group=params.user_group,
+ owner=params.hbase_user
+ )
diff --git a/app-packages/hbase-win/package/scripts/hbase_master.py b/app-packages/hbase-win/package/scripts/hbase_master.py
new file mode 100644
index 0000000..47b2409
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/hbase_master.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseMaster(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='master')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'master',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'master',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid")
+ check_process_status(pid_file)
+
+
+if __name__ == "__main__":
+ HbaseMaster().execute()
diff --git a/app-packages/hbase-win/package/scripts/hbase_regionserver.py b/app-packages/hbase-win/package/scripts/hbase_regionserver.py
new file mode 100644
index 0000000..daa5732
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/hbase_regionserver.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseRegionServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='regionserver')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'regionserver',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'regionserver',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid")
+ check_process_status(pid_file)
+
+
+if __name__ == "__main__":
+ HbaseRegionServer().execute()
diff --git a/app-packages/hbase-win/package/scripts/hbase_rest.py b/app-packages/hbase-win/package/scripts/hbase_rest.py
new file mode 100644
index 0000000..36b51f9
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/hbase_rest.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseRest(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='rest')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'rest',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'rest',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-rest.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseRest().execute()
diff --git a/app-packages/hbase-win/package/scripts/hbase_service.py b/app-packages/hbase-win/package/scripts/hbase_service.py
new file mode 100644
index 0000000..e269531
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/hbase_service.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+
+def hbase_service(
+ name,
+ action='start'): # 'start' or 'stop' or 'status'
+
+ import params
+
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-{name}.pid")
+ custom_port = None
+ custom_info_port = None
+ heap_size = params.master_heapsize
+ main_class = "org.apache.hadoop.hbase.master.HMaster"
+ if name == "regionserver":
+ heap_size = params.regionserver_heapsize
+ main_class = "org.apache.hadoop.hbase.regionserver.HRegionServer"
+ if name == "rest":
+ heap_size = params.restserver_heapsize
+ main_class = "org.apache.hadoop.hbase.rest.RESTServer"
+ custom_port = params.rest_port
+ if name == "thrift":
+ heap_size = params.thriftserver_heapsize
+ main_class = "org.apache.hadoop.hbase.thrift.ThriftServer"
+ custom_port = params.thrift_port
+ custom_info_port = params.thrift_info_port
+ if name == "thrift2":
+ heap_size = params.thrift2server_heapsize
+ main_class = "org.apache.hadoop.hbase.thrift2.ThriftServer"
+ custom_port = params.thrift2_port
+ custom_info_port = params.thrift2_info_port
+
+ role_user = format("{hbase_user}-{name}")
+
+ rest_of_the_command = InlineTemplate(params.hbase_env_sh_template, [], heap_size=heap_size, role_user=role_user)()
+
+ process_cmd = format("{java64_home}\\bin\\java {rest_of_the_command} {main_class} {action}")
+
+ if custom_port:
+ process_cmd = format("{process_cmd} -p {custom_port}")
+
+ if custom_info_port:
+ process_cmd = format("{process_cmd} --infoport {custom_info_port}")
+
+ Execute(process_cmd,
+ logoutput=False,
+ wait_for_finish=False,
+ pid_file=pid_file
+ )
\ No newline at end of file
diff --git a/app-packages/hbase-win/package/scripts/hbase_thrift.py b/app-packages/hbase-win/package/scripts/hbase_thrift.py
new file mode 100644
index 0000000..84bfc62
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/hbase_thrift.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseThrift(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='thrift')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'thrift',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'thrift',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseThrift().execute()
diff --git a/app-packages/hbase-win/package/scripts/hbase_thrift2.py b/app-packages/hbase-win/package/scripts/hbase_thrift2.py
new file mode 100644
index 0000000..b72196c
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/hbase_thrift2.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseThrift2(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='thrift2')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'thrift2',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'thrift2',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift2.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseThrift2().execute()
diff --git a/app-packages/hbase-win/package/scripts/params.py b/app-packages/hbase-win/package/scripts/params.py
new file mode 100644
index 0000000..5a54e25
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/params.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from functions import calc_xmn_from_xms
+from resource_management import *
+import status_params
+
+# server configurations
+config = Script.get_config()
+
+java64_home = config['hostLevelParams']['java_home']
+hbase_root = config['configurations']['global']['app_root']
+hbase_instance_name = config['configurations']['global']['hbase_instance_name']
+hbase_user = status_params.hbase_user
+user_group = config['configurations']['global']['user_group']
+
+pid_dir = status_params.pid_dir
+tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
+local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'],
+ config['configurations']['hbase-site'])
+conf_dir = format("{hbase_root}/conf")
+log_dir = config['configurations']['global']['app_log_dir']
+input_conf_files_dir = config['configurations']['global']['app_input_conf_dir']
+
+hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']
+
+"""
+Read various ports
+"""
+rest_port = default("/configurations/global/hbase_rest_port", 1700)
+thrift_port = default("/configurations/global/hbase_thrift_port", 9090)
+thrift2_port = default("/configurations/global/hbase_thrift2_port", 9091)
+thrift_info_port = default("/configurations/global/hbase_info_thrift_port", 9095)
+thrift2_info_port = default("/configurations/global/hbase_info_thrift2_port", 9096)
+
+"""
+Compute or read various heap sizes
+"""
+master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize']
+regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize']
+regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max']
+regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio']
+regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
+
+restserver_heapsize = default("/configurations/hbase-env/hbase_restserver_heapsize", "512m")
+thriftserver_heapsize = default("/configurations/hbase-env/hbase_thriftserver_heapsize", "512m")
+thrift2server_heapsize = default("/configurations/hbase-env/hbase_thrift2server_heapsize", "512m")
+
+hbase_env_sh_template = config['configurations']['hbase-env']['content']
+java_library_path = config['configurations']['global']['java_library_path']
+hbase_additional_cp = config['configurations']['global']['hbase_additional_cp']
+
+# log4j.properties
+if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])):
+ log4j_props = config['configurations']['hbase-log4j']['content']
+else:
+ log4j_props = None
\ No newline at end of file
diff --git a/app-packages/hbase-win/package/scripts/status_params.py b/app-packages/hbase-win/package/scripts/status_params.py
new file mode 100644
index 0000000..c18cbb9
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/status_params.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+pid_dir = config['configurations']['global']['app_pid_dir']
+hbase_user = config['configurations']['global']['app_user']
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java b/app-packages/hbase-win/package/templates/hbase_client_jaas.conf.j2
similarity index 83%
rename from slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java
rename to app-packages/hbase-win/package/templates/hbase_client_jaas.conf.j2
index a286ba4..bb4279c 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java
+++ b/app-packages/hbase-win/package/templates/hbase_client_jaas.conf.j2
@@ -15,11 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.slider.core.registry.info;
-
-public class CommonRegistryConstants {
-
- public static final String WEB_UI = "org.apache.http.UI";
-
-}
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=false
+useTicketCache=true;
+};
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java b/app-packages/hbase-win/package/templates/hbase_master_jaas.conf.j2
similarity index 80%
copy from slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java
copy to app-packages/hbase-win/package/templates/hbase_master_jaas.conf.j2
index a286ba4..91ce3ef 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java
+++ b/app-packages/hbase-win/package/templates/hbase_master_jaas.conf.j2
@@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.slider.core.registry.info;
-
-public class CommonRegistryConstants {
-
- public static final String WEB_UI = "org.apache.http.UI";
-
-}
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{master_keytab_path}}"
+principal="{{master_jaas_princ}}";
+};
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java b/app-packages/hbase-win/package/templates/hbase_regionserver_jaas.conf.j2
similarity index 79%
copy from slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java
copy to app-packages/hbase-win/package/templates/hbase_regionserver_jaas.conf.j2
index a286ba4..2a9b9f3 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java
+++ b/app-packages/hbase-win/package/templates/hbase_regionserver_jaas.conf.j2
@@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.slider.core.registry.info;
-
-public class CommonRegistryConstants {
-
- public static final String WEB_UI = "org.apache.http.UI";
-
-}
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{regionserver_keytab_path}}"
+principal="{{regionserver_jaas_princ}}";
+};
diff --git a/app-packages/hbase-win/pom.xml b/app-packages/hbase-win/pom.xml
new file mode 100644
index 0000000..55c4c53
--- /dev/null
+++ b/app-packages/hbase-win/pom.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.60.0-incubating</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-hbase-app-win-package</artifactId>
+ <packaging>pom</packaging>
+ <name>Slider HBase App Package for Windows</name>
+ <description>Slider HBase App Package for Windows</description>
+ <version>${pkg.version}</version>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ <app.package.name>${project.artifactId}-${project.version}</app.package.name>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>hbase-app-package-win</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}"/>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/hbase.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ </build>
+
+ <dependencies>
+ </dependencies>
+
+</project>
diff --git a/app-packages/hbase/resources.json b/app-packages/hbase-win/resources-default.json
similarity index 69%
rename from app-packages/hbase/resources.json
rename to app-packages/hbase-win/resources-default.json
index d2fdbd8..93dc17c 100644
--- a/app-packages/hbase/resources.json
+++ b/app-packages/hbase-win/resources-default.json
@@ -3,34 +3,37 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": ""
},
"components": {
"HBASE_MASTER": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"slider-appmaster": {
+ "yarn.memory": "1024"
},
"HBASE_REGIONSERVER": {
"yarn.role.priority": "2",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"HBASE_REST": {
"yarn.role.priority": "3",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"HBASE_THRIFT": {
"yarn.role.priority": "4",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"HBASE_THRIFT2": {
"yarn.role.priority": "5",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.component.instances": "0",
+ "yarn.memory": "512"
}
}
}
diff --git a/app-packages/hbase-win/src/assembly/hbase.xml b/app-packages/hbase-win/src/assembly/hbase.xml
new file mode 100644
index 0000000..a94c827
--- /dev/null
+++ b/app-packages/hbase-win/src/assembly/hbase.xml
@@ -0,0 +1,68 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>hbase_v${hbase.version}</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>appConfig-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>metainfo.xml</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>pom.xml</exclude>
+ <exclude>src/**</exclude>
+ <exclude>target/**</exclude>
+ <exclude>appConfig-default.json</exclude>
+ <exclude>metainfo.xml</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ </fileSets>
+</assembly>
diff --git a/app-packages/hbase/README.md b/app-packages/hbase/README.md
new file mode 100644
index 0000000..2d52fc9
--- /dev/null
+++ b/app-packages/hbase/README.md
@@ -0,0 +1,84 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Create Slider App Package for HBase
+
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
+
+OPTION-I: Use a downloaded tarball
+OPTION-II: Use the tarball from the local repository
+
+## OPTION - I
+
+To create the app package you will need the HBase tarball and invoke mvn command
+with appropriate parameters.
+
+Command:
+
+ mvn clean package -Phbase-app-package -Dpkg.version=<version>
+ -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
+
+Example:
+
+ mvn clean package -Phbase-app-package -Dpkg.version=0.98.5-hadoop2
+ -Dpkg.name=hbase-0.98.5-hadoop2-bin.tar.gz
+ -Dpkg.src=/Users/user1/Downloads/0.98.5-hadoop2
+
+App package can be found in
+ app-packages/hbase/target/slider-hbase-app-package-${pkg.version}.zip
+
+## OPTION - II
+
+You need the HBase version available on local maven repo to create the Slider App Package for HBase.
+
+Download the tarball for HBase:
+ e.g. path to tarball `~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz`
+
+The version of HBase used for the app package can be adjusted by adding a
+flag such as
+
+ -Dhbase.version=0.98.3-hadoop2
+
+Use the following command to install HBase tarball locally (under local workspace of HBase repo):
+
+ mvn install:install-file -Dfile=<path-to-tarball> -DgroupId=org.apache.hbase -DartifactId=hbase -Dversion=0.98.3-hadoop2 -Dclassifier=bin -Dpackaging=tar.gz
+
+You may need to copy the hbase tarball to the following location if the above step doesn't publish the tarball:
+
+ ~/.m2/repository/org/apache/hbase/hbase/0.98.3-hadoop2/
+
+After HBase tarball is published locally in maven repository, you can use the following command:
+
+ mvn clean package -DskipTests -Phbase-app-package
+
+App package can be found in
+
+ app-packages/hbase/target/apache-slider-hbase-${hbase.version}-app-package-${slider.version}.zip
+
+If an HBase version older than 0.98.3 is desired, it must be installed in the local maven repo.
+
+A less descriptive file name can be specified with
+
+ -Dapp.package.name=HBase_98dot3 which would create a file HBase_98dot3.zip.
+
+## Verifying the content
+
+Verify the content using
+
+ zip -Tv apache-slider-hbase-*.zip
diff --git a/app-packages/hbase/README.txt b/app-packages/hbase/README.txt
deleted file mode 100644
index 1d5c4bb..0000000
--- a/app-packages/hbase/README.txt
+++ /dev/null
@@ -1,75 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-Create Slider App Package for HBase
-
-While appConfig.json and resources.json are not required for the package they
-work well as the default configuration for Slider apps. So it is advisable that
-when you create an application package for Slider, include sample/default
-resources.json and appConfig.json for a minimal Yarn cluster.
-
-OPTION-I: Use mvn command
-OPTION-II: Manual
-
-****** OPTION - I (use mvn command) **
-You need the HBase version available on local maven repo to create the Slider App Package for HBase.
-
-Download the tarball for HBase:
- e.g. path to tarball ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz
-
-The version of HBase used for the app package can be adjusted by adding a
-flag such as
- -Dhbase.version=0.98.3-hadoop2
-
-Use the following command to install HBase tarball locally (under local workspace of HBase repo):
- mvn install:install-file -Dfile=<path-to-tarball> -DgroupId=org.apache.hbase -DartifactId=hbase -Dversion=0.98.3-hadoop2 -Dclassifier=bin -Dpackaging=tar.gz
-
-You may need to copy the hbase tarball to the following location if the above step doesn't publish the tarball:
-~/.m2/repository/org/apache/hbase/hbase/0.98.3-hadoop2/
-
-After HBase tarball is published locally in maven repository, you can use the following command:
- mvn clean package -DskipTests -Phbase-app-package
-
-App package can be found in
- app-packages/hbase/target/apache-slider-hbase-${hbase.version}-app-package-${slider.version}.zip
-
-Verify the content using
- zip -Tv apache-slider-hbase-*.zip
-
-If an HBase version older than 0.98.3 is desired, it must be installed in the local maven repo.
-
-A less descriptive file name can be specified with
- -Dapp.package.name=HBase_98dot3 which would create a file HBase_98dot3.zip.
-
-****** OPTION - II (manual) **
-The Slider App Package for HBase can also be created manually.
-
-Download the tarball for HBase:
- e.g. path to tarball ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz
-
-Copy the hbase tarball to package/files
- cp ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz package/files
-
-Edit appConfig.json/metainfo.xml
- Replace 4 occurrences of "${hbase.version}" with the hbase version values such as "0.98.3-hadoop2"
- Replace 1 occurrence of "${app.package.name}" with the desired app package name, e.g. "hbase-v098"
-
-Create a zip package at the root of the package (<slider enlistment>/app-packages/hbase/)
- zip -r hbase-v098.zip .
-
-Verify the content using
- zip -Tv hbase-v098.zip
diff --git a/app-packages/hbase/appConfig-default.json b/app-packages/hbase/appConfig-default.json
new file mode 100644
index 0000000..52587e5
--- /dev/null
+++ b/app-packages/hbase/appConfig-default.json
@@ -0,0 +1,46 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/HBASE/slider-hbase-app-package-${pkg.version}.zip",
+ "create.default.zookeeper.node": "true",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
+ "system_configs": "core-site",
+
+ "site.global.app_user": "yarn",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
+
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_port": "8667",
+ "site.global.ganglia_server_id": "Application1",
+ "site.global.ganglia_enabled":"true",
+
+ "site.global.hbase_instance_name": "instancename",
+ "site.global.hbase_root_password": "secret",
+ "site.global.user_group": "hadoop",
+ "site.global.monitor_protocol": "http",
+ "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
+ "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
+ "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+
+ "site.hbase-env.hbase_master_heapsize": "1024m",
+ "site.hbase-env.hbase_regionserver_heapsize": "1024m",
+
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
+ "site.hbase-site.hbase.superuser": "${USER_NAME}",
+ "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+ "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+ "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
+ "site.hbase-site.zookeeper.znode.parent": "${DEFAULT_ZK_PATH}",
+ "site.hbase-site.hbase.regionserver.info.port": "0",
+ "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
+ "site.hbase-site.hbase.regionserver.port": "0",
+ "site.hbase-site.hbase.master.port": "0"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "1024M"
+ }
+ }
+}
diff --git a/app-packages/hbase/appConfig-secured-default.json b/app-packages/hbase/appConfig-secured-default.json
new file mode 100644
index 0000000..2a2b08f
--- /dev/null
+++ b/app-packages/hbase/appConfig-secured-default.json
@@ -0,0 +1,63 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/HBASE/slider-hbase-app-package-${pkg.version}.zip",
+ "create.default.zookeeper.node": "true",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
+ "system_configs": "core-site,hdfs-site",
+
+ "site.global.app_user": "${USER_NAME}",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
+
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_port": "8667",
+ "site.global.ganglia_server_id": "Application1",
+ "site.global.ganglia_enabled":"true",
+
+ "site.global.hbase_instance_name": "instancename",
+ "site.global.hbase_root_password": "secret",
+ "site.global.user_group": "hadoop",
+ "site.global.monitor_protocol": "http",
+ "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
+ "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
+ "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+
+ "site.hbase-env.hbase_master_heapsize": "1024m",
+ "site.hbase-env.hbase_regionserver_heapsize": "1024m",
+
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
+ "site.hbase-site.hbase.superuser": "${USER_NAME}",
+ "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+ "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+ "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
+ "site.hbase-site.zookeeper.znode.parent": "${DEFAULT_ZK_PATH}",
+ "site.hbase-site.hbase.regionserver.info.port": "0",
+ "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
+ "site.hbase-site.hbase.regionserver.port": "0",
+ "site.hbase-site.hbase.master.port": "0",
+
+ "site.hbase-site.hbase.security.authentication": "kerberos",
+ "site.hbase-site.hbase.security.authorization": "true",
+ "site.hbase-site.hbase.security.access.early_out": "true",
+ "site.hbase-site.hbase.coprocessor.master.classes": "org.apache.hadoop.hbase.security.access.AccessController",
+ "site.hbase-site.hbase.coprocessor.region.classes": "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.AccessController",
+ "site.hbase-site.hbase.regionserver.kerberos.principal": "${USER_NAME}/_HOST@EXAMPLE.COM",
+ "site.hbase-site.hbase.regionserver.keytab.file": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.HBASE.service.keytab",
+ "site.hbase-site.hbase.master.kerberos.principal": "${USER_NAME}/_HOST@EXAMPLE.COM",
+ "site.hbase-site.hbase.master.keytab.file": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.HBASE.service.keytab",
+ "site.hbase-site.hbase.rest.kerberos.principal": "${USER_NAME}/_HOST@EXAMPLE.COM",
+ "site.hbase-site.hbase.rest.keytab.file": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.HBASE.service.keytab",
+ "site.hbase-site.hbase.thrift.kerberos.principal": "${USER_NAME}/_HOST@EXAMPLE.COM",
+ "site.hbase-site.hbase.thrift.keytab.file": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.HBASE.service.keytab"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "1024M",
+ "slider.hdfs.keytab.dir": ".slider/keytabs/hbase",
+ "slider.am.login.keytab.name": "${USER_NAME}.headless.keytab",
+ "slider.keytab.principal.name": "${USER_NAME}"
+ }
+ }
+}
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
deleted file mode 100644
index d00ae6d..0000000
--- a/app-packages/hbase/appConfig.json
+++ /dev/null
@@ -1,70 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": "${app.package.name}.zip",
- "create.default.zookeeper.node": "true",
- "config_types": "core-site,hdfs-site,hbase-site",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/hbase-${hbase.version}-bin.tar.gz",
- "site.global.app_user": "yarn",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.global.hbase_master_heapsize": "1024m",
- "site.global.hbase_regionserver_heapsize": "1024m",
- "site.global.hbase_instance_name": "instancename",
- "site.global.hbase_root_password": "secret",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.global.monitor_protocol": "http",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
- "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
- "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
- "site.hbase-site.hbase.hstore.flush.retries.number": "120",
- "site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
- "site.hbase-site.hbase.hstore.compactionThreshold": "3",
- "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data",
- "site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging",
- "site.hbase-site.hbase.regionserver.handler.count": "60",
- "site.hbase-site.hbase.regionserver.global.memstore.lowerLimit": "0.38",
- "site.hbase-site.hbase.hregion.memstore.block.multiplier": "2",
- "site.hbase-site.hbase.hregion.memstore.flush.size": "134217728",
- "site.hbase-site.hbase.superuser": "yarn",
- "site.hbase-site.hbase.zookeeper.property.clientPort": "2181",
- "site.hbase-site.hbase.regionserver.global.memstore.upperLimit": "0.4",
- "site.hbase-site.zookeeper.session.timeout": "30000",
- "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
- "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
- "site.hbase-site.hbase.hregion.max.filesize": "10737418240",
- "site.hbase-site.hfile.block.cache.size": "0.40",
- "site.hbase-site.hbase.security.authentication": "simple",
- "site.hbase-site.hbase.defaults.for.version.skip": "true",
- "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
- "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
- "site.hbase-site.hbase.hstore.blockingStoreFiles": "10",
- "site.hbase-site.hbase.hregion.majorcompaction": "86400000",
- "site.hbase-site.hbase.security.authorization": "false",
- "site.hbase-site.hbase.cluster.distributed": "true",
- "site.hbase-site.hbase.hregion.memstore.mslab.enabled": "true",
- "site.hbase-site.hbase.client.scanner.caching": "100",
- "site.hbase-site.hbase.zookeeper.useMulti": "true",
- "site.hbase-site.hbase.regionserver.info.port": "0",
- "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
- "site.hbase-site.hbase.regionserver.port": "0"
- },
- "components": {
- "HBASE_MASTER": {
- },
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- },
- "HBASE_REGIONSERVER": {
- }
- }
-}
diff --git a/app-packages/hbase/configuration/global.xml b/app-packages/hbase/configuration/global.xml
deleted file mode 100644
index b2c57bd..0000000
--- a/app-packages/hbase/configuration/global.xml
+++ /dev/null
@@ -1,160 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <property>
- <name>hbasemaster_host</name>
- <value></value>
- <description>HBase Master Host.</description>
- </property>
- <property>
- <name>regionserver_hosts</name>
- <value></value>
- <description>Region Server Hosts</description>
- </property>
- <property>
- <name>hbase_log_dir</name>
- <value>/var/log/hbase</value>
- <description>Log Directories for HBase.</description>
- </property>
- <property>
- <name>hbase_pid_dir</name>
- <value>/var/run/hbase</value>
- <description>Log Directories for HBase.</description>
- </property>
- <property>
- <name>hbase_log_dir</name>
- <value>/var/log/hbase</value>
- <description>Log Directories for HBase.</description>
- </property>
- <property>
- <name>hbase_regionserver_heapsize</name>
- <value>1024</value>
- <description>Log Directories for HBase.</description>
- </property>
- <property>
- <name>hbase_master_heapsize</name>
- <value>1024</value>
- <description>HBase Master Heap Size</description>
- </property>
- <property>
- <name>hstore_compactionthreshold</name>
- <value>3</value>
- <description>HBase HStore compaction threshold.</description>
- </property>
- <property>
- <name>hfile_blockcache_size</name>
- <value>0.40</value>
- <description>HFile block cache size.</description>
- </property>
- <property>
- <name>hstorefile_maxsize</name>
- <value>10737418240</value>
- <description>Maximum HStoreFile Size</description>
- </property>
- <property>
- <name>regionserver_handlers</name>
- <value>60</value>
- <description>HBase RegionServer Handler</description>
- </property>
- <property>
- <name>hregion_majorcompaction</name>
- <value>604800000</value>
- <description>The time between major compactions of all HStoreFiles in a region. Set to 0 to disable automated major compactions.</description>
- </property>
- <property>
- <name>hregion_blockmultiplier</name>
- <value>2</value>
- <description>HBase Region Block Multiplier</description>
- </property>
- <property>
- <name>hregion_memstoreflushsize</name>
- <value></value>
- <description>HBase Region MemStore Flush Size.</description>
- </property>
- <property>
- <name>client_scannercaching</name>
- <value>100</value>
- <description>Base Client Scanner Caching</description>
- </property>
- <property>
- <name>zookeeper_sessiontimeout</name>
- <value>30000</value>
- <description>ZooKeeper Session Timeout</description>
- </property>
- <property>
- <name>hfile_max_keyvalue_size</name>
- <value>10485760</value>
- <description>HBase Client Maximum key-value Size</description>
- </property>
- <property>
- <name>hbase_hdfs_root_dir</name>
- <value>/apps/hbase/data</value>
- <description>HBase Relative Path to HDFS.</description>
- </property>
- <property>
- <name>hbase_conf_dir</name>
- <value>/etc/hbase</value>
- <description>Config Directory for HBase.</description>
- </property>
- <property>
- <name>hdfs_enable_shortcircuit_read</name>
- <value>true</value>
- <description>HDFS Short Circuit Read</description>
- </property>
- <property>
- <name>hdfs_support_append</name>
- <value>true</value>
- <description>HDFS append support</description>
- </property>
- <property>
- <name>hstore_blockingstorefiles</name>
- <value>10</value>
- <description>HStore blocking storefiles.</description>
- </property>
- <property>
- <name>regionserver_memstore_lab</name>
- <value>true</value>
- <description>Region Server memstore.</description>
- </property>
- <property>
- <name>regionserver_memstore_lowerlimit</name>
- <value>0.38</value>
- <description>Region Server memstore lower limit.</description>
- </property>
- <property>
- <name>regionserver_memstore_upperlimit</name>
- <value>0.4</value>
- <description>Region Server memstore upper limit.</description>
- </property>
- <property>
- <name>hbase_conf_dir</name>
- <value>/etc/hbase</value>
- <description>HBase conf dir.</description>
- </property>
- <property>
- <name>hbase_user</name>
- <value>hbase</value>
- <description>HBase User Name.</description>
- </property>
-
-</configuration>
diff --git a/app-packages/hbase/package/templates/hbase-env.sh.j2 b/app-packages/hbase/configuration/hbase-env.xml
similarity index 62%
rename from app-packages/hbase/package/templates/hbase-env.sh.j2
rename to app-packages/hbase/configuration/hbase-env.xml
index 4aa79ad..554c3c5 100644
--- a/app-packages/hbase/package/templates/hbase-env.sh.j2
+++ b/app-packages/hbase/configuration/hbase-env.xml
@@ -1,20 +1,52 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>hbase_regionserver_heapsize</name>
+ <value>1024</value>
+ <description>HBase RegionServer Heap Size.</description>
+ </property>
+ <property>
+ <name>hbase_regionserver_xmn_max</name>
+ <value>512</value>
+ <description>HBase RegionServer maximum value for minimum heap size.</description>
+ </property>
+ <property>
+ <name>hbase_regionserver_xmn_ratio</name>
+ <value>0.2</value>
+ <description>HBase RegionServer minimum heap size is calculated as a percentage of max heap size.</description>
+ </property>
+ <property>
+ <name>hbase_master_heapsize</name>
+ <value>1024</value>
+ <description>HBase Master Heap Size</description>
+ </property>
+
+ <!-- hbase-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for hbase-env.sh file</description>
+ <value>
# Set environment variables here.
# The java implementation to use. Java 1.6 required.
@@ -79,3 +111,7 @@
export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}"
export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}"
{% endif %}
+ </value>
+ </property>
+
+</configuration>
diff --git a/app-packages/hbase/configuration/hbase-log4j.xml b/app-packages/hbase/configuration/hbase-log4j.xml
index 3bbc549..d488c4e 100644
--- a/app-packages/hbase/configuration/hbase-log4j.xml
+++ b/app-packages/hbase/configuration/hbase-log4j.xml
@@ -24,6 +24,7 @@
<property>
<name>content</name>
+ <description>Custom log4j.properties</description>
<value>
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
diff --git a/app-packages/hbase/configuration/hbase-site.xml b/app-packages/hbase/configuration/hbase-site.xml
index cf9416e..a9711d3 100644
--- a/app-packages/hbase/configuration/hbase-site.xml
+++ b/app-packages/hbase/configuration/hbase-site.xml
@@ -43,6 +43,11 @@
</description>
</property>
<property>
+ <name>hbase.master.port</name>
+ <value>60000</value>
+ <description>The port the HBase Master should bind to.</description>
+ </property>
+ <property>
<name>hbase.tmp.dir</name>
<value>/hadoop/hbase</value>
<description>Temporary directory on the local filesystem.
@@ -59,18 +64,18 @@
</property>
<property>
<name>hbase.master.info.bindAddress</name>
- <value></value>
+ <value>0.0.0.0</value>
<description>The bind address for the HBase Master web UI
</description>
</property>
<property>
<name>hbase.master.info.port</name>
- <value></value>
+ <value>60010</value>
<description>The port for the HBase Master web UI.</description>
</property>
<property>
<name>hbase.regionserver.info.port</name>
- <value></value>
+ <value>60030</value>
<description>The port for the HBase RegionServer web UI.</description>
</property>
<property>
@@ -222,14 +227,14 @@
values, included here for documentation purposes -->
<property>
<name>hbase.master.keytab.file</name>
- <value></value>
+ <value>/etc/security/keytabs/hbase.service.keytab</value>
<description>Full path to the kerberos keytab file to use for logging in
the configured HMaster server principal.
</description>
</property>
<property>
<name>hbase.master.kerberos.principal</name>
- <value></value>
+ <value>hbase/_HOST@EXAMPLE.COM</value>
<description>Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name
that should be used to run the HMaster process. The principal name should
be in the form: user/hostname@DOMAIN. If "_HOST" is used as the hostname
@@ -239,14 +244,14 @@
</property>
<property>
<name>hbase.regionserver.keytab.file</name>
- <value></value>
+ <value>/etc/security/keytabs/hbase.service.keytab</value>
<description>Full path to the kerberos keytab file to use for logging in
the configured HRegionServer server principal.
</description>
</property>
<property>
<name>hbase.regionserver.kerberos.principal</name>
- <value></value>
+ <value>hbase/_HOST@EXAMPLE.COM</value>
<description>Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name
that should be used to run the HRegionServer process. The principal name
should be in the form: user/hostname@DOMAIN. If "_HOST" is used as the
diff --git a/app-packages/hbase/ganglia_metrics.json b/app-packages/hbase/ganglia_metrics.json
new file mode 100644
index 0000000..da73d48
--- /dev/null
+++ b/app-packages/hbase/ganglia_metrics.json
@@ -0,0 +1,38 @@
+{
+ "Component": {
+ "HBASE_REGIONSERVER": {
+ "readRequestsCount": {
+ "metric": "regionserver.Server.readRequestCount",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "regions": {
+ "metric": "regionserver.Server.regionCount",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "flushQueueSize": {
+ "metric": "regionserver.Server.flushQueueLength",
+ "pointInTime": false,
+ "temporal": true
+ }
+ },
+ "HBASE_MASTER": {
+ "cluster_requests": {
+ "metric": "master.Server.clusterRequests",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "splitTime_avg_time": {
+ "metric": "master.FileSystem.HlogSplitTime_mean",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "splitSize_avg_time": {
+ "metric": "master.FileSystem.HlogSplitSize_mean",
+ "pointInTime": false,
+ "temporal": true
+ }
+ }
+ }
+}
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index aae048d..d5e07a7 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -25,9 +25,10 @@
1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner.
2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance.
</comment>
- <version>${hbase.version}</version>
+ <version>${pkg.version}</version>
<type>YARN-APP</type>
<minHadoopVersion>2.1.0</minHadoopVersion>
+ <exportedConfigs>hbase-site</exportedConfigs>
<exportGroups>
<exportGroup>
<name>QuickLinks</name>
@@ -53,11 +54,11 @@
<value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
</export>
<export>
- <name>app.metrics</name>
+ <name>org.apache.slider.metrics</name>
<value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
</export>
<export>
- <name>app.ganglia</name>
+ <name>org.apache.slider.metrics.ui</name>
<value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
</export>
</exports>
@@ -80,15 +81,14 @@
<name>HBASE_MASTER</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>2</maxInstanceCount>
- <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-app.metrics,QuickLinks-app.ganglia</appExports>
+ <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics,QuickLinks-org.apache.slider.metrics.ui</appExports>
<componentExports>
<componentExport>
- <name>app.jmx</name>
+ <name>org.apache.slider.jmx</name>
<value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
</componentExport>
<componentExport>
- <name>app.monitor</name>
+ <name>org.apache.slider.monitor</name>
<value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
</componentExport>
</componentExports>
@@ -112,7 +112,6 @@
<component>
<name>HBASE_REST</name>
<category>MASTER</category>
- <minInstanceCount>0</minInstanceCount>
<appExports>QuickLinks-org.apache.slider.hbase.rest</appExports>
<commandScript>
<script>scripts/hbase_rest.py</script>
@@ -123,7 +122,6 @@
<component>
<name>HBASE_THRIFT</name>
<category>MASTER</category>
- <minInstanceCount>0</minInstanceCount>
<appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports>
<commandScript>
<script>scripts/hbase_thrift.py</script>
@@ -145,7 +143,6 @@
<component>
<name>HBASE_CLIENT</name>
<category>CLIENT</category>
- <minInstanceCount>0</minInstanceCount>
<commandScript>
<script>scripts/hbase_client.py</script>
<scriptType>PYTHON</scriptType>
@@ -159,11 +156,34 @@
<packages>
<package>
<type>tarball</type>
- <name>files/hbase-${hbase.version}-bin.tar.gz</name>
+ <name>files/hbase-${pkg.version}.tar.gz</name>
</package>
</packages>
</osSpecific>
</osSpecifics>
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>hbase-site.xml</fileName>
+ <dictionaryName>hbase-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>hbase-env.sh</fileName>
+ <dictionaryName>hbase-env</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>hbase-log4j.properties</fileName>
+ <dictionaryName>hbase-log4j</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>hbase-policy.xml</fileName>
+ <dictionaryName>hbase-policy</dictionaryName>
+ </configFile>
+ </configFiles>
+
</application>
</metainfo>
diff --git a/app-packages/hbase/package/scripts/hbase.py b/app-packages/hbase/package/scripts/hbase.py
index ed6ec51..35897df 100644
--- a/app-packages/hbase/package/scripts/hbase.py
+++ b/app-packages/hbase/package/scripts/hbase.py
@@ -21,11 +21,13 @@
from resource_management import *
import sys
+import shutil
def hbase(name=None # 'master' or 'regionserver' or 'client'
):
import params
+ """
if name in ["master","regionserver"]:
params.HdfsDirectory(params.hbase_hdfs_root_dir,
action="create_delayed"
@@ -35,10 +37,13 @@
mode=0711
)
params.HdfsDirectory(None, action="create")
+ """
+
Directory( params.conf_dir,
owner = params.hbase_user,
group = params.user_group,
- recursive = True
+ recursive = True,
+ content = params.input_conf_files_dir
)
Directory (params.tmp_dir,
@@ -60,18 +65,13 @@
group = params.user_group
)
- XmlConfig( "hdfs-site.xml",
- conf_dir = params.conf_dir,
- configurations = params.config['configurations']['hdfs-site'],
- owner = params.hbase_user,
- group = params.user_group
- )
-
+
if 'hbase-policy' in params.config['configurations']:
XmlConfig( "hbase-policy.xml",
- configurations = params.config['configurations']['hbase-policy'],
- owner = params.hbase_user,
- group = params.user_group
+ conf_dir = params.conf_dir,
+ configurations = params.config['configurations']['hbase-policy'],
+ owner = params.hbase_user,
+ group = params.user_group
)
# Manually overriding ownership of file installed by hadoop package
else:
@@ -80,8 +80,10 @@
group = params.user_group
)
- hbase_TemplateConfig( 'hbase-env.sh')
-
+ File(format("{conf_dir}/hbase-env.sh"),
+ owner = params.hbase_user,
+ content=InlineTemplate(params.hbase_env_sh_template)
+ )
hbase_TemplateConfig( params.metric_prop_file_name,
tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS'
)
@@ -107,7 +109,7 @@
owner=params.hbase_user,
content=params.log4j_props
)
- elif (os.path.exists(format("{params.conf_dir}/log4j.properties"))):
+ elif (os.path.exists(format("{conf_dir}/log4j.properties"))):
File(format("{params.conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
diff --git a/app-packages/hbase/package/scripts/hbase_service.py b/app-packages/hbase/package/scripts/hbase_service.py
index 96add84..db663b8 100644
--- a/app-packages/hbase/package/scripts/hbase_service.py
+++ b/app-packages/hbase/package/scripts/hbase_service.py
@@ -34,7 +34,7 @@
no_op_test = None
if action == 'start':
- daemon_cmd = format("{cmd} start {role}")
+ daemon_cmd = format("env HBASE_IDENT_STRING={hbase_user} {cmd} start {role}")
if name == 'rest':
daemon_cmd = format("{daemon_cmd} -p {rest_port}")
elif name == 'thrift':
@@ -43,7 +43,7 @@
daemon_cmd = format("{daemon_cmd} -p {thrift2_port}")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
elif action == 'stop':
- daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
+ daemon_cmd = format("env HBASE_IDENT_STRING={hbase_user} {cmd} stop {role} && rm -f {pid_file}")
if daemon_cmd is not None:
Execute ( daemon_cmd,
diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py
index 1f25f68..33e2bbf 100644
--- a/app-packages/hbase/package/scripts/params.py
+++ b/app-packages/hbase/package/scripts/params.py
@@ -41,14 +41,17 @@
java64_home = config['hostLevelParams']['java_home']
log_dir = config['configurations']['global']['app_log_dir']
-master_heapsize = config['configurations']['global']['hbase_master_heapsize']
+master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize']
-regionserver_heapsize = config['configurations']['global']['hbase_regionserver_heapsize']
-regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
+regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize']
+regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max']
+regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio']
+regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
pid_dir = status_params.pid_dir
tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'], config['configurations']['hbase-site'])
+input_conf_files_dir = config['configurations']['global']['app_input_conf_dir']
client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf_dir}/hbase_client_jaas.conf"))
master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf"))
@@ -62,20 +65,10 @@
thrift2_port = config['configurations']['global']['hbase_thrift2_port']
if security_enabled:
-
- _use_hostname_in_principal = default('instance_name', True)
- _master_primary_name = config['configurations']['global']['hbase_master_primary_name']
_hostname_lowercase = config['hostname'].lower()
- _kerberos_domain = config['configurations']['global']['kerberos_domain']
- _master_principal_name = config['configurations']['global']['hbase_master_principal_name']
- _regionserver_primary_name = config['configurations']['global']['hbase_regionserver_primary_name']
-
- if _use_hostname_in_principal:
- master_jaas_princ = format("{_master_primary_name}/{_hostname_lowercase}@{_kerberos_domain}")
- regionserver_jaas_princ = format("{_regionserver_primary_name}/{_hostname_lowercase}@{_kerberos_domain}")
- else:
- master_jaas_princ = format("{_master_principal_name}@{_kerberos_domain}")
- regionserver_jaas_princ = format("{_regionserver_primary_name}@{_kerberos_domain}")
+ master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
+ regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
+
master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
@@ -91,6 +84,7 @@
else:
log4j_props = None
+hbase_env_sh_template = config['configurations']['hbase-env']['content']
hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']
hbase_staging_dir = config['configurations']['hbase-site']['hbase.stagingdir']
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 7dede6c..6caef05 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -1,5 +1,6 @@
<?xml version="1.0"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -19,17 +20,17 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.41.0-incubating-SNAPSHOT</version>
+ <version>0.60.0-incubating</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>slider-hbase-app-package</artifactId>
- <packaging>jar</packaging>
+ <packaging>pom</packaging>
<name>Slider HBase App Package</name>
<description>Slider HBase App Package</description>
<properties>
<work.dir>package-tmp</work.dir>
- <app.package.name>apache-slider-hbase-${hbase.version}-app-package-${project.version}</app.package.name>
+ <app.package.name>${project.artifactId}-${project.version}</app.package.name>
</properties>
<profiles>
@@ -39,6 +40,63 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>${maven-antrun-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}"/>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/hbase.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>hbase-app-package-it</id>
+ <properties>
+ <pkg.version>${hbase.version}</pkg.version>
+ <pkg.src>${project.build.directory}/${work.dir}</pkg.src>
+ <pkg.name>hbase-${hbase.version}-bin.tar.gz</pkg.name>
+ </properties>
+ <build>
+ <resources>
+ <resource>
+ <directory>src/test/resources</directory>
+ <filtering>true</filtering>
+ <targetPath>${project.build.directory}/test-config</targetPath>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptor>src/assembly/hbase.xml</descriptor>
@@ -95,151 +153,143 @@
<java.awt.headless>true</java.awt.headless>
<!-- this property must be supplied-->
<slider.conf.dir>${slider.conf.dir}</slider.conf.dir>
- <slider.bin.dir>../../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
+ <slider.bin.dir>../../slider-assembly/target/slider-${project.version}-all/slider-${project.version}
+ </slider.bin.dir>
<test.app.pkg.dir>target</test.app.pkg.dir>
<test.app.pkg.file>${app.package.name}.zip</test.app.pkg.file>
- <test.app.resource>target/test-config/resources.json</test.app.resource>
- <test.app.template>target/${app.package.name}/appConfig.json</test.app.template>
+ <test.app.resource>target/test-config/resources-default.json</test.app.resource>
+ <test.app.template>target/${app.package.name}/appConfig-default.json</test.app.template>
</systemPropertyVariables>
</configuration>
</plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>${groovy-eclipse-compiler.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-batch</artifactId>
+ <version>${groovy-eclipse-batch.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!-- can't figure out how to get the surefire plugin not to pick up the ITs, so skip it entirely -->
+ <skip>true</skip>
+ </configuration>
+ </plugin>
</plugins>
</build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>${hbase.version}</version>
+ <classifier>bin</classifier>
+ <type>tar.gz</type>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-protocol</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-funtest</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
</profile>
</profiles>
<build>
- <!-- resources are filtered for dynamic updates. This gets build info in-->
- <resources>
- <resource>
- <directory>src/test/resources</directory>
- <filtering>true</filtering>
- <targetPath>${project.build.directory}/test-config</targetPath>
- </resource>
- </resources>
-
- <plugins>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy-eclipse-compiler</artifactId>
- <version>${groovy-eclipse-compiler.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy-eclipse-batch</artifactId>
- <version>${groovy-eclipse-batch.version}</version>
- </dependency>
- </dependencies>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <!-- can't figure out how to get the surefire plugin not to pick up the ITs, so skip it entirely -->
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
</build>
<dependencies>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <version>${hbase.version}</version>
- <classifier>bin</classifier>
- <type>tar.gz</type>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-protocol</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-it</artifactId>
- <classifier>tests</classifier>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop-compat</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop2-compat</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.slider</groupId>
- <artifactId>slider-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.slider</groupId>
- <artifactId>slider-funtest</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy-all</artifactId>
- <scope>test</scope>
- </dependency>
</dependencies>
</project>
diff --git a/app-packages/hbase/resources.json b/app-packages/hbase/resources-default.json
similarity index 69%
copy from app-packages/hbase/resources.json
copy to app-packages/hbase/resources-default.json
index d2fdbd8..b1da1f7 100644
--- a/app-packages/hbase/resources.json
+++ b/app-packages/hbase/resources-default.json
@@ -3,34 +3,37 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": ""
},
"components": {
"HBASE_MASTER": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "1500"
},
"slider-appmaster": {
+ "yarn.memory": "1024"
},
"HBASE_REGIONSERVER": {
"yarn.role.priority": "2",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "1500"
},
"HBASE_REST": {
"yarn.role.priority": "3",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "556"
},
"HBASE_THRIFT": {
"yarn.role.priority": "4",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.component.instances": "0",
+ "yarn.memory": "556"
},
"HBASE_THRIFT2": {
"yarn.role.priority": "5",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "556"
}
}
}
diff --git a/app-packages/hbase/src/assembly/hbase.xml b/app-packages/hbase/src/assembly/hbase.xml
index ff1c395..a74304c 100644
--- a/app-packages/hbase/src/assembly/hbase.xml
+++ b/app-packages/hbase/src/assembly/hbase.xml
@@ -30,7 +30,13 @@
<files>
<file>
- <source>appConfig.json</source>
+ <source>appConfig-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>appConfig-secured-default.json</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
<fileMode>0755</fileMode>
@@ -41,6 +47,12 @@
<filtered>true</filtered>
<fileMode>0755</fileMode>
</file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
</files>
<fileSets>
@@ -51,22 +63,13 @@
<exclude>pom.xml</exclude>
<exclude>src/**</exclude>
<exclude>target/**</exclude>
- <exclude>appConfig.json</exclude>
+ <exclude>appConfig-default.json</exclude>
+ <exclude>appConfig-secured-default.json</exclude>
<exclude>metainfo.xml</exclude>
</excludes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
- <fileSet>
- <directory>${project.build.directory}/${work.dir}</directory>
- <outputDirectory>package/files</outputDirectory>
- <includes>
- <include>hbase-${hbase.version}-bin.tar.gz</include>
- </includes>
- <fileMode>0755</fileMode>
- <directoryMode>0755</directoryMode>
- </fileSet>
-
</fileSets>
</assembly>
diff --git a/app-packages/hbase/src/test/resources/appConfig_monitor_ssl.json b/app-packages/hbase/src/test/resources/appConfig_monitor_ssl.json
index 37d72d0..73b33ed 100644
--- a/app-packages/hbase/src/test/resources/appConfig_monitor_ssl.json
+++ b/app-packages/hbase/src/test/resources/appConfig_monitor_ssl.json
@@ -19,7 +19,6 @@
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
"site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
"site.global.monitor_protocol": "https",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_port": "8667",
diff --git a/app-packages/hbase/src/test/resources/resources.json b/app-packages/hbase/src/test/resources/resources-default.json
similarity index 84%
rename from app-packages/hbase/src/test/resources/resources.json
rename to app-packages/hbase/src/test/resources/resources-default.json
index e0ff26f..4fedf01 100644
--- a/app-packages/hbase/src/test/resources/resources.json
+++ b/app-packages/hbase/src/test/resources/resources-default.json
@@ -3,6 +3,8 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": ""
},
"components": {
"HBASE_MASTER": {
diff --git a/app-packages/memcached-win/README.txt b/app-packages/memcached-win/README.txt
index 4d93b91..84d2728 100644
--- a/app-packages/memcached-win/README.txt
+++ b/app-packages/memcached-win/README.txt
@@ -30,7 +30,6 @@
Verify the content using
unzip -l "$@" jmemcached-1.0.0.zip
-While appConfig.json and resources.json are not required for the package they work
-well as the default configuration for Slider apps. So its advisable that when you
-create an application package for Slider, include sample/default resources.json and
-appConfig.json for a minimal Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
diff --git a/app-packages/memcached-win/appConfig-default.json b/app-packages/memcached-win/appConfig-default.json
new file mode 100644
index 0000000..8a5ffd0
--- /dev/null
+++ b/app-packages/memcached-win/appConfig-default.json
@@ -0,0 +1,21 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/MEMCACHED/jmemcached-1.0.0.zip",
+ "java_home": "C:\\java",
+
+ "site.global.additional_cp": "C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
+ "site.global.xmx_val": "256m",
+ "site.global.xms_val": "128m",
+ "site.global.memory_val": "200M",
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{PER_CONTAINER}"
+
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
diff --git a/app-packages/memcached-win/appConfig.json b/app-packages/memcached-win/appConfig.json
deleted file mode 100644
index b76ecde..0000000
--- a/app-packages/memcached-win/appConfig.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": "/slider/jmemcached-1.0.0.zip",
- "java_home": "C:\\java",
-
- "site.global.app_user": "hadoop",
- "site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
- "site.global.pid_file": "${AGENT_WORK_ROOT}\\app\\run\\component.pid",
- "site.global.additional_cp": "C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
- "site.global.xmx_val": "256m",
- "site.global.xms_val": "128m",
- "site.global.memory_val": "200M",
- "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
-
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- },
- "MEMCACHED": {
- }
- }
-}
diff --git a/app-packages/memcached-win/metainfo.xml b/app-packages/memcached-win/metainfo.xml
index d056c0a..093001b 100644
--- a/app-packages/memcached-win/metainfo.xml
+++ b/app-packages/memcached-win/metainfo.xml
@@ -23,17 +23,23 @@
<comment>Memcache is a network accessible key/value storage system, often used as a distributed cache.</comment>
<version>1.0.0</version>
<exportedConfigs>None</exportedConfigs>
+ <exportGroups>
+ <exportGroup>
+ <name>Servers</name>
+ <exports>
+ <export>
+ <name>host_port</name>
+ <value>${MEMCACHED_HOST}:${site.global.listen_port}</value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
<components>
<component>
<name>MEMCACHED</name>
<category>MASTER</category>
- <exports>
- <export>
- <name>host_port</name>
- <value>${THIS_HOST}:${site.global.listen_port}</value>
- </export>
- </exports>
+ <compExports>Servers-host_port</compExports>
<commandScript>
<script>scripts/memcached.py</script>
<scriptType>PYTHON</scriptType>
diff --git a/app-packages/memcached-win/package/scripts/memcached.py b/app-packages/memcached-win/package/scripts/memcached.py
index bc9905d..c272e47 100644
--- a/app-packages/memcached-win/package/scripts/memcached.py
+++ b/app-packages/memcached-win/package/scripts/memcached.py
@@ -37,10 +37,10 @@
process_cmd = format("{java64_home}\\bin\\java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}\\*;{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val} --port={port}")
Execute(process_cmd,
- user=params.app_user,
logoutput=False,
wait_for_finish=False,
- pid_file=params.pid_file
+ pid_file=params.pid_file,
+ poll_after = 5
)
def stop(self, env):
@@ -50,8 +50,7 @@
def status(self, env):
import params
env.set_params(params)
- #Check process status need to be changed for Windows
- #check_process_status(params.pid_file)
+ check_process_status(params.pid_file)
if __name__ == "__main__":
Memcached().execute()
diff --git a/app-packages/memcached-win/package/scripts/params.py b/app-packages/memcached-win/package/scripts/params.py
index fab3714..056a3b9 100644
--- a/app-packages/memcached-win/package/scripts/params.py
+++ b/app-packages/memcached-win/package/scripts/params.py
@@ -25,8 +25,8 @@
app_root = config['configurations']['global']['app_root']
java64_home = config['hostLevelParams']['java_home']
-app_user = config['configurations']['global']['app_user']
pid_file = config['configurations']['global']['pid_file']
+
additional_cp = config['configurations']['global']['additional_cp']
xmx_val = config['configurations']['global']['xmx_val']
xms_val = config['configurations']['global']['xms_val']
diff --git a/app-packages/memcached-win/resources.json b/app-packages/memcached-win/resources-default.json
similarity index 100%
rename from app-packages/memcached-win/resources.json
rename to app-packages/memcached-win/resources-default.json
diff --git a/app-packages/memcached/README.txt b/app-packages/memcached/README.txt
index eed2954..fc0e4f3 100644
--- a/app-packages/memcached/README.txt
+++ b/app-packages/memcached/README.txt
@@ -19,7 +19,16 @@
To create the app package you will need the Memcached tarball copied to a specific location.
-Replace the placeholder tarball for JMemcached.
+Replace the placeholder tarball for JMemcached. The tarball must have all the jar files at the
+root directory.
+Example:
+ tar -tvf jmemcached-1.0.0.tar
+ -rw-r--r-- ./jmemcached-cli-1.0.0.jar
+ -rwxr-xr-x ./jmemcached-core-1.0.0.jar
+
+If not modify, appConfig.json to have correct application install root.
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/my_sub_root_for_jars",
+
cp ~/Downloads/jmemcached-1.0.0.tar package/files/
rm package/files/jmemcached-1.0.0.tar.REPLACE
@@ -29,7 +38,6 @@
Verify the content using
unzip -l "$@" jmemcached-1.0.0.zip
-While appConfig.json and resources.json are not required for the package they work
-well as the default configuration for Slider apps. So its advisable that when you
-create an application package for Slider, include sample/default resources.json and
-appConfig.json for a minimal Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
diff --git a/app-packages/memcached/appConfig-default.json b/app-packages/memcached/appConfig-default.json
new file mode 100644
index 0000000..16dd931
--- /dev/null
+++ b/app-packages/memcached/appConfig-default.json
@@ -0,0 +1,20 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/MEMCACHED/jmemcached-1.0.0.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
+
+ "site.global.additional_cp": "/usr/lib/hadoop/lib/*",
+ "site.global.xmx_val": "256m",
+ "site.global.xms_val": "128m",
+ "site.global.memory_val": "200M",
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{PER_CONTAINER}"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
deleted file mode 100644
index 5f32030..0000000
--- a/app-packages/memcached/appConfig.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": "package/jmemcached-1.0.0.zip",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
-
- "site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/jmemcached-1.0.0",
- "site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/component.pid",
-
- "site.global.additional_cp": "/usr/lib/hadoop/lib/*",
- "site.global.xmx_val": "256m",
- "site.global.xms_val": "128m",
- "site.global.memory_val": "200M",
- "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- },
- "MEMCACHED": {
- }
- }
-}
diff --git a/app-packages/memcached/metainfo.xml b/app-packages/memcached/metainfo.xml
index 525816e..0984dc9 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -23,17 +23,23 @@
<comment>Memcache is a network accessible key/value storage system, often used as a distributed cache.</comment>
<version>1.0.0</version>
<exportedConfigs>None</exportedConfigs>
+ <exportGroups>
+ <exportGroup>
+ <name>Servers</name>
+ <exports>
+ <export>
+ <name>host_port</name>
+ <value>${MEMCACHED_HOST}:${site.global.listen_port}</value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
<components>
<component>
<name>MEMCACHED</name>
<category>MASTER</category>
- <exports>
- <export>
- <name>host_port</name>
- <value>${THIS_HOST}:${site.global.listen_port}</value>
- </export>
- </exports>
+ <compExports>Servers-host_port</compExports>
<commandScript>
<script>scripts/memcached.py</script>
<scriptType>PYTHON</scriptType>
diff --git a/app-packages/memcached/package/scripts/memcached.py b/app-packages/memcached/package/scripts/memcached.py
index 6e14e86..986b61e 100644
--- a/app-packages/memcached/package/scripts/memcached.py
+++ b/app-packages/memcached/package/scripts/memcached.py
@@ -37,10 +37,10 @@
process_cmd = format("{java64_home}/bin/java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}/*:{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val} --port={port}")
Execute(process_cmd,
- user=params.app_user,
logoutput=False,
wait_for_finish=False,
- pid_file=params.pid_file
+ pid_file=params.pid_file,
+ poll_after = 5
)
def stop(self, env):
diff --git a/app-packages/memcached/package/scripts/params.py b/app-packages/memcached/package/scripts/params.py
index 25b4055..056a3b9 100644
--- a/app-packages/memcached/package/scripts/params.py
+++ b/app-packages/memcached/package/scripts/params.py
@@ -25,7 +25,6 @@
app_root = config['configurations']['global']['app_root']
java64_home = config['hostLevelParams']['java_home']
-app_user = config['configurations']['global']['app_user']
pid_file = config['configurations']['global']['pid_file']
additional_cp = config['configurations']['global']['additional_cp']
diff --git a/app-packages/memcached/resources.json b/app-packages/memcached/resources-default.json
similarity index 100%
rename from app-packages/memcached/resources.json
rename to app-packages/memcached/resources-default.json
diff --git a/app-packages/storm-win/README.txt b/app-packages/storm-win/README.txt
new file mode 100644
index 0000000..8631714
--- /dev/null
+++ b/app-packages/storm-win/README.txt
@@ -0,0 +1,36 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+How to create a Slider app package for Storm?
+
+To create the app package you will need the Storm tarball and invoke mvn command
+with appropriate parameters.
+
+Command:
+mvn clean package -Pstorm-app-package-win -Dpkg.version=<version>
+ -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
+
+Example:
+mvn clean package -Pstorm-app-package-win -Dpkg.version=0.9.3
+ -Dpkg.name=storm-0.9.3.zip -Dpkg.src=/Users/user1/Downloads
+
+App package can be found in
+ app-packages/storm-win/target/slider-storm-app-win-package-${pkg.version}.zip
+
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
diff --git a/app-packages/storm-win/appConfig-default.json b/app-packages/storm-win/appConfig-default.json
new file mode 100644
index 0000000..a77f00d
--- /dev/null
+++ b/app-packages/storm-win/appConfig-default.json
@@ -0,0 +1,39 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/STORM/slider-storm-app-win-package-${pkg.version}.zip",
+ "java_home": "C:\\java",
+ "create.default.zookeeper.node": "true",
+
+ "site.global.app_user": "hadoop",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/storm-${pkg.version}",
+ "site.global.user_group": "hadoop",
+
+ "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
+ "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
+ "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}",
+ "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
+ "site.storm-site.transactional.zookeeper.root": "/transactional",
+ "site.storm-site.storm.zookeeper.port": "2181",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m",
+ "site.storm-site.worker.childopts": "-Xmx768m",
+ "site.storm-site.ui.childopts": "-Xmx768m",
+ "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
+ "site.storm-site.drpc.invocations.port": "0",
+ "site.storm-site.storm.zookeeper.root": "${DEFAULT_ZK_PATH}",
+ "site.storm-site.transactional.zookeeper.port": "null",
+ "site.storm-site.nimbus.host": "${NIMBUS_HOST}",
+ "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
+ "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER},${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}]",
+ "site.storm-site.supervisor.childopts": "-Xmx256m",
+ "site.storm-site.drpc.port": "0",
+ "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
diff --git a/app-packages/storm-win/configuration/storm-env.xml b/app-packages/storm-win/configuration/storm-env.xml
new file mode 100644
index 0000000..091c08d
--- /dev/null
+++ b/app-packages/storm-win/configuration/storm-env.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+ <property>
+ <name>kerberos_domain</name>
+ <value></value>
+ <description>The kerberos domain to be used for this Storm cluster</description>
+ </property>
+ <property>
+ <name>storm_client_principal_name</name>
+ <value></value>
+ <description>The principal name for the Storm client to be used to communicate with Nimbus and Zookeeper</description>
+ </property>
+ <property>
+ <name>storm_server_principal_name</name>
+ <value></value>
+ <description>The principal name for the Storm server to be used by Nimbus</description>
+ </property>
+ <property>
+ <name>storm_client_keytab</name>
+ <value></value>
+ <description>The keytab file path for Storm client</description>
+ </property>
+ <property>
+ <name>storm_server_keytab</name>
+ <value></value>
+ <description>The keytab file path for Storm server</description>
+ </property>
+ <!-- storm-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for storm-env.sh file</description>
+ <value>
+#!/bin/bash
+
+# Set Storm specific environment variables here.
+
+# The java implementation to use.
+export JAVA_HOME={{java_home}}
+
+# export STORM_CONF_DIR=""
+ </value>
+ </property>
+</configuration>
diff --git a/app-packages/storm-win/configuration/storm-site.xml b/app-packages/storm-win/configuration/storm-site.xml
new file mode 100644
index 0000000..c09e29b
--- /dev/null
+++ b/app-packages/storm-win/configuration/storm-site.xml
@@ -0,0 +1,580 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>java.library.path</name>
+ <value>/usr/local/lib:/opt/local/lib:/usr/lib</value>
+ <description>This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers)
+ for the java.library.path value. java.library.path tells the JVM where
+ to look for native libraries. It is necessary to set this config correctly since
+ Storm uses the ZeroMQ and JZMQ native libs. </description>
+ </property>
+ <property>
+ <name>storm.local.dir</name>
+ <value>/hadoop/storm</value>
+ <description>A directory on the local filesystem used by Storm for any local
+ filesystem usage it needs. The directory must exist and the Storm daemons must
+ have permission to read/write from this location.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.servers</name>
+ <value>['localhost']</value>
+ <description>A list of hosts of ZooKeeper servers used to manage the cluster.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.port</name>
+ <value>2181</value>
+ <description>The port Storm will use to connect to each of the ZooKeeper servers.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.root</name>
+ <value>/storm</value>
+ <description>The root location at which Storm stores data in ZooKeeper.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.session.timeout</name>
+ <value>20000</value>
+ <description>The session timeout for clients to ZooKeeper.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.connection.timeout</name>
+ <value>15000</value>
+ <description>The connection timeout for clients to ZooKeeper.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.retry.times</name>
+ <value>5</value>
+ <description>The number of times to retry a Zookeeper operation.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.retry.interval</name>
+ <value>1000</value>
+ <description>The interval between retries of a Zookeeper operation.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.retry.intervalceiling.millis</name>
+ <value>30000</value>
+ <description>The ceiling of the interval between retries of a Zookeeper operation.</description>
+ </property>
+ <property>
+ <name>storm.cluster.mode</name>
+ <value>distributed</value>
+ <description>The mode this Storm cluster is running in. Either "distributed" or "local".</description>
+ </property>
+ <property>
+ <name>storm.local.mode.zmq</name>
+ <value>false</value>
+ <description>Whether or not to use ZeroMQ for messaging in local mode. If this is set
+ to false, then Storm will use a pure-Java messaging system. The purpose
+ of this flag is to make it easy to run Storm in local mode by eliminating
+ the need for native dependencies, which can be difficult to install.
+ </description>
+ </property>
+ <property>
+ <name>storm.thrift.transport</name>
+ <value>backtype.storm.security.auth.SimpleTransportPlugin</value>
+ <description>The transport plug-in for Thrift client/server communication.</description>
+ </property>
+ <property>
+ <name>storm.messaging.transport</name>
+ <value>backtype.storm.messaging.netty.Context</value>
+ <description>The transporter for communication among Storm tasks.</description>
+ </property>
+ <property>
+ <name>nimbus.host</name>
+ <value>localhost</value>
+ <description>The host that the master server is running on.</description>
+ </property>
+ <property>
+ <name>nimbus.thrift.port</name>
+ <value>6627</value>
+ <description> Which port the Thrift interface of Nimbus should run on. Clients should
+ connect to this port to upload jars and submit topologies.</description>
+ </property>
+ <property>
+ <name>nimbus.thrift.max_buffer_size</name>
+ <value>1048576</value>
+ <description>The maximum buffer size thrift should use when reading messages.</description>
+ </property>
+ <property>
+ <name>nimbus.childopts</name>
+ <value>-Xmx1024m</value>
+ <description>This parameter is used by the storm-deploy project to configure the jvm options for the nimbus daemon.</description>
+ </property>
+ <property>
+ <name>nimbus.task.timeout.secs</name>
+ <value>30</value>
+ <description>How long without heartbeating a task can go before nimbus will consider the task dead and reassign it to another location.</description>
+ </property>
+ <property>
+ <name>nimbus.supervisor.timeout.secs</name>
+ <value>60</value>
+ <description>How long before a supervisor can go without heartbeating before nimbus considers it dead and stops assigning new work to it.</description>
+ </property>
+ <property>
+ <name>nimbus.monitor.freq.secs</name>
+ <value>10</value>
+ <description>
+ How often nimbus should wake up to check heartbeats and do reassignments. Note
+ that if a machine ever goes down Nimbus will immediately wake up and take action.
+ This parameter is for checking for failures when there's no explicit event like that occuring.
+ </description>
+ </property>
+ <property>
+ <name>nimbus.cleanup.inbox.freq.secs</name>
+ <value>600</value>
+ <description>How often nimbus should wake the cleanup thread to clean the inbox.</description>
+ </property>
+ <property>
+ <name>nimbus.inbox.jar.expiration.secs</name>
+ <value>3600</value>
+ <description>
+ The length of time a jar file lives in the inbox before being deleted by the cleanup thread.
+
+ Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS.
+ Note that the time it takes to delete an inbox jar file is going to be somewhat more than
+ NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS (depending on how often NIMBUS_CLEANUP_FREQ_SECS is set to).
+ </description>
+ </property>
+ <property>
+ <name>nimbus.task.launch.secs</name>
+ <value>120</value>
+ <description>A special timeout used when a task is initially launched. During launch, this is the timeout
+ used until the first heartbeat, overriding nimbus.task.timeout.secs.</description>
+ </property>
+ <property>
+ <name>nimbus.reassign</name>
+ <value>true</value>
+ <description>Whether or not nimbus should reassign tasks if it detects that a task goes down.
+ Defaults to true, and it's not recommended to change this value.</description>
+ </property>
+ <property>
+ <name>nimbus.file.copy.expiration.secs</name>
+ <value>600</value>
+ <description>During upload/download with the master, how long an upload or download connection is idle
+ before nimbus considers it dead and drops the connection.</description>
+ </property>
+ <property>
+ <name>nimbus.topology.validator</name>
+ <value>backtype.storm.nimbus.DefaultTopologyValidator</value>
+ <description>A custom class that implements ITopologyValidator that is run whenever a
+ topology is submitted. Can be used to provide business-specific logic for
+ whether topologies are allowed to run or not.</description>
+ </property>
+ <property>
+ <name>ui.port</name>
+ <value>8744</value>
+ <description>Storm UI binds to this port.</description>
+ </property>
+ <property>
+ <name>ui.childopts</name>
+ <value>-Xmx768m</value>
+ <description>Childopts for Storm UI Java process.</description>
+ </property>
+ <property>
+ <name>logviewer.port</name>
+ <value>8000</value>
+ <description>HTTP UI port for log viewer.</description>
+ </property>
+ <property>
+ <name>logviewer.childopts</name>
+ <value>-Xmx128m</value>
+ <description>Childopts for log viewer java process.</description>
+ </property>
+ <property>
+ <name>logviewer.appender.name</name>
+ <value>A1</value>
+ <description>Appender name used by log viewer to determine log directory.</description>
+ </property>
+ <property>
+ <name>drpc.port</name>
+ <value>3772</value>
+ <description>This port is used by Storm DRPC for receiving DPRC requests from clients.</description>
+ </property>
+ <property>
+ <name>drpc.worker.threads</name>
+ <value>64</value>
+ <description>DRPC thrift server worker threads.</description>
+ </property>
+ <property>
+ <name>drpc.queue.size</name>
+ <value>128</value>
+ <description>DRPC thrift server queue size.</description>
+ </property>
+ <property>
+ <name>drpc.invocations.port</name>
+ <value>3773</value>
+ <description>This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back.</description>
+ </property>
+ <property>
+ <name>drpc.request.timeout.secs</name>
+ <value>600</value>
+ <description>The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also
+ timeout based on the socket timeout on the DRPC client, and separately based on the topology message
+ timeout for the topology implementing the DRPC function.</description>
+ </property>
+ <property>
+ <name>drpc.childopts</name>
+ <value>-Xmx768m</value>
+ <description>Childopts for Storm DRPC Java process.</description>
+ </property>
+ <property>
+ <name>transactional.zookeeper.root</name>
+ <value>/transactional</value>
+ <description>The root directory in ZooKeeper for metadata about TransactionalSpouts.</description>
+ </property>
+ <property>
+ <name>transactional.zookeeper.servers</name>
+ <value>null</value>
+ <description>The list of zookeeper servers in which to keep the transactional state. If null (which is default),
+ will use storm.zookeeper.servers</description>
+ </property>
+ <property>
+ <name>transactional.zookeeper.port</name>
+ <value>null</value>
+ <description>The port to use to connect to the transactional zookeeper servers. If null (which is default),
+ will use storm.zookeeper.port</description>
+ </property>
+ <property>
+ <name>supervisor.slots.ports</name>
+ <value>[6700, 6701]</value>
+ <description>A list of ports that can run workers on this supervisor. Each worker uses one port, and
+ the supervisor will only run one worker per port. Use this configuration to tune
+ how many workers run on each machine.</description>
+ </property>
+ <property>
+ <name>supervisor.childopts</name>
+ <value>-Xmx256m</value>
+ <description>This parameter is used by the storm-deploy project to configure the jvm options for the supervisor daemon.</description>
+ </property>
+ <property>
+ <name>supervisor.worker.start.timeout.secs</name>
+ <value>120</value>
+ <description>How long a worker can go without heartbeating during the initial launch before
+ the supervisor tries to restart the worker process. This value override
+ supervisor.worker.timeout.secs during launch because there is additional
+ overhead to starting and configuring the JVM on launch.</description>
+ </property>
+ <property>
+ <name>supervisor.worker.timeout.secs</name>
+ <value>30</value>
+ <description>How long a worker can go without heartbeating before the supervisor tries to restart the worker process.</description>
+ </property>
+ <property>
+ <name>supervisor.monitor.frequency.secs</name>
+ <value>3</value>
+ <description>How often the supervisor checks the worker heartbeats to see if any of them need to be restarted.</description>
+ </property>
+ <property>
+ <name>supervisor.heartbeat.frequency.secs</name>
+ <value>5</value>
+ <description>How often the supervisor sends a heartbeat to the master.</description>
+ </property>
+ <property>
+ <name>worker.childopts</name>
+ <value>-Xmx768m</value>
+ <description>The jvm opts provided to workers launched by this supervisor. All \"%ID%\" substrings are replaced with an identifier for this worker.</description>
+ </property>
+ <property>
+ <name>worker.heartbeat.frequency.secs</name>
+ <value>1</value>
+ <description>How often this worker should heartbeat to the supervisor.</description>
+ </property>
+ <property>
+ <name>task.heartbeat.frequency.secs</name>
+ <value>3</value>
+ <description>How often a task should heartbeat its status to the master.</description>
+ </property>
+ <property>
+ <name>task.refresh.poll.secs</name>
+ <value>10</value>
+ <description>How often a task should sync its connections with other tasks (if a task is
+ reassigned, the other tasks sending messages to it need to refresh their connections).
+ In general though, when a reassignment happens other tasks will be notified
+ almost immediately. This configuration is here just in case that notification doesn't
+ come through.</description>
+ </property>
+ <property>
+ <name>zmq.threads</name>
+ <value>1</value>
+ <description>The number of threads that should be used by the zeromq context in each worker process.</description>
+ </property>
+ <property>
+ <name>zmq.linger.millis</name>
+ <value>5000</value>
+ <description>How long a connection should retry sending messages to a target host when
+ the connection is closed. This is an advanced configuration and can almost
+ certainly be ignored.</description>
+ </property>
+ <property>
+ <name>zmq.hwm</name>
+ <value>0</value>
+ <description>The high water for the ZeroMQ push sockets used for networking. Use this config to prevent buffer explosion
+ on the networking layer.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.server_worker_threads</name>
+ <value>1</value>
+ <description>Netty based messaging: The # of worker threads for the server.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.client_worker_threads</name>
+ <value>1</value>
+ <description>Netty based messaging: The # of worker threads for the client.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.buffer_size</name>
+ <value>5242880</value>
+ <description>Netty based messaging: The buffer size for send/recv buffer.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.max_retries</name>
+ <value>30</value>
+ <description>Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.max_wait_ms</name>
+ <value>1000</value>
+ <description>Netty based messaging: The max # of milliseconds that a peer will wait.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.min_wait_ms</name>
+ <value>100</value>
+ <description>Netty based messaging: The min # of milliseconds that a peer will wait.</description>
+ </property>
+ <property>
+ <name>topology.enable.message.timeouts</name>
+ <value>true</value>
+ <description>True if Storm should timeout messages or not. Defaults to true. This is meant to be used
+ in unit tests to prevent tuples from being accidentally timed out during the test.</description>
+ </property>
+ <property>
+ <name>topology.debug</name>
+ <value>false</value>
+ <description>When set to true, Storm will log every message that's emitted.</description>
+ </property>
+ <property>
+ <name>topology.optimize</name>
+ <value>true</value>
+ <description>Whether or not the master should optimize topologies by running multiple tasks in a single thread where appropriate.</description>
+ </property>
+ <property>
+ <name>topology.workers</name>
+ <value>1</value>
+ <description>How many processes should be spawned around the cluster to execute this
+ topology. Each process will execute some number of tasks as threads within
+ them. This parameter should be used in conjunction with the parallelism hints
+ on each component in the topology to tune the performance of a topology.</description>
+ </property>
+ <property>
+ <name>topology.acker.executors</name>
+ <value>null</value>
+ <description>How many executors to spawn for ackers.
+
+ If this is set to 0, then Storm will immediately ack tuples as soon
+ as they come off the spout, effectively disabling reliability.
+ </description>
+ </property>
+ <property>
+ <name>topology.message.timeout.secs</name>
+ <value>30</value>
+ <description>The maximum amount of time given to the topology to fully process a message
+ emitted by a spout. If the message is not acked within this time frame, Storm
+ will fail the message on the spout. Some spouts implementations will then replay
+ the message at a later time.</description>
+ </property>
+ <property>
+ <name>topology.skip.missing.kryo.registrations</name>
+ <value>false</value>
+ <description> Whether or not Storm should skip the loading of kryo registrations for which it
+ does not know the class or have the serializer implementation. Otherwise, the task will
+ fail to load and will throw an error at runtime. The use case of this is if you want to
+ declare your serializations on the storm.yaml files on the cluster rather than every single
+ time you submit a topology. Different applications may use different serializations and so
+ a single application may not have the code for the other serializers used by other apps.
+ By setting this config to true, Storm will ignore that it doesn't have those other serializations
+ rather than throw an error.</description>
+ </property>
+ <property>
+ <name>topology.max.task.parallelism</name>
+ <value>null</value>
+ <description>The maximum parallelism allowed for a component in this topology. This configuration is
+ typically used in testing to limit the number of threads spawned in local mode.</description>
+ </property>
+ <property>
+ <name>topology.max.spout.pending</name>
+ <value>null</value>
+ <description>The maximum number of tuples that can be pending on a spout task at any given time.
+ This config applies to individual tasks, not to spouts or topologies as a whole.
+
+ A pending tuple is one that has been emitted from a spout but has not been acked or failed yet.
+ Note that this config parameter has no effect for unreliable spouts that don't tag
+ their tuples with a message id.</description>
+ </property>
+ <property>
+ <name>topology.state.synchronization.timeout.secs</name>
+ <value>60</value>
+ <description>The maximum amount of time a component gives a source of state to synchronize before it requests
+ synchronization again.</description>
+ </property>
+ <property>
+ <name>topology.stats.sample.rate</name>
+ <value>0.05</value>
+ <description>The percentage of tuples to sample to produce stats for a task.</description>
+ </property>
+ <property>
+ <name>topology.builtin.metrics.bucket.size.secs</name>
+ <value>60</value>
+ <description>The time period that builtin metrics data in bucketed into.</description>
+ </property>
+ <property>
+ <name>topology.fall.back.on.java.serialization</name>
+ <value>true</value>
+ <description>Whether or not to use Java serialization in a topology.</description>
+ </property>
+ <property>
+ <name>topology.worker.childopts</name>
+ <value>null</value>
+ <description>Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS.</description>
+ </property>
+ <property>
+ <name>topology.executor.receive.buffer.size</name>
+ <value>1024</value>
+ <description>The size of the Disruptor receive queue for each executor. Must be a power of 2.</description>
+ </property>
+ <property>
+ <name>topology.executor.send.buffer.size</name>
+ <value>1024</value>
+ <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description>
+ </property>
+ <property>
+ <name>topology.receiver.buffer.size</name>
+ <value>8</value>
+ <description>The maximum number of messages to batch from the thread receiving off the network to the
+ executor queues. Must be a power of 2.</description>
+ </property>
+ <property>
+ <name>topology.transfer.buffer.size</name>
+ <value>1024</value>
+ <description>The size of the Disruptor transfer queue for each worker.</description>
+ </property>
+ <property>
+ <name>topology.tick.tuple.freq.secs</name>
+ <value>null</value>
+ <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent
+ to tasks. Meant to be used as a component-specific configuration.</description>
+ </property>
+ <property>
+ <name>topology.worker.shared.thread.pool.size</name>
+ <value>4</value>
+ <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed
+ via the TopologyContext.</description>
+ </property>
+ <property>
+ <name>topology.disruptor.wait.strategy</name>
+ <value>com.lmax.disruptor.BlockingWaitStrategy</value>
+ <description>Configure the wait strategy used for internal queuing. Can be used to tradeoff latency
+ vs. throughput.</description>
+ </property>
+ <property>
+ <name>topology.executor.send.buffer.size</name>
+ <value>1024</value>
+ <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description>
+ </property>
+ <property>
+ <name>topology.receiver.buffer.size</name>
+ <value>8</value>
+ <description>The maximum number of messages to batch from the thread receiving off the network to the
+ executor queues. Must be a power of 2.</description>
+ </property>
+ <property>
+ <name>topology.transfer.buffer.size</name>
+ <value>1024</value>
+ <description>The size of the Disruptor transfer queue for each worker.</description>
+ </property>
+ <property>
+ <name>topology.tick.tuple.freq.secs</name>
+ <value>null</value>
+ <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent
+ to tasks. Meant to be used as a component-specific configuration.</description>
+ </property>
+ <property>
+ <name>topology.worker.shared.thread.pool.size</name>
+ <value>4</value>
+ <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed
+ via the TopologyContext.</description>
+ </property>
+ <property>
+ <name>topology.spout.wait.strategy</name>
+ <value>backtype.storm.spout.SleepSpoutWaitStrategy</value>
+ <description>A class that implements a strategy for what to do when a spout needs to wait. Waiting is
+ triggered in one of two conditions:
+
+ 1. nextTuple emits no tuples
+ 2. The spout has hit maxSpoutPending and can't emit any more tuples</description>
+ </property>
+ <property>
+ <name>topology.sleep.spout.wait.strategy.time.ms</name>
+ <value>1</value>
+ <description>The amount of milliseconds the SleepEmptyEmitStrategy should sleep for.</description>
+ </property>
+ <property>
+ <name>topology.error.throttle.interval.secs</name>
+ <value>10</value>
+ <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example,
+ an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be
+ reported to Zookeeper per task for every 10 second interval of time.</description>
+ </property>
+ <property>
+ <name>topology.max.error.report.per.interval</name>
+ <value>5</value>
+ <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example,
+ an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be
+ reported to Zookeeper per task for every 10 second interval of time.</description>
+ </property>
+ <property>
+ <name>topology.kryo.factory</name>
+ <value>backtype.storm.serialization.DefaultKryoFactory</value>
+ <description>Class that specifies how to create a Kryo instance for serialization. Storm will then apply
+ topology.kryo.register and topology.kryo.decorators on top of this. The default implementation
+ implements topology.fall.back.on.java.serialization and turns references off.</description>
+ </property>
+ <property>
+ <name>topology.tuple.serializer</name>
+ <value>backtype.storm.serialization.types.ListDelegateSerializer</value>
+ <description>The serializer class for ListDelegate (tuple payload).
+ The default serializer will be ListDelegateSerializer</description>
+ </property>
+ <property>
+ <name>topology.trident.batch.emit.interval.millis</name>
+ <value>500</value>
+ <description>How often a batch can be emitted in a Trident topology.</description>
+ </property>
+ <property>
+ <name>dev.zookeeper.path</name>
+ <value>/tmp/dev-storm-zookeeper</value>
+ <description>The path to use as the zookeeper dir when running a zookeeper server via
+ "storm dev-zookeeper". This zookeeper instance is only intended for development;
+ it is not a production grade zookeeper setup.</description>
+ </property>
+</configuration>
diff --git a/app-packages/storm-win/metainfo.xml b/app-packages/storm-win/metainfo.xml
new file mode 100644
index 0000000..1e9e7ab
--- /dev/null
+++ b/app-packages/storm-win/metainfo.xml
@@ -0,0 +1,150 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <application>
+ <name>STORM</name>
+ <comment>Apache Hadoop Stream processing framework</comment>
+ <version>${pkg.version}</version>
+ <exportedConfigs>storm-site</exportedConfigs>
+
+ <exportGroups>
+ <exportGroup>
+ <name>QuickLinks</name>
+ <exports>
+ <export>
+ <name>org.apache.slider.jmx</name>
+ <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}/api/v1/cluster/summary</value>
+ </export>
+ <export>
+ <name>org.apache.slider.monitor</name>
+ <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
+ </export>
+ <export>
+ <name>nimbus.host_port</name>
+ <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
+
+ <commandOrders>
+ <commandOrder>
+ <command>NIMBUS-START</command>
+ <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED
+ </requires>
+ </commandOrder>
+ <commandOrder>
+ <command>SUPERVISOR-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>DRPC_SERVER-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>STORM_UI_SERVER-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ </commandOrders>
+
+ <components>
+
+ <component>
+ <name>NIMBUS</name>
+ <category>MASTER</category>
+ <publishConfig>true</publishConfig>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <appExports>QuickLinks-nimbus.host_port</appExports>
+ <commandScript>
+ <script>scripts/nimbus.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>SUPERVISOR</name>
+ <category>SLAVE</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <componentExports>
+ <componentExport>
+ <name>log_viewer_port</name>
+ <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value>
+ </componentExport>
+ </componentExports>
+ <commandScript>
+ <script>scripts/supervisor.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_UI_SERVER</name>
+ <category>MASTER</category>
+ <publishConfig>true</publishConfig>
+ <appExports>QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.jmx</appExports>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <commandScript>
+ <script>scripts/ui_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>DRPC_SERVER</name>
+ <category>MASTER</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <commandScript>
+ <script>scripts/drpc_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>zip</type>
+ <name>files/${pkg.name}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <configFiles>
+ <configFile>
+ <type>yaml</type>
+ <fileName>storm.yaml</fileName>
+ <dictionaryName>storm-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>storm-env.sh</fileName>
+ <dictionaryName>storm-env</dictionaryName>
+ </configFile>
+ </configFiles>
+
+ </application>
+</metainfo>
diff --git a/app-packages/storm-win/package/scripts/drpc_server.py b/app-packages/storm-win/package/scripts/drpc_server.py
new file mode 100644
index 0000000..779854a
--- /dev/null
+++ b/app-packages/storm-win/package/scripts/drpc_server.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+
+class DrpcServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("drpc", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("drpc", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ #check_process_status(status_params.pid_drpc)
+
+if __name__ == "__main__":
+ DrpcServer().execute()
diff --git a/app-packages/storm-win/package/scripts/nimbus.py b/app-packages/storm-win/package/scripts/nimbus.py
new file mode 100644
index 0000000..c7c3120
--- /dev/null
+++ b/app-packages/storm-win/package/scripts/nimbus.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+
+class Nimbus(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("nimbus", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("nimbus", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.pid_nimbus)
+
+if __name__ == "__main__":
+ Nimbus().execute()
diff --git a/app-packages/storm-win/package/scripts/params.py b/app-packages/storm-win/package/scripts/params.py
new file mode 100644
index 0000000..21e5c65
--- /dev/null
+++ b/app-packages/storm-win/package/scripts/params.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+import status_params
+
+# server configurations
+config = Script.get_config()
+
+app_root = config['configurations']['global']['app_root']
+conf_dir = format("{app_root}/conf")
+storm_user = config['configurations']['global']['app_user']
+log_dir = config['configurations']['global']['app_log_dir']
+pid_dir = status_params.pid_dir
+local_dir = config['configurations']['storm-site']['storm.local.dir']
+user_group = config['configurations']['global']['user_group']
+java64_home = config['hostLevelParams']['java_home']
+nimbus_host = config['configurations']['storm-site']['nimbus.host']
+nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
+rest_api_conf_file = format("{conf_dir}/config.yaml")
+rest_lib_dir = format("{app_root}/external/storm-rest")
+storm_bin = format("{app_root}/bin/storm.cmd")
diff --git a/app-packages/storm-win/package/scripts/rest_api.py b/app-packages/storm-win/package/scripts/rest_api.py
new file mode 100644
index 0000000..33d8924
--- /dev/null
+++ b/app-packages/storm-win/package/scripts/rest_api.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+
+
+class StormRestApi(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("rest_api", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("rest_api", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.pid_rest_api)
+
+if __name__ == "__main__":
+ StormRestApi().execute()
diff --git a/app-packages/storm-win/package/scripts/service.py b/app-packages/storm-win/package/scripts/service.py
new file mode 100644
index 0000000..aa7d339
--- /dev/null
+++ b/app-packages/storm-win/package/scripts/service.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+
+from resource_management import *
+import os
+import sys
+import xml.etree.ElementTree as et
+
+"""
+Slider package uses jps as pgrep does not list the whole process start command
+"""
+def service(
+ name,
+ action='start'):
+ import params
+ import status_params
+
+ pid_file = status_params.pid_files[name]
+ backtype = format("backtype.storm.daemon.{name}")
+
+ if action == "start":
+ os.environ['STORM_LOG_DIR'] = params.log_dir
+ os.environ['STORM_HOME'] = params.app_root
+ os.environ['STORM_CONF_DIR'] = params.conf_dir
+
+ generate_xml = format("{storm_bin} --service {name} > {log_dir}/{name}.cmd")
+
+ Execute(generate_xml,
+ logoutput=True,
+ wait_for_finish=True
+ )
+
+ tree = et.parse(format("{log_dir}/{name}.cmd"))
+ root = tree.getroot()
+ cmd_part = None
+ for child in root:
+ if child.tag == "arguments":
+ cmd_part = child.text
+
+ if cmd_part:
+ cmd = format("{java64_home}\\bin\\java {cmd_part}")
+
+ Execute(cmd,
+ logoutput=False,
+ wait_for_finish=False,
+ pid_file=pid_file
+ )
+ else:
+ Logger.warn("Valid command file did not get generated at " + format("{log_dir}/{name}.cmd"))
+
+ elif action == "stop":
+ pid = format("`cat {pid_file}` >/dev/null 2>&1")
+ Execute(format("kill {pid}")
+ )
+ Execute(format("kill -9 {pid}"),
+ ignore_failures=True
+ )
+ Execute(format("rm -f {pid_file}"))
diff --git a/app-packages/storm-win/package/scripts/status_params.py b/app-packages/storm-win/package/scripts/status_params.py
new file mode 100644
index 0000000..2bf6870
--- /dev/null
+++ b/app-packages/storm-win/package/scripts/status_params.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management import *
+
+config = Script.get_config()
+
+container_id = config['configurations']['global']['app_container_id']
+pid_dir = config['configurations']['global']['app_pid_dir']
+pid_nimbus = format("{pid_dir}/nimbus.pid")
+pid_supervisor = format("{pid_dir}/supervisor.pid")
+pid_drpc = format("{pid_dir}/drpc.pid")
+pid_ui = format("{pid_dir}/ui.pid")
+pid_logviewer = format("{pid_dir}/logviewer.pid")
+pid_files = {"logviewer":pid_logviewer,
+ "ui": pid_ui,
+ "nimbus": pid_nimbus,
+ "supervisor": pid_supervisor,
+ "drpc": pid_drpc}
\ No newline at end of file
diff --git a/app-packages/storm-win/package/scripts/storm.py b/app-packages/storm-win/package/scripts/storm.py
new file mode 100644
index 0000000..e2e6465
--- /dev/null
+++ b/app-packages/storm-win/package/scripts/storm.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from yaml_config import yaml_config
+import sys
+
+def storm():
+ import params
+
+ Directory([params.log_dir, params.pid_dir, params.local_dir, params.conf_dir],
+ owner=params.storm_user,
+ group=params.user_group,
+ recursive=True
+ )
+
+ File(format("{conf_dir}/config.yaml"),
+ content=Template("config.yaml.j2"),
+ owner = params.storm_user,
+ group = params.user_group
+ )
+
+ yaml_config( "storm.yaml",
+ conf_dir = params.conf_dir,
+ configurations = params.config['configurations']['storm-site'],
+ owner = params.storm_user,
+ group = params.user_group
+ )
diff --git a/app-packages/storm-win/package/scripts/supervisor.py b/app-packages/storm-win/package/scripts/supervisor.py
new file mode 100644
index 0000000..47c20c9
--- /dev/null
+++ b/app-packages/storm-win/package/scripts/supervisor.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from yaml_config import yaml_config
+from storm import storm
+from service import service
+
+
+class Supervisor(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("supervisor", action="start")
+ service("logviewer", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("supervisor", action="stop")
+ service("logviewer", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ check_process_status(status_params.pid_supervisor)
+
+
+if __name__ == "__main__":
+ Supervisor().execute()
+
diff --git a/app-packages/storm-win/package/scripts/ui_server.py b/app-packages/storm-win/package/scripts/ui_server.py
new file mode 100644
index 0000000..0fe7cd2
--- /dev/null
+++ b/app-packages/storm-win/package/scripts/ui_server.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+
+class UiServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("ui", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("ui", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.pid_ui)
+
+if __name__ == "__main__":
+ UiServer().execute()
diff --git a/app-packages/storm-win/package/scripts/yaml_config.py b/app-packages/storm-win/package/scripts/yaml_config.py
new file mode 100644
index 0000000..5f763cc
--- /dev/null
+++ b/app-packages/storm-win/package/scripts/yaml_config.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import re
+import socket
+from resource_management import *
+
+def escape_yaml_propetry(value):
+ # pre-process value for any "_HOST" tokens
+ value = value.replace('_HOST', socket.getfqdn())
+
+ unquouted = False
+ unquouted_values = ["null","Null","NULL","true","True","TRUE","false","False","FALSE","YES","Yes","yes","NO","No","no","ON","On","on","OFF","Off","off"]
+
+ if value in unquouted_values:
+ unquouted = True
+
+ # if is list [a,b,c]
+ if re.match('^\w*\[.+\]\w*$', value):
+ unquouted = True
+
+ # if is map {'a':'b'}
+ if re.match('^\w*\{.+\}\w*$', value):
+ unquouted = True
+
+ try:
+ int(value)
+ unquouted = True
+ except ValueError:
+ pass
+
+ try:
+ float(value)
+ unquouted = True
+ except ValueError:
+ pass
+
+ if not unquouted:
+ value = value.replace("'","''")
+ value = "'"+value+"'"
+
+ return value
+
+def yaml_inline_template(configurations):
+ return source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }}
+{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry])
+
+def yaml_config(
+ filename,
+ configurations = None,
+ conf_dir = None,
+ mode = None,
+ owner = None,
+ group = None
+):
+ config_content = yaml_inline_template(configurations)
+
+ File (format("{conf_dir}/{filename}"),
+ content = config_content,
+ owner = owner,
+ group = group,
+ mode = mode
+ )
diff --git a/app-packages/storm-win/package/templates/config.yaml.j2 b/app-packages/storm-win/package/templates/config.yaml.j2
new file mode 100644
index 0000000..c3dd542
--- /dev/null
+++ b/app-packages/storm-win/package/templates/config.yaml.j2
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nimbusHost: {{nimbus_host}}
+nimbusPort: {{nimbus_port}}
+
+enableGanglia: false
+
+# ganglia configuration
+ganglia:
+
+ # how often to report to ganglia metrics (in seconds)
+ reportInterval: 600
+
+ # the hostname of the gmond server where storm cluster metrics will be sent
+ host: localhost
+ port: 8649
+
+ # address mode
+ # default is MULTICAST
+ addressMode: "UNICAST"
+
+ # an <IP>:<HOSTNAME> pair to spoof
+ # this allows us to simulate storm cluster metrics coming from a specific host
+ #spoof: "192.168.1.1:storm"
\ No newline at end of file
diff --git a/app-packages/storm-win/package/templates/storm_jaas.conf.j2 b/app-packages/storm-win/package/templates/storm_jaas.conf.j2
new file mode 100644
index 0000000..a1ba6ea
--- /dev/null
+++ b/app-packages/storm-win/package/templates/storm_jaas.conf.j2
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+StormServer {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{storm_server_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ principal="{{storm_jaas_server_principal}}";
+};
+StormClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{storm_client_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="{{storm_jaas_stormclient_servicename}}"
+ debug=true
+ principal="{{storm_jaas_client_principal}}";
+};
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{storm_client_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="zookeeper"
+ principal="{{storm_jaas_client_principal}}";
+};
diff --git a/app-packages/storm-win/pom.xml b/app-packages/storm-win/pom.xml
new file mode 100644
index 0000000..3b4cb9d
--- /dev/null
+++ b/app-packages/storm-win/pom.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.60.0-incubating</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-storm-app-win-package</artifactId>
+ <packaging>pom</packaging>
+ <name>Slider Storm App Package</name>
+ <description>Slider Storm App Package</description>
+ <version>${pkg.version}</version>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>storm-app-package-win</id>
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/storm.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ </build>
+
+ <dependencies>
+ </dependencies>
+
+</project>
diff --git a/app-packages/storm/resources.json b/app-packages/storm-win/resources-default.json
similarity index 83%
rename from app-packages/storm/resources.json
rename to app-packages/storm-win/resources-default.json
index b184a40..a36f005 100644
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm-win/resources-default.json
@@ -3,6 +3,8 @@
"metadata" : {
},
"global" : {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": ""
},
"components": {
"slider-appmaster": {
@@ -11,21 +13,17 @@
"yarn.role.priority": "1",
"yarn.component.instances": "1"
},
- "STORM_REST_API": {
+ "STORM_UI_SERVER": {
"yarn.role.priority": "2",
"yarn.component.instances": "1"
},
- "STORM_UI_SERVER": {
+ "DRPC_SERVER": {
"yarn.role.priority": "3",
"yarn.component.instances": "1"
},
- "DRPC_SERVER": {
- "yarn.role.priority": "4",
- "yarn.component.instances": "1"
- },
"SUPERVISOR": {
- "yarn.role.priority": "5",
+ "yarn.role.priority": "4",
"yarn.component.instances": "1"
}
}
-}
\ No newline at end of file
+}
diff --git a/app-packages/storm-win/src/assembly/storm.xml b/app-packages/storm-win/src/assembly/storm.xml
new file mode 100644
index 0000000..2ee7d31
--- /dev/null
+++ b/app-packages/storm-win/src/assembly/storm.xml
@@ -0,0 +1,68 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>storm_v${storm.version}</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>appConfig-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>metainfo.xml</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>pom.xml</exclude>
+ <exclude>src/**</exclude>
+ <exclude>target/**</exclude>
+ <exclude>appConfig-default.json</exclude>
+ <exclude>metainfo.xml</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ </fileSets>
+</assembly>
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index 971cf14..12dc140 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -17,21 +17,20 @@
How to create a Slider app package for Storm?
-To create the app package you will need the Storm tarball copied to a specific location.
-Various configurations provided in this sample are customized for apache-storm-0.9.1.2.1.1.0-237.tar.gz.
-So if you use a different version you may need to edit a few config values.
+To create the app package you will need the Storm tarball and invoke mvn command
+with appropriate parameters.
-Replace the placeholder tarball for Storm.
- cp ~/Downloads/apache-storm-0.9.1.2.1.1.0-237.tar.gz package/files/
- rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
+Command:
+mvn clean package -Pstorm-app-package -Dpkg.version=<version>
+ -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
-Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/)
- zip -r storm_v091.zip .
+Example:
+mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578
+ -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz -Dpkg.src=/Users/user1/Downloads
-Verify the content using
- unzip -l "$@" storm_v091.zip
+App package can be found in
+ app-packages/storm/target/slider-storm-app-package-${pkg.version}.zip
-While appConfig.json and resources.json are not required for the package they work
-well as the default configuration for Slider apps. So its advisable that when you
-create an application package for Slider, include sample/default resources.json and
-appConfig.json for a minimal Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
diff --git a/app-packages/storm/appConfig-default.json b/app-packages/storm/appConfig-default.json
new file mode 100644
index 0000000..d7908a3
--- /dev/null
+++ b/app-packages/storm/appConfig-default.json
@@ -0,0 +1,43 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/STORM/slider-storm-app-package-${pkg.version}.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
+ "create.default.zookeeper.node": "true",
+
+ "site.global.app_user": "${USER_NAME}",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
+ "site.global.user_group": "hadoop",
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_id": "Application2",
+ "site.global.ganglia_enabled":"true",
+ "site.global.ganglia_server_port": "8668",
+
+ "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
+ "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
+ "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}",
+ "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
+ "site.storm-site.transactional.zookeeper.root": "/transactional",
+ "site.storm-site.storm.zookeeper.port": "2181",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.ui.childopts": "-Xmx768m",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
+ "site.storm-site.drpc.invocations.port": "0",
+ "site.storm-site.storm.zookeeper.root": "${DEFAULT_ZK_PATH}",
+ "site.storm-site.transactional.zookeeper.port": "null",
+ "site.storm-site.nimbus.host": "${NIMBUS_HOST}",
+ "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
+ "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER},${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}]",
+ "site.storm-site.drpc.port": "0",
+ "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
diff --git a/app-packages/storm/appConfig-secured-default.json b/app-packages/storm/appConfig-secured-default.json
new file mode 100644
index 0000000..4c40ddf
--- /dev/null
+++ b/app-packages/storm/appConfig-secured-default.json
@@ -0,0 +1,67 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/STORM/slider-storm-app-package-${pkg.version}.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
+ "create.default.zookeeper.node": "true",
+
+ "site.global.app_user": "${USER_NAME}",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
+ "site.global.user_group": "hadoop",
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_id": "Application2",
+ "site.global.ganglia_enabled":"true",
+ "site.global.ganglia_server_port": "8668",
+
+ "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
+ "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
+ "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}",
+ "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
+ "site.storm-site.transactional.zookeeper.root": "/transactional",
+ "site.storm-site.storm.zookeeper.port": "2181",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -Djava.security.auth.login.config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/conf/storm_jaas.conf -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Djava.security.auth.login.config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/conf/storm_jaas.conf -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.ui.childopts": "-Xmx768m -Djava.security.auth.login.config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/conf/storm_jaas.conf",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
+ "site.storm-site.drpc.invocations.port": "0",
+ "site.storm-site.storm.zookeeper.root": "${DEFAULT_ZK_PATH}",
+ "site.storm-site.transactional.zookeeper.port": "null",
+ "site.storm-site.nimbus.host": "${NIMBUS_HOST}",
+ "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
+ "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER},${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}]",
+ "site.storm-site.drpc.port": "0",
+ "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}",
+
+ "site.storm-site.nimbus.authorizer": "backtype.storm.security.auth.authorizer.SimpleACLAuthorizer",
+ "site.storm-site.storm.thrift.transport": "backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin",
+ "site.storm-site.java.security.auth.login.config": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/conf/storm_jaas.conf",
+ "site.storm-site.storm.principal.tolocal": "backtype.storm.security.auth.KerberosPrincipalToLocal",
+ "site.storm-site.storm.zookeeper.superACL": "sasl:${USER_NAME}",
+ "site.storm-site.nimbus.admins": "['${USER_NAME}']",
+ "site.storm-site.nimbus.users": "['${USER_NAME}']",
+ "site.storm-site.nimbus.supervisor.users": "['${USER_NAME}']",
+ "site.storm-site.nimubs.authorizer": "backtype.storm.security.auth.authorizer.SimpleACLAuthorizer",
+ "site.storm-site.storm.thrift.transport": "backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin",
+ "site.storm-site.storm.principal.tolocal": "backtype.storm.security.auth.KerberosPrincipalToLocal",
+ "site.storm-site.ui.filter": "org.apache.hadoop.security.authentication.server.AuthenticationFilter",
+ "site.storm-site.ui.filter.params": "{'type': 'kerberos', 'kerberos.principal': 'HTTP/_HOST', 'kerberos.keytab': '/etc/security/keytabs/spnego.service.keytab', 'kerberos.name.rules': 'RULE:[2:$1@$0]([jt]t@.*EXAMPLE.COM)s/.*/$MAPRED_USER/ RULE:[2:$1@$0]([nd]n@.*EXAMPLE.COM)s/.*/$HDFS_USER/DEFAULT'}",
+
+ "site.storm-env.kerberos_domain": "EXAMPLE.COM",
+ "site.storm-env.storm_client_principal_name": "${USER_NAME}@EXAMPLE.COM",
+ "site.storm-env.storm_server_principal_name": "${USER_NAME}/_HOST@EXAMPLE.COM",
+ "site.storm-env.storm_client_keytab": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.STORM.client.keytab",
+ "site.storm-env.storm_server_keytab": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.STORM.nimbus.keytab"
+
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M",
+ "slider.hdfs.keytab.dir": ".slider/keytabs/storm",
+ "slider.am.login.keytab.name": "${USER_NAME}.headless.keytab",
+ "slider.keytab.principal.name": "${USER_NAME}"
+ }
+ }
+}
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
deleted file mode 100644
index 24078cf..0000000
--- a/app-packages/storm/appConfig.json
+++ /dev/null
@@ -1,126 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": "package/storm_v091.zip",
- "config_types": "storm-site",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz",
- "create.default.zookeeper.node": "true",
- "site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_id": "Application2",
- "site.global.ganglia_enabled":"true",
- "site.global.ganglia_server_port": "8668",
- "site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
- "site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
- "site.storm-site.topology.tuple.serializer": "backtype.storm.serialization.types.ListDelegateSerializer",
- "site.storm-site.topology.workers": "1",
- "site.storm-site.drpc.worker.threads": "64",
- "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
- "site.storm-site.supervisor.heartbeat.frequency.secs": "5",
- "site.storm-site.topology.executor.send.buffer.size": "1024",
- "site.storm-site.drpc.childopts": "-Xmx768m",
- "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}",
- "site.storm-site.storm.zookeeper.retry.intervalceiling.millis": "30000",
- "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
- "site.storm-site.topology.receiver.buffer.size": "8",
- "site.storm-site.storm.messaging.netty.client_worker_threads": "1",
- "site.storm-site.transactional.zookeeper.root": "/transactional",
- "site.storm-site.drpc.request.timeout.secs": "600",
- "site.storm-site.topology.skip.missing.kryo.registrations": "false",
- "site.storm-site.worker.heartbeat.frequency.secs": "1",
- "site.storm-site.zmq.hwm": "0",
- "site.storm-site.storm.zookeeper.connection.timeout": "15000",
- "site.storm-site.topology.max.error.report.per.interval": "5",
- "site.storm-site.storm.messaging.netty.server_worker_threads": "1",
- "site.storm-site.supervisor.worker.start.timeout.secs": "120",
- "site.storm-site.zmq.threads": "1",
- "site.storm-site.topology.acker.executors": "null",
- "site.storm-site.storm.local.mode.zmq": "false",
- "site.storm-site.topology.max.task.parallelism": "null",
- "site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
- "site.storm-site.drpc.queue.size": "128",
- "site.storm-site.storm.zookeeper.retry.times": "5",
- "site.storm-site.nimbus.monitor.freq.secs": "10",
- "site.storm-site.storm.cluster.mode": "distributed",
- "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
- "site.storm-site.drpc.invocations.port": "0",
- "site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
- "site.storm-site.logviewer.childopts": "-Xmx128m",
- "site.storm-site.transactional.zookeeper.port": "null",
- "site.storm-site.topology.worker.childopts": "null",
- "site.storm-site.topology.max.spout.pending": "null",
- "site.storm-site.nimbus.cleanup.inbox.freq.secs": "600",
- "site.storm-site.storm.messaging.netty.min_wait_ms": "100",
- "site.storm-site.nimbus.task.timeout.secs": "30",
- "site.storm-site.nimbus.thrift.max_buffer_size": "1048576",
- "site.storm-site.topology.sleep.spout.wait.strategy.time.ms": "1",
- "site.storm-site.topology.optimize": "true",
- "site.storm-site.nimbus.reassign": "true",
- "site.storm-site.storm.messaging.transport": "backtype.storm.messaging.netty.Context",
- "site.storm-site.logviewer.appender.name": "A1",
- "site.storm-site.nimbus.host": "${NIMBUS_HOST}",
- "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
- "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.nimbus.file.copy.expiration.secs": "600",
- "site.storm-site.supervisor.monitor.frequency.secs": "3",
- "site.storm-site.transactional.zookeeper.servers": "null",
- "site.storm-site.zmq.linger.millis": "5000",
- "site.storm-site.topology.error.throttle.interval.secs": "10",
- "site.storm-site.topology.worker.shared.thread.pool.size": "4",
- "site.storm-site.java.library.path": "/usr/local/lib:/opt/local/lib:/usr/lib",
- "site.storm-site.topology.spout.wait.strategy": "backtype.storm.spout.SleepSpoutWaitStrategy",
- "site.storm-site.task.heartbeat.frequency.secs": "3",
- "site.storm-site.topology.transfer.buffer.size": "1024",
- "site.storm-site.storm.zookeeper.session.timeout": "20000",
- "site.storm-site.topology.executor.receive.buffer.size": "1024",
- "site.storm-site.topology.stats.sample.rate": "0.05",
- "site.storm-site.topology.fall.back.on.java.serialization": "true",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
- "site.storm-site.topology.enable.message.timeouts": "true",
- "site.storm-site.storm.messaging.netty.max_wait_ms": "1000",
- "site.storm-site.nimbus.topology.validator": "backtype.storm.nimbus.DefaultTopologyValidator",
- "site.storm-site.nimbus.supervisor.timeout.secs": "60",
- "site.storm-site.topology.disruptor.wait.strategy": "com.lmax.disruptor.BlockingWaitStrategy",
- "site.storm-site.nimbus.inbox.jar.expiration.secs": "3600",
- "site.storm-site.drpc.port": "0",
- "site.storm-site.topology.kryo.factory": "backtype.storm.serialization.DefaultKryoFactory",
- "site.storm-site.storm.zookeeper.retry.interval": "1000",
- "site.storm-site.storm.messaging.netty.max_retries": "30",
- "site.storm-site.topology.tick.tuple.freq.secs": "null",
- "site.storm-site.supervisor.enable": "true",
- "site.storm-site.nimbus.task.launch.secs": "120",
- "site.storm-site.task.refresh.poll.secs": "10",
- "site.storm-site.topology.message.timeout.secs": "30",
- "site.storm-site.storm.messaging.netty.buffer_size": "5242880",
- "site.storm-site.topology.state.synchronization.timeout.secs": "60",
- "site.storm-site.supervisor.worker.timeout.secs": "30",
- "site.storm-site.topology.trident.batch.emit.interval.millis": "500",
- "site.storm-site.topology.builtin.metrics.bucket.size.secs": "60",
- "site.storm-site.storm.thrift.transport": "backtype.storm.security.auth.SimpleTransportPlugin",
- "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}",
- "site.storm-site.topology.debug": "false"
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- },
- "NIMBUS": {
- },
- "STORM_REST_API": {
- },
- "STORM_UI_SERVER": {
- },
- "DRPC_SERVER": {
- },
- "SUPERVISOR": {
- }
- }
-}
diff --git a/app-packages/storm/configuration/global.xml b/app-packages/storm/configuration/global.xml
deleted file mode 100644
index 5cc9170..0000000
--- a/app-packages/storm/configuration/global.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <property>
- <name>storm_user</name>
- <value>storm</value>
- <description></description>
- </property>
- <property>
- <name>storm_log_dir</name>
- <value>/var/log/storm</value>
- <description></description>
- </property>
- <property>
- <name>storm_pid_dir</name>
- <value>/var/run/storm</value>
- <description></description>
- </property>
-</configuration>
diff --git a/app-packages/storm/configuration/storm-env.xml b/app-packages/storm/configuration/storm-env.xml
new file mode 100644
index 0000000..091c08d
--- /dev/null
+++ b/app-packages/storm/configuration/storm-env.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+ <property>
+ <name>kerberos_domain</name>
+ <value></value>
+ <description>The kerberos domain to be used for this Storm cluster</description>
+ </property>
+ <property>
+ <name>storm_client_principal_name</name>
+ <value></value>
+ <description>The principal name for the Storm client to be used to communicate with Nimbus and Zookeeper</description>
+ </property>
+ <property>
+ <name>storm_server_principal_name</name>
+ <value></value>
+ <description>The principal name for the Storm server to be used by Nimbus</description>
+ </property>
+ <property>
+ <name>storm_client_keytab</name>
+ <value></value>
+ <description>The keytab file path for Storm client</description>
+ </property>
+ <property>
+ <name>storm_server_keytab</name>
+ <value></value>
+ <description>The keytab file path for Storm server</description>
+ </property>
+ <!-- storm-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for storm-env.sh file</description>
+ <value>
+#!/bin/bash
+
+# Set Storm specific environment variables here.
+
+# The java implementation to use.
+export JAVA_HOME={{java_home}}
+
+# export STORM_CONF_DIR=""
+ </value>
+ </property>
+</configuration>
diff --git a/app-packages/storm/configuration/storm-site.xml b/app-packages/storm/configuration/storm-site.xml
index 6eca8f9..b3cce6a 100644
--- a/app-packages/storm/configuration/storm-site.xml
+++ b/app-packages/storm/configuration/storm-site.xml
@@ -118,7 +118,7 @@
</property>
<property>
<name>nimbus.childopts</name>
- <value>-Xmx1024m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8649,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM</value>
+ <value>-Xmx1024m -Djava.security.auth.login.config=/etc/storm/conf/storm_jaas.conf -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8649,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM</value>
<description>This parameter is used by the storm-deploy project to configure the jvm options for the nimbus daemon.</description>
</property>
<property>
@@ -188,7 +188,7 @@
</property>
<property>
<name>ui.childopts</name>
- <value>-Xmx768m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf</value>
+ <value>-Xmx768m -Djava.security.auth.login.config=/etc/storm/conf/storm_jaas.conf</value>
<description>Childopts for Storm UI Java process.</description>
</property>
<property>
@@ -264,7 +264,7 @@
</property>
<property>
<name>supervisor.childopts</name>
- <value>-Xmx256m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=56431 -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM</value>
+ <value>-Xmx256m -Djava.security.auth.login.config=/etc/storm/conf/storm_jaas.conf -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=56431 -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM</value>
<description>This parameter is used by the storm-deploy project to configure the jvm options for the supervisor daemon.</description>
</property>
<property>
@@ -291,15 +291,8 @@
<description>How often the supervisor sends a heartbeat to the master.</description>
</property>
<property>
- <name>supervisor.enable</name>
- <value>true</value>
- <description>Whether or not the supervisor should launch workers assigned to it. Defaults
- to true -- and you should probably never change this value. This configuration
- is used in the Storm unit tests.</description>
- </property>
- <property>
<name>worker.childopts</name>
- <value>-Xmx768m -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM</value>
+ <value>-Xmx768m -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM</value>
<description>The jvm opts provided to workers launched by this supervisor. All \"%ID%\" substrings are replaced with an identifier for this worker.</description>
</property>
<property>
diff --git a/app-packages/storm/jmx_metrics.json b/app-packages/storm/jmx_metrics.json
index f7d4e60..b0816b1 100644
--- a/app-packages/storm/jmx_metrics.json
+++ b/app-packages/storm/jmx_metrics.json
@@ -2,17 +2,17 @@
"Component": {
"NIMBUS": {
"FreeSlots": {
- "metric": "$['slots.free']",
+ "metric": "$['slotsFree']",
"pointInTime": true,
"temporal": false
},
"Tasks": {
- "metric": "$['tasks.total']",
+ "metric": "$['tasksTotal']",
"pointInTime": true,
"temporal": false
},
"Executors": {
- "metric": "$['executors.total']",
+ "metric": "$['executorsTotal']",
"pointInTime": true,
"temporal": false
},
@@ -22,7 +22,7 @@
"temporal": false
},
"NimbusUptime": {
- "metric": "$['nimbus.uptime']",
+ "metric": "$['nimbusUptime']",
"pointInTime": true,
"temporal": false
}
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index dbe8549..28b0e9b 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -21,7 +21,7 @@
<application>
<name>STORM</name>
<comment>Apache Hadoop Stream processing framework</comment>
- <version>0.9.1.2.1</version>
+ <version>${pkg.version}</version>
<exportedConfigs>storm-site</exportedConfigs>
<exportGroups>
@@ -29,19 +29,19 @@
<name>QuickLinks</name>
<exports>
<export>
- <name>app.jmx</name>
- <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
+ <name>org.apache.slider.jmx</name>
+ <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}/api/v1/cluster/summary</value>
</export>
<export>
- <name>app.monitor</name>
+ <name>org.apache.slider.monitor</name>
<value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
</export>
<export>
- <name>app.metrics</name>
+ <name>org.apache.slider.metrics</name>
<value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
</export>
<export>
- <name>ganglia.ui</name>
+ <name>org.apache.slider.metrics.ui</name>
<value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
</export>
<export>
@@ -55,8 +55,7 @@
<commandOrders>
<commandOrder>
<command>NIMBUS-START</command>
- <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED
- </requires>
+ <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED</requires>
</commandOrder>
<commandOrder>
<command>SUPERVISOR-START</command>
@@ -67,10 +66,6 @@
<requires>NIMBUS-STARTED</requires>
</commandOrder>
<commandOrder>
- <command>STORM_REST_API-START</command>
- <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires>
- </commandOrder>
- <commandOrder>
<command>STORM_UI_SERVER-START</command>
<requires>NIMBUS-STARTED</requires>
</commandOrder>
@@ -81,8 +76,10 @@
<component>
<name>NIMBUS</name>
<category>MASTER</category>
+ <publishConfig>true</publishConfig>
<autoStartOnFailure>true</autoStartOnFailure>
- <appExports>QuickLinks-nimbus.host_port,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
+ <appExports>QuickLinks-nimbus.host_port,QuickLinks-org.apache.slider.metrics.ui,QuickLinks-org.apache.slider.metrics</appExports>
+ <maxInstanceCount>1</maxInstanceCount>
<commandScript>
<script>scripts/nimbus.py</script>
<scriptType>PYTHON</scriptType>
@@ -91,18 +88,6 @@
</component>
<component>
- <name>STORM_REST_API</name>
- <category>MASTER</category>
- <autoStartOnFailure>true</autoStartOnFailure>
- <appExports>QuickLinks-app.jmx</appExports>
- <commandScript>
- <script>scripts/rest_api.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
<name>SUPERVISOR</name>
<category>SLAVE</category>
<autoStartOnFailure>true</autoStartOnFailure>
@@ -123,7 +108,8 @@
<name>STORM_UI_SERVER</name>
<category>MASTER</category>
<publishConfig>true</publishConfig>
- <appExports>QuickLinks-app.monitor</appExports>
+ <appExports>QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.jmx</appExports>
+ <maxInstanceCount>1</maxInstanceCount>
<autoStartOnFailure>true</autoStartOnFailure>
<commandScript>
<script>scripts/ui_server.py</script>
@@ -136,6 +122,7 @@
<name>DRPC_SERVER</name>
<category>MASTER</category>
<autoStartOnFailure>true</autoStartOnFailure>
+ <maxInstanceCount>1</maxInstanceCount>
<commandScript>
<script>scripts/drpc_server.py</script>
<scriptType>PYTHON</scriptType>
@@ -150,10 +137,24 @@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ <name>files/apache-storm-${pkg.version}.tar.gz</name>
</package>
</packages>
</osSpecific>
</osSpecifics>
+
+ <configFiles>
+ <configFile>
+ <type>yaml</type>
+ <fileName>storm.yaml</fileName>
+ <dictionaryName>storm-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>storm-env.sh</fileName>
+ <dictionaryName>storm-env</dictionaryName>
+ </configFile>
+ </configFiles>
+
</application>
</metainfo>
diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py
index cf21b27..1ccba5e 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -34,12 +34,10 @@
java64_home = config['hostLevelParams']['java_home']
nimbus_host = config['configurations']['storm-site']['nimbus.host']
nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
-nimbus_host = config['configurations']['storm-site']['nimbus.host']
-rest_api_port = config['configurations']['global']['rest_api_port']
-rest_api_admin_port = config['configurations']['global']['rest_api_admin_port']
rest_api_conf_file = format("{conf_dir}/config.yaml")
-rest_lib_dir = format("{app_root}/contrib/storm-rest")
-storm_bin = format("{app_root}/bin/storm")
+rest_lib_dir = format("{app_root}/external/storm-rest")
+storm_bin = format("{app_root}/bin/storm.py")
+storm_env_sh_template = config['configurations']['storm-env']['content']
ganglia_installed = config['configurations']['global']['ganglia_enabled']
if ganglia_installed:
@@ -47,12 +45,17 @@
ganglia_server = config['configurations']['global']['ganglia_server_host']
ganglia_port = config['configurations']['global']['ganglia_server_port']
-_authentication = config['configurations']['core-site']['hadoop.security.authentication']
-security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
+security_enabled = config['configurations']['global']['security_enabled']
if security_enabled:
_hostname_lowercase = config['hostname'].lower()
- _kerberos_domain = config['configurations']['global']['kerberos_domain']
- _storm_principal_name = config['configurations']['global']['storm_principal_name']
- storm_jaas_principal = _storm_principal_name.replace('_HOST', _hostname_lowercase)
- storm_keytab_path = config['configurations']['global']['storm_keytab']
+ _kerberos_domain = config['configurations']['storm-env']['kerberos_domain']
+ _storm_client_principal_name = config['configurations']['storm-env']['storm_client_principal_name']
+ _storm_server_principal_name = config['configurations']['storm-env']['storm_server_principal_name']
+
+ storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase)
+ storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab']
+ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST',nimbus_host.lower())
+ storm_jaas_stormclient_servicename = storm_jaas_server_principal.split("/")[0]
+ storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab']
+ kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index 13fcef2..7a7dbdf 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -21,6 +21,8 @@
from resource_management import *
import time
+import os
+import sys
"""
Slider package uses jps as pgrep does not list the whole process start command
@@ -31,6 +33,7 @@
import params
import status_params
+ python_binary = os.environ['PYTHON_EXE'] if 'PYTHON_EXE' in os.environ else sys.executable
pid_file = status_params.pid_files[name]
container_id = status_params.container_id
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
@@ -50,18 +53,16 @@
if name == "rest_api":
cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
else:
- cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
+ cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
Execute(cmd,
not_if=no_op_test,
- user=params.storm_user,
logoutput=False,
wait_for_finish=False
)
if name == "rest_api":
Execute(crt_pid_cmd,
- user=params.storm_user,
logoutput=True,
tries=6,
try_sleep=10
@@ -70,14 +71,13 @@
content = None
for i in xrange(12):
Execute(crt_pid_cmd,
- user=params.storm_user,
logoutput=True
)
with open(pid_file) as f:
content = f.readline().strip()
if content.isdigit():
break;
- File(pid_file, action = "delete")
+ File(pid_file, action="delete")
time.sleep(10)
pass
diff --git a/app-packages/storm/package/scripts/status_params.py b/app-packages/storm/package/scripts/status_params.py
index 5907446..7dda158 100644
--- a/app-packages/storm/package/scripts/status_params.py
+++ b/app-packages/storm/package/scripts/status_params.py
@@ -33,5 +33,5 @@
"ui": pid_ui,
"nimbus": pid_nimbus,
"supervisor": pid_supervisor,
- "drpc": pid_drpc,
- "rest_api": pid_rest_api}
\ No newline at end of file
+ "rest_api": pid_rest_api,
+ "drpc": pid_drpc}
\ No newline at end of file
diff --git a/app-packages/storm/package/scripts/storm.py b/app-packages/storm/package/scripts/storm.py
index bce272b..8ecb3a1 100644
--- a/app-packages/storm/package/scripts/storm.py
+++ b/app-packages/storm/package/scripts/storm.py
@@ -43,8 +43,16 @@
owner = params.storm_user,
group = params.user_group
)
-
+
+ File(format("{conf_dir}/storm-env.sh"),
+ owner=params.storm_user,
+ content=InlineTemplate(params.storm_env_sh_template)
+ )
+
if params.security_enabled:
- TemplateConfig( format("{conf_dir}/storm_jaas.conf"),
- owner = params.storm_user
- )
\ No newline at end of file
+ File(format("{conf_dir}/storm_jaas.conf"),
+ content=Template("storm_jaas.conf.j2"),
+ owner = params.storm_user,
+ group = params.user_group
+ )
+
diff --git a/app-packages/storm/package/scripts/yaml_config.py b/app-packages/storm/package/scripts/yaml_config.py
index 39261be..5f763cc 100644
--- a/app-packages/storm/package/scripts/yaml_config.py
+++ b/app-packages/storm/package/scripts/yaml_config.py
@@ -19,9 +19,13 @@
"""
import re
+import socket
from resource_management import *
def escape_yaml_propetry(value):
+ # pre-process value for any "_HOST" tokens
+ value = value.replace('_HOST', socket.getfqdn())
+
unquouted = False
unquouted_values = ["null","Null","NULL","true","True","TRUE","false","False","FALSE","YES","Yes","yes","NO","No","no","ON","On","on","OFF","Off","off"]
@@ -31,7 +35,11 @@
# if is list [a,b,c]
if re.match('^\w*\[.+\]\w*$', value):
unquouted = True
-
+
+ # if is map {'a':'b'}
+ if re.match('^\w*\{.+\}\w*$', value):
+ unquouted = True
+
try:
int(value)
unquouted = True
@@ -50,6 +58,10 @@
return value
+def yaml_inline_template(configurations):
+ return source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }}
+{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry])
+
def yaml_config(
filename,
configurations = None,
@@ -58,8 +70,7 @@
owner = None,
group = None
):
- config_content = source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }}
-{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry])
+ config_content = yaml_inline_template(configurations)
File (format("{conf_dir}/{filename}"),
content = config_content,
diff --git a/app-packages/storm/package/templates/config.yaml.j2 b/app-packages/storm/package/templates/config.yaml.j2
index 32d2c99..aa4ec46 100644
--- a/app-packages/storm/package/templates/config.yaml.j2
+++ b/app-packages/storm/package/templates/config.yaml.j2
@@ -16,15 +16,6 @@
nimbusHost: {{nimbus_host}}
nimbusPort: {{nimbus_port}}
-# HTTP-specific options.
-http:
-
- # The port on which the HTTP server listens for service requests.
- port: {{rest_api_port}}
-
- # The port on which the HTTP server listens for administrative requests.
- adminPort: {{rest_api_admin_port}}
-
{% if ganglia_installed %}
enableGanglia: {{ganglia_installed}}
diff --git a/app-packages/storm/package/templates/storm_jaas.conf.j2 b/app-packages/storm/package/templates/storm_jaas.conf.j2
index 4031d22..a1ba6ea 100644
--- a/app-packages/storm/package/templates/storm_jaas.conf.j2
+++ b/app-packages/storm/package/templates/storm_jaas.conf.j2
@@ -15,12 +15,30 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+StormServer {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{storm_server_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ principal="{{storm_jaas_server_principal}}";
+};
+StormClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{storm_client_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="{{storm_jaas_stormclient_servicename}}"
+ debug=true
+ principal="{{storm_jaas_client_principal}}";
+};
Client {
com.sun.security.auth.module.Krb5LoginModule required
useKeyTab=true
- keyTab="{{storm_keytab_path}}"
+ keyTab="{{storm_client_keytab_path}}"
storeKey=true
useTicketCache=false
serviceName="zookeeper"
- principal="{{storm_jaas_principal}}";
+ principal="{{storm_jaas_client_principal}}";
};
diff --git a/app-packages/storm/pom.xml b/app-packages/storm/pom.xml
new file mode 100644
index 0000000..00ec044
--- /dev/null
+++ b/app-packages/storm/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.60.0-incubating</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-storm-app-package</artifactId>
+ <packaging>pom</packaging>
+ <name>Slider Storm App Package</name>
+ <description>Slider Storm App Package</description>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>storm-app-package</id>
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/storm.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ </build>
+
+ <dependencies>
+ </dependencies>
+
+</project>
diff --git a/app-packages/storm/resources-default.json b/app-packages/storm/resources-default.json
new file mode 100644
index 0000000..65e6579
--- /dev/null
+++ b/app-packages/storm/resources-default.json
@@ -0,0 +1,34 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": ""
+ },
+ "components": {
+ "slider-appmaster": {
+ "yarn.memory": "512"
+ },
+ "NIMBUS": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "2048"
+ },
+ "STORM_UI_SERVER": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": "1",
+ "yarn.memory": "1278"
+ },
+ "DRPC_SERVER": {
+ "yarn.role.priority": "3",
+ "yarn.component.instances": "1",
+ "yarn.memory": "1278"
+ },
+ "SUPERVISOR": {
+ "yarn.role.priority": "4",
+ "yarn.component.instances": "1",
+ "yarn.memory": "3072"
+ }
+ }
+}
diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml
new file mode 100644
index 0000000..f7dcf13
--- /dev/null
+++ b/app-packages/storm/src/assembly/storm.xml
@@ -0,0 +1,75 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>slider-storm-v${storm.version}</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>appConfig-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>appConfig-secured-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>metainfo.xml</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>pom.xml</exclude>
+ <exclude>src/**</exclude>
+ <exclude>target/**</exclude>
+ <exclude>appConfig-default.json</exclude>
+ <exclude>appConfig-secured-default.json</exclude>
+ <exclude>metainfo.xml</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ </fileSets>
+</assembly>
diff --git a/pom.xml b/pom.xml
index fb27aba..6cf66ab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,4 +1,4 @@
-<!--
+ <!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
@@ -19,7 +19,7 @@
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
<name>Slider</name>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
<packaging>pom</packaging>
<description>
@@ -38,7 +38,6 @@
<module>app-packages/command-logger/slider-pkg</module>
<module>slider-core</module>
<module>slider-agent</module>
- <module>app-packages/accumulo</module>
<module>slider-assembly</module>
<module>slider-funtest</module>
<module>slider-providers/hbase/slider-hbase-provider</module>
@@ -72,6 +71,19 @@
<url>http://slider.incubator.apache.org/</url>
</site>
<downloadUrl>http://git-wip-us.apache.org/repos/asf/incubator-slider.git</downloadUrl>
+ <repository>
+ <id>${distMgmtStagingId}</id>
+ <name>${distMgmtStagingName}</name>
+ <url>${distMgmtStagingUrl}</url>
+ </repository>
+ <snapshotRepository>
+ <id>${distMgmtSnapshotsId}</id>
+ <name>${distMgmtSnapshotsName}</name>
+ <url>${distMgmtSnapshotsUrl}</url>
+ </snapshotRepository>
+
+
+
</distributionManagement>
<mailingLists>
@@ -92,7 +104,12 @@
</mailingLists>
<properties>
-
+ <distMgmtSnapshotsId>apache.snapshots.https</distMgmtSnapshotsId>
+ <distMgmtSnapshotsName>Apache Development Snapshot Repository</distMgmtSnapshotsName>
+ <distMgmtSnapshotsUrl>https://repository.apache.org/content/repositories/snapshots</distMgmtSnapshotsUrl>
+ <distMgmtStagingId>apache.staging.https</distMgmtStagingId>
+ <distMgmtStagingName>Apache Release Distribution Repository</distMgmtStagingName>
+ <distMgmtStagingUrl>https://repository.apache.org/service/local/staging/deploy/maven2</distMgmtStagingUrl>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -114,14 +131,16 @@
<test.failIfNoTests>true</test.failIfNoTests>
<test.funtests.failIfNoTests>false</test.funtests.failIfNoTests>
<test.forkMode>always</test.forkMode>
+ <!-- path environment variable -->
+ <test.env.path>${env.PATH}</test.env.path>
<!--
core artifacts
-->
- <hadoop.version>2.4.1</hadoop.version>
+ <hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
<hbase.version>0.98.4-hadoop2</hbase.version>
- <accumulo.version>1.6.0</accumulo.version>
+ <accumulo.version>1.6.1</accumulo.version>
<!--
artifact versions
@@ -150,7 +169,7 @@
<servlet-api.version>2.5</servlet-api.version>
<jsr311-api.version>1.1.1</jsr311-api.version>
<jaxb-api.version>2.2.7</jaxb-api.version>
-
+ <jsp.version>2.1</jsp.version>
<junit.version>4.11</junit.version>
<log4j.version>1.2.17</log4j.version>
<metrics.version>3.0.1</metrics.version>
@@ -162,7 +181,7 @@
<slf4j.version>1.7.5</slf4j.version>
<stringtemplate.version>2.4.1</stringtemplate.version>
- <zookeeper.version>3.4.5</zookeeper.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
<!-- Plugin versions -->
@@ -175,6 +194,7 @@
<maven.version.range>[3.0.0,)</maven.version.range>
+ <maven-antrun-plugin.version>1.7</maven-antrun-plugin.version>
<maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
<maven.cobertura.version>2.5.2</maven.cobertura.version>
<maven-compiler-plugin.version>3.1</maven-compiler-plugin.version>
@@ -182,6 +202,7 @@
<maven-deploy-plugin.version>2.7</maven-deploy-plugin.version>
<maven-doxia-module-markdown.version>1.4</maven-doxia-module-markdown.version>
<maven-enforcer-plugin.version>1.0</maven-enforcer-plugin.version>
+ <maven-exec-plugin.version>1.2.1</maven-exec-plugin.version>
<maven-jar-plugin.version>2.3.1</maven-jar-plugin.version>
<maven.javadoc.version>2.8</maven.javadoc.version>
<maven.project.version>2.4</maven.project.version>
@@ -213,6 +234,16 @@
<id>ASF Staging</id>
<url>https://repository.apache.org/content/groups/staging/</url>
</repository>
+ <repository>
+ <id>ASF Snapshots</id>
+ <url>https://repository.apache.org/content/repositories/snapshots/</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ </repository>
</repositories>
@@ -320,36 +351,7 @@
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>**/*.tar</exclude>
- <exclude>**/build.properties</exclude>
- <exclude>**/regionservers</exclude>
- <exclude>**/slaves</exclude>
- <exclude>**/httpfs-signature.secret</exclude>
- <exclude>**/dfs.exclude</exclude>
- <exclude>**/*.iml</exclude>
- <exclude>**/rat.txt</exclude>
- <exclude>DISCLAIMER</exclude>
- <exclude>app-packages/hbase/target/**</exclude>
- <exclude>target/*</exclude>
- </excludes>
- </configuration>
- </plugin>
-
+
</plugins>
</build>
@@ -524,6 +526,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
<version>${hadoop.version}</version>
</dependency>
@@ -598,6 +606,12 @@
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>${httpclient.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- ======================================================== -->
@@ -931,6 +945,18 @@
<groupId>org.junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -1127,6 +1153,12 @@
</dependency>
<dependency>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ <version>${jsp.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<version>${jersey.version}</version>
@@ -1216,24 +1248,126 @@
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<version>${jetty.version}</version>
+ <exclusions>
+ <!-- cut the jetty version of the servlet API —Hadoop ships with one-->
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
+
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>${jetty.version}</version>
</dependency>
+
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
<version>${jetty.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ <version>1.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-core</artifactId>
+ <version>1.5</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-reflect</artifactId>
+ <version>1.5</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-easymock</artifactId>
+ <version>1.5</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>1.5</version>
+ </dependency>
+
</dependencies>
+
</dependencyManagement>
<profiles>
<profile>
+ <id>Non-Windows</id>
+ <activation>
+ <os>
+ <family>!windows</family>
+ </os>
+ </activation>
+ <modules>
+ <module>app-packages/accumulo</module>
+ <module>app-packages/hbase</module>
+ <module>app-packages/storm</module>
+ </modules>
+ </profile>
+
+ <profile>
+ <id>Windows</id>
+ <activation>
+ <os>
+ <family>windows</family>
+ </os>
+ </activation>
+ <modules>
+ <module>app-packages/hbase-win</module>
+ <module>app-packages/storm-win</module>
+ </modules>
+ </profile>
+ <profile>
+ <id>rat</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/*.tar</exclude>
+ <exclude>**/build.properties</exclude>
+ <exclude>**/regionservers</exclude>
+ <exclude>**/slaves</exclude>
+ <exclude>**/httpfs-signature.secret</exclude>
+ <exclude>**/dfs.exclude</exclude>
+ <exclude>**/*.iml</exclude>
+ <exclude>**/rat.txt</exclude>
+ <exclude>DISCLAIMER</exclude>
+ <exclude>app-packages/hbase/target/**</exclude>
+ <exclude>target/*</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>apache-release</id>
<build>
<plugins>
@@ -1271,39 +1405,54 @@
</profile>
<profile>
- <!-- local builds of everything -->
- <id>local</id>
+ <!-- 2.6 snapshots -->
+ <id>branch-2.6</id>
<properties>
- <hadoop.version>2.4.1-SNAPSHOT</hadoop.version>
- <hbase.version>0.98.4-SNAPSHOT</hbase.version>
- <accumulo.version>1.6.0-SNAPSHOT</accumulo.version>
+ <hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <!-- 2.6 snapshots -->
+ <id>release-2.6</id>
+ <properties>
+ <hadoop.version>2.6.0</hadoop.version>
</properties>
</profile>
<profile>
<!-- hadoop branch-2 builds -->
- <id>hadoop-2.4.1</id>
- <properties>
- <hadoop.version>2.4.1</hadoop.version>
- </properties>
- </profile>
- <profile>
- <!-- hadoop branch-2 builds -->
<id>branch-2</id>
<properties>
- <hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
+ <hadoop.version>2.7.0-SNAPSHOT</hadoop.version>
</properties>
</profile>
<profile>
- <!-- hadoop branch-2 builds -->
- <id>hadoop-trunk</id>
+ <!-- hadoop trunk builds -->
+ <id>trunk</id>
<properties>
<hadoop.version>3.0.0-SNAPSHOT</hadoop.version>
</properties>
</profile>
<profile>
+ <!-- Java 7 build -->
+ <id>java7</id>
+ <properties>
+ <project.java.src.version>7</project.java.src.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <!-- Java 8 build -->
+ <id>java8</id>
+ <properties>
+ <project.java.src.version>8</project.java.src.version>
+ </properties>
+ </profile>
+
+ <profile>
<!-- anything for a jenkins build -->
<id>jenkins</id>
<properties>
@@ -1387,7 +1536,49 @@
</build>
</profile>
+ <profile>
+ <id>sign</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>sign-artifacts</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>sign</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>private-repo</id>
+ <!-- this profile is for pulling custom app versions from a private maven repo -->
+ <activation>
+ <property>
+ <name>private.repo.url</name>
+ </property>
+ </activation>
+ <repositories>
+ <repository>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ <id>private-repo</id>
+ <name>Private Repo</name>
+ <url>${private.repo.url}</url>
+ </repository>
+ </repositories>
+ </profile>
</profiles>
diff --git a/slider-agent/conf/agent.ini b/slider-agent/conf/agent.ini
index 7b9d57d..48113e3 100644
--- a/slider-agent/conf/agent.ini
+++ b/slider-agent/conf/agent.ini
@@ -43,6 +43,7 @@
[command]
max_retries=2
sleep_between_retries=1
+auto_restart=5,5
[security]
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
index 09f2dae..332d5d1 100644
--- a/slider-agent/pom.xml
+++ b/slider-agent/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>slider-agent</artifactId>
@@ -32,8 +32,9 @@
<package.release>1</package.release>
<skipTests>false</skipTests>
<python.ver>python >= 2.6</python.ver>
+ <executable.python>${project.basedir}/../slider-agent/src/test/python/python-wrap</executable.python>
+ <python.path.l>${project.basedir}/src/main/python/jinja2:${project.basedir}/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/agent:${project.basedir}/src/main/python/resource_management:${project.basedir}/src/test/python/agent:${project.basedir}/src/test/python/resource_management:${project.basedir}/src/main/python/kazoo</python.path.l>
</properties>
-
<build>
<plugins>
@@ -59,17 +60,17 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2</version>
+ <version>${maven-exec-plugin.version}</version>
<executions>
<execution>
<configuration>
- <executable>${project.basedir}/src/test/python/python-wrap</executable>
+ <executable>${executable.python}</executable>
<workingDirectory>src/test/python</workingDirectory>
<arguments>
<argument>unitTests.py</argument>
</arguments>
<environmentVariables>
- <PYTHONPATH>${project.basedir}/src/main/python/jinja2:${project.basedir}/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/agent:${project.basedir}/src/main/python/resource_management:${project.basedir}/src/test/python/agent:${project.basedir}/src/test/python/resource_management:${project.basedir}/src/main/python/kazoo</PYTHONPATH>
+ <PYTHONPATH>${python.path.l}</PYTHONPATH>
</environmentVariables>
<skip>${skipTests}</skip>
</configuration>
@@ -82,32 +83,9 @@
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>src/test/python/agent/dummy_output_error.txt</exclude>
- <exclude>src/test/python/agent/dummy_output_good.txt</exclude>
- <!-- jinja2 files (BSD license) -->
- <exclude>src/main/python/jinja2/**</exclude>
- <!-- mock files (BSD license) -->
- <exclude>src/test/python/mock/**</exclude>
- <!-- kazoo files (Apache License, Version 2.0) -->
- <exclude>src/main/python/kazoo/**</exclude>
- </excludes>
- </configuration>
- </plugin>
+
</plugins>
+
<extensions>
<extension>
<groupId>org.apache.maven.wagon</groupId>
@@ -115,4 +93,60 @@
</extension>
</extensions>
</build>
+
+ <profiles>
+ <profile>
+ <id>Windows</id>
+ <activation>
+ <os><family>windows</family></os>
+ </activation>
+ <properties>
+ <executable.python>python</executable.python>
+ <python.path.l>${project.basedir}\src\main\python\jinja2;${project.basedir}\src\test\python;${project.basedir}\src\main\python;${project.basedir}\src\main\python\agent;${project.basedir}\src\main\python\resource_management;${project.basedir}\src\test\python\agent;${project.basedir}\src\test\python\resource_management;${project.basedir}\src\main\python\kazoo</python.path.l>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>Linux</id>
+ <activation>
+ <os><family>!windows</family></os>
+ </activation>
+ <properties>
+ <executable.python>${project.basedir}/../slider-agent/src/test/python/python-wrap</executable.python>
+ <python.path.l>${project.basedir}/src/main/python/jinja2:${project.basedir}/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/agent:${project.basedir}/src/main/python/resource_management:${project.basedir}/src/test/python/agent:${project.basedir}/src/test/python/resource_management:${project.basedir}/src/main/python/kazoo</python.path.l>
+ </properties>
+ </profile>
+ <profile>
+ <id>rat</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>src/test/python/agent/dummy_output_error.txt</exclude>
+ <exclude>src/test/python/agent/dummy_output_good.txt</exclude>
+ <!-- jinja2 files (BSD license) -->
+ <exclude>src/main/python/jinja2/**</exclude>
+ <!-- mock files (BSD license) -->
+ <exclude>src/test/python/mock/**</exclude>
+ <!-- kazoo files (Apache License, Version 2.0) -->
+ <exclude>src/main/python/kazoo/**</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index 4c45a76..4cb5de7 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -27,6 +27,7 @@
import time
from AgentConfig import AgentConfig
+from AgentToggleLogger import AgentToggleLogger
from CommandStatusDict import CommandStatusDict
from CustomServiceOrchestrator import CustomServiceOrchestrator
import Constants
@@ -51,8 +52,10 @@
STORE_APPLIED_CONFIG = 'record_config'
AUTO_RESTART = 'auto_restart'
- def __init__(self, config, controller):
+ def __init__(self, config, controller, agentToggleLogger):
super(ActionQueue, self).__init__()
+ self.queueOutAgentToggleLogger = agentToggleLogger
+ self.queueInAgentToggleLogger = AgentToggleLogger("info")
self.commandQueue = Queue.Queue()
self.commandStatuses = CommandStatusDict(callback_action=
self.status_update_callback)
@@ -61,7 +64,8 @@
self._stop = threading.Event()
self.tmpdir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.customServiceOrchestrator = CustomServiceOrchestrator(config,
- controller)
+ controller,
+ self.queueOutAgentToggleLogger)
def stop(self):
@@ -72,9 +76,12 @@
def put(self, commands):
for command in commands:
- logger.info("Adding " + command['commandType'] + " for service " + \
- command['serviceName'] + " of cluster " + \
- command['clusterName'] + " to the queue.")
+ self.queueInAgentToggleLogger.adjustLogLevelAtStart(command['commandType'])
+ message = "Adding " + command['commandType'] + " for service " + \
+ command['serviceName'] + " of cluster " + \
+ command['clusterName'] + " to the queue."
+ self.queueInAgentToggleLogger.log(message)
+ self.queueInAgentToggleLogger.adjustLogLevelAtEnd(command['commandType'])
logger.debug(pprint.pformat(command))
self.commandQueue.put(command)
@@ -86,7 +93,9 @@
while not self.stopped():
time.sleep(2)
command = self.commandQueue.get() # Will block if queue is empty
+ self.queueOutAgentToggleLogger.adjustLogLevelAtStart(command['commandType'])
self.process_command(command)
+ self.queueOutAgentToggleLogger.adjustLogLevelAtEnd(command['commandType'])
logger.info("ActionQueue stopped.")
@@ -142,9 +151,10 @@
store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
store_command = False
if 'roleParams' in command and ActionQueue.AUTO_RESTART in command['roleParams']:
- logger.info("Component has indicated auto-restart. Saving details from START command.")
store_command = 'true' == command['roleParams'][ActionQueue.AUTO_RESTART]
+ if store_command:
+ logger.info("Component has indicated auto-restart. Saving details from START command.")
# running command
commandresult = self.customServiceOrchestrator.runCommand(command,
@@ -154,6 +164,12 @@
'tmperr'],
True,
store_config or store_command)
+ # If command is STOP then set flag to indicate stop has been triggered.
+ # In future we might check status of STOP command and take other measures
+ # if graceful STOP fails (like force kill the processes)
+ if command['roleCommand'] == 'STOP':
+ self.controller.appGracefulStopTriggered = True
+
# dumping results
status = self.COMPLETED_STATUS
if commandresult[Constants.EXIT_CODE] != 0:
diff --git a/slider-agent/src/main/python/agent/AgentConfig.py b/slider-agent/src/main/python/agent/AgentConfig.py
index e45ba23..86925b1 100644
--- a/slider-agent/src/main/python/agent/AgentConfig.py
+++ b/slider-agent/src/main/python/agent/AgentConfig.py
@@ -61,6 +61,7 @@
[command]
max_retries=2
sleep_between_retries=1
+auto_restart=5,5
[security]
keysdir=security/keys
@@ -109,6 +110,8 @@
# agent version file
VERSION_FILE = "version_file"
+ AUTO_RESTART = "auto_restart"
+
FOLDER_MAPPING = {
APP_PACKAGE_DIR: "WORK",
APP_INSTALL_DIR: "WORK",
@@ -164,6 +167,17 @@
return ""
return command
+ # return max, window - max failures within window minutes
+ def getErrorWindow(self):
+ window = config.get(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART)
+ if window != None:
+ parts = window.split(',')
+ if len(parts) == 2:
+ if parts[0].isdigit() and parts[1].isdigit():
+ return (int(parts[0]), int(parts[1]))
+ pass
+ return (0, 0)
+
def set(self, category, name, value):
global config
return config.set(category, name, value)
diff --git a/slider-agent/src/main/python/agent/AgentToggleLogger.py b/slider-agent/src/main/python/agent/AgentToggleLogger.py
new file mode 100644
index 0000000..9a0ae3f
--- /dev/null
+++ b/slider-agent/src/main/python/agent/AgentToggleLogger.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import logging
+
+logger = logging.getLogger()
+
+'''
+Create a singleton instance of this class for every loop that either
+writes to or reads from the action queue, takes action based on the
+commandType and dumps logs along the way. It's target is to keep the
+verbosity level of agent logs to zero during non-interesting heartbeats
+like STATUS_COMMAND, and to ensure that it starts logging at info level
+again, the moment a non-STATUS_COMMAND shows up during its path into or
+out of the action queue.
+'''
+class AgentToggleLogger:
+ def __init__(self, logLevel="info"):
+ self.logLevel = logLevel
+
+ def log(self, message, *args, **kwargs):
+ if self.logLevel == "info":
+ logger.info(message, *args, **kwargs)
+ else:
+ logger.debug(message, *args, **kwargs)
+
+ '''
+ The methods adjustLogLevelAtStart and adjustLogLevelAtEnd work hand
+ in hand to do the following :
+ - STATUS related info should be logged at least once before the agent
+ enters into the STATUS loop
+ - If a non STATUS command shows up in the queue the logger switches
+ to info level
+ - STATUS will be logged at least once every time the log level toggles
+ back to info level when a non STATUS command shows up
+ '''
+
+ # Call this method at the start of the loop over action queue,
+ # right after reading from or writing to the queue
+ def adjustLogLevelAtStart(self, commandType):
+ from ActionQueue import ActionQueue
+ if self.logLevel != "info" and commandType != ActionQueue.STATUS_COMMAND:
+ self.logLevel = "info"
+
+ # Call this method as the last statement in the loop over action queue
+ def adjustLogLevelAtEnd(self, commandType):
+ from ActionQueue import ActionQueue
+ if commandType == ActionQueue.STATUS_COMMAND:
+ self.logLevel = "debug"
+ else:
+ self.logLevel = "info"
+
diff --git a/slider-agent/src/main/python/agent/Constants.py b/slider-agent/src/main/python/agent/Constants.py
index 2975266..f120b94 100644
--- a/slider-agent/src/main/python/agent/Constants.py
+++ b/slider-agent/src/main/python/agent/Constants.py
@@ -33,3 +33,4 @@
ZK_QUORUM="zk_quorum"
ZK_REG_PATH="zk_reg_path"
AUTO_GENERATED="auto_generated"
+MAX_AM_CONNECT_RETRIES = 10
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index 1e27efa..387bc7e 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -27,9 +27,11 @@
import threading
import urllib2
import pprint
+import math
from random import randint
from AgentConfig import AgentConfig
+from AgentToggleLogger import AgentToggleLogger
from Heartbeat import Heartbeat
from Register import Register
from ActionQueue import ActionQueue
@@ -86,7 +88,13 @@
self.statusCommand = None
self.failureCount = 0
self.heartBeatRetryCount = 0
- self.autoRestart = False
+ self.autoRestartFailures = 0
+ self.autoRestartTrackingSince = 0
+ self.terminateAgent = False
+ self.stopCommand = None
+ self.appGracefulStopQueued = False
+ self.appGracefulStopTriggered = False
+ self.tags = ""
def __del__(self):
@@ -122,34 +130,42 @@
self.componentActualState,
self.componentExpectedState,
self.actionQueue.customServiceOrchestrator.allocated_ports,
+ self.actionQueue.customServiceOrchestrator.log_folders,
+ self.tags,
id))
logger.info("Registering with the server at " + self.registerUrl +
" with data " + pprint.pformat(data))
response = self.sendRequest(self.registerUrl, data)
- ret = json.loads(response)
+ regResp = json.loads(response)
exitstatus = 0
- # exitstatus is a code of error which was rised on server side.
+ # exitstatus is a code of error which was raised on server side.
# exitstatus = 0 (OK - Default)
# exitstatus = 1 (Registration failed because
# different version of agent and server)
- if 'exitstatus' in ret.keys():
- exitstatus = int(ret['exitstatus'])
- # log - message, which will be printed to agents log
- if 'log' in ret.keys():
- log = ret['log']
+ if 'exitstatus' in regResp.keys():
+ exitstatus = int(regResp['exitstatus'])
+
+ # log - message, which will be printed to agents log
+ if 'log' in regResp.keys():
+ log = regResp['log']
+
+ # container may be associated with tags
+ if 'tags' in regResp.keys():
+ self.tags = regResp['tags']
+
if exitstatus == 1:
logger.error(log)
self.isRegistered = False
self.repeatRegistration = False
- return ret
- logger.info("Registered with the server with " + pprint.pformat(ret))
+ return regResp
+ logger.info("Registered with the server with " + pprint.pformat(regResp))
print("Registered with the server")
- self.responseId = int(ret['responseId'])
+ self.responseId = int(regResp['responseId'])
self.isRegistered = True
- if 'statusCommands' in ret.keys():
+ if 'statusCommands' in regResp.keys():
logger.info("Got status commands on registration " + pprint.pformat(
- ret['statusCommands']))
- self.addToQueue(ret['statusCommands'])
+ regResp['statusCommands']))
+ self.addToQueue(regResp['statusCommands'])
pass
else:
self.hasMappedComponents = False
@@ -166,7 +182,7 @@
time.sleep(delay)
pass
pass
- return ret
+ return regResp
def addToQueue(self, commands):
@@ -187,15 +203,46 @@
def shouldStopAgent(self):
'''
- If component has failed after start then stop the agent
+ Stop the agent if:
+ - Component has failed after start
+ - AM sent terminate agent command
'''
+ shouldStopAgent = False
if (self.componentActualState == State.FAILED) \
and (self.componentExpectedState == State.STARTED) \
and (self.failureCount >= Controller.MAX_FAILURE_COUNT_TO_STOP):
- return True
- else:
- return False
- pass
+ logger.info("Component instance has stopped, stopping the agent ...")
+ shouldStopAgent = True
+ if self.terminateAgent:
+ logger.info("Terminate agent command received from AM, stopping the agent ...")
+ shouldStopAgent = True
+ return shouldStopAgent
+
+ def isAppGracefullyStopped(self):
+ '''
+ If an app graceful stop command was queued then it is considered stopped if:
+ - app stop was triggered
+
+ Note: We should enhance this method by checking if the app is stopped
+ successfully and if not, then take alternate measures (like kill
+ processes). For now if stop is triggered it is considered stopped.
+ '''
+ isAppStopped = False
+ if self.appGracefulStopTriggered:
+ isAppStopped = True
+ return isAppStopped
+
+ def stopApp(self):
+ '''
+ Stop the app if:
+ - the app is currently in STARTED state and
+ a valid stop command is provided
+ '''
+ if (self.componentActualState == State.STARTED) and (not self.stopCommand == None):
+ # Try to do graceful stop
+ self.addToQueue([self.stopCommand])
+ self.appGracefulStopQueued = True
+ logger.info("Attempting to gracefully stop the application ...")
def heartbeatWithServer(self):
self.DEBUG_HEARTBEAT_RETRIES = 0
@@ -207,12 +254,14 @@
while not self.DEBUG_STOP_HEARTBEATING:
- if self.shouldStopAgent():
- logger.info("Component instance has stopped, stopping the agent ...")
- ProcessHelper.stopAgent()
-
commandResult = {}
try:
+ if self.appGracefulStopQueued and not self.isAppGracefullyStopped():
+ # Continue to wait until app is stopped
+ continue
+ if self.shouldStopAgent():
+ ProcessHelper.stopAgent()
+
if not retry:
data = json.dumps(
self.heartbeat.build(commandResult,
@@ -229,11 +278,24 @@
serverId = int(response['responseId'])
+ if 'restartAgent' in response.keys():
+ restartAgent = response['restartAgent']
+ if restartAgent:
+ logger.error("Got restartAgent command")
+ self.restartAgent()
+ if 'terminateAgent' in response.keys():
+ self.terminateAgent = response['terminateAgent']
+ if self.terminateAgent:
+ logger.error("Got terminateAgent command")
+ self.stopApp()
+ # Continue will add some wait time
+ continue
+
restartEnabled = False
if 'restartEnabled' in response:
restartEnabled = response['restartEnabled']
if restartEnabled:
- logger.info("Component auto-restart is enabled.")
+ logger.debug("Component auto-restart is enabled.")
if 'hasMappedComponents' in response.keys():
self.hasMappedComponents = response['hasMappedComponents'] != False
@@ -254,17 +316,18 @@
else:
self.responseId = serverId
+ commandSentFromAM = False
if 'executionCommands' in response.keys():
self.updateStateBasedOnCommand(response['executionCommands'])
self.addToQueue(response['executionCommands'])
+ commandSentFromAM = True
pass
if 'statusCommands' in response.keys() and len(response['statusCommands']) > 0:
self.addToQueue(response['statusCommands'])
+ commandSentFromAM = True
pass
- if "true" == response['restartAgent']:
- logger.error("Got restartAgent command")
- self.restartAgent()
- else:
+
+ if not commandSentFromAM:
logger.info("No commands sent from the Server.")
pass
@@ -274,7 +337,7 @@
stored_command = self.actionQueue.customServiceOrchestrator.stored_command
if len(stored_command) > 0:
auto_start_command = self.create_start_command(stored_command)
- if auto_start_command:
+ if auto_start_command and self.shouldAutoRestart():
logger.info("Automatically adding a start command.")
logger.debug("Auto start command: " + pprint.pformat(auto_start_command))
self.updateStateBasedOnCommand([auto_start_command], False)
@@ -330,8 +393,7 @@
zk_quorum = self.config.get(AgentConfig.SERVER_SECTION, Constants.ZK_QUORUM)
zk_reg_path = self.config.get(AgentConfig.SERVER_SECTION, Constants.ZK_REG_PATH)
registry = Registry(zk_quorum, zk_reg_path)
- amHost, amSecuredPort = registry.readAMHostPort()
- logger.info("Read from ZK registry: AM host = %s, AM secured port = %s" % (amHost, amSecuredPort))
+ amHost, amUnsecuredPort, amSecuredPort = registry.readAMHostPort()
self.hostname = amHost
self.secured_port = amSecuredPort
self.config.set(AgentConfig.SERVER_SECTION, "hostname", self.hostname)
@@ -342,13 +404,14 @@
return
self.cachedconnect = None # Previous connection is broken now
retry = True
- # Sleep for some time
- timeout = self.netutil.HEARTBEAT_IDDLE_INTERVAL_SEC \
- - self.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS
- self.heartbeat_wait_event.wait(timeout=timeout)
- # Sleep a bit more to allow STATUS_COMMAND results to be collected
- # and sent in one heartbeat. Also avoid server overload with heartbeats
- time.sleep(self.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS)
+ finally:
+ # Sleep for some time
+ timeout = self.netutil.HEARTBEAT_IDDLE_INTERVAL_SEC \
+ - self.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS
+ self.heartbeat_wait_event.wait(timeout=timeout)
+ # Sleep a bit more to allow STATUS_COMMAND results to be collected
+ # and sent in one heartbeat. Also avoid server overload with heartbeats
+ time.sleep(self.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS)
pass
logger.info("Controller stopped heart-beating.")
@@ -364,6 +427,17 @@
def updateStateBasedOnCommand(self, commands, createStatus=True):
+ # A STOP command is paired with the START command to provide agents the
+ # capability to gracefully stop the app if possible. The STOP command needs
+ # to be stored since the AM might not be able to provide it since it could
+ # have lost the container state for whatever reasons. The STOP command has
+ # no other role to play in the Agent state transition so it is removed from
+ # the commands list.
+ index = 0
+ deleteIndex = 0
+ delete = False
+ # break only if an INSTALL command is found, since we might get a STOP
+ # command for a START command
for command in commands:
if command["roleCommand"] == "START":
self.componentExpectedState = State.STARTED
@@ -372,12 +446,22 @@
if createStatus:
self.statusCommand = self.createStatusCommand(command)
+ # The STOP command index is stored to be deleted
+ if command["roleCommand"] == "STOP":
+ self.stopCommand = command
+ delete = True
+ deleteIndex = index
+
if command["roleCommand"] == "INSTALL":
self.componentExpectedState = State.INSTALLED
self.componentActualState = State.INSTALLING
self.failureCount = 0
- break;
+ break;
+ index += 1
+ # Delete the STOP command
+ if delete:
+ del commands[deleteIndex]
def updateStateBasedOnResult(self, commandResult):
if len(commandResult) > 0:
@@ -432,10 +516,11 @@
def run(self):
- self.actionQueue = ActionQueue(self.config, controller=self)
+ self.agentToggleLogger = AgentToggleLogger("info")
+ self.actionQueue = ActionQueue(self.config, controller=self, agentToggleLogger=self.agentToggleLogger)
self.actionQueue.start()
self.register = Register(self.config)
- self.heartbeat = Heartbeat(self.actionQueue, self.config)
+ self.heartbeat = Heartbeat(self.actionQueue, self.config, self.agentToggleLogger)
opener = urllib2.build_opener()
urllib2.install_opener(opener)
@@ -486,6 +571,35 @@
return {'exitstatus': 1, 'log': err_msg}
+ # Basic window that only counts failures till the window duration expires
+ def shouldAutoRestart(self):
+ max, window = self.config.getErrorWindow()
+ if max <= 0 or window <= 0:
+ return True
+
+ seconds_now = time.time()
+ if self.autoRestartTrackingSince == 0:
+ self.autoRestartTrackingSince = seconds_now
+ self.autoRestartFailures = 1
+ return True
+
+ self.autoRestartFailures += 1
+ minutes = math.floor((seconds_now - self.autoRestartTrackingSince) / 60)
+ if self.autoRestartFailures > max:
+ logger.info("Auto restart not allowed due to " + str(self.autoRestartFailures) + " failures in " + str(minutes) +
+ " minutes. Max restarts allowed is " + str(max) + " in " + str(window) + " minutes.")
+ return False
+
+ if minutes > window:
+ logger.info("Resetting window as number of minutes passed is " + str(minutes))
+ self.autoRestartTrackingSince = seconds_now
+ self.autoRestartFailures = 1
+ return True
+ return True
+
+ pass
+
+
def main(argv=None):
# Allow Ctrl-C
signal.signal(signal.SIGINT, signal.SIG_DFL)
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 15f1664..119c926 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -22,16 +22,19 @@
import os
import json
import pprint
+import random
import sys
import socket
import posixpath
import platform
+import copy
from AgentConfig import AgentConfig
from AgentException import AgentException
from PythonExecutor import PythonExecutor
import hostname
import Constants
+MAX_ATTEMPTS = 5
logger = logging.getLogger()
@@ -47,10 +50,10 @@
LIVE_STATUS = "STARTED"
DEAD_STATUS = "INSTALLED"
- def __init__(self, config, controller):
+ def __init__(self, config, controller, agentToggleLogger):
self.config = config
self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
- self.python_executor = PythonExecutor(self.tmp_dir, config)
+ self.python_executor = PythonExecutor(self.tmp_dir, config, agentToggleLogger)
self.status_commands_stdout = os.path.realpath(posixpath.join(self.tmp_dir,
'status_command_stdout.txt'))
self.status_commands_stderr = os.path.realpath(posixpath.join(self.tmp_dir,
@@ -58,6 +61,7 @@
self.public_fqdn = hostname.public_hostname()
self.stored_command = {}
self.allocated_ports = {}
+ self.log_folders = {}
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
@@ -133,15 +137,17 @@
}
if Constants.EXIT_CODE in ret and ret[Constants.EXIT_CODE] == 0:
- ret[Constants.ALLOCATED_PORTS] = allocated_ports
- self.allocated_ports = allocated_ports
+ ret[Constants.ALLOCATED_PORTS] = copy.deepcopy(allocated_ports)
+ ## Generally all ports are allocated at once but just in case
+ self.allocated_ports.update(allocated_ports)
# Irrespective of the outcome report the folder paths
if command_name == 'INSTALL':
- ret[Constants.FOLDERS] = {
+ self.log_folders = {
Constants.AGENT_LOG_ROOT: self.config.getLogPath(),
Constants.AGENT_WORK_ROOT: self.config.getWorkRootPath()
}
+ ret[Constants.FOLDERS] = copy.deepcopy(self.log_folders)
return ret
@@ -212,8 +218,11 @@
"""
# Perform few modifications to stay compatible with the way in which
# site.pp files are generated by manifestGenerator.py
- public_fqdn = self.public_fqdn
- command['public_hostname'] = public_fqdn
+ command['public_hostname'] = self.public_fqdn
+ if 'hostname' in command:
+ command['appmaster_hostname'] = command['hostname']
+ command['hostname'] = self.public_fqdn
+
# Now, dump the json file
command_type = command['commandType']
from ActionQueue import ActionQueue # To avoid cyclic dependency
@@ -226,13 +235,13 @@
task_id = command['taskId']
file_path = os.path.realpath(posixpath.join(self.tmp_dir, "command-{0}.json".format(task_id)))
# Json may contain passwords, that's why we need proper permissions
- if os.path.isfile(file_path):
+ if os.path.isfile(file_path) and os.path.exists(file_path):
os.unlink(file_path)
self.finalize_command(command, store_command, allocated_ports)
with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT,
- 0600), 'w') as f:
+ 0644), 'w') as f:
content = json.dumps(command, sort_keys=False, indent=4)
f.write(content)
return file_path
@@ -242,16 +251,16 @@
${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath()
${AGENT_LOG_ROOT} -> AgentConfig.getLogPath()
ALLOCATED_PORT is a hint to allocate port. It works as follows:
- Its of the form {component_name.ALLOCATED_PORT}[{DEFAULT_default_port}][{DO_NOT_PROPAGATE}]
+ Its of the form {component_name.ALLOCATED_PORT}[{DEFAULT_default_port}][{PER_CONTAINER}]
Either a port gets allocated or if not then just set the value to "0"
"""
-
def finalize_command(self, command, store_command, allocated_ports):
component = command['componentName']
allocated_for_this_component_format = "${{{0}.ALLOCATED_PORT}}"
allocated_for_any = ".ALLOCATED_PORT}"
port_allocation_req = allocated_for_this_component_format.format(component)
+ allowed_ports = self.get_allowed_ports(command)
if 'configurations' in command:
for key in command['configurations']:
if len(command['configurations'][key]) > 0:
@@ -262,7 +271,7 @@
value = value.replace("${AGENT_LOG_ROOT}",
self.config.getLogPath())
if port_allocation_req in value:
- value = self.allocate_ports(value, port_allocation_req)
+ value = self.allocate_ports(value, port_allocation_req, allowed_ports)
allocated_ports[key + "." + k] = value
elif allocated_for_any in value:
## All unallocated ports should be set to 0
@@ -285,7 +294,7 @@
All unallocated ports should be set to 0
Look for "${SOME_COMPONENT_NAME.ALLOCATED_PORT}"
or "${component.ALLOCATED_PORT}{DEFAULT_port}"
- or "${component.ALLOCATED_PORT}{DEFAULT_port}{DO_NOT_PROPAGATE}"
+ or "${component.ALLOCATED_PORT}{DEFAULT_port}{PER_CONTAINER}"
"""
def set_all_unallocated_ports(self, value):
@@ -314,11 +323,11 @@
Port allocation can asks for multiple dynamic ports
port_req_pattern is of type ${component_name.ALLOCATED_PORT}
append {DEFAULT_ and find the default value
- append {DO_NOT_PROPAGATE} if it exists
+ append {PER_CONTAINER} if it exists
"""
- def allocate_ports(self, value, port_req_pattern):
+ def allocate_ports(self, value, port_req_pattern, allowed_ports=None):
default_port_pattern = "{DEFAULT_"
- do_not_propagate_pattern = "{DO_NOT_PROPAGATE}"
+ do_not_propagate_pattern = "{PER_CONTAINER}"
index = value.find(port_req_pattern)
while index != -1:
replaced_pattern = port_req_pattern
@@ -338,7 +347,7 @@
if index == value.find(replaced_pattern + do_not_propagate_pattern):
replaced_pattern = replaced_pattern + do_not_propagate_pattern
pass
- port = self.allocate_port(def_port)
+ port = self.allocate_port(def_port, allowed_ports)
value = value.replace(replaced_pattern, str(port), 1)
logger.info("Allocated port " + str(port) + " for " + replaced_pattern)
index = value.find(port_req_pattern)
@@ -347,24 +356,28 @@
pass
- def allocate_port(self, default_port=None):
+ def allocate_port(self, default_port=None, allowed_ports=None):
if default_port != None:
if self.is_port_available(default_port):
return default_port
- MAX_ATTEMPT = 5
- iter = 0
+ port_list = [0] * MAX_ATTEMPTS
+ if allowed_ports != None:
+ port_list = allowed_ports
+
+ i = 0
port = -1
- while iter < MAX_ATTEMPT:
- iter = iter + 1
+ itor = iter(port_list)
+ while i < min(len(port_list), MAX_ATTEMPTS):
try:
sock = socket.socket()
- sock.bind(('', 0))
+ sock.bind(('', itor.next()))
port = sock.getsockname()[1]
except Exception, err:
- logger.info("Encountered error while trying to opening socket - " + str(err))
+ logger.info("Encountered error while trying to open socket - " + str(err))
finally:
sock.close()
+ i = i + 1
pass
logger.info("Allocated dynamic port: " + str(port))
return port
@@ -380,3 +393,43 @@
return False
+ def get_allowed_ports(self, command):
+ allowed_ports = None
+ global_config = command['configurations'].get('global')
+ if global_config != None:
+ allowed_ports_value = global_config.get("slider.allowed.ports")
+ if allowed_ports_value:
+ allowed_ports = self.get_allowed_port_list(allowed_ports_value)
+
+ return allowed_ports
+
+
+ def get_allowed_port_list(self, allowedPortsOptionValue,
+ num_values=MAX_ATTEMPTS):
+ selection = set()
+ invalid = set()
+ # tokens are comma seperated values
+ tokens = [x.strip() for x in allowedPortsOptionValue.split(',')]
+ for i in tokens:
+ try:
+ selection.add(int(i))
+ except:
+ # should be a range
+ try:
+ token = [int(k.strip()) for k in i.split('-')]
+ if len(token) > 1:
+ token.sort()
+ first = token[0]
+ last = token[len(token)-1]
+ for x in range(first, last+1):
+ selection.add(x)
+ except:
+ # not an int and not a range...
+ invalid.add(i)
+ selection = random.sample(selection, min (len(selection), num_values))
+ # Report invalid tokens before returning valid selection
+ logger.info("Allowed port values: " + str(selection))
+ logger.warning("Invalid port range values: " + str(invalid))
+ return selection
+
+
diff --git a/slider-agent/src/main/python/agent/Heartbeat.py b/slider-agent/src/main/python/agent/Heartbeat.py
index b107d92..e157ce3 100644
--- a/slider-agent/src/main/python/agent/Heartbeat.py
+++ b/slider-agent/src/main/python/agent/Heartbeat.py
@@ -31,16 +31,17 @@
logger = logging.getLogger()
class Heartbeat:
- def __init__(self, actionQueue, config=None):
+ def __init__(self, actionQueue, config=None, agentToggleLogger=None):
self.actionQueue = actionQueue
self.config = config
self.reports = []
+ self.agentToggleLogger = agentToggleLogger
def build(self, commandResult, id='-1',
componentsMapped=False):
timestamp = int(time.time() * 1000)
queueResult = self.actionQueue.result()
- logger.info("Queue result: " + pformat(queueResult))
+ self.agentToggleLogger.log("Queue result: " + pformat(queueResult))
nodeStatus = {"status": "HEALTHY",
"cause": "NONE"}
@@ -93,10 +94,11 @@
if len(componentStatuses) > 0:
heartbeat['componentStatus'] = componentStatuses
- logger.info("Sending heartbeat with response id: " + str(id) + " and "
- "timestamp: " + str(timestamp) +
- ". Command(s) in progress: " + repr(commandsInProgress) +
- ". Components mapped: " + repr(componentsMapped))
+ self.agentToggleLogger.log(
+ "Sending heartbeat with response id: " + str(id) + " and "
+ "timestamp: " + str(timestamp) +
+ ". Command(s) in progress: " + repr(commandsInProgress) +
+ ". Components mapped: " + repr(componentsMapped))
logger.debug("Heartbeat : " + pformat(heartbeat))
return heartbeat
diff --git a/slider-agent/src/main/python/agent/ProcessHelper.py b/slider-agent/src/main/python/agent/ProcessHelper.py
index 467c4d8..81737f4 100644
--- a/slider-agent/src/main/python/agent/ProcessHelper.py
+++ b/slider-agent/src/main/python/agent/ProcessHelper.py
@@ -24,11 +24,12 @@
import sys
import posixpath
from shell import getTempFiles
+import Constants
logger = logging.getLogger()
-if 'AGENT_WORK_ROOT' in os.environ:
- pidfile = os.path.realpath(posixpath.join(os.environ['AGENT_WORK_ROOT'], "infra", "run", "agent.pid"))
+if Constants.AGENT_WORK_ROOT in os.environ:
+ pidfile = os.path.realpath(posixpath.join(os.environ[Constants.AGENT_WORK_ROOT], "infra", "run", "agent.pid"))
else:
pidfile = None
diff --git a/slider-agent/src/main/python/agent/PythonExecutor.py b/slider-agent/src/main/python/agent/PythonExecutor.py
index 54ce247..985d75f 100644
--- a/slider-agent/src/main/python/agent/PythonExecutor.py
+++ b/slider-agent/src/main/python/agent/PythonExecutor.py
@@ -30,6 +30,7 @@
import sys
import platform
import Constants
+from AgentToggleLogger import AgentToggleLogger
logger = logging.getLogger()
@@ -47,9 +48,10 @@
event = threading.Event()
python_process_has_been_killed = False
- def __init__(self, tmpDir, config):
+ def __init__(self, tmpDir, config, agentToggleLogger):
self.tmpDir = tmpDir
self.config = config
+ self.agentToggleLogger = agentToggleLogger
pass
def run_file(self, script, script_params, tmpoutfile, tmperrfile, timeout,
@@ -81,7 +83,7 @@
script_params += [tmpstructedoutfile, logger_level]
pythonCommand = self.python_command(script, script_params)
- logger.info("Running command " + pprint.pformat(pythonCommand))
+ self.agentToggleLogger.log("Running command " + pprint.pformat(pythonCommand))
process = self.launch_python_subprocess(pythonCommand, tmpout, tmperr,
environment_vars)
logger.debug("Launching watchdog thread")
@@ -99,24 +101,23 @@
out = open(tmpoutfile, 'r').read()
error = open(tmperrfile, 'r').read()
+ structured_out = {}
try:
with open(tmpstructedoutfile, 'r') as fp:
structured_out = json.load(fp)
- except Exception:
+ except Exception as e:
if os.path.exists(tmpstructedoutfile):
- errMsg = 'Unable to read structured output from ' + tmpstructedoutfile
+ errMsg = 'Unable to read structured output from ' + tmpstructedoutfile + ' ' + str(e)
structured_out = {
'msg': errMsg
}
logger.warn(structured_out)
- else:
- structured_out = {}
if self.python_process_has_been_killed:
error = str(error) + "\n Python script has been killed due to timeout"
returncode = 999
result = self.condenseOutput(out, error, returncode, structured_out)
- logger.info("Result: %s" % result)
+ self.agentToggleLogger.log("Result: %s" % result)
return result
@@ -130,7 +131,7 @@
env = os.environ.copy()
if environment_vars:
for k, v in environment_vars:
- logger.info("Setting env: %s to %s", k, v)
+ self.agentToggleLogger.log("Setting env: %s to %s", k, v)
env[k] = v
return subprocess.Popen(command,
stdout=tmpout,
diff --git a/slider-agent/src/main/python/agent/Register.py b/slider-agent/src/main/python/agent/Register.py
index b59154f..c5197fd 100644
--- a/slider-agent/src/main/python/agent/Register.py
+++ b/slider-agent/src/main/python/agent/Register.py
@@ -29,19 +29,21 @@
def __init__(self, config):
self.config = config
- def build(self, actualState, expectedState, allocated_ports, id='-1'):
+ def build(self, actualState, expectedState, allocated_ports, log_folders, tags="", id='-1'):
timestamp = int(time.time() * 1000)
version = self.read_agent_version()
register = {'responseId': int(id),
'timestamp': timestamp,
- 'hostname': self.config.getLabel(),
+ 'label': self.config.getLabel(),
'publicHostname': hostname.public_hostname(),
'agentVersion': version,
'actualState': actualState,
'expectedState': expectedState,
- 'allocatedPorts': allocated_ports
+ 'allocatedPorts': allocated_ports,
+ 'logFolders': log_folders,
+ 'tags': tags
}
return register
diff --git a/slider-agent/src/main/python/agent/Registry.py b/slider-agent/src/main/python/agent/Registry.py
index 37736fe..e0bc5da 100644
--- a/slider-agent/src/main/python/agent/Registry.py
+++ b/slider-agent/src/main/python/agent/Registry.py
@@ -24,14 +24,16 @@
logger = logging.getLogger()
-class Registry:
+class Registry(object):
def __init__(self, zk_quorum, zk_reg_path):
self.zk_quorum = zk_quorum
self.zk_reg_path = zk_reg_path
def readAMHostPort(self):
+ logger.debug("Trying to connect to ZK...")
amHost = ""
amSecuredPort = ""
+ amUnsecuredPort = ""
zk = None
try:
zk = KazooClient(hosts=self.zk_quorum, read_only=True)
@@ -39,19 +41,30 @@
data, stat = zk.get(self.zk_reg_path)
logger.debug("Registry Data: %s" % (data.decode("utf-8")))
sliderRegistry = json.loads(data)
- amUrl = sliderRegistry["payload"]["internalView"]["endpoints"]["org.apache.slider.agents"]["address"]
- amHost = amUrl.split("/")[2].split(":")[0]
- amSecuredPort = amUrl.split(":")[2].split("/")[0]
- # the port needs to be utf-8 encoded
+ internalAttr = sliderRegistry["internal"]
+ for internal in internalAttr:
+ if internal["api"] == "classpath:org.apache.slider.agents.secure":
+ address0 = internal["addresses"][0]
+ amUrl = address0["uri"]
+ amHost = amUrl.split("/")[2].split(":")[0]
+ amSecuredPort = amUrl.split(":")[2].split("/")[0]
+ if internal["api"] == "classpath:org.apache.slider.agents.oneway":
+ address0 = internal["addresses"][0]
+ amUnsecureUrl = address0["uri"]
+ amHost = amUnsecureUrl.split("/")[2].split(":")[0]
+ amUnsecuredPort = amUnsecureUrl.split(":")[2].split("/")[0]
+
+ # the ports needs to be utf-8 encoded
amSecuredPort = amSecuredPort.encode('utf8', 'ignore')
- except Exception:
+ amUnsecuredPort = amUnsecuredPort.encode('utf8', 'ignore')
+ except Exception, e:
# log and let empty strings be returned
- logger.error("Could not connect to zk registry at %s in quorum %s" %
- (self.zk_reg_path, self.zk_quorum))
+ logger.error("Could not connect to zk registry at %s in quorum %s. Error: %s" %
+ (self.zk_reg_path, self.zk_quorum, str(e)))
pass
finally:
- if not zk == None:
+ if not zk is None:
zk.stop()
zk.close()
- logger.info("AM Host = %s, AM Secured Port = %s" % (amHost, amSecuredPort))
- return amHost, amSecuredPort
+ logger.info("AM Host = %s, AM Secured Port = %s, ping port = %s" % (amHost, amSecuredPort, amUnsecuredPort))
+ return amHost, amUnsecuredPort, amSecuredPort
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
index f68db04..3a75cb1 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -43,7 +43,7 @@
agentPid = os.getpid()
configFileRelPath = "infra/conf/agent.ini"
-logFileName = "agent.log"
+logFileName = "slider-agent.log"
SERVER_STATUS_URL="https://{0}:{1}{2}"
@@ -172,7 +172,8 @@
if pid == -1:
print ("Agent process is not running")
else:
- os.kill(pid, signal.SIGKILL)
+ if not IS_WINDOWS:
+ os.kill(pid, signal.SIGKILL)
os._exit(1)
@@ -185,9 +186,14 @@
parser.add_option("--debug", dest="debug", help="Agent debug hint", default="")
(options, args) = parser.parse_args()
- if not 'AGENT_WORK_ROOT' in os.environ:
- parser.error("AGENT_WORK_ROOT environment variable must be set.")
- options.root_folder = os.environ['AGENT_WORK_ROOT']
+ if not Constants.AGENT_WORK_ROOT in os.environ and not 'PWD' in os.environ:
+ parser.error("AGENT_WORK_ROOT environment variable or PWD must be set.")
+ if Constants.AGENT_WORK_ROOT in os.environ:
+ options.root_folder = os.environ[Constants.AGENT_WORK_ROOT]
+ else:
+ # some launch environments do not end up setting all environment variables
+ options.root_folder = os.environ['PWD']
+
if not 'AGENT_LOG_ROOT' in os.environ:
parser.error("AGENT_LOG_ROOT environment variable must be set.")
options.log_folder = os.environ['AGENT_LOG_ROOT']
@@ -217,15 +223,6 @@
if options.debug:
agentConfig.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, options.debug)
- # Extract the AM hostname and secured port from ZK registry
- registry = Registry(options.zk_quorum, options.zk_reg_path)
- amHost, amSecuredPort = registry.readAMHostPort()
- if amHost:
- agentConfig.set(AgentConfig.SERVER_SECTION, "hostname", amHost)
-
- if amSecuredPort:
- agentConfig.set(AgentConfig.SERVER_SECTION, "secured_port", amSecuredPort)
-
# set the security directory to a subdirectory of the run dir
secDir = posixpath.join(agentConfig.getResolvedPath(AgentConfig.RUN_DIR), "security")
logger.info("Security/Keys directory: " + secDir)
@@ -248,16 +245,44 @@
if len(all_log_folders) > 1:
logger.info("Selected log folder from available: " + ",".join(all_log_folders))
- server_url = SERVER_STATUS_URL.format(
- agentConfig.get(AgentConfig.SERVER_SECTION, 'hostname'),
- agentConfig.get(AgentConfig.SERVER_SECTION, 'secured_port'),
- agentConfig.get(AgentConfig.SERVER_SECTION, 'check_path'))
- print("Connecting to the server at " + server_url + "...")
- logger.info('Connecting to the server at: ' + server_url)
+ # Extract the AM hostname and secured port from ZK registry
+ zk_lookup_tries = 0
+ while zk_lookup_tries < Constants.MAX_AM_CONNECT_RETRIES:
+ registry = Registry(options.zk_quorum, options.zk_reg_path)
+ amHost, amUnsecuredPort, amSecuredPort = registry.readAMHostPort()
- # Wait until server is reachable
- netutil = NetUtil()
- netutil.try_to_connect(server_url, -1, logger)
+ tryConnect = True
+ if not amHost or not amSecuredPort or not amUnsecuredPort:
+ logger.info("Unable to extract AM host details from ZK, retrying ...")
+ tryConnect = False
+ time.sleep(NetUtil.CONNECT_SERVER_RETRY_INTERVAL_SEC)
+
+ if tryConnect:
+ if amHost:
+ agentConfig.set(AgentConfig.SERVER_SECTION, "hostname", amHost)
+
+ if amSecuredPort:
+ agentConfig.set(AgentConfig.SERVER_SECTION, "secured_port", amSecuredPort)
+
+ if amUnsecuredPort:
+ agentConfig.set(AgentConfig.SERVER_SECTION, "port", amUnsecuredPort)
+
+ server_url = SERVER_STATUS_URL.format(
+ agentConfig.get(AgentConfig.SERVER_SECTION, 'hostname'),
+ agentConfig.get(AgentConfig.SERVER_SECTION, 'port'),
+ agentConfig.get(AgentConfig.SERVER_SECTION, 'check_path'))
+ print("Connecting to the server at " + server_url + "...")
+ logger.info('Connecting to the server at: ' + server_url)
+
+ # Wait until server is reachable and continue to query ZK
+ netutil = NetUtil()
+ retries = netutil.try_to_connect(server_url, 3, logger)
+ if retries < 3:
+ break;
+ pass
+ pass
+ zk_lookup_tries += 1
+ pass
# Launch Controller communication
controller = Controller(agentConfig)
diff --git a/slider-agent/src/main/python/jinja2/ext/Vim/htmljinja.vim b/slider-agent/src/main/python/jinja2/ext/Vim/htmljinja.vim
deleted file mode 100644
index 3f9cba4..0000000
--- a/slider-agent/src/main/python/jinja2/ext/Vim/htmljinja.vim
+++ /dev/null
@@ -1,27 +0,0 @@
-" Vim syntax file
-" Language: Jinja HTML template
-" Maintainer: Armin Ronacher <armin.ronacher@active-4.com>
-" Last Change: 2007 Apr 8
-
-" For version 5.x: Clear all syntax items
-" For version 6.x: Quit when a syntax file was already loaded
-if version < 600
- syntax clear
-elseif exists("b:current_syntax")
- finish
-endif
-
-if !exists("main_syntax")
- let main_syntax = 'html'
-endif
-
-if version < 600
- so <sfile>:p:h/jinja.vim
- so <sfile>:p:h/html.vim
-else
- runtime! syntax/jinja.vim
- runtime! syntax/html.vim
- unlet b:current_syntax
-endif
-
-let b:current_syntax = "htmljinja"
diff --git a/slider-agent/src/main/python/jinja2/ext/Vim/jinja.vim b/slider-agent/src/main/python/jinja2/ext/Vim/jinja.vim
deleted file mode 100644
index 919954b..0000000
--- a/slider-agent/src/main/python/jinja2/ext/Vim/jinja.vim
+++ /dev/null
@@ -1,113 +0,0 @@
-" Vim syntax file
-" Language: Jinja template
-" Maintainer: Armin Ronacher <armin.ronacher@active-4.com>
-" Last Change: 2008 May 9
-" Version: 1.1
-"
-" Known Bugs:
-" because of odd limitations dicts and the modulo operator
-" appear wrong in the template.
-"
-" Changes:
-"
-" 2008 May 9: Added support for Jinja2 changes (new keyword rules)
-
-" For version 5.x: Clear all syntax items
-" For version 6.x: Quit when a syntax file was already loaded
-if version < 600
- syntax clear
-elseif exists("b:current_syntax")
- finish
-endif
-
-syntax case match
-
-" Jinja template built-in tags and parameters (without filter, macro, is and raw, they
-" have special threatment)
-syn keyword jinjaStatement containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained and if else in not or recursive as import
-
-syn keyword jinjaStatement containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained is filter skipwhite nextgroup=jinjaFilter
-syn keyword jinjaStatement containedin=jinjaTagBlock contained macro skipwhite nextgroup=jinjaFunction
-syn keyword jinjaStatement containedin=jinjaTagBlock contained block skipwhite nextgroup=jinjaBlockName
-
-" Variable Names
-syn match jinjaVariable containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/
-syn keyword jinjaSpecial containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained false true none False True None loop super caller varargs kwargs
-
-" Filters
-syn match jinjaOperator "|" containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained nextgroup=jinjaFilter
-syn match jinjaFilter contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/
-syn match jinjaFunction contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/
-syn match jinjaBlockName contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/
-
-" Jinja template constants
-syn region jinjaString containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained start=/"/ skip=/\\"/ end=/"/
-syn region jinjaString containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained start=/'/ skip=/\\'/ end=/'/
-syn match jinjaNumber containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[0-9]\+\(\.[0-9]\+\)\?/
-
-" Operators
-syn match jinjaOperator containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[+\-*\/<>=!,:]/
-syn match jinjaPunctuation containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[()\[\]]/
-syn match jinjaOperator containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /\./ nextgroup=jinjaAttribute
-syn match jinjaAttribute contained /[a-zA-Z_][a-zA-Z0-9_]*/
-
-" Jinja template tag and variable blocks
-syn region jinjaNested matchgroup=jinjaOperator start="(" end=")" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained
-syn region jinjaNested matchgroup=jinjaOperator start="\[" end="\]" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained
-syn region jinjaNested matchgroup=jinjaOperator start="{" end="}" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained
-syn region jinjaTagBlock matchgroup=jinjaTagDelim start=/{%-\?/ end=/-\?%}/ skipwhite containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment
-
-syn region jinjaVarBlock matchgroup=jinjaVarDelim start=/{{-\?/ end=/-\?}}/ containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment
-
-" Jinja template 'raw' tag
-syn region jinjaRaw matchgroup=jinjaRawDelim start="{%\s*raw\s*%}" end="{%\s*endraw\s*%}" containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaString,jinjaComment
-
-" Jinja comments
-syn region jinjaComment matchgroup=jinjaCommentDelim start="{#" end="#}" containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaString
-
-" Block start keywords. A bit tricker. We only highlight at the start of a
-" tag block and only if the name is not followed by a comma or equals sign
-" which usually means that we have to deal with an assignment.
-syn match jinjaStatement containedin=jinjaTagBlock contained skipwhite /\({%-\?\s*\)\@<=\<[a-zA-Z_][a-zA-Z0-9_]*\>\(\s*[,=]\)\@!/
-
-" and context modifiers
-syn match jinjaStatement containedin=jinjaTagBlock contained /\<with\(out\)\?\s\+context\>/ skipwhite
-
-
-" Define the default highlighting.
-" For version 5.7 and earlier: only when not done already
-" For version 5.8 and later: only when an item doesn't have highlighting yet
-if version >= 508 || !exists("did_jinja_syn_inits")
- if version < 508
- let did_jinja_syn_inits = 1
- command -nargs=+ HiLink hi link <args>
- else
- command -nargs=+ HiLink hi def link <args>
- endif
-
- HiLink jinjaPunctuation jinjaOperator
- HiLink jinjaAttribute jinjaVariable
- HiLink jinjaFunction jinjaFilter
-
- HiLink jinjaTagDelim jinjaTagBlock
- HiLink jinjaVarDelim jinjaVarBlock
- HiLink jinjaCommentDelim jinjaComment
- HiLink jinjaRawDelim jinja
-
- HiLink jinjaSpecial Special
- HiLink jinjaOperator Normal
- HiLink jinjaRaw Normal
- HiLink jinjaTagBlock PreProc
- HiLink jinjaVarBlock PreProc
- HiLink jinjaStatement Statement
- HiLink jinjaFilter Function
- HiLink jinjaBlockName Function
- HiLink jinjaVariable Identifier
- HiLink jinjaString Constant
- HiLink jinjaNumber Constant
- HiLink jinjaComment Comment
-
- delcommand HiLink
-endif
-
-let b:current_syntax = "jinja"
diff --git a/slider-agent/src/main/python/jinja2/ext/django2jinja/django2jinja.py b/slider-agent/src/main/python/jinja2/ext/django2jinja/django2jinja.py
deleted file mode 100644
index 6d9e76c..0000000
--- a/slider-agent/src/main/python/jinja2/ext/django2jinja/django2jinja.py
+++ /dev/null
@@ -1,768 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
- Django to Jinja
- ~~~~~~~~~~~~~~~
-
- Helper module that can convert django templates into Jinja2 templates.
-
- This file is not intended to be used as stand alone application but to
- be used as library. To convert templates you basically create your own
- writer, add extra conversion logic for your custom template tags,
- configure your django environment and run the `convert_templates`
- function.
-
- Here a simple example::
-
- # configure django (or use settings.configure)
- import os
- os.environ['DJANGO_SETTINGS_MODULE'] = 'yourapplication.settings'
- from yourapplication.foo.templatetags.bar import MyNode
-
- from django2jinja import Writer, convert_templates
-
- def write_my_node(writer, node):
- writer.start_variable()
- writer.write('myfunc(')
- for idx, arg in enumerate(node.args):
- if idx:
- writer.write(', ')
- writer.node(arg)
- writer.write(')')
- writer.end_variable()
-
- writer = Writer()
- writer.node_handlers[MyNode] = write_my_node
- convert_templates('/path/to/output/folder', writer=writer)
-
- Here is an example hos to automatically translate your django
- variables to jinja2::
-
- import re
- # List of tuple (Match pattern, Replace pattern, Exclusion pattern)
-
- var_re = ((re.compile(r"(u|user)\.is_authenticated"), r"\1.is_authenticated()", None),
- (re.compile(r"\.non_field_errors"), r".non_field_errors()", None),
- (re.compile(r"\.label_tag"), r".label_tag()", None),
- (re.compile(r"\.as_dl"), r".as_dl()", None),
- (re.compile(r"\.as_table"), r".as_table()", None),
- (re.compile(r"\.as_widget"), r".as_widget()", None),
- (re.compile(r"\.as_hidden"), r".as_hidden()", None),
-
- (re.compile(r"\.get_([0-9_\w]+)_url"), r".get_\1_url()", None),
- (re.compile(r"\.url"), r".url()", re.compile(r"(form|calendar).url")),
- (re.compile(r"\.get_([0-9_\w]+)_display"), r".get_\1_display()", None),
- (re.compile(r"loop\.counter"), r"loop.index", None),
- (re.compile(r"loop\.revcounter"), r"loop.revindex", None),
- (re.compile(r"request\.GET\.([0-9_\w]+)"), r"request.GET.get('\1', '')", None),
- (re.compile(r"request\.get_host"), r"request.get_host()", None),
-
- (re.compile(r"\.all(?!_)"), r".all()", None),
- (re.compile(r"\.all\.0"), r".all()[0]", None),
- (re.compile(r"\.([0-9])($|\s+)"), r"[\1]\2", None),
- (re.compile(r"\.items"), r".items()", None),
- )
- writer = Writer(var_re=var_re)
-
- For details about the writing process have a look at the module code.
-
- :copyright: (c) 2009 by the Jinja Team.
- :license: BSD.
-"""
-import re
-import os
-import sys
-from jinja2.defaults import *
-from django.conf import settings
-from django.template import defaulttags as core_tags, loader, TextNode, \
- FilterExpression, libraries, Variable, loader_tags, TOKEN_TEXT, \
- TOKEN_VAR
-from django.template.debug import DebugVariableNode as VariableNode
-from django.templatetags import i18n as i18n_tags
-from StringIO import StringIO
-
-
-_node_handlers = {}
-_resolved_filters = {}
-_newline_re = re.compile(r'(?:\r\n|\r|\n)')
-
-
-# Django stores an itertools object on the cycle node. Not only is this
-# thread unsafe but also a problem for the converter which needs the raw
-# string values passed to the constructor to create a jinja loop.cycle()
-# call from it.
-_old_cycle_init = core_tags.CycleNode.__init__
-def _fixed_cycle_init(self, cyclevars, variable_name=None):
- self.raw_cycle_vars = map(Variable, cyclevars)
- _old_cycle_init(self, cyclevars, variable_name)
-core_tags.CycleNode.__init__ = _fixed_cycle_init
-
-
-def node(cls):
- def proxy(f):
- _node_handlers[cls] = f
- return f
- return proxy
-
-
-def convert_templates(output_dir, extensions=('.html', '.txt'), writer=None,
- callback=None):
- """Iterates over all templates in the template dirs configured and
- translates them and writes the new templates into the output directory.
- """
- if writer is None:
- writer = Writer()
-
- def filter_templates(files):
- for filename in files:
- ifilename = filename.lower()
- for extension in extensions:
- if ifilename.endswith(extension):
- yield filename
-
- def translate(f, loadname):
- template = loader.get_template(loadname)
- original = writer.stream
- writer.stream = f
- writer.body(template.nodelist)
- writer.stream = original
-
- if callback is None:
- def callback(template):
- print template
-
- for directory in settings.TEMPLATE_DIRS:
- for dirname, _, files in os.walk(directory):
- dirname = dirname[len(directory) + 1:]
- for filename in filter_templates(files):
- source = os.path.normpath(os.path.join(dirname, filename))
- target = os.path.join(output_dir, dirname, filename)
- basetarget = os.path.dirname(target)
- if not os.path.exists(basetarget):
- os.makedirs(basetarget)
- callback(source)
- f = file(target, 'w')
- try:
- translate(f, source)
- finally:
- f.close()
-
-
-class Writer(object):
- """The core writer class."""
-
- def __init__(self, stream=None, error_stream=None,
- block_start_string=BLOCK_START_STRING,
- block_end_string=BLOCK_END_STRING,
- variable_start_string=VARIABLE_START_STRING,
- variable_end_string=VARIABLE_END_STRING,
- comment_start_string=COMMENT_START_STRING,
- comment_end_string=COMMENT_END_STRING,
- initial_autoescape=True,
- use_jinja_autoescape=False,
- custom_node_handlers=None,
- var_re=[],
- env=None):
- if stream is None:
- stream = sys.stdout
- if error_stream is None:
- error_stream = sys.stderr
- self.stream = stream
- self.error_stream = error_stream
- self.block_start_string = block_start_string
- self.block_end_string = block_end_string
- self.variable_start_string = variable_start_string
- self.variable_end_string = variable_end_string
- self.comment_start_string = comment_start_string
- self.comment_end_string = comment_end_string
- self.autoescape = initial_autoescape
- self.spaceless = False
- self.use_jinja_autoescape = use_jinja_autoescape
- self.node_handlers = dict(_node_handlers,
- **(custom_node_handlers or {}))
- self._loop_depth = 0
- self._filters_warned = set()
- self.var_re = var_re
- self.env = env
-
- def enter_loop(self):
- """Increments the loop depth so that write functions know if they
- are in a loop.
- """
- self._loop_depth += 1
-
- def leave_loop(self):
- """Reverse of enter_loop."""
- self._loop_depth -= 1
-
- @property
- def in_loop(self):
- """True if we are in a loop."""
- return self._loop_depth > 0
-
- def write(self, s):
- """Writes stuff to the stream."""
- self.stream.write(s.encode(settings.FILE_CHARSET))
-
- def print_expr(self, expr):
- """Open a variable tag, write to the string to the stream and close."""
- self.start_variable()
- self.write(expr)
- self.end_variable()
-
- def _post_open(self):
- if self.spaceless:
- self.write('- ')
- else:
- self.write(' ')
-
- def _pre_close(self):
- if self.spaceless:
- self.write(' -')
- else:
- self.write(' ')
-
- def start_variable(self):
- """Start a variable."""
- self.write(self.variable_start_string)
- self._post_open()
-
- def end_variable(self, always_safe=False):
- """End a variable."""
- if not always_safe and self.autoescape and \
- not self.use_jinja_autoescape:
- self.write('|e')
- self._pre_close()
- self.write(self.variable_end_string)
-
- def start_block(self):
- """Starts a block."""
- self.write(self.block_start_string)
- self._post_open()
-
- def end_block(self):
- """Ends a block."""
- self._pre_close()
- self.write(self.block_end_string)
-
- def tag(self, name):
- """Like `print_expr` just for blocks."""
- self.start_block()
- self.write(name)
- self.end_block()
-
- def variable(self, name):
- """Prints a variable. This performs variable name transformation."""
- self.write(self.translate_variable_name(name))
-
- def literal(self, value):
- """Writes a value as literal."""
- value = repr(value)
- if value[:2] in ('u"', "u'"):
- value = value[1:]
- self.write(value)
-
- def filters(self, filters, is_block=False):
- """Dumps a list of filters."""
- want_pipe = not is_block
- for filter, args in filters:
- name = self.get_filter_name(filter)
- if name is None:
- self.warn('Could not find filter %s' % name)
- continue
- if name not in DEFAULT_FILTERS and \
- name not in self._filters_warned:
- self._filters_warned.add(name)
- self.warn('Filter %s probably doesn\'t exist in Jinja' %
- name)
- if not want_pipe:
- want_pipe = True
- else:
- self.write('|')
- self.write(name)
- if args:
- self.write('(')
- for idx, (is_var, value) in enumerate(args):
- if idx:
- self.write(', ')
- if is_var:
- self.node(value)
- else:
- self.literal(value)
- self.write(')')
-
- def get_location(self, origin, position):
- """Returns the location for an origin and position tuple as name
- and lineno.
- """
- if hasattr(origin, 'source'):
- source = origin.source
- name = '<unknown source>'
- else:
- source = origin.loader(origin.loadname, origin.dirs)[0]
- name = origin.loadname
- lineno = len(_newline_re.findall(source[:position[0]])) + 1
- return name, lineno
-
- def warn(self, message, node=None):
- """Prints a warning to the error stream."""
- if node is not None and hasattr(node, 'source'):
- filename, lineno = self.get_location(*node.source)
- message = '[%s:%d] %s' % (filename, lineno, message)
- print >> self.error_stream, message
-
- def translate_variable_name(self, var):
- """Performs variable name translation."""
- if self.in_loop and var == 'forloop' or var.startswith('forloop.'):
- var = var[3:]
-
- for reg, rep, unless in self.var_re:
- no_unless = unless and unless.search(var) or True
- if reg.search(var) and no_unless:
- var = reg.sub(rep, var)
- break
- return var
-
- def get_filter_name(self, filter):
- """Returns the filter name for a filter function or `None` if there
- is no such filter.
- """
- if filter not in _resolved_filters:
- for library in libraries.values():
- for key, value in library.filters.iteritems():
- _resolved_filters[value] = key
- return _resolved_filters.get(filter, None)
-
- def node(self, node):
- """Invokes the node handler for a node."""
- for cls, handler in self.node_handlers.iteritems():
- if type(node) is cls or type(node).__name__ == cls:
- handler(self, node)
- break
- else:
- self.warn('Untranslatable node %s.%s found' % (
- node.__module__,
- node.__class__.__name__
- ), node)
-
- def body(self, nodes):
- """Calls node() for every node in the iterable passed."""
- for node in nodes:
- self.node(node)
-
-
-@node(TextNode)
-def text_node(writer, node):
- writer.write(node.s)
-
-
-@node(Variable)
-def variable(writer, node):
- if node.translate:
- writer.warn('i18n system used, make sure to install translations', node)
- writer.write('_(')
- if node.literal is not None:
- writer.literal(node.literal)
- else:
- writer.variable(node.var)
- if node.translate:
- writer.write(')')
-
-
-@node(VariableNode)
-def variable_node(writer, node):
- writer.start_variable()
- if node.filter_expression.var.var == 'block.super' \
- and not node.filter_expression.filters:
- writer.write('super()')
- else:
- writer.node(node.filter_expression)
- writer.end_variable()
-
-
-@node(FilterExpression)
-def filter_expression(writer, node):
- writer.node(node.var)
- writer.filters(node.filters)
-
-
-@node(core_tags.CommentNode)
-def comment_tag(writer, node):
- pass
-
-
-@node(core_tags.DebugNode)
-def comment_tag(writer, node):
- writer.warn('Debug tag detected. Make sure to add a global function '
- 'called debug to the namespace.', node=node)
- writer.print_expr('debug()')
-
-
-@node(core_tags.ForNode)
-def for_loop(writer, node):
- writer.start_block()
- writer.write('for ')
- for idx, var in enumerate(node.loopvars):
- if idx:
- writer.write(', ')
- writer.variable(var)
- writer.write(' in ')
- if node.is_reversed:
- writer.write('(')
- writer.node(node.sequence)
- if node.is_reversed:
- writer.write(')|reverse')
- writer.end_block()
- writer.enter_loop()
- writer.body(node.nodelist_loop)
- writer.leave_loop()
- writer.tag('endfor')
-
-
-@node(core_tags.IfNode)
-def if_condition(writer, node):
- writer.start_block()
- writer.write('if ')
- join_with = 'and'
- if node.link_type == core_tags.IfNode.LinkTypes.or_:
- join_with = 'or'
-
- for idx, (ifnot, expr) in enumerate(node.bool_exprs):
- if idx:
- writer.write(' %s ' % join_with)
- if ifnot:
- writer.write('not ')
- writer.node(expr)
- writer.end_block()
- writer.body(node.nodelist_true)
- if node.nodelist_false:
- writer.tag('else')
- writer.body(node.nodelist_false)
- writer.tag('endif')
-
-
-@node(core_tags.IfEqualNode)
-def if_equal(writer, node):
- writer.start_block()
- writer.write('if ')
- writer.node(node.var1)
- if node.negate:
- writer.write(' != ')
- else:
- writer.write(' == ')
- writer.node(node.var2)
- writer.end_block()
- writer.body(node.nodelist_true)
- if node.nodelist_false:
- writer.tag('else')
- writer.body(node.nodelist_false)
- writer.tag('endif')
-
-
-@node(loader_tags.BlockNode)
-def block(writer, node):
- writer.tag('block ' + node.name.replace('-', '_').rstrip('_'))
- node = node
- while node.parent is not None:
- node = node.parent
- writer.body(node.nodelist)
- writer.tag('endblock')
-
-
-@node(loader_tags.ExtendsNode)
-def extends(writer, node):
- writer.start_block()
- writer.write('extends ')
- if node.parent_name_expr:
- writer.node(node.parent_name_expr)
- else:
- writer.literal(node.parent_name)
- writer.end_block()
- writer.body(node.nodelist)
-
-
-@node(loader_tags.ConstantIncludeNode)
-@node(loader_tags.IncludeNode)
-def include(writer, node):
- writer.start_block()
- writer.write('include ')
- if hasattr(node, 'template'):
- writer.literal(node.template.name)
- else:
- writer.node(node.template_name)
- writer.end_block()
-
-
-@node(core_tags.CycleNode)
-def cycle(writer, node):
- if not writer.in_loop:
- writer.warn('Untranslatable free cycle (cycle outside loop)', node=node)
- return
- if node.variable_name is not None:
- writer.start_block()
- writer.write('set %s = ' % node.variable_name)
- else:
- writer.start_variable()
- writer.write('loop.cycle(')
- for idx, var in enumerate(node.raw_cycle_vars):
- if idx:
- writer.write(', ')
- writer.node(var)
- writer.write(')')
- if node.variable_name is not None:
- writer.end_block()
- else:
- writer.end_variable()
-
-
-@node(core_tags.FilterNode)
-def filter(writer, node):
- writer.start_block()
- writer.write('filter ')
- writer.filters(node.filter_expr.filters, True)
- writer.end_block()
- writer.body(node.nodelist)
- writer.tag('endfilter')
-
-
-@node(core_tags.AutoEscapeControlNode)
-def autoescape_control(writer, node):
- original = writer.autoescape
- writer.autoescape = node.setting
- writer.body(node.nodelist)
- writer.autoescape = original
-
-
-@node(core_tags.SpacelessNode)
-def spaceless(writer, node):
- original = writer.spaceless
- writer.spaceless = True
- writer.warn('entering spaceless mode with different semantics', node)
- # do the initial stripping
- nodelist = list(node.nodelist)
- if nodelist:
- if isinstance(nodelist[0], TextNode):
- nodelist[0] = TextNode(nodelist[0].s.lstrip())
- if isinstance(nodelist[-1], TextNode):
- nodelist[-1] = TextNode(nodelist[-1].s.rstrip())
- writer.body(nodelist)
- writer.spaceless = original
-
-
-@node(core_tags.TemplateTagNode)
-def template_tag(writer, node):
- tag = {
- 'openblock': writer.block_start_string,
- 'closeblock': writer.block_end_string,
- 'openvariable': writer.variable_start_string,
- 'closevariable': writer.variable_end_string,
- 'opencomment': writer.comment_start_string,
- 'closecomment': writer.comment_end_string,
- 'openbrace': '{',
- 'closebrace': '}'
- }.get(node.tagtype)
- if tag:
- writer.start_variable()
- writer.literal(tag)
- writer.end_variable()
-
-
-@node(core_tags.URLNode)
-def url_tag(writer, node):
- writer.warn('url node used. make sure to provide a proper url() '
- 'function', node)
- if node.asvar:
- writer.start_block()
- writer.write('set %s = ' % node.asvar)
- else:
- writer.start_variable()
- autoescape = writer.autoescape
- writer.write('url(')
- writer.literal(node.view_name)
- for arg in node.args:
- writer.write(', ')
- writer.node(arg)
- for key, arg in node.kwargs.items():
- writer.write(', %s=' % key)
- writer.node(arg)
- writer.write(')')
- if node.asvar:
- writer.end_block()
- else:
- writer.end_variable()
-
-
-@node(core_tags.WidthRatioNode)
-def width_ratio(writer, node):
- writer.warn('widthratio expanded into formula. You may want to provide '
- 'a helper function for this calculation', node)
- writer.start_variable()
- writer.write('(')
- writer.node(node.val_expr)
- writer.write(' / ')
- writer.node(node.max_expr)
- writer.write(' * ')
- writer.write(str(int(node.max_width)))
- writer.write(')|round|int')
- writer.end_variable(always_safe=True)
-
-
-@node(core_tags.WithNode)
-def with_block(writer, node):
- writer.warn('with block expanded into set statement. This could cause '
- 'variables following that block to be overriden.', node)
- writer.start_block()
- writer.write('set %s = ' % node.name)
- writer.node(node.var)
- writer.end_block()
- writer.body(node.nodelist)
-
-
-@node(core_tags.RegroupNode)
-def regroup(writer, node):
- if node.expression.var.literal:
- writer.warn('literal in groupby filter used. Behavior in that '
- 'situation is undefined and translation is skipped.', node)
- return
- elif node.expression.filters:
- writer.warn('filters in groupby filter used. Behavior in that '
- 'situation is undefined which is most likely a bug '
- 'in your code. Filters were ignored.', node)
- writer.start_block()
- writer.write('set %s = ' % node.var_name)
- writer.node(node.target)
- writer.write('|groupby(')
- writer.literal(node.expression.var.var)
- writer.write(')')
- writer.end_block()
-
-
-@node(core_tags.LoadNode)
-def warn_load(writer, node):
- writer.warn('load statement used which was ignored on conversion', node)
-
-
-@node(i18n_tags.GetAvailableLanguagesNode)
-def get_available_languages(writer, node):
- writer.warn('make sure to provide a get_available_languages function', node)
- writer.tag('set %s = get_available_languages()' %
- writer.translate_variable_name(node.variable))
-
-
-@node(i18n_tags.GetCurrentLanguageNode)
-def get_current_language(writer, node):
- writer.warn('make sure to provide a get_current_language function', node)
- writer.tag('set %s = get_current_language()' %
- writer.translate_variable_name(node.variable))
-
-
-@node(i18n_tags.GetCurrentLanguageBidiNode)
-def get_current_language_bidi(writer, node):
- writer.warn('make sure to provide a get_current_language_bidi function', node)
- writer.tag('set %s = get_current_language_bidi()' %
- writer.translate_variable_name(node.variable))
-
-
-@node(i18n_tags.TranslateNode)
-def simple_gettext(writer, node):
- writer.warn('i18n system used, make sure to install translations', node)
- writer.start_variable()
- writer.write('_(')
- writer.node(node.value)
- writer.write(')')
- writer.end_variable()
-
-
-@node(i18n_tags.BlockTranslateNode)
-def translate_block(writer, node):
- first_var = []
- variables = set()
-
- def touch_var(name):
- variables.add(name)
- if not first_var:
- first_var.append(name)
-
- def dump_token_list(tokens):
- for token in tokens:
- if token.token_type == TOKEN_TEXT:
- writer.write(token.contents)
- elif token.token_type == TOKEN_VAR:
- writer.print_expr(token.contents)
- touch_var(token.contents)
-
- writer.warn('i18n system used, make sure to install translations', node)
- writer.start_block()
- writer.write('trans')
- idx = -1
- for idx, (key, var) in enumerate(node.extra_context.items()):
- if idx:
- writer.write(',')
- writer.write(' %s=' % key)
- touch_var(key)
- writer.node(var.filter_expression)
-
- have_plural = False
- plural_var = None
- if node.plural and node.countervar and node.counter:
- have_plural = True
- plural_var = node.countervar
- if plural_var not in variables:
- if idx > -1:
- writer.write(',')
- touch_var(plural_var)
- writer.write(' %s=' % plural_var)
- writer.node(node.counter)
-
- writer.end_block()
- dump_token_list(node.singular)
- if node.plural and node.countervar and node.counter:
- writer.start_block()
- writer.write('pluralize')
- if node.countervar != first_var[0]:
- writer.write(' ' + node.countervar)
- writer.end_block()
- dump_token_list(node.plural)
- writer.tag('endtrans')
-
-@node("SimpleNode")
-def simple_tag(writer, node):
- """Check if the simple tag exist as a filter in """
- name = node.tag_name
- if writer.env and \
- name not in writer.env.filters and \
- name not in writer._filters_warned:
- writer._filters_warned.add(name)
- writer.warn('Filter %s probably doesn\'t exist in Jinja' %
- name)
-
- if not node.vars_to_resolve:
- # No argument, pass the request
- writer.start_variable()
- writer.write('request|')
- writer.write(name)
- writer.end_variable()
- return
-
- first_var = node.vars_to_resolve[0]
- args = node.vars_to_resolve[1:]
- writer.start_variable()
-
- # Copied from Writer.filters()
- writer.node(first_var)
-
- writer.write('|')
- writer.write(name)
- if args:
- writer.write('(')
- for idx, var in enumerate(args):
- if idx:
- writer.write(', ')
- if var.var:
- writer.node(var)
- else:
- writer.literal(var.literal)
- writer.write(')')
- writer.end_variable()
-
-# get rid of node now, it shouldn't be used normally
-del node
diff --git a/slider-agent/src/main/python/jinja2/ext/django2jinja/example.py b/slider-agent/src/main/python/jinja2/ext/django2jinja/example.py
deleted file mode 100644
index 2d4ab9a..0000000
--- a/slider-agent/src/main/python/jinja2/ext/django2jinja/example.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from django.conf import settings
-settings.configure(TEMPLATE_DIRS=['templates'], TEMPLATE_DEBUG=True)
-
-from django2jinja import convert_templates, Writer
-
-writer = Writer(use_jinja_autoescape=True)
-convert_templates('converted', writer=writer)
diff --git a/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/index.html b/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/index.html
deleted file mode 100644
index d0fbe38..0000000
--- a/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/index.html
+++ /dev/null
@@ -1,58 +0,0 @@
-{% extends "layout.html" %}
-{% load i18n %}
-{% block title %}Foo{% endblock %}
-{% block page-body %}
- {{ block.super }}
- Hello {{ name|cut:"d" }}!
-
- {% for item in seq reversed %}
- {% if forloop.index|divisibleby:2 %}
- <li class="{% cycle 'a' 'b' %}">{{ item }}</li>
- {% endif %}
- {% endfor %}
- {% ifequal foo bar %}
- haha
- {% else %}
- hmm
- {% endifequal %}
- {% filter upper %}
- {% include "subtemplate.html" %}
- {% include foo %}
- {% endfilter %}
- {% spaceless %}
- Hello World
- {{ foo }}
- Hmm
- {% endspaceless %}
- {% templatetag opencomment %}...{% templatetag closecomment %}
- {% url foo a, b, c=d %}
- {% url foo a, b, c=d as hmm %}
-
- {% with object.value as value %}
- <img src='bar.gif' height='10' width='{% widthratio value 200 100 %}'>
- {% endwith %}
-
- <pre>{% debug %}</pre>
-
- {% blocktrans with book|title as book_t and author|title as author_t %}
- This is {{ book_t }} by {{ author_t }}
- {% endblocktrans %}
-
- {% blocktrans count list|length as counter %}
- There is only one {{ name }} object.
- {% plural %}
- There are {{ counter }} {{ name }} objects.
- {% endblocktrans %}
-
- {% blocktrans with name|escape as name count list|length as counter %}
- There is only one {{ name }} object.
- {% plural %}
- There are {{ counter }} {{ name }} objects.
- {% endblocktrans %}
-
- {% blocktrans %}This string will have {{ value }} inside.{% endblocktrans %}
-
- <p>{% trans "This is the title." %}</p>
-
- {% regroup people by gender as grouped %}
-{% endblock %}
diff --git a/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/layout.html b/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/layout.html
deleted file mode 100644
index 3f21a12..0000000
--- a/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/layout.html
+++ /dev/null
@@ -1,4 +0,0 @@
-<title>{% block title %}{% endblock %}</title>
-<div class="body">
- {% block page-body %}{% endblock %}
-</div>
diff --git a/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/subtemplate.html b/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/subtemplate.html
deleted file mode 100644
index 980a0d5..0000000
--- a/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/subtemplate.html
+++ /dev/null
@@ -1 +0,0 @@
-Hello World!
diff --git a/slider-agent/src/main/python/jinja2/ext/djangojinja2.py b/slider-agent/src/main/python/jinja2/ext/djangojinja2.py
deleted file mode 100644
index d24d164..0000000
--- a/slider-agent/src/main/python/jinja2/ext/djangojinja2.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
- djangojinja2
- ~~~~~~~~~~~~
-
- Adds support for Jinja2 to Django.
-
- Configuration variables:
-
- ======================= =============================================
- Key Description
- ======================= =============================================
- `JINJA2_TEMPLATE_DIRS` List of template folders
- `JINJA2_EXTENSIONS` List of Jinja2 extensions to use
- `JINJA2_CACHE_SIZE` The size of the Jinja2 template cache.
- ======================= =============================================
-
- :copyright: (c) 2009 by the Jinja Team.
- :license: BSD.
-"""
-from itertools import chain
-from django.conf import settings
-from django.http import HttpResponse
-from django.core.exceptions import ImproperlyConfigured
-from django.template.context import get_standard_processors
-from django.template import TemplateDoesNotExist
-from jinja2 import Environment, FileSystemLoader, TemplateNotFound
-from jinja2.defaults import DEFAULT_NAMESPACE
-
-
-# the environment is unconfigured until the first template is loaded.
-_jinja_env = None
-
-
-def get_env():
- """Get the Jinja2 env and initialize it if necessary."""
- global _jinja_env
- if _jinja_env is None:
- _jinja_env = create_env()
- return _jinja_env
-
-
-def create_env():
- """Create a new Jinja2 environment."""
- searchpath = list(settings.JINJA2_TEMPLATE_DIRS)
- return Environment(loader=FileSystemLoader(searchpath),
- auto_reload=settings.TEMPLATE_DEBUG,
- cache_size=getattr(settings, 'JINJA2_CACHE_SIZE', 50),
- extensions=getattr(settings, 'JINJA2_EXTENSIONS', ()))
-
-
-def get_template(template_name, globals=None):
- """Load a template."""
- try:
- return get_env().get_template(template_name, globals=globals)
- except TemplateNotFound, e:
- raise TemplateDoesNotExist(str(e))
-
-
-def select_template(templates, globals=None):
- """Try to load one of the given templates."""
- env = get_env()
- for template in templates:
- try:
- return env.get_template(template, globals=globals)
- except TemplateNotFound:
- continue
- raise TemplateDoesNotExist(', '.join(templates))
-
-
-def render_to_string(template_name, context=None, request=None,
- processors=None):
- """Render a template into a string."""
- context = dict(context or {})
- if request is not None:
- context['request'] = request
- for processor in chain(get_standard_processors(), processors or ()):
- context.update(processor(request))
- return get_template(template_name).render(context)
-
-
-def render_to_response(template_name, context=None, request=None,
- processors=None, mimetype=None):
- """Render a template into a response object."""
- return HttpResponse(render_to_string(template_name, context, request,
- processors), mimetype=mimetype)
diff --git a/slider-agent/src/main/python/jinja2/ext/inlinegettext.py b/slider-agent/src/main/python/jinja2/ext/inlinegettext.py
deleted file mode 100644
index cf4ed5e..0000000
--- a/slider-agent/src/main/python/jinja2/ext/inlinegettext.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
- Inline Gettext
- ~~~~~~~~~~~~~~
-
- An example extension for Jinja2 that supports inline gettext calls.
- Requires the i18n extension to be loaded.
-
- :copyright: (c) 2009 by the Jinja Team.
- :license: BSD.
-"""
-import re
-from jinja2.ext import Extension
-from jinja2.lexer import Token, count_newlines
-from jinja2.exceptions import TemplateSyntaxError
-
-
-_outside_re = re.compile(r'\\?(gettext|_)\(')
-_inside_re = re.compile(r'\\?[()]')
-
-
-class InlineGettext(Extension):
- """This extension implements support for inline gettext blocks::
-
- <h1>_(Welcome)</h1>
- <p>_(This is a paragraph)</p>
-
- Requires the i18n extension to be loaded and configured.
- """
-
- def filter_stream(self, stream):
- paren_stack = 0
-
- for token in stream:
- if token.type is not 'data':
- yield token
- continue
-
- pos = 0
- lineno = token.lineno
-
- while 1:
- if not paren_stack:
- match = _outside_re.search(token.value, pos)
- else:
- match = _inside_re.search(token.value, pos)
- if match is None:
- break
- new_pos = match.start()
- if new_pos > pos:
- preval = token.value[pos:new_pos]
- yield Token(lineno, 'data', preval)
- lineno += count_newlines(preval)
- gtok = match.group()
- if gtok[0] == '\\':
- yield Token(lineno, 'data', gtok[1:])
- elif not paren_stack:
- yield Token(lineno, 'block_begin', None)
- yield Token(lineno, 'name', 'trans')
- yield Token(lineno, 'block_end', None)
- paren_stack = 1
- else:
- if gtok == '(' or paren_stack > 1:
- yield Token(lineno, 'data', gtok)
- paren_stack += gtok == ')' and -1 or 1
- if not paren_stack:
- yield Token(lineno, 'block_begin', None)
- yield Token(lineno, 'name', 'endtrans')
- yield Token(lineno, 'block_end', None)
- pos = match.end()
-
- if pos < len(token.value):
- yield Token(lineno, 'data', token.value[pos:])
-
- if paren_stack:
- raise TemplateSyntaxError('unclosed gettext expression',
- token.lineno, stream.name,
- stream.filename)
diff --git a/slider-agent/src/main/python/jinja2/ext/jinja.el b/slider-agent/src/main/python/jinja2/ext/jinja.el
deleted file mode 100644
index 401ba29..0000000
--- a/slider-agent/src/main/python/jinja2/ext/jinja.el
+++ /dev/null
@@ -1,213 +0,0 @@
-;;; jinja.el --- Jinja mode highlighting
-;;
-;; Author: Georg Brandl
-;; Copyright: (c) 2009 by the Jinja Team
-;; Last modified: 2008-05-22 23:04 by gbr
-;;
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;;
-;;; Commentary:
-;;
-;; Mostly ripped off django-mode by Lennart Borgman.
-;;
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;;
-;; This program is free software; you can redistribute it and/or
-;; modify it under the terms of the GNU General Public License as
-;; published by the Free Software Foundation; either version 2, or
-;; (at your option) any later version.
-;;
-;; This program is distributed in the hope that it will be useful,
-;; but WITHOUT ANY WARRANTY; without even the implied warranty of
-;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-;; General Public License for more details.
-;;
-;; You should have received a copy of the GNU General Public License
-;; along with this program; see the file COPYING. If not, write to
-;; the Free Software Foundation, Inc., 51 Franklin Street, Fifth
-;; Floor, Boston, MA 02110-1301, USA.
-;;
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;;
-;;; Code:
-
-(defconst jinja-font-lock-keywords
- (list
-; (cons (rx "{% comment %}" (submatch (0+ anything))
-; "{% endcomment %}") (list 1 font-lock-comment-face))
- '("{# ?\\(.*?\\) ?#}" . (1 font-lock-comment-face))
- '("{%-?\\|-?%}\\|{{\\|}}" . font-lock-preprocessor-face)
- '("{#\\|#}" . font-lock-comment-delimiter-face)
- ;; first word in a block is a command
- '("{%-?[ \t\n]*\\([a-zA-Z_]+\\)" . (1 font-lock-keyword-face))
- ;; variables
- '("\\({{ ?\\)\\([^|]*?\\)\\(|.*?\\)? ?}}" . (1 font-lock-variable-name-face))
- ;; keywords and builtins
- (cons (rx word-start
- (or "in" "as" "recursive" "not" "and" "or" "if" "else"
- "import" "with" "without" "context")
- word-end)
- font-lock-keyword-face)
- (cons (rx word-start
- (or "true" "false" "none" "loop" "self" "super")
- word-end)
- font-lock-builtin-face)
- ;; tests
- '("\\(is\\)[ \t]*\\(not\\)[ \t]*\\([a-zA-Z_]+\\)"
- (1 font-lock-keyword-face) (2 font-lock-keyword-face)
- (3 font-lock-function-name-face))
- ;; builtin filters
- (cons (rx
- "|" (* space)
- (submatch
- (or "abs" "batch" "capitalize" "capture" "center" "count" "default"
- "dformat" "dictsort" "e" "escape" "filesizeformat" "first"
- "float" "format" "getattribute" "getitem" "groupby" "indent"
- "int" "join" "jsonencode" "last" "length" "lower" "markdown"
- "pprint" "random" "replace" "reverse" "round" "rst" "slice"
- "sort" "string" "striptags" "sum" "textile" "title" "trim"
- "truncate" "upper" "urlencode" "urlize" "wordcount" "wordwrap"
- "xmlattr")))
- (list 1 font-lock-builtin-face))
- )
- "Minimal highlighting expressions for Jinja mode")
-
-(define-derived-mode jinja-mode nil "Jinja"
- "Simple Jinja mode for use with `mumamo-mode'.
-This mode only provides syntax highlighting."
- ;;(set (make-local-variable 'comment-start) "{#")
- ;;(set (make-local-variable 'comment-end) "#}")
- (setq font-lock-defaults '(jinja-font-lock-keywords)))
-
-;; mumamo stuff
-
-(when (require 'mumamo nil t)
-
- (defun mumamo-chunk-jinja3(pos min max)
- "Find {# ... #}. Return range and `jinja-mode'.
-See `mumamo-find-possible-chunk' for POS, MIN and MAX."
- (mumamo-find-possible-chunk pos min max
- 'mumamo-search-bw-exc-start-jinja3
- 'mumamo-search-bw-exc-end-jinja3
- 'mumamo-search-fw-exc-start-jinja3
- 'mumamo-search-fw-exc-end-jinja3))
-
- (defun mumamo-chunk-jinja2(pos min max)
- "Find {{ ... }}. Return range and `jinja-mode'.
-See `mumamo-find-possible-chunk' for POS, MIN and MAX."
- (mumamo-find-possible-chunk pos min max
- 'mumamo-search-bw-exc-start-jinja2
- 'mumamo-search-bw-exc-end-jinja2
- 'mumamo-search-fw-exc-start-jinja2
- 'mumamo-search-fw-exc-end-jinja2))
-
- (defun mumamo-chunk-jinja (pos min max)
- "Find {% ... %}. Return range and `jinja-mode'.
-See `mumamo-find-possible-chunk' for POS, MIN and MAX."
- (mumamo-find-possible-chunk pos min max
- 'mumamo-search-bw-exc-start-jinja
- 'mumamo-search-bw-exc-end-jinja
- 'mumamo-search-fw-exc-start-jinja
- 'mumamo-search-fw-exc-end-jinja))
-
- (defun mumamo-search-bw-exc-start-jinja (pos min)
- "Helper for `mumamo-chunk-jinja'.
-POS is where to start search and MIN is where to stop."
- (let ((exc-start (mumamo-chunk-start-bw-str-inc pos min "{%")))
- (and exc-start
- (<= exc-start pos)
- (cons exc-start 'jinja-mode))))
-
- (defun mumamo-search-bw-exc-start-jinja2(pos min)
- "Helper for `mumamo-chunk-jinja2'.
-POS is where to start search and MIN is where to stop."
- (let ((exc-start (mumamo-chunk-start-bw-str-inc pos min "{{")))
- (and exc-start
- (<= exc-start pos)
- (cons exc-start 'jinja-mode))))
-
- (defun mumamo-search-bw-exc-start-jinja3(pos min)
- "Helper for `mumamo-chunk-jinja3'.
-POS is where to start search and MIN is where to stop."
- (let ((exc-start (mumamo-chunk-start-bw-str-inc pos min "{#")))
- (and exc-start
- (<= exc-start pos)
- (cons exc-start 'jinja-mode))))
-
- (defun mumamo-search-bw-exc-end-jinja (pos min)
- "Helper for `mumamo-chunk-jinja'.
-POS is where to start search and MIN is where to stop."
- (mumamo-chunk-end-bw-str-inc pos min "%}"))
-
- (defun mumamo-search-bw-exc-end-jinja2(pos min)
- "Helper for `mumamo-chunk-jinja2'.
-POS is where to start search and MIN is where to stop."
- (mumamo-chunk-end-bw-str-inc pos min "}}"))
-
- (defun mumamo-search-bw-exc-end-jinja3(pos min)
- "Helper for `mumamo-chunk-jinja3'.
-POS is where to start search and MIN is where to stop."
- (mumamo-chunk-end-bw-str-inc pos min "#}"))
-
- (defun mumamo-search-fw-exc-start-jinja (pos max)
- "Helper for `mumamo-chunk-jinja'.
-POS is where to start search and MAX is where to stop."
- (mumamo-chunk-start-fw-str-inc pos max "{%"))
-
- (defun mumamo-search-fw-exc-start-jinja2(pos max)
- "Helper for `mumamo-chunk-jinja2'.
-POS is where to start search and MAX is where to stop."
- (mumamo-chunk-start-fw-str-inc pos max "{{"))
-
- (defun mumamo-search-fw-exc-start-jinja3(pos max)
- "Helper for `mumamo-chunk-jinja3'.
-POS is where to start search and MAX is where to stop."
- (mumamo-chunk-start-fw-str-inc pos max "{#"))
-
- (defun mumamo-search-fw-exc-end-jinja (pos max)
- "Helper for `mumamo-chunk-jinja'.
-POS is where to start search and MAX is where to stop."
- (mumamo-chunk-end-fw-str-inc pos max "%}"))
-
- (defun mumamo-search-fw-exc-end-jinja2(pos max)
- "Helper for `mumamo-chunk-jinja2'.
-POS is where to start search and MAX is where to stop."
- (mumamo-chunk-end-fw-str-inc pos max "}}"))
-
- (defun mumamo-search-fw-exc-end-jinja3(pos max)
- "Helper for `mumamo-chunk-jinja3'.
-POS is where to start search and MAX is where to stop."
- (mumamo-chunk-end-fw-str-inc pos max "#}"))
-
-;;;###autoload
- (define-mumamo-multi-major-mode jinja-html-mumamo
- "Turn on multiple major modes for Jinja with main mode `html-mode'.
-This also covers inlined style and javascript."
- ("Jinja HTML Family" html-mode
- (mumamo-chunk-jinja
- mumamo-chunk-jinja2
- mumamo-chunk-jinja3
- mumamo-chunk-inlined-style
- mumamo-chunk-inlined-script
- mumamo-chunk-style=
- mumamo-chunk-onjs=
- )))
-
-;;;###autoload
- (define-mumamo-multi-major-mode jinja-nxhtml-mumamo
- "Turn on multiple major modes for Jinja with main mode `nxhtml-mode'.
-This also covers inlined style and javascript."
- ("Jinja nXhtml Family" nxhtml-mode
- (mumamo-chunk-jinja
- mumamo-chunk-jinja2
- mumamo-chunk-jinja3
- mumamo-chunk-inlined-style
- mumamo-chunk-inlined-script
- mumamo-chunk-style=
- mumamo-chunk-onjs=
- )))
- )
-
-(provide 'jinja)
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;;; jinja.el ends here
diff --git a/slider-agent/src/main/python/kazoo/client.py b/slider-agent/src/main/python/kazoo/client.py
index a315489..47545ee 100644
--- a/slider-agent/src/main/python/kazoo/client.py
+++ b/slider-agent/src/main/python/kazoo/client.py
@@ -220,7 +220,6 @@
elif type(command_retry) is KazooRetry:
self.retry = command_retry
-
if type(self._conn_retry) is KazooRetry:
if self.handler.sleep_func != self._conn_retry.sleep_func:
raise ConfigurationError("Retry handler and event handler "
@@ -228,19 +227,21 @@
if type(self.retry) is KazooRetry:
if self.handler.sleep_func != self.retry.sleep_func:
- raise ConfigurationError("Command retry handler and event "
- "handler must use the same sleep func")
+ raise ConfigurationError(
+ "Command retry handler and event handler "
+ "must use the same sleep func")
if self.retry is None or self._conn_retry is None:
old_retry_keys = dict(_RETRY_COMPAT_DEFAULTS)
for key in old_retry_keys:
try:
old_retry_keys[key] = kwargs.pop(key)
- warnings.warn('Passing retry configuration param %s to the'
- ' client directly is deprecated, please pass a'
- ' configured retry object (using param %s)' % (
- key, _RETRY_COMPAT_MAPPING[key]),
- DeprecationWarning, stacklevel=2)
+ warnings.warn(
+ 'Passing retry configuration param %s to the '
+ 'client directly is deprecated, please pass a '
+ 'configured retry object (using param %s)' % (
+ key, _RETRY_COMPAT_MAPPING[key]),
+ DeprecationWarning, stacklevel=2)
except KeyError:
pass
@@ -258,12 +259,13 @@
**retry_keys)
self._conn_retry.interrupt = lambda: self._stopped.is_set()
- self._connection = ConnectionHandler(self, self._conn_retry.copy(),
- logger=self.logger)
+ self._connection = ConnectionHandler(
+ self, self._conn_retry.copy(), logger=self.logger)
# Every retry call should have its own copy of the retry helper
# to avoid shared retry counts
self._retry = self.retry
+
def _retry(*args, **kwargs):
return self._retry.copy()(*args, **kwargs)
self.retry = _retry
@@ -282,7 +284,7 @@
self.Semaphore = partial(Semaphore, self)
self.ShallowParty = partial(ShallowParty, self)
- # If we got any unhandled keywords, complain like python would
+ # If we got any unhandled keywords, complain like Python would
if kwargs:
raise TypeError('__init__() got unexpected keyword arguments: %s'
% (kwargs.keys(),))
@@ -433,7 +435,8 @@
return
if state in (KeeperState.CONNECTED, KeeperState.CONNECTED_RO):
- self.logger.info("Zookeeper connection established, state: %s", state)
+ self.logger.info("Zookeeper connection established, "
+ "state: %s", state)
self._live.set()
self._make_state_change(KazooState.CONNECTED)
elif state in LOST_STATES:
@@ -510,12 +513,12 @@
self._queue.append((request, async_object))
# wake the connection, guarding against a race with close()
- write_pipe = self._connection._write_pipe
- if write_pipe is None:
+ write_sock = self._connection._write_sock
+ if write_sock is None:
async_object.set_exception(ConnectionClosedError(
"Connection has been closed"))
try:
- os.write(write_pipe, b'\0')
+ write_sock.send(b'\0')
except:
async_object.set_exception(ConnectionClosedError(
"Connection has been closed"))
@@ -585,7 +588,7 @@
self._stopped.set()
self._queue.append((CloseInstance, None))
- os.write(self._connection._write_pipe, b'\0')
+ self._connection._write_sock.send(b'\0')
self._safe_close()
def restart(self):
@@ -622,7 +625,7 @@
if not self._live.is_set():
raise ConnectionLoss("No connection to server")
- peer = self._connection._socket.getpeername()
+ peer = self._connection._socket.getpeername()[:2]
sock = self.handler.create_connection(
peer, timeout=self._session_timeout / 1000.0)
sock.sendall(cmd)
@@ -786,7 +789,7 @@
"""
acl = acl or self.default_acl
return self.create_async(path, value, acl=acl, ephemeral=ephemeral,
- sequence=sequence, makepath=makepath).get()
+ sequence=sequence, makepath=makepath).get()
def create_async(self, path, value=b"", acl=None, ephemeral=False,
sequence=False, makepath=False):
@@ -828,7 +831,8 @@
@capture_exceptions(async_result)
def do_create():
- result = self._create_async_inner(path, value, acl, flags, trailing=sequence)
+ result = self._create_async_inner(
+ path, value, acl, flags, trailing=sequence)
result.rawlink(create_completion)
@capture_exceptions(async_result)
@@ -867,10 +871,13 @@
return async_result
def ensure_path(self, path, acl=None):
- """Recursively create a path if it doesn't exist.
+ """Recursively create a path if it doesn't exist. Also return value indicates
+ if path already existed or had to be created.
:param path: Path of node.
:param acl: Permissions for node.
+ :returns `True` if path existed, `False` otherwise.
+ :rtype: bool
"""
return self.ensure_path_async(path, acl).get()
@@ -1291,6 +1298,13 @@
Transactions are not thread-safe and should not be accessed from
multiple threads at once.
+ .. note::
+
+ The ``committed`` attribute only indicates whether this
+ transaction has been sent to Zookeeper and is used to prevent
+ duplicate commits of the same transaction. The result should be
+ checked to determine if the transaction executed as desired.
+
.. versionadded:: 0.6
Requires Zookeeper 3.4+
diff --git a/slider-agent/src/main/python/kazoo/handlers/threading.py b/slider-agent/src/main/python/kazoo/handlers/threading.py
index 3ca9a8f..684a6b0 100644
--- a/slider-agent/src/main/python/kazoo/handlers/threading.py
+++ b/slider-agent/src/main/python/kazoo/handlers/threading.py
@@ -35,7 +35,7 @@
log = logging.getLogger(__name__)
-class TimeoutError(Exception):
+class KazooTimeoutError(Exception):
pass
@@ -104,7 +104,7 @@
raise self._exception
# if we get to this point we timeout
- raise TimeoutError()
+ raise KazooTimeoutError()
def get_nowait(self):
"""Return the value or raise the exception without blocking.
@@ -174,7 +174,7 @@
"""
name = "sequential_threading_handler"
- timeout_exception = TimeoutError
+ timeout_exception = KazooTimeoutError
sleep_func = staticmethod(time.sleep)
queue_impl = Queue.Queue
queue_empty = Queue.Empty
diff --git a/slider-agent/src/main/python/kazoo/handlers/utils.py b/slider-agent/src/main/python/kazoo/handlers/utils.py
index 60d6404..93cfdb5 100644
--- a/slider-agent/src/main/python/kazoo/handlers/utils.py
+++ b/slider-agent/src/main/python/kazoo/handlers/utils.py
@@ -8,7 +8,9 @@
HAS_FNCTL = False
import functools
import os
-
+import sys
+import socket
+import errno
def _set_fd_cloexec(fd):
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
@@ -21,18 +23,43 @@
_set_fd_cloexec(sock)
return sock
+def create_socket_pair(port=0):
+ """Create socket pair.
-def create_pipe():
- """Create a non-blocking read/write pipe.
+ If socket.socketpair isn't available, we emulate it.
"""
- r, w = os.pipe()
- if HAS_FNCTL:
- fcntl.fcntl(r, fcntl.F_SETFL, os.O_NONBLOCK)
- fcntl.fcntl(w, fcntl.F_SETFL, os.O_NONBLOCK)
- _set_fd_cloexec(r)
- _set_fd_cloexec(w)
- return r, w
+ # See if socketpair() is available.
+ have_socketpair = hasattr(socket, 'socketpair')
+ if have_socketpair:
+ client_sock, srv_sock = socket.socketpair()
+ return client_sock, srv_sock
+ # Create a non-blocking temporary server socket
+ temp_srv_sock = socket.socket()
+ temp_srv_sock.setblocking(False)
+ temp_srv_sock.bind(('', port))
+ port = temp_srv_sock.getsockname()[1]
+ temp_srv_sock.listen(1)
+
+ # Create non-blocking client socket
+ client_sock = socket.socket()
+ client_sock.setblocking(False)
+ try:
+ client_sock.connect(('localhost', port))
+ except socket.error as err:
+ # EWOULDBLOCK is not an error, as the socket is non-blocking
+ if err.errno != errno.EWOULDBLOCK:
+ raise
+
+ # Use select to wait for connect() to succeed.
+ import select
+ timeout = 1
+ readable = select.select([temp_srv_sock], [], [], timeout)[0]
+ if temp_srv_sock not in readable:
+ raise Exception('Client socket not connected in {} second(s)'.format(timeout))
+ srv_sock, _ = temp_srv_sock.accept()
+
+ return client_sock, srv_sock
def create_tcp_socket(module):
"""Create a TCP socket with the CLOEXEC flag set.
diff --git a/slider-agent/src/main/python/kazoo/protocol/connection.py b/slider-agent/src/main/python/kazoo/protocol/connection.py
index 3cbb87f..6b89c18 100644
--- a/slider-agent/src/main/python/kazoo/protocol/connection.py
+++ b/slider-agent/src/main/python/kazoo/protocol/connection.py
@@ -17,7 +17,7 @@
SessionExpiredError,
NoNodeError
)
-from kazoo.handlers.utils import create_pipe
+from kazoo.handlers.utils import create_socket_pair
from kazoo.loggingsupport import BLATHER
from kazoo.protocol.serialization import (
Auth,
@@ -146,8 +146,8 @@
self.connection_stopped.set()
self.ping_outstanding = client.handler.event_object()
- self._read_pipe = None
- self._write_pipe = None
+ self._read_sock = None
+ self._write_sock = None
self._socket = None
self._xid = None
@@ -169,7 +169,7 @@
def start(self):
"""Start the connection up"""
if self.connection_closed.is_set():
- self._read_pipe, self._write_pipe = create_pipe()
+ self._read_sock, self._write_sock = create_socket_pair()
self.connection_closed.clear()
if self._connection_routine:
raise Exception("Unable to start, connection routine already "
@@ -192,12 +192,12 @@
if not self.connection_stopped.is_set():
raise Exception("Cannot close connection until it is stopped")
self.connection_closed.set()
- wp, rp = self._write_pipe, self._read_pipe
- self._write_pipe = self._read_pipe = None
- if wp is not None:
- os.close(wp)
- if rp is not None:
- os.close(rp)
+ ws, rs = self._write_sock, self._read_sock
+ self._write_sock = self._read_sock = None
+ if ws is not None:
+ ws.close()
+ if rs is not None:
+ rs.close()
def _server_pinger(self):
"""Returns a server pinger iterable, that will ping the next
@@ -238,8 +238,8 @@
if xid:
header, buffer, offset = self._read_header(timeout)
if header.xid != xid:
- raise RuntimeError('xids do not match, expected %r received %r',
- xid, header.xid)
+ raise RuntimeError('xids do not match, expected %r '
+ 'received %r', xid, header.xid)
if header.zxid > 0:
zxid = header.zxid
if header.err:
@@ -257,8 +257,9 @@
try:
obj, _ = request.deserialize(msg, 0)
except Exception:
- self.logger.exception("Exception raised during deserialization"
- " of request: %s", request)
+ self.logger.exception(
+ "Exception raised during deserialization "
+ "of request: %s", request)
# raise ConnectionDropped so connect loop will retry
raise ConnectionDropped('invalid server response')
@@ -276,8 +277,9 @@
if request.type:
b.extend(int_struct.pack(request.type))
b += request.serialize()
- self.logger.log((BLATHER if isinstance(request, Ping) else logging.DEBUG),
- "Sending request(xid=%s): %s", xid, request)
+ self.logger.log(
+ (BLATHER if isinstance(request, Ping) else logging.DEBUG),
+ "Sending request(xid=%s): %s", xid, request)
self._write(int_struct.pack(len(b)) + b, timeout)
def _write(self, msg, timeout):
@@ -358,8 +360,9 @@
try:
response = request.deserialize(buffer, offset)
except Exception as exc:
- self.logger.exception("Exception raised during deserialization"
- " of request: %s", request)
+ self.logger.exception(
+ "Exception raised during deserialization "
+ "of request: %s", request)
async_object.set_exception(exc)
return
self.logger.debug(
@@ -415,11 +418,11 @@
except IndexError:
# Not actually something on the queue, this can occur if
# something happens to cancel the request such that we
- # don't clear the pipe below after sending
+ # don't clear the socket below after sending
try:
# Clear possible inconsistence (no request in the queue
- # but have data in the read pipe), which causes cpu to spin.
- os.read(self._read_pipe, 1)
+ # but have data in the read socket), which causes cpu to spin.
+ self._read_sock.recv(1)
except OSError:
pass
return
@@ -440,7 +443,7 @@
self._submit(request, connect_timeout, xid)
client._queue.popleft()
- os.read(self._read_pipe, 1)
+ self._read_sock.recv(1)
client._pending.append((request, async_object, xid))
def _send_ping(self, connect_timeout):
@@ -492,7 +495,7 @@
def _connect_attempt(self, host, port, retry):
client = self.client
- TimeoutError = self.handler.timeout_exception
+ KazooTimeoutError = self.handler.timeout_exception
close_connection = False
self._socket = None
@@ -519,7 +522,7 @@
jitter_time = random.randint(0, 40) / 100.0
# Ensure our timeout is positive
timeout = max([read_timeout / 2.0 - jitter_time, jitter_time])
- s = self.handler.select([self._socket, self._read_pipe],
+ s = self.handler.select([self._socket, self._read_sock],
[], [], timeout)[0]
if not s:
@@ -537,7 +540,7 @@
self.logger.info('Closing connection to %s:%s', host, port)
client._session_callback(KeeperState.CLOSED)
return STOP_CONNECTING
- except (ConnectionDropped, TimeoutError) as e:
+ except (ConnectionDropped, KazooTimeoutError) as e:
if isinstance(e, ConnectionDropped):
self.logger.warning('Connection dropped: %s', e)
else:
@@ -570,9 +573,9 @@
self.logger.info('Connecting to %s:%s', host, port)
self.logger.log(BLATHER,
- ' Using session_id: %r session_passwd: %s',
- client._session_id,
- hexlify(client._session_passwd))
+ ' Using session_id: %r session_passwd: %s',
+ client._session_id,
+ hexlify(client._session_passwd))
with self._socket_error_handling():
self._socket = self.handler.create_connection(
@@ -584,7 +587,8 @@
client._session_id or 0, client._session_passwd,
client.read_only)
- connect_result, zxid = self._invoke(client._session_timeout, connect)
+ connect_result, zxid = self._invoke(
+ client._session_timeout / 1000.0, connect)
if connect_result.time_out <= 0:
raise SessionExpiredError("Session has expired")
@@ -601,13 +605,13 @@
client._session_passwd = connect_result.passwd
self.logger.log(BLATHER,
- 'Session created, session_id: %r session_passwd: %s\n'
- ' negotiated session timeout: %s\n'
- ' connect timeout: %s\n'
- ' read timeout: %s', client._session_id,
- hexlify(client._session_passwd),
- negotiated_session_timeout, connect_timeout,
- read_timeout)
+ 'Session created, session_id: %r session_passwd: %s\n'
+ ' negotiated session timeout: %s\n'
+ ' connect timeout: %s\n'
+ ' read timeout: %s', client._session_id,
+ hexlify(client._session_passwd),
+ negotiated_session_timeout, connect_timeout,
+ read_timeout)
if connect_result.read_only:
client._session_callback(KeeperState.CONNECTED_RO)
@@ -618,7 +622,7 @@
for scheme, auth in client.auth_data:
ap = Auth(0, scheme, auth)
- zxid = self._invoke(connect_timeout, ap, xid=AUTH_XID)
+ zxid = self._invoke(connect_timeout / 1000.0, ap, xid=AUTH_XID)
if zxid:
client.last_zxid = zxid
return read_timeout, connect_timeout
diff --git a/slider-agent/src/main/python/kazoo/testing/__init__.py b/slider-agent/src/main/python/kazoo/testing/__init__.py
deleted file mode 100644
index 660546b..0000000
--- a/slider-agent/src/main/python/kazoo/testing/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-"""license: Apache License 2.0, see LICENSE for more details."""
-from kazoo.testing.harness import KazooTestCase
-from kazoo.testing.harness import KazooTestHarness
-
-
-__all__ = ('KazooTestHarness', 'KazooTestCase', )
diff --git a/slider-agent/src/main/python/kazoo/testing/common.py b/slider-agent/src/main/python/kazoo/testing/common.py
deleted file mode 100644
index b497a8e..0000000
--- a/slider-agent/src/main/python/kazoo/testing/common.py
+++ /dev/null
@@ -1,284 +0,0 @@
-"""license: Apache License 2.0, see LICENSE for more details."""
-#
-# Copyright (C) 2010-2011, 2011 Canonical Ltd. All Rights Reserved
-#
-# This file was originally taken from txzookeeper and modified later.
-#
-# Authors:
-# Kapil Thangavelu and the Kazoo team
-#
-# txzookeeper is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# txzookeeper is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with txzookeeper. If not, see <http://www.gnu.org/licenses/>.
-
-
-import code
-import os
-import os.path
-import shutil
-import signal
-import subprocess
-import tempfile
-import traceback
-
-from itertools import chain
-from collections import namedtuple
-from glob import glob
-
-
-def debug(sig, frame):
- """Interrupt running process, and provide a python prompt for
- interactive debugging."""
- d = {'_frame': frame} # Allow access to frame object.
- d.update(frame.f_globals) # Unless shadowed by global
- d.update(frame.f_locals)
-
- i = code.InteractiveConsole(d)
- message = "Signal recieved : entering python shell.\nTraceback:\n"
- message += ''.join(traceback.format_stack(frame))
- i.interact(message)
-
-
-def listen():
- if os.name != 'nt': # SIGUSR1 is not supported on Windows
- signal.signal(signal.SIGUSR1, debug) # Register handler
-listen()
-
-
-def to_java_compatible_path(path):
- if os.name == 'nt':
- path = path.replace('\\', '/')
- return path
-
-ServerInfo = namedtuple(
- "ServerInfo", "server_id client_port election_port leader_port")
-
-
-class ManagedZooKeeper(object):
- """Class to manage the running of a ZooKeeper instance for testing.
-
- Note: no attempt is made to probe the ZooKeeper instance is
- actually available, or that the selected port is free. In the
- future, we may want to do that, especially when run in a
- Hudson/Buildbot context, to ensure more test robustness."""
-
- def __init__(self, software_path, server_info, peers=(), classpath=None):
- """Define the ZooKeeper test instance.
-
- @param install_path: The path to the install for ZK
- @param port: The port to run the managed ZK instance
- """
- self.install_path = software_path
- self._classpath = classpath
- self.server_info = server_info
- self.host = "127.0.0.1"
- self.peers = peers
- self.working_path = tempfile.mkdtemp()
- self._running = False
-
- def run(self):
- """Run the ZooKeeper instance under a temporary directory.
-
- Writes ZK log messages to zookeeper.log in the current directory.
- """
- if self.running:
- return
- config_path = os.path.join(self.working_path, "zoo.cfg")
- log_path = os.path.join(self.working_path, "log")
- log4j_path = os.path.join(self.working_path, "log4j.properties")
- data_path = os.path.join(self.working_path, "data")
-
- # various setup steps
- if not os.path.exists(self.working_path):
- os.mkdir(self.working_path)
- if not os.path.exists(log_path):
- os.mkdir(log_path)
- if not os.path.exists(data_path):
- os.mkdir(data_path)
-
- with open(config_path, "w") as config:
- config.write("""
-tickTime=2000
-dataDir=%s
-clientPort=%s
-maxClientCnxns=0
-""" % (to_java_compatible_path(data_path), self.server_info.client_port))
-
- # setup a replicated setup if peers are specified
- if self.peers:
- servers_cfg = []
- for p in chain((self.server_info,), self.peers):
- servers_cfg.append("server.%s=localhost:%s:%s" % (
- p.server_id, p.leader_port, p.election_port))
-
- with open(config_path, "a") as config:
- config.write("""
-initLimit=4
-syncLimit=2
-%s
-""" % ("\n".join(servers_cfg)))
-
- # Write server ids into datadir
- with open(os.path.join(data_path, "myid"), "w") as myid_file:
- myid_file.write(str(self.server_info.server_id))
-
- with open(log4j_path, "w") as log4j:
- log4j.write("""
-# DEFAULT: console appender only
-log4j.rootLogger=INFO, ROLLINGFILE
-log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
-log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
-log4j.appender.ROLLINGFILE.Threshold=DEBUG
-log4j.appender.ROLLINGFILE.File=""" + to_java_compatible_path(
- self.working_path + os.sep + "zookeeper.log\n"))
-
- self.process = subprocess.Popen(
- args=["java",
- "-cp", self.classpath,
- "-Dreadonlymode.enabled=true",
- "-Dzookeeper.log.dir=%s" % log_path,
- "-Dzookeeper.root.logger=INFO,CONSOLE",
- "-Dlog4j.configuration=file:%s" % log4j_path,
- # "-Dlog4j.debug",
- "org.apache.zookeeper.server.quorum.QuorumPeerMain",
- config_path])
- self._running = True
-
- @property
- def classpath(self):
- """Get the classpath necessary to run ZooKeeper."""
-
- if self._classpath:
- return self._classpath
-
- # Two possibilities, as seen in zkEnv.sh:
- # Check for a release - top-level zookeeper-*.jar?
- jars = glob((os.path.join(
- self.install_path, 'zookeeper-*.jar')))
- if jars:
- # Release build (`ant package`)
- jars.extend(glob(os.path.join(
- self.install_path,
- "lib/*.jar")))
- # support for different file locations on Debian/Ubuntu
- jars.extend(glob(os.path.join(
- self.install_path,
- "log4j-*.jar")))
- jars.extend(glob(os.path.join(
- self.install_path,
- "slf4j-api-*.jar")))
- jars.extend(glob(os.path.join(
- self.install_path,
- "slf4j-log4j-*.jar")))
- else:
- # Development build (plain `ant`)
- jars = glob((os.path.join(
- self.install_path, 'build/zookeeper-*.jar')))
- jars.extend(glob(os.path.join(
- self.install_path,
- "build/lib/*.jar")))
-
- return os.pathsep.join(jars)
-
- @property
- def address(self):
- """Get the address of the ZooKeeper instance."""
- return "%s:%s" % (self.host, self.client_port)
-
- @property
- def running(self):
- return self._running
-
- @property
- def client_port(self):
- return self.server_info.client_port
-
- def reset(self):
- """Stop the zookeeper instance, cleaning out its on disk-data."""
- self.stop()
- shutil.rmtree(os.path.join(self.working_path, "data"))
- os.mkdir(os.path.join(self.working_path, "data"))
- with open(os.path.join(self.working_path, "data", "myid"), "w") as fh:
- fh.write(str(self.server_info.server_id))
-
- def stop(self):
- """Stop the Zookeeper instance, retaining on disk state."""
- if not self.running:
- return
- self.process.terminate()
- self.process.wait()
- self._running = False
-
- def destroy(self):
- """Stop the ZooKeeper instance and destroy its on disk-state"""
- # called by at exit handler, reimport to avoid cleanup race.
- import shutil
- self.stop()
-
- shutil.rmtree(self.working_path)
-
-
-class ZookeeperCluster(object):
-
- def __init__(self, install_path=None, classpath=None, size=3, port_offset=20000):
- self._install_path = install_path
- self._classpath = classpath
- self._servers = []
-
- # Calculate ports and peer group
- port = port_offset
- peers = []
-
- for i in range(size):
- info = ServerInfo(i + 1, port, port + 1, port + 2)
- peers.append(info)
- port += 10
-
- # Instantiate Managed ZK Servers
- for i in range(size):
- server_peers = list(peers)
- server_info = server_peers.pop(i)
- self._servers.append(
- ManagedZooKeeper(
- self._install_path, server_info, server_peers, classpath=self._classpath))
-
- def __getitem__(self, k):
- return self._servers[k]
-
- def __iter__(self):
- return iter(self._servers)
-
- def start(self):
- # Zookeeper client expresses a preference for either lower ports or
- # lexicographical ordering of hosts, to ensure that all servers have a
- # chance to startup, start them in reverse order.
- for server in reversed(list(self)):
- server.run()
- # Giving the servers a moment to start, decreases the overall time
- # required for a client to successfully connect (2s vs. 4s without
- # the sleep).
- import time
- time.sleep(2)
-
- def stop(self):
- for server in self:
- server.stop()
- self._servers = []
-
- def terminate(self):
- for server in self:
- server.destroy()
-
- def reset(self):
- for server in self:
- server.reset()
diff --git a/slider-agent/src/main/python/kazoo/testing/harness.py b/slider-agent/src/main/python/kazoo/testing/harness.py
deleted file mode 100644
index 93cc744..0000000
--- a/slider-agent/src/main/python/kazoo/testing/harness.py
+++ /dev/null
@@ -1,181 +0,0 @@
-"""license: Apache License 2.0, see LICENSE for more details."""
-"""Kazoo testing harnesses"""
-import atexit
-import logging
-import os
-import uuid
-import threading
-import unittest
-
-from kazoo.client import KazooClient
-from kazoo.exceptions import NotEmptyError
-from kazoo.protocol.states import (
- KazooState
-)
-from kazoo.testing.common import ZookeeperCluster
-from kazoo.protocol.connection import _SESSION_EXPIRED
-
-log = logging.getLogger(__name__)
-
-CLUSTER = None
-
-
-def get_global_cluster():
- global CLUSTER
- if CLUSTER is None:
- ZK_HOME = os.environ.get("ZOOKEEPER_PATH")
- ZK_CLASSPATH = os.environ.get("ZOOKEEPER_CLASSPATH")
- ZK_PORT_OFFSET = int(os.environ.get("ZOOKEEPER_PORT_OFFSET", 20000))
-
- assert ZK_HOME or ZK_CLASSPATH, (
- "either ZOOKEEPER_PATH or ZOOKEEPER_CLASSPATH environment variable "
- "must be defined.\n"
- "For deb package installations this is /usr/share/java")
-
- CLUSTER = ZookeeperCluster(
- install_path=ZK_HOME,
- classpath=ZK_CLASSPATH,
- port_offset=ZK_PORT_OFFSET,
- )
- atexit.register(lambda cluster: cluster.terminate(), CLUSTER)
- return CLUSTER
-
-
-class KazooTestHarness(unittest.TestCase):
- """Harness for testing code that uses Kazoo
-
- This object can be used directly or as a mixin. It supports starting
- and stopping a complete ZooKeeper cluster locally and provides an
- API for simulating errors and expiring sessions.
-
- Example::
-
- class MyTestCase(KazooTestHarness):
- def setUp(self):
- self.setup_zookeeper()
-
- # additional test setup
-
- def tearDown(self):
- self.teardown_zookeeper()
-
- def test_something(self):
- something_that_needs_a_kazoo_client(self.client)
-
- def test_something_else(self):
- something_that_needs_zk_servers(self.servers)
-
- """
-
- def __init__(self, *args, **kw):
- super(KazooTestHarness, self).__init__(*args, **kw)
- self.client = None
- self._clients = []
-
- @property
- def cluster(self):
- return get_global_cluster()
-
- @property
- def servers(self):
- return ",".join([s.address for s in self.cluster])
-
- def _get_nonchroot_client(self):
- return KazooClient(self.servers)
-
- def _get_client(self, **kwargs):
- c = KazooClient(self.hosts, **kwargs)
- try:
- self._clients.append(c)
- except AttributeError:
- self._client = [c]
- return c
-
- def expire_session(self, client_id=None):
- """Force ZK to expire a client session
-
- :param client_id: id of client to expire. If unspecified, the id of
- self.client will be used.
-
- """
- client_id = client_id or self.client.client_id
-
- lost = threading.Event()
- safe = threading.Event()
-
- def watch_loss(state):
- if state == KazooState.LOST:
- lost.set()
- if lost.is_set() and state == KazooState.CONNECTED:
- safe.set()
- return True
-
- self.client.add_listener(watch_loss)
-
- self.client._call(_SESSION_EXPIRED, None)
-
- lost.wait(5)
- if not lost.isSet():
- raise Exception("Failed to get notified of session loss")
-
- # Wait for the reconnect now
- safe.wait(15)
- if not safe.isSet():
- raise Exception("Failed to see client reconnect")
- self.client.retry(self.client.get_async, '/')
-
- def setup_zookeeper(self, **client_options):
- """Create a ZK cluster and chrooted :class:`KazooClient`
-
- The cluster will only be created on the first invocation and won't be
- fully torn down until exit.
- """
- if not self.cluster[0].running:
- self.cluster.start()
- namespace = "/kazootests" + uuid.uuid4().hex
- self.hosts = self.servers + namespace
-
- if 'timeout' not in client_options:
- client_options['timeout'] = 0.8
- self.client = self._get_client(**client_options)
- self.client.start()
- self.client.ensure_path("/")
-
- def teardown_zookeeper(self):
- """Clean up any ZNodes created during the test
- """
- if not self.cluster[0].running:
- self.cluster.start()
-
- tries = 0
- if self.client and self.client.connected:
- while tries < 3:
- try:
- self.client.retry(self.client.delete, '/', recursive=True)
- break
- except NotEmptyError:
- pass
- tries += 1
- self.client.stop()
- self.client.close()
- del self.client
- else:
- client = self._get_client()
- client.start()
- client.retry(client.delete, '/', recursive=True)
- client.stop()
- client.close()
- del client
-
- for client in self._clients:
- client.stop()
- del client
- self._clients = None
-
-
-class KazooTestCase(KazooTestHarness):
- def setUp(self):
- self.setup_zookeeper()
-
- def tearDown(self):
- self.teardown_zookeeper()
diff --git a/slider-agent/src/main/python/kazoo/tests/test_client.py b/slider-agent/src/main/python/kazoo/tests/test_client.py
index eb19ef5..f851b63 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_client.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_client.py
@@ -72,7 +72,8 @@
def test_invalid_handler(self):
from kazoo.handlers.threading import SequentialThreadingHandler
- self.assertRaises(ConfigurationError,
+ self.assertRaises(
+ ConfigurationError,
self._makeOne, handler=SequentialThreadingHandler)
def test_chroot(self):
@@ -84,14 +85,14 @@
hosts='127.0.0.1:2181,127.0.0.1:2182/a/b').chroot, '/a/b')
def test_connection_timeout(self):
- from kazoo.handlers.threading import TimeoutError
+ from kazoo.handlers.threading import KazooTimeoutError
client = self._makeOne(hosts='127.0.0.1:9')
- self.assertTrue(client.handler.timeout_exception is TimeoutError)
- self.assertRaises(TimeoutError, client.start, 0.1)
+ self.assertTrue(client.handler.timeout_exception is KazooTimeoutError)
+ self.assertRaises(KazooTimeoutError, client.start, 0.1)
def test_ordered_host_selection(self):
client = self._makeOne(hosts='127.0.0.1:9,127.0.0.2:9/a',
- randomize_hosts=False)
+ randomize_hosts=False)
hosts = [h for h in client.hosts]
eq_(hosts, [('127.0.0.1', 9), ('127.0.0.2', 9)])
@@ -371,29 +372,29 @@
client = self.client
client.stop()
- write_pipe = client._connection._write_pipe
+ write_sock = client._connection._write_sock
- # close the connection to free the pipe
+ # close the connection to free the socket
client.close()
- eq_(client._connection._write_pipe, None)
+ eq_(client._connection._write_sock, None)
# sneak in and patch client to simulate race between a thread
# calling stop(); close() and one running a command
oldstate = client._state
client._state = KeeperState.CONNECTED
- client._connection._write_pipe = write_pipe
+ client._connection._write_sock = write_sock
try:
- # simulate call made after write pipe is closed
+ # simulate call made after write socket is closed
self.assertRaises(ConnectionClosedError, client.exists, '/')
- # simualte call made after write pipe is set to None
- client._connection._write_pipe = None
+ # simulate call made after write socket is set to None
+ client._connection._write_sock = None
self.assertRaises(ConnectionClosedError, client.exists, '/')
finally:
# reset for teardown
client._state = oldstate
- client._connection._write_pipe = None
+ client._connection._write_sock = None
class TestClient(KazooTestCase):
@@ -544,9 +545,9 @@
client = self.client
client.create("/1", b"ephemeral", ephemeral=True)
self.assertRaises(NoChildrenForEphemeralsError,
- client.create, "/1/2", b"val1")
+ client.create, "/1/2", b"val1")
self.assertRaises(NoChildrenForEphemeralsError,
- client.create, "/1/2", b"val1", ephemeral=True)
+ client.create, "/1/2", b"val1", ephemeral=True)
def test_create_sequence(self):
client = self.client
@@ -560,8 +561,8 @@
def test_create_ephemeral_sequence(self):
basepath = "/" + uuid.uuid4().hex
- realpath = self.client.create(basepath, b"sandwich", sequence=True,
- ephemeral=True)
+ realpath = self.client.create(basepath, b"sandwich",
+ sequence=True, ephemeral=True)
self.assertTrue(basepath != realpath and realpath.startswith(basepath))
data, stat = self.client.get(realpath)
eq_(data, b"sandwich")
@@ -575,33 +576,35 @@
data, stat = self.client.get("/1/2/3/4/5")
eq_(data, b"val2")
- self.assertRaises(NodeExistsError, self.client.create, "/1/2/3/4/5",
- b"val2", makepath=True)
+ self.assertRaises(NodeExistsError, self.client.create,
+ "/1/2/3/4/5", b"val2", makepath=True)
def test_create_makepath_incompatible_acls(self):
from kazoo.client import KazooClient
from kazoo.security import make_digest_acl_credential, CREATOR_ALL_ACL
credential = make_digest_acl_credential("username", "password")
- alt_client = KazooClient(self.cluster[0].address + self.client.chroot,
+ alt_client = KazooClient(
+ self.cluster[0].address + self.client.chroot,
max_retries=5, auth_data=[("digest", credential)])
alt_client.start()
alt_client.create("/1/2", b"val2", makepath=True, acl=CREATOR_ALL_ACL)
try:
- self.assertRaises(NoAuthError, self.client.create, "/1/2/3/4/5",
- b"val2", makepath=True)
+ self.assertRaises(NoAuthError, self.client.create,
+ "/1/2/3/4/5", b"val2", makepath=True)
finally:
alt_client.delete('/', recursive=True)
alt_client.stop()
def test_create_no_makepath(self):
- self.assertRaises(NoNodeError, self.client.create, "/1/2", b"val1")
- self.assertRaises(NoNodeError, self.client.create, "/1/2", b"val1",
- makepath=False)
+ self.assertRaises(NoNodeError, self.client.create,
+ "/1/2", b"val1")
+ self.assertRaises(NoNodeError, self.client.create,
+ "/1/2", b"val1", makepath=False)
self.client.create("/1/2", b"val1", makepath=True)
- self.assertRaises(NoNodeError, self.client.create, "/1/2/3/4", b"val1",
- makepath=False)
+ self.assertRaises(NoNodeError, self.client.create,
+ "/1/2/3/4", b"val1", makepath=False)
def test_create_exists(self):
from kazoo.exceptions import NodeExistsError
@@ -844,7 +847,7 @@
client = self.client
self.assertRaises(NoNodeError, client.get_children, '/none')
self.assertRaises(NoNodeError, client.get_children,
- '/none', include_data=True)
+ '/none', include_data=True)
def test_get_children_invalid_path(self):
client = self.client
@@ -855,7 +858,7 @@
self.assertRaises(TypeError, client.get_children, ('a', 'b'))
self.assertRaises(TypeError, client.get_children, 'a', watch=True)
self.assertRaises(TypeError, client.get_children,
- 'a', include_data='yes')
+ 'a', include_data='yes')
def test_invalid_auth(self):
from kazoo.exceptions import AuthFailedError
diff --git a/slider-agent/src/main/python/kazoo/tests/test_connection.py b/slider-agent/src/main/python/kazoo/tests/test_connection.py
index c764b03..c8c4581 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_connection.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_connection.py
@@ -1,5 +1,6 @@
"""license: Apache License 2.0, see LICENSE for more details."""
from collections import namedtuple
+import sys
import os
import errno
import threading
@@ -42,8 +43,9 @@
class TestConnectionHandler(KazooTestCase):
def test_bad_deserialization(self):
async_object = self.client.handler.async_result()
- self.client._queue.append((Delete(self.client.chroot, -1), async_object))
- os.write(self.client._connection._write_pipe, b'\0')
+ self.client._queue.append(
+ (Delete(self.client.chroot, -1), async_object))
+ self.client._connection._write_sock.send(b'\0')
@raises(ValueError)
def testit():
@@ -150,7 +152,7 @@
deserialize_ev = threading.Event()
- def bad_deserialize(bytes, offset):
+ def bad_deserialize(_bytes, offset):
deserialize_ev.set()
raise struct.error()
@@ -184,63 +186,51 @@
# should be able to restart
self.client.start()
- def test_connection_pipe(self):
+ def test_connection_sock(self):
client = self.client
- read_pipe = client._connection._read_pipe
- write_pipe = client._connection._write_pipe
+ read_sock = client._connection._read_sock
+ write_sock = client._connection._write_sock
- assert read_pipe is not None
- assert write_pipe is not None
+ assert read_sock is not None
+ assert write_sock is not None
- # stop client and pipe should not yet be closed
+ # stop client and socket should not yet be closed
client.stop()
- assert read_pipe is not None
- assert write_pipe is not None
- os.fstat(read_pipe)
- os.fstat(write_pipe)
+ assert read_sock is not None
+ assert write_sock is not None
+
+ read_sock.getsockname()
+ write_sock.getsockname()
- # close client, and pipes should be
+ # close client, and sockets should be closed
client.close()
- try:
- os.fstat(read_pipe)
- except OSError as e:
- if not e.errno == errno.EBADF:
- raise
- else:
- self.fail("Expected read_pipe to be closed")
-
- try:
- os.fstat(write_pipe)
- except OSError as e:
- if not e.errno == errno.EBADF:
- raise
- else:
- self.fail("Expected write_pipe to be closed")
-
- # start client back up. should get a new, valid pipe
+ # Todo check socket closing
+
+ # start client back up. should get a new, valid socket
client.start()
- read_pipe = client._connection._read_pipe
- write_pipe = client._connection._write_pipe
+ read_sock = client._connection._read_sock
+ write_sock = client._connection._write_sock
- assert read_pipe is not None
- assert write_pipe is not None
- os.fstat(read_pipe)
- os.fstat(write_pipe)
+ assert read_sock is not None
+ assert write_sock is not None
+ read_sock.getsockname()
+ write_sock.getsockname()
+
- def test_dirty_pipe(self):
+ def test_dirty_sock(self):
client = self.client
- read_pipe = client._connection._read_pipe
- write_pipe = client._connection._write_pipe
+ read_sock = client._connection._read_sock
+ write_sock = client._connection._write_sock
- # add a stray byte to the pipe and ensure that doesn't
+ # add a stray byte to the socket and ensure that doesn't
# blow up client. simulates case where some error leaves
- # a byte in the pipe which doesn't correspond to the
+ # a byte in the socket which doesn't correspond to the
# request queue.
- os.write(write_pipe, b'\0')
+ write_sock.send(b'\0')
- # eventually this byte should disappear from pipe
- wait(lambda: client.handler.select([read_pipe], [], [], 0)[0] == [])
+ # eventually this byte should disappear from socket
+ wait(lambda: client.handler.select([read_sock], [], [], 0)[0] == [])
class TestConnectionDrop(KazooTestCase):
diff --git a/slider-agent/src/main/python/resource_management/core/logger.py b/slider-agent/src/main/python/resource_management/core/logger.py
index b80042a..5d6e414 100644
--- a/slider-agent/src/main/python/resource_management/core/logger.py
+++ b/slider-agent/src/main/python/resource_management/core/logger.py
@@ -29,7 +29,15 @@
# unprotected_strings : protected_strings map
sensitive_strings = {}
-
+
+ @staticmethod
+ def error(text):
+ Logger.logger.error(Logger.get_protected_text(text))
+
+ @staticmethod
+ def warning(text):
+ Logger.logger.warning(Logger.get_protected_text(text))
+
@staticmethod
def info(text):
Logger.logger.info(Logger.get_protected_text(text))
@@ -39,6 +47,14 @@
Logger.logger.debug(Logger.get_protected_text(text))
@staticmethod
+ def error_resource(resource):
+ Logger.error(Logger.get_protected_text(Logger._get_resource_repr(resource)))
+
+ @staticmethod
+ def warning_resource(resource):
+ Logger.warning(Logger.get_protected_text(Logger._get_resource_repr(resource)))
+
+ @staticmethod
def info_resource(resource):
Logger.info(Logger.get_protected_text(Logger._get_resource_repr(resource)))
@@ -92,4 +108,4 @@
if arguments_str:
arguments_str = arguments_str[:-2]
- return "{0} {{{1}}}".format(resource, arguments_str)
\ No newline at end of file
+ return unicode("{0} {{{1}}}").format(resource, arguments_str)
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/core/providers/__init__.py b/slider-agent/src/main/python/resource_management/core/providers/__init__.py
index 630183b..8ebeb77 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/__init__.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/__init__.py
@@ -42,18 +42,20 @@
PROVIDERS = dict(
redhat=dict(
- Package="resource_management.core.providers.package.yumrpm.YumProvider",
+ Package="resource_management.core.providers.package.yumrpm.YumProvider"
),
suse=dict(
- Package="resource_management.core.providers.package.zypper.ZypperProvider",
+ Package="resource_management.core.providers.package.zypper.ZypperProvider"
),
debian=dict(
- Package="resource_management.core.providers.package.apt.AptProvider",
+ Package="resource_management.core.providers.package.apt.AptProvider"
),
winsrv=dict(
Service="resource_management.core.providers.windows.service.ServiceProvider",
Execute="resource_management.core.providers.windows.system.ExecuteProvider",
- File="resource_management.core.providers.windows.system.FileProvider"
+ File="resource_management.core.providers.windows.system.FileProvider",
+ Directory="resource_management.core.providers.windows.system.DirectoryProvider",
+ Tarball="resource_management.core.providers.windows.tarball.TarballProvider"
),
default=dict(
File="resource_management.core.providers.system.FileProvider",
@@ -65,21 +67,14 @@
User="resource_management.core.providers.accounts.UserProvider",
Group="resource_management.core.providers.accounts.GroupProvider",
Service="resource_management.core.providers.service.ServiceProvider",
- Tarball="resource_management.core.providers.tarball.TarballProvider",
- ),
+ Tarball="resource_management.core.providers.tarball.TarballProvider"
+ )
)
def find_provider(env, resource, class_path=None):
if not class_path:
- providers = [PROVIDERS, LIBRARY_PROVIDERS]
- for provider in providers:
- if resource in provider[env.system.os_family]:
- class_path = provider[env.system.os_family][resource]
- break
- if resource in provider["default"]:
- class_path = provider["default"][resource]
- break
+ class_path = get_class_path(env.system.os_family, resource)
try:
mod_path, class_name = class_path.rsplit('.', 1)
@@ -87,3 +82,13 @@
raise Fail("Unable to find provider for %s as %s" % (resource, class_path))
mod = __import__(mod_path, {}, {}, [class_name])
return getattr(mod, class_name)
+
+def get_class_path(os, resource):
+ providers = [PROVIDERS, LIBRARY_PROVIDERS]
+ for provider in providers:
+ if os in provider:
+ if resource in provider[os]:
+ return provider[os][resource]
+ if resource in provider["default"]:
+ return provider["default"][resource]
+ return None
diff --git a/slider-agent/src/main/python/resource_management/core/providers/system.py b/slider-agent/src/main/python/resource_management/core/providers/system.py
index 6969c62..6f56967 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/system.py
@@ -76,19 +76,21 @@
path, existing_mode, mode))
os.chmod(path, mode)
- if user:
- uid = _coerce_uid(user)
- if stat.st_uid != uid:
- Logger.info(
- "Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
- os.chown(path, uid, -1)
+ ## Slider apps should have no need to chown the uid or gid
+ ## Keeping the code around as a reference
+ ## if user:
+ ## uid = _coerce_uid(user)
+ ## if stat.st_uid != uid:
+ ## Logger.info(
+ ## "Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
+ ## os.chown(path, uid, -1)
- if group:
- gid = _coerce_gid(group)
- if stat.st_gid != gid:
- Logger.info(
- "Changing group for %s from %d to %s" % (path, stat.st_gid, group))
- os.chown(path, -1, gid)
+ ## if group:
+ ## gid = _coerce_gid(group)
+ ## if stat.st_gid != gid:
+ ## Logger.info(
+ ## "Changing group for %s from %d to %s" % (path, stat.st_gid, group))
+ ## os.chown(path, -1, gid)
class FileProvider(Provider):
@@ -166,6 +168,14 @@
_ensure_metadata(path, self.resource.owner, self.resource.group,
mode=self.resource.mode)
+ if self.resource.content and os.path.isdir(self.resource.content):
+ Logger.info("Copying from " + self.resource.content + " to " + path)
+ for item in os.listdir(self.resource.content):
+ src = os.path.join(self.resource.content, item)
+ dst = os.path.join(path, item)
+ if not os.path.isdir(src):
+ Logger.info("Copying " + src + " as " + dst)
+ shutil.copy2(src, dst)
def action_delete(self):
path = self.resource.path
@@ -243,7 +253,7 @@
cwd=self.resource.cwd, env=self.resource.environment,
preexec_fn=_preexec_fn(self.resource), user=self.resource.user,
wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout,
- pid_file=self.resource.pid_file)
+ pid_file=self.resource.pid_file, poll_after=self.resource.poll_after)
break
except Fail as ex:
if i == self.resource.tries-1: # last try
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
index f0d4825..6167977 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -28,41 +28,100 @@
import os
import subprocess
import shutil
+from resource_management.libraries.script import Script
-def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None, pid_file_name=None):
- # TODO implement logoutput
+def _merge_env(env1, env2, merge_keys=['PYTHONPATH']):
+ """
+ Merge env2 into env1. Also current python instance variables from merge_keys list taken into account and they will be
+ merged with equivalent keys from env1 and env2 using system path separator.
+ :param env1: first environment, usually returned by CreateEnvironmentBlock
+ :param env2: custom environment
+ :param merge_keys: env variables to merge as PATH
+ :return: merged environment
+ """
+ env1 = dict(env1) # copy to new dict in case env1 is os.environ
+ if env2:
+ for key, value in env2.iteritems():
+ if not key in merge_keys:
+ env1[key] = value
+ # strnsform keys and values to str(windows can not accept unicode)
+ result_env = {}
+ for key, value in env1.iteritems():
+ if not key in merge_keys:
+ result_env[str(key)] = str(value)
+ #merge keys from merge_keys
+ def put_values(key, env, result):
+ if env and key in env:
+ result.extend(env[key].split(os.pathsep))
+
+ for key in merge_keys:
+ all_values = []
+ for env in [env1, env2, os.environ]:
+ put_values(key, env, all_values)
+ result_env[str(key)] = str(os.pathsep.join(set(all_values)))
+ return result_env
+
+
+# Execute command. As windows stack heavily relies on proper environment it is better to reload fresh environment
+# on every execution. env variable will me merged with fresh environment for user.
+def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None, user=None,
+ pid_file_name=None, poll_after=None):
+ # TODO implement user
Logger.info("Executing %s" % (command))
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=cwd, env=env, shell=False)
+ code = None
if not wait_for_finish:
+ Logger.debug("No need to wait for the process to exit. Will leave the process running ...")
+ code = 0
+ logAnyway = False
if pid_file_name:
+ Logger.debug("Writing the process id %s to file %s" % (str(proc.pid), pid_file_name))
pidfile = open(pid_file_name, 'w')
pidfile.write(str(proc.pid))
pidfile.close()
- return None, None
+ Logger.info("Wrote the process id to file %s" % pid_file_name)
+
+ ## wait poll_after seconds and poll
+ if poll_after:
+ time.sleep(poll_after)
+ if proc.poll() is None:
+ return code, None, None # if still running then return
+ else:
+ logAnyway = True # assume failure and log
+ Logger.warning("Process is not up after the polling interval " + str(poll_after) + " seconds.")
+ else:
+ return code, None, None
if timeout:
q = Queue()
- t = threading.Timer( timeout, on_timeout, [proc, q] )
+ t = threading.Timer(timeout, on_timeout, [proc, q])
t.start()
- out = proc.communicate()[0].strip()
+ out, err = proc.communicate()
code = proc.returncode
- if logoutput and out:
- Logger.info(out)
- return code, out
+
+ if logoutput or logAnyway:
+ if out:
+ Logger.info("Out: " + str(out))
+ if err:
+ Logger.info("Err: " + str(err))
+ if code:
+ Logger.info("Ret Code: " + str(code))
+
+ return code, out, err
# see msdn Icacls doc for rights
def _set_file_acl(file, user, rights):
acls_modify_cmd = "icacls {0} /grant {1}:{2}".format(file, user, rights)
acls_remove_cmd = "icacls {0} /remove {1}".format(file, user)
- code, out = _call_command(acls_remove_cmd)
+ code, out, err = _call_command(acls_remove_cmd)
if code != 0:
raise Fail("Can not remove rights for path {0} and user {1}".format(file, user))
- code, out = _call_command(acls_modify_cmd)
+ code, out, err = _call_command(acls_modify_cmd)
if code != 0:
- raise Fail("Can not set rights {0} for path {1} and user {2}".format(file, user))
+ raise Fail("Can not set rights {0} for path {1} and user {2}".format(rights, file, user))
else:
return
@@ -137,12 +196,16 @@
for i in range(0, self.resource.tries):
try:
- _call_command(self.resource.command, logoutput=self.resource.logoutput,
- cwd=self.resource.cwd, env=self.resource.environment,
- wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout,
- pid_file_name=self.resource.pid_file)
+ code, _, _ = _call_command(self.resource.command, logoutput=self.resource.logoutput,
+ cwd=self.resource.cwd, env=self.resource.environment,
+ wait_for_finish=self.resource.wait_for_finish,
+ timeout=self.resource.timeout, user=self.resource.user,
+ pid_file_name=self.resource.pid_file, poll_after=self.resource.poll_after)
+ if code != 0 and not self.resource.ignore_failures:
+ raise Fail("Failed to execute " + self.resource.command)
break
except Fail as ex:
+ Logger.info("Error raised: %s" % str(ex))
if i == self.resource.tries - 1: # last try
raise ex
else:
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/tarball.py b/slider-agent/src/main/python/resource_management/core/providers/windows/tarball.py
new file mode 100644
index 0000000..35f195b
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/tarball.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from __future__ import with_statement
+
+from resource_management.core import shell
+from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
+import os
+import zipfile
+
+class TarballProvider(Provider):
+ def action_install(self):
+ package_name = self.resource.package_name
+ location = self.resource.location
+ if package_name.lower().endswith("zip"):
+ if not self._check_existence(package_name, location):
+ zf = zipfile.ZipFile(package_name)
+ path = location
+ for member in zf.infolist():
+ zf.extract(member, path)
+ else:
+ Logger.info("Unsupported archive %s" % (package_name,))
+
+ def _check_existence(self, name, location):
+ return False
+
diff --git a/slider-agent/src/main/python/resource_management/core/resources/system.py b/slider-agent/src/main/python/resource_management/core/resources/system.py
index a63d993..f751d52 100644
--- a/slider-agent/src/main/python/resource_management/core/resources/system.py
+++ b/slider-agent/src/main/python/resource_management/core/resources/system.py
@@ -45,6 +45,7 @@
mode = ResourceArgument()
owner = ResourceArgument()
group = ResourceArgument()
+ content = ResourceArgument()
recursive = BooleanArgument(default=False) # this work for 'create', 'delete' is anyway recursive
actions = Resource.actions + ["create", "delete"]
@@ -105,6 +106,7 @@
if wait_for_finish is True then optionally the caller can ask for the pid to be written
"""
pid_file = ResourceArgument()
+ poll_after = ResourceArgument() #seconds
class ExecuteScript(Resource):
diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py
index fb2c946..95d18fc 100644
--- a/slider-agent/src/main/python/resource_management/core/shell.py
+++ b/slider-agent/src/main/python/resource_management/core/shell.py
@@ -29,17 +29,18 @@
from exceptions import Fail
from exceptions import ExecuteTimeoutException
from resource_management.core.logger import Logger
+import time
def checked_call(command, logoutput=False,
- cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None):
- return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout, pid_file)
+ cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None, poll_after=None):
+ return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout, pid_file, poll_after)
def call(command, logoutput=False,
- cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None):
- return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout, pid_file)
+ cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None, poll_after=None):
+ return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout, pid_file, poll_after)
def _call(command, logoutput=False, throw_on_failure=True,
- cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file_name=None):
+ cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file_name=None, poll_after=None):
"""
Execute shell command
@@ -66,13 +67,25 @@
cwd=cwd, env=env, shell=False,
preexec_fn=preexec_fn)
+ logAnyway = False
if not wait_for_finish:
if pid_file_name:
pidfile = open(pid_file_name, 'w')
pidfile.write(str(proc.pid))
pidfile.close()
- return None, None
-
+
+ ## wait poll_after seconds and poll
+ if poll_after:
+ time.sleep(poll_after)
+ if proc.poll() is None:
+ return None, None #if still running then return
+ else:
+ logAnyway = True #assume failure and log
+ Logger.warning("Process is not up after the polling interval " + str(poll_after) + " seconds.")
+ else:
+ return None, None
+
+
if timeout:
q = Queue()
t = threading.Timer( timeout, on_timeout, [proc, q] )
@@ -89,7 +102,7 @@
code = proc.returncode
- if logoutput and out:
+ if (logoutput or logAnyway) and out:
Logger.info(out)
if throw_on_failure and code:
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
index e8ed9f0..8f10455 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
@@ -25,6 +25,11 @@
__all__ = ["check_process_status"]
import os
+import subprocess
+import platform
+
+IS_WINDOWS = platform.system() == "Windows"
+
def check_process_status(pid_file):
"""
@@ -36,22 +41,44 @@
@param pid_file: path to service pid file
"""
if not pid_file or not os.path.isfile(pid_file):
+ if not pid_file:
+ Logger.warning("pid_file is not valid")
+ else:
+ Logger.info("pid file does not exist {0}".format(pid_file))
raise ComponentIsNotRunning()
+
with open(pid_file, "r") as f:
try:
pid = int(f.read())
except:
Logger.debug("Pid file {0} does not exist".format(pid_file))
raise ComponentIsNotRunning()
- try:
- # Kill will not actually kill the process
- # From the doc:
- # If sig is 0, then no signal is sent, but error checking is still
- # performed; this can be used to check for the existence of a
- # process ID or process group ID.
- os.kill(pid, 0)
- except OSError:
- Logger.debug("Process with pid {0} is not running. Stale pid file"
- " at {1}".format(pid, pid_file))
- raise ComponentIsNotRunning()
+
+ if IS_WINDOWS:
+ not_running = True
+ try:
+ ps = subprocess.Popen(r'tasklist.exe /NH /FI "PID eq %d"' % (pid),
+ shell=True, stdout=subprocess.PIPE)
+ output = ps.stdout.read()
+ ps.stdout.close()
+ ps.wait()
+ not_running = str(pid) not in output
+ except OSError, e:
+ Logger.debug("Error {0}".format(str(e)))
+ Logger.info("Process with pid {0} is not running. Stale pid file"
+ " at {1}".format(pid, pid_file))
+ if not_running:
+ raise ComponentIsNotRunning()
+ else:
+ try:
+ # Kill will not actually kill the process
+ # From the doc:
+ # If sig is 0, then no signal is sent, but error checking is still
+ # performed; this can be used to check for the existence of a
+ # process ID or process group ID.
+ os.kill(pid, 0)
+ except OSError:
+ Logger.info("Process with pid {0} is not running. Stale pid file"
+ " at {1}".format(pid, pid_file))
+ raise ComponentIsNotRunning()
pass
diff --git a/slider-agent/src/main/python/resource_management/libraries/script/script.py b/slider-agent/src/main/python/resource_management/libraries/script/script.py
index 00b80b4..8c53fb6 100644
--- a/slider-agent/src/main/python/resource_management/libraries/script/script.py
+++ b/slider-agent/src/main/python/resource_management/libraries/script/script.py
@@ -170,7 +170,7 @@
for package in package_list:
name = package['name']
type = package['type']
- if type.lower() == "tarball":
+ if type.lower() == "tarball" or type.lower() == "zip":
if name.startswith(os.path.sep):
tarball = name
else:
diff --git a/slider-agent/src/main/python/setup.py b/slider-agent/src/main/python/setup.py
index 421b5f9..56969b6 100644
--- a/slider-agent/src/main/python/setup.py
+++ b/slider-agent/src/main/python/setup.py
@@ -17,7 +17,7 @@
setup(
name = "slider-agent",
- version = "0.31.0-incubating-SNAPSHOT",
+ version = "0.51.0-incubating-SNAPSHOT",
packages = ['agent'],
# metadata for upload to PyPI
author = "Apache Software Foundation",
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
index 8071ee8..48260a0 100644
--- a/slider-agent/src/test/python/agent/TestActionQueue.py
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -37,6 +37,9 @@
from CustomServiceOrchestrator import CustomServiceOrchestrator
from PythonExecutor import PythonExecutor
from CommandStatusDict import CommandStatusDict
+from AgentToggleLogger import AgentToggleLogger
+import platform
+IS_WINDOWS = platform.system() == "Windows"
class TestActionQueue(TestCase):
@@ -45,6 +48,7 @@
sys.stdout = out
# save original open() method for later use
self.original_open = open
+ self.agentToggleLogger = AgentToggleLogger("info")
def tearDown(self):
@@ -104,7 +108,7 @@
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
config = MagicMock()
- actionQueue = ActionQueue(config, dummy_controller)
+ actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
actionQueue.start()
time.sleep(3)
actionQueue.stop()
@@ -118,7 +122,7 @@
def test_process_command(self,
execute_command_mock, print_exc_mock):
dummy_controller = MagicMock()
- actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
+ actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
execution_command = {
'commandType': ActionQueue.EXECUTION_COMMAND,
}
@@ -167,7 +171,7 @@
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
- actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
+ actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
requestComponentStatus_mock.return_value = {'exitcode': 'dummy report'}
actionQueue.execute_status_command(self.status_command)
@@ -193,7 +197,7 @@
csoMocks[0].status_commands_stdout = None
csoMocks[0].status_commands_stderr = None
dummy_controller = MagicMock()
- actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
+ actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
runCommand_mock.return_value = {'configurations': {}}
actionQueue.execute_status_command(self.status_command_with_config)
@@ -212,7 +216,7 @@
def test_process_command2(self, execute_status_command_mock,
execute_command_mock, print_exc_mock):
dummy_controller = MagicMock()
- actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
+ actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
execution_command = {
'commandType': ActionQueue.EXECUTION_COMMAND,
}
@@ -294,7 +298,7 @@
resolve_script_path_mock.return_value = "abc.py"
dummy_controller = MagicMock()
- actionQueue = ActionQueue(config, dummy_controller)
+ actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
unfreeze_flag = threading.Event()
python_execution_result_dict = {
'stdout': 'out',
@@ -345,6 +349,19 @@
'taskId': 3,
'exitcode': 777,
'reportResult': True}
+ if IS_WINDOWS:
+ expected = {'status': 'IN_PROGRESS',
+ 'stderr': 'Read from {0}\\errors-3.txt'.format(tempdir),
+ 'stdout': 'Read from {0}\\output-3.txt'.format(tempdir),
+ 'structuredOut': '',
+ 'clusterName': u'cc',
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HBASE',
+ 'role': u'HBASE_MASTER',
+ 'actionId': '1-1',
+ 'taskId': 3,
+ 'exitcode': 777,
+ 'reportResult': True}
self.assertEqual(report['reports'][0], expected)
# Continue command execution
unfreeze_flag.set()
diff --git a/slider-agent/src/test/python/agent/TestAgentToggleLogger.py b/slider-agent/src/test/python/agent/TestAgentToggleLogger.py
new file mode 100644
index 0000000..fc07f30
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestAgentToggleLogger.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import unittest
+from AgentToggleLogger import AgentToggleLogger
+import logging
+
+class TestAgentToggleLogger(unittest.TestCase):
+ def setUp(self):
+ self.agentToggleLogger = AgentToggleLogger("info")
+
+ def test_adjustLogLevel(self):
+ from ActionQueue import ActionQueue
+ # log level is set to info here (during initialization)
+ # run an execution command first
+ self.agentToggleLogger.adjustLogLevelAtStart(ActionQueue.EXECUTION_COMMAND)
+ assert self.agentToggleLogger.logLevel == "info"
+ self.agentToggleLogger.adjustLogLevelAtEnd(ActionQueue.EXECUTION_COMMAND)
+ assert self.agentToggleLogger.logLevel == "info"
+
+ # run a status command now
+ self.agentToggleLogger.adjustLogLevelAtStart(ActionQueue.STATUS_COMMAND)
+ assert self.agentToggleLogger.logLevel == "info"
+ self.agentToggleLogger.adjustLogLevelAtEnd(ActionQueue.STATUS_COMMAND)
+ assert self.agentToggleLogger.logLevel == "debug"
+
+ # run a status command again
+ self.agentToggleLogger.adjustLogLevelAtStart(ActionQueue.STATUS_COMMAND)
+ assert self.agentToggleLogger.logLevel == "debug"
+ self.agentToggleLogger.adjustLogLevelAtEnd(ActionQueue.STATUS_COMMAND)
+ assert self.agentToggleLogger.logLevel == "debug"
+
+ # now an execution command shows up
+ self.agentToggleLogger.adjustLogLevelAtStart(ActionQueue.EXECUTION_COMMAND)
+ assert self.agentToggleLogger.logLevel == "info"
+ self.agentToggleLogger.adjustLogLevelAtEnd(ActionQueue.EXECUTION_COMMAND)
+ assert self.agentToggleLogger.logLevel == "info"
+
+
+if __name__ == "__main__":
+ logging.basicConfig(format='%(asctime)s %(message)s',level=logging.DEBUG)
+ unittest.main()
+
diff --git a/slider-agent/src/test/python/agent/TestController.py b/slider-agent/src/test/python/agent/TestController.py
index 401d69a..2d4a2bb 100644
--- a/slider-agent/src/test/python/agent/TestController.py
+++ b/slider-agent/src/test/python/agent/TestController.py
@@ -25,12 +25,14 @@
from agent import Controller, ActionQueue
from agent import hostname
import sys
+import time
from Controller import AGENT_AUTO_RESTART_EXIT_CODE
from Controller import State
from AgentConfig import AgentConfig
from mock.mock import patch, MagicMock, call, Mock
import logging
from threading import Event
+from AgentToggleLogger import AgentToggleLogger
class TestController(unittest.TestCase):
@@ -55,7 +57,8 @@
self.controller = Controller.Controller(config)
self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS = 0.1
self.controller.netutil.HEARTBEAT_NOT_IDDLE_INTERVAL_SEC = 0.1
- self.controller.actionQueue = ActionQueue.ActionQueue(config, self.controller)
+ self.agentToggleLogger = AgentToggleLogger("info")
+ self.controller.actionQueue = ActionQueue.ActionQueue(config, self.controller, self.agentToggleLogger)
@patch("json.dumps")
@@ -255,6 +258,68 @@
self.assertTrue(os_exit_mock.call_args[0][0] == AGENT_AUTO_RESTART_EXIT_CODE)
+ @patch("time.time")
+ def test_failure_window(self, mock_time):
+ config = AgentConfig("", "")
+ original_config = config.get(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART)
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '2,1')
+ ## The behavior of side_effect is different when you run tests in command line and when you do it through IDE
+ ## So few extra items are there in the list
+ mock_time.side_effect = [200, 500, 500]
+ controller5 = Controller.Controller(config)
+
+ try:
+ self.assertTrue(controller5.shouldAutoRestart())
+ self.assertTrue(controller5.shouldAutoRestart())
+ finally:
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, original_config)
+
+
+ @patch("time.time")
+ def test_failure_window(self, mock_time):
+ config = AgentConfig("", "")
+ original_config = config.get(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART)
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '3,1')
+ ## The behavior of side_effect is different when you run tests in command line and when you do it through IDE
+ ## So few extra items are there in the list
+ mock_time.side_effect = [200, 210, 220, 230, 240, 250]
+ controller5 = Controller.Controller(config)
+
+ try:
+ self.assertTrue(controller5.shouldAutoRestart())
+ self.assertTrue(controller5.shouldAutoRestart())
+ self.assertTrue(controller5.shouldAutoRestart())
+ self.assertFalse(controller5.shouldAutoRestart())
+ finally:
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, original_config)
+
+
+ def test_failure_window2(self):
+ config = MagicMock()
+ config.getErrorWindow.return_value = (0, 0)
+ controller = Controller.Controller(config)
+
+ self.assertTrue(controller.shouldAutoRestart())
+
+ config.getErrorWindow.return_value = (0, 1)
+ self.assertTrue(controller.shouldAutoRestart())
+
+ config.getErrorWindow.return_value = (1, 0)
+ self.assertTrue(controller.shouldAutoRestart())
+
+ config.getErrorWindow.return_value = (-1, -1)
+ self.assertTrue(controller.shouldAutoRestart())
+
+ config.getErrorWindow.return_value = (1, 1)
+ self.assertTrue(controller.shouldAutoRestart())
+
+ #second failure within a minute
+ self.assertFalse(controller.shouldAutoRestart())
+
+ #do not reset unless window expires
+ self.assertFalse(controller.shouldAutoRestart())
+
+
@patch("urllib2.urlopen")
def test_sendRequest(self, requestMock):
@@ -325,7 +390,7 @@
self.assertEqual(1, self.controller.DEBUG_SUCCESSFULL_HEARTBEATS)
# retry registration
- response["registrationCommand"] = "true"
+ response["registrationCommand"] = {"command": "register"}
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
@@ -333,7 +398,7 @@
self.assertTrue(self.controller.repeatRegistration)
# components are not mapped
- response["registrationCommand"] = "false"
+ response["registrationCommand"] = {"command": "register"}
response["hasMappedComponents"] = False
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
@@ -358,7 +423,7 @@
self.assertTrue(self.controller.hasMappedComponents)
# wrong responseId => restart
- response = {"responseId":"2", "restartAgent":"false"}
+ response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
restartAgent = MagicMock(name="restartAgent")
@@ -384,7 +449,7 @@
# just status command when state = STARTED
self.controller.responseId = 1
- response = {"responseId":"2", "restartAgent":"false"}
+ response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
@@ -398,7 +463,7 @@
# just status command when state = FAILED
self.controller.responseId = 1
- response = {"responseId":"2", "restartAgent":"false"}
+ response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
@@ -412,7 +477,7 @@
# no status command when state = STARTING
self.controller.responseId = 1
- response = {"responseId":"2", "restartAgent":"false"}
+ response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
@@ -434,7 +499,7 @@
# restartAgent command
self.controller.responseId = 1
self.controller.DEBUG_STOP_HEARTBEATING = False
- response["restartAgent"] = "true"
+ response["restartAgent"] = True
restartAgent = MagicMock(name="restartAgent")
self.controller.restartAgent = restartAgent
self.controller.heartbeatWithServer()
@@ -445,7 +510,7 @@
self.controller.responseId = 1
self.controller.DEBUG_STOP_HEARTBEATING = False
actionQueue.isIdle.return_value = False
- response["restartAgent"] = "false"
+ response["restartAgent"] = False
self.controller.heartbeatWithServer()
sleepMock.assert_called_with(
@@ -453,11 +518,138 @@
sys.stdout = sys.__stdout__
self.controller.sendRequest = Controller.Controller.sendRequest
- self.controller.sendRequest = Controller.Controller.addToQueue
+ self.controller.addToQueue = Controller.Controller.addToQueue
self.controller.config = original_value
pass
+ @patch.object(threading._Event, "wait")
+ @patch("time.sleep")
+ @patch("json.loads")
+ @patch("json.dumps")
+ def test_heartbeatWithServerTerminateAgent(self, dumpsMock, loadsMock, sleepMock, event_mock):
+ original_value = self.controller.config
+ self.controller.config = AgentConfig("", "")
+ out = StringIO.StringIO()
+ sys.stdout = out
+
+ hearbeat = MagicMock()
+ self.controller.heartbeat = hearbeat
+
+ dumpsMock.return_value = "data"
+
+ sendRequest = MagicMock(name="sendRequest")
+ self.controller.sendRequest = sendRequest
+
+ self.controller.responseId = 1
+ response = {"responseId":"2", "restartAgent": False}
+ loadsMock.return_value = response
+
+ def one_heartbeat(*args, **kwargs):
+ self.controller.DEBUG_STOP_HEARTBEATING = True
+ return "data"
+
+ sendRequest.side_effect = one_heartbeat
+
+ actionQueue = MagicMock()
+ actionQueue.isIdle.return_value = True
+
+ # one successful request, after stop
+ self.controller.actionQueue = actionQueue
+ self.controller.heartbeatWithServer()
+ self.assertTrue(sendRequest.called)
+
+ calls = []
+ def retry(*args, **kwargs):
+ if len(calls) == 0:
+ calls.append(1)
+ response["responseId"] = "3"
+ raise Exception()
+ if len(calls) > 0:
+ self.controller.DEBUG_STOP_HEARTBEATING = True
+ return "data"
+
+ # exception, retry, successful and stop
+ sendRequest.side_effect = retry
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ self.assertEqual(1, self.controller.DEBUG_SUCCESSFULL_HEARTBEATS)
+
+ original_stopApp = self.controller.stopApp
+
+ # terminateAgent command - test 1
+ self.controller.responseId = 1
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ response = {"responseId":"2", "terminateAgent": True}
+ loadsMock.return_value = response
+ stopApp = MagicMock(name="stopApp")
+ self.controller.stopApp = stopApp
+ self.controller.heartbeatWithServer()
+ stopApp.assert_called_once_with()
+
+ # reset for next test
+ self.controller.terminateAgent = False
+
+ # terminateAgent command - test 2
+ self.controller.responseId = 1
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ response = {"responseId":"2", "terminateAgent": True}
+ loadsMock.return_value = response
+ self.controller.stopApp = original_stopApp
+ stopCommand = {"roleCommand": "STOP"}
+ self.controller.stopCommand = stopCommand
+ addToQueue = MagicMock(name="addToQueue")
+ self.controller.addToQueue = addToQueue
+ self.controller.componentActualState = State.STARTED
+ self.controller.heartbeatWithServer()
+ self.assertTrue(self.controller.terminateAgent)
+ self.assertTrue(self.controller.appGracefulStopQueued)
+ addToQueue.assert_has_calls([call([stopCommand])])
+
+ # reset for next test
+ self.controller.terminateAgent = False
+ self.controller.appGracefulStopQueued = False
+
+ # terminateAgent command - test 3
+ self.controller.responseId = 1
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ # set stopCommand to None and let it get set by updateStateBasedOnCommand
+ self.controller.stopCommand = None
+ # in this heartbeat don't send terminateAgent signal
+ response = {"responseId":"2", "terminateAgent": False}
+ stopCommand = {"roleCommand": "STOP"}
+ response["executionCommands"] = [stopCommand]
+ loadsMock.return_value = response
+ # terminateAgent is False - make STOP in commands set the stopCommand
+ self.controller.heartbeatWithServer()
+ self.assertFalse(self.controller.terminateAgent)
+ assert not self.controller.stopCommand == None
+
+ # now no need to have STOP command in response, just send terminateAgent
+ self.controller.responseId = 2
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ response = {"responseId":"3", "terminateAgent": True}
+ loadsMock.return_value = response
+ addToQueue = MagicMock(name="addToQueue")
+ self.controller.addToQueue = addToQueue
+ self.controller.stopApp = original_stopApp
+ self.controller.componentActualState = State.STARTED
+ self.controller.heartbeatWithServer()
+ self.assertTrue(self.controller.terminateAgent)
+ self.assertTrue(self.controller.appGracefulStopQueued)
+ addToQueue.assert_has_calls([call([stopCommand])])
+ self.controller.terminateAgent = False
+
+ sleepMock.assert_called_with(
+ self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS)
+
+ sys.stdout = sys.__stdout__
+ self.controller.sendRequest = Controller.Controller.sendRequest
+ self.controller.addToQueue = Controller.Controller.addToQueue
+
+ self.controller.config = original_value
+ pass
@patch.object(Controller.Controller, "createStatusCommand")
def test_updateStateBasedOnResult(self, mock_createStatusCommand):
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index e545afe..30c8d7a 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -35,7 +35,9 @@
import StringIO
import sys
from socket import socket
-
+from AgentToggleLogger import AgentToggleLogger
+import platform
+IS_WINDOWS = platform.system() == "Windows"
class TestCustomServiceOrchestrator(TestCase):
@@ -45,16 +47,17 @@
sys.stdout = out
# generate sample config
tmpdir = tempfile.gettempdir()
+ self.agentToggleLogger = AgentToggleLogger("info")
@patch("hostname.public_hostname")
@patch("os.path.isfile")
- @patch("os.unlink")
- def test_dump_command_to_json(self, unlink_mock,
+ def test_dump_command_to_json(self,
isfile_mock, hostname_mock):
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
+ 'hostname' : 'host1',
'componentName': 'NAMENODE',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
@@ -78,26 +81,40 @@
config.getLogPath.return_value = tempdir
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
isfile_mock.return_value = True
+ self.assertEquals(command['hostname'], "host1")
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command, {})
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
- self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ if IS_WINDOWS:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0666')
+ else:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0644')
self.assertTrue(json_file.endswith("command-3.json"))
os.unlink(json_file)
+
+ # Testing side effect of dump_command_to_json
+ self.assertEquals(command['public_hostname'], "test.hst")
+ self.assertEquals(command['hostname'], "test.hst")
+ self.assertEquals(command['appmaster_hostname'], "host1")
+
# Test dumping STATUS_COMMAND
command['commandType'] = 'STATUS_COMMAND'
json_file = orchestrator.dump_command_to_json(command, {})
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
- self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ if IS_WINDOWS:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0666')
+ else:
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0644')
self.assertTrue(json_file.endswith("status_command.json"))
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
- self.assertTrue(unlink_mock.called)
+ self.assertEquals(command['hostname'], "test.hst")
+ self.assertEquals(command['appmaster_hostname'], "test.hst")
@patch.object(CustomServiceOrchestrator, "resolve_script_path")
@@ -132,7 +149,7 @@
resolve_script_path_mock.return_value = "/basedir/scriptpath"
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
# normal run case
run_file_mock.return_value = {
'stdout': 'sss',
@@ -218,7 +235,7 @@
resolve_script_path_mock.return_value = "/basedir/scriptpath"
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
# normal run case
run_file_mock.return_value = {
'stdout': 'sss',
@@ -247,7 +264,7 @@
config.getLogPath.return_value = tempdir
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
ret = orchestrator.allocate_port(10)
self.assertEqual(ret, 10)
@@ -265,7 +282,7 @@
config.getLogPath.return_value = tempdir
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
socket_getsockname_mock.return_value = [100, 101]
ret = orchestrator.allocate_port(10)
self.assertEqual(ret, 101)
@@ -281,7 +298,7 @@
config.getLogPath.return_value = tempdir
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
socket_getsockname_mock.return_value = [100, 102]
ret = orchestrator.allocate_port()
self.assertEqual(ret, 102)
@@ -298,7 +315,7 @@
config.getLogPath.return_value = tempdir
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
is_port_available_mock.return_value = False
allocate_port_mock.side_effect = [101, 102, 103, 104, 105, 106]
@@ -310,9 +327,9 @@
self.assertEqual(ret, "102,103")
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_0}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "104")
- ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_0}{DO_NOT_PROPAGATE}", "${A.ALLOCATED_PORT}")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_0}{PER_CONTAINER}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "105")
- ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DO_NOT_PROPAGATE}", "${A.ALLOCATED_PORT}")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{PER_CONTAINER}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "106")
@@ -326,7 +343,7 @@
config.getLogPath.return_value = tempdir
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
is_port_available_mock.return_value = True
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1005}", "${A.ALLOCATED_PORT}")
@@ -336,7 +353,7 @@
"${A.ALLOCATED_PORT}")
self.assertEqual(ret, "1005-1006")
- ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1006}{DO_NOT_PROPAGATE}",
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1006}{PER_CONTAINER}",
"${A.ALLOCATED_PORT}")
self.assertEqual(ret, "1006")
@@ -399,7 +416,7 @@
resolve_script_path_mock.return_value = "/basedir/scriptpath"
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
# normal run case
run_file_mock.return_value = {
'stdout': 'sss',
@@ -449,7 +466,7 @@
config.getWorkRootPath.return_value = tempdir
config.getLogPath.return_value = tempdir
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
# Test alive case
runCommand_mock.return_value = {
"exitcode": 0
@@ -476,7 +493,7 @@
config.getLogPath.return_value = tempdir
mock_allocate_ports.return_value = "10023"
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
command = {}
command['componentName'] = "HBASE_MASTER"
command['configurations'] = {}
@@ -489,8 +506,8 @@
command['configurations']['oozie-site']['log_root'] = "${AGENT_LOG_ROOT}"
command['configurations']['oozie-site']['a_port'] = "${HBASE_MASTER.ALLOCATED_PORT}"
command['configurations']['oozie-site']['ignore_port1'] = "[${HBASE_RS.ALLOCATED_PORT}]"
- command['configurations']['oozie-site']['ignore_port2'] = "[${HBASE_RS.ALLOCATED_PORT},${HBASE_REST.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]"
- command['configurations']['oozie-site']['ignore_port3'] = "[${HBASE_RS.ALLOCATED_PORT}{a}{b}{c},${A.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${A.ALLOCATED_PORT}{DEFAULT_3}{DO_NOT_PROPAGATE}]"
+ command['configurations']['oozie-site']['ignore_port2'] = "[${HBASE_RS.ALLOCATED_PORT},${HBASE_REST.ALLOCATED_PORT}{PER_CONTAINER}]"
+ command['configurations']['oozie-site']['ignore_port3'] = "[${HBASE_RS.ALLOCATED_PORT}{a}{b}{c},${A.ALLOCATED_PORT}{PER_CONTAINER},${A.ALLOCATED_PORT}{DEFAULT_3}{PER_CONTAINER}]"
command['configurations']['oozie-site']['ignore_port4'] = "${HBASE_RS}{a}{b}{c}"
allocated_ports = {}
@@ -530,11 +547,39 @@
config.getWorkRootPath.return_value = tempWorkDir
config.getLogPath.return_value = tempdir
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
port = orchestrator.allocate_port()
self.assertFalse(port == -1)
self.assertTrue(port > 0)
+
+ def test_parse_allowed_port_values(self):
+ dummy_controller = MagicMock()
+ tempdir = tempfile.gettempdir()
+ tempWorkDir = tempdir + "W"
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempWorkDir
+ config.getLogPath.return_value = tempdir
+
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
+ port_range = "48000-48005"
+ port_range_full_list = [48000, 48001, 48002, 48003, 48004, 48005]
+ allowed_ports = orchestrator.get_allowed_port_list(port_range, 3)
+ self.assertTrue(set(allowed_ports).issubset(port_range_full_list))
+
+ port_range = "48000 , 48005"
+ port_range_full_list = [48000, 48005]
+ allowed_ports = orchestrator.get_allowed_port_list(port_range, 1)
+ self.assertTrue(set(allowed_ports).issubset(port_range_full_list))
+
+ port_range = "48000 , 48004-48005"
+ port_range_full_list = [48000, 48004, 48005]
+ allowed_ports = orchestrator.get_allowed_port_list(port_range, 2)
+ self.assertTrue(set(allowed_ports).issubset(port_range_full_list))
+
+
def tearDown(self):
# enable stdout
sys.stdout = sys.__stdout__
diff --git a/slider-agent/src/test/python/agent/TestGrep.py b/slider-agent/src/test/python/agent/TestGrep.py
index 351befb..c11a6e4 100644
--- a/slider-agent/src/test/python/agent/TestGrep.py
+++ b/slider-agent/src/test/python/agent/TestGrep.py
@@ -23,6 +23,8 @@
import socket
import os, sys
import logging
+import platform
+IS_WINDOWS = platform.system() == "Windows"
class TestGrep(TestCase):
@@ -32,8 +34,12 @@
grep = Grep()
def setUp(self):
- self.string_good = open('agent' + os.sep + 'dummy_output_good.txt', 'r').read().replace("\n", os.linesep)
- self.string_bad = open('agent' + os.sep + 'dummy_output_error.txt', 'r').read().replace("\n", os.linesep)
+ if IS_WINDOWS:
+ self.string_good = open('agent' + os.sep + 'dummy_output_good.txt', 'r').read().replace("\r\n", os.linesep)
+ self.string_bad = open('agent' + os.sep + 'dummy_output_error.txt', 'r').read().replace("\r\n", os.linesep)
+ else:
+ self.string_good = open('agent' + os.sep + 'dummy_output_good.txt', 'r').read().replace("\n", os.linesep)
+ self.string_bad = open('agent' + os.sep + 'dummy_output_error.txt', 'r').read().replace("\n", os.linesep)
pass
def test_grep_many_lines(self):
@@ -43,6 +49,7 @@
def test_grep_few_lines(self):
+ self.assertEqual.__self__.maxDiff = None
fragment = self.grep.grep(self.string_bad, "Err", 3, 3)
desired = """
debug: /Schedule[never]: Skipping device resources because running on a host
@@ -77,6 +84,7 @@
self.assertEquals(fragment, desired, "Grep tail function should return all lines if there are less lines than n")
def test_tail_few_lines(self):
+ self.assertEqual.__self__.maxDiff = None
fragment = self.grep.tail(self.string_good, 3)
desired = """
debug: Finishing transaction 70060456663980
@@ -106,6 +114,7 @@
pass
def test_cleanByTemplate(self):
+ self.assertEqual.__self__.maxDiff = None
fragment = self.grep.cleanByTemplate(self.string_bad, "debug")
desired = """
info: Applying configuration version '1352127563'
diff --git a/slider-agent/src/test/python/agent/TestHeartbeat.py b/slider-agent/src/test/python/agent/TestHeartbeat.py
index b012218..9898444 100644
--- a/slider-agent/src/test/python/agent/TestHeartbeat.py
+++ b/slider-agent/src/test/python/agent/TestHeartbeat.py
@@ -31,12 +31,15 @@
import sys
import logging
from Controller import State
+from AgentToggleLogger import AgentToggleLogger
+
class TestHeartbeat(TestCase):
def setUp(self):
# disable stdout
out = StringIO.StringIO()
sys.stdout = out
+ self.agentToggleLogger = AgentToggleLogger("info")
def tearDown(self):
@@ -48,8 +51,8 @@
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
- actionQueue = ActionQueue(config, dummy_controller)
- heartbeat = Heartbeat(actionQueue, config)
+ actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
+ heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
result = heartbeat.build({}, 100)
print "Heartbeat: " + str(result)
self.assertEquals(result['hostname'] != '', True,
@@ -74,7 +77,7 @@
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
- actionQueue = ActionQueue(config, dummy_controller)
+ actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
result_mock.return_value = {
'reports': [{'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
@@ -144,7 +147,7 @@
{'status': 'UNHEALTHY', 'componentName': 'HBASE_MASTER', 'reportResult' : False},
],
}
- heartbeat = Heartbeat(actionQueue, config)
+ heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
# State.STARTED results in agentState to be set to 4 (enum order)
hb = heartbeat.build({}, 10)
hb['hostname'] = 'hostname'
@@ -182,7 +185,7 @@
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
- actionQueue = ActionQueue(config, dummy_controller)
+ actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
result_mock.return_value = {
'reports': [{'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
@@ -198,7 +201,7 @@
],
'componentStatus': []
}
- heartbeat = Heartbeat(actionQueue, config)
+ heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
commandResult = {}
hb = heartbeat.build(commandResult, 10)
@@ -219,7 +222,7 @@
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
- actionQueue = ActionQueue(config, dummy_controller)
+ actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
result_mock.return_value = {
'reports': [{'status': 'COMPLETED',
'stderr': 'Read from /tmp/errors-3.txt',
@@ -235,7 +238,7 @@
],
'componentStatus': []
}
- heartbeat = Heartbeat(actionQueue, config)
+ heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
commandResult = {}
hb = heartbeat.build(commandResult, 10)
diff --git a/slider-agent/src/test/python/agent/TestMain.py b/slider-agent/src/test/python/agent/TestMain.py
index bc68582..537ab1f 100644
--- a/slider-agent/src/test/python/agent/TestMain.py
+++ b/slider-agent/src/test/python/agent/TestMain.py
@@ -31,7 +31,11 @@
import os
import tempfile
from Controller import Controller
+from Registry import Registry
from optparse import OptionParser
+import platform
+
+IS_WINDOWS = platform.system() == "Windows"
logger = logging.getLogger()
@@ -46,7 +50,6 @@
# enable stdout
sys.stdout = sys.__stdout__
-
@patch("os._exit")
@patch("os.getpid")
@patch.object(ProcessHelper, "stopAgent")
@@ -116,15 +119,15 @@
main.update_log_level(config, tmpoutfile)
setLevel_mock.assert_called_with(logging.INFO)
-
- @patch("signal.signal")
- def test_bind_signal_handlers(self, signal_mock):
- main.bind_signal_handlers()
- # Check if on SIGINT/SIGTERM agent is configured to terminate
- signal_mock.assert_any_call(signal.SIGINT, main.signal_handler)
- signal_mock.assert_any_call(signal.SIGTERM, main.signal_handler)
- # Check if on SIGUSR1 agent is configured to fall into debug
- signal_mock.assert_any_call(signal.SIGUSR1, main.debug)
+ if not IS_WINDOWS:
+ @patch("signal.signal")
+ def test_bind_signal_handlers(self, signal_mock):
+ main.bind_signal_handlers()
+ # Check if on SIGINT/SIGTERM agent is configured to terminate
+ signal_mock.assert_any_call(signal.SIGINT, main.signal_handler)
+ signal_mock.assert_any_call(signal.SIGTERM, main.signal_handler)
+ # Check if on SIGUSR1 agent is configured to fall into debug
+ signal_mock.assert_any_call(signal.SIGUSR1, main.debug)
@patch("os.path.exists")
@@ -180,45 +183,45 @@
main.perform_prestart_checks(main.config)
self.assertFalse(exit_mock.called)
-
- @patch("time.sleep")
- @patch("os.kill")
- @patch("os._exit")
- @patch("os.path.exists")
- def test_daemonize_and_stop(self, exists_mock, _exit_mock, kill_mock,
+ if not IS_WINDOWS:
+ @patch("time.sleep")
+ @patch("os.kill")
+ @patch("os._exit")
+ @patch("os.path.exists")
+ def test_daemonize_and_stop(self, exists_mock, _exit_mock, kill_mock,
sleep_mock):
- oldpid = ProcessHelper.pidfile
- pid = str(os.getpid())
- _, tmpoutfile = tempfile.mkstemp()
- ProcessHelper.pidfile = tmpoutfile
+ oldpid = ProcessHelper.pidfile
+ pid = str(os.getpid())
+ _, tmpoutfile = tempfile.mkstemp()
+ ProcessHelper.pidfile = tmpoutfile
- # Test daemonization
- main.write_pid()
- saved = open(ProcessHelper.pidfile, 'r').read()
- self.assertEqual(pid, saved)
+ # Test daemonization
+ main.write_pid()
+ saved = open(ProcessHelper.pidfile, 'r').read()
+ self.assertEqual(pid, saved)
- # Reuse pid file when testing agent stop
- # Testing normal exit
- exists_mock.return_value = False
- main.stop_agent()
- kill_mock.assert_called_with(int(pid), signal.SIGTERM)
+ # Reuse pid file when testing agent stop
+ # Testing normal exit
+ exists_mock.return_value = False
+ main.stop_agent()
+ kill_mock.assert_called_with(int(pid), signal.SIGTERM)
- # Restore
- kill_mock.reset_mock()
- _exit_mock.reset_mock()
+ # Restore
+ kill_mock.reset_mock()
+ _exit_mock.reset_mock()
- # Testing exit when failed to remove pid file
- exists_mock.return_value = True
- main.stop_agent()
- kill_mock.assert_any_call(int(pid), signal.SIGTERM)
- kill_mock.assert_any_call(int(pid), signal.SIGKILL)
- _exit_mock.assert_called_with(1)
+ # Testing exit when failed to remove pid file
+ exists_mock.return_value = True
+ main.stop_agent()
+ kill_mock.assert_any_call(int(pid), signal.SIGTERM)
+ kill_mock.assert_any_call(int(pid), signal.SIGKILL)
+ _exit_mock.assert_called_with(1)
- # Restore
- ProcessHelper.pidfile = oldpid
- os.remove(tmpoutfile)
+ # Restore
+ ProcessHelper.pidfile = oldpid
+ os.remove(tmpoutfile)
-
+ @patch.object(Registry, "readAMHostPort")
@patch.object(main, "setup_logging")
@patch.object(main, "bind_signal_handlers")
@patch.object(main, "update_config_from_file")
@@ -236,27 +239,31 @@
update_log_level_mock, write_pid_mock,
perform_prestart_checks_mock,
update_config_from_file_mock,
- bind_signal_handlers_mock, setup_logging_mock):
+ bind_signal_handlers_mock, setup_logging_mock,
+ readAMHostPort_mock):
Controller_init_mock.return_value = None
isAlive_mock.return_value = False
options = MagicMock()
parse_args_mock.return_value = (options, MagicMock)
+ readAMHostPort_mock.return_value = ("host1", 101, 100)
tmpdir = tempfile.gettempdir()
#testing call without command-line arguments
os.environ["AGENT_WORK_ROOT"] = os.path.join(tmpdir, "work")
os.environ["AGENT_LOG_ROOT"] = ",".join([os.path.join(tmpdir, "log"),os.path.join(tmpdir, "log2")])
+ try_to_connect_mock.return_value = 1
main.main()
self.assertTrue(setup_logging_mock.called)
- self.assertTrue(bind_signal_handlers_mock.called)
+ if not IS_WINDOWS:
+ self.assertTrue(bind_signal_handlers_mock.called)
self.assertTrue(update_config_from_file_mock.called)
self.assertTrue(perform_prestart_checks_mock.called)
self.assertTrue(write_pid_mock.called)
self.assertTrue(update_log_level_mock.called)
self.assertTrue(options.log_folder == os.path.join(tmpdir, "log"))
- try_to_connect_mock.assert_called_once_with(ANY, -1, ANY)
+ try_to_connect_mock.assert_called_once_with('https://host1:101/ws/v1/slider/agents/', 3, ANY)
self.assertTrue(start_mock.called)
class AgentOptions:
@@ -267,6 +274,8 @@
self.verbose = verbose
self.debug = debug
+ @patch.object(Registry, "readAMHostPort")
+ @patch("time.sleep")
@patch.object(main, "setup_logging")
@patch.object(main, "bind_signal_handlers")
@patch.object(main, "stop_agent")
@@ -287,22 +296,63 @@
update_log_level_mock, write_pid_mock,
perform_prestart_checks_mock,
update_config_from_file_mock, stop_mock,
- bind_signal_handlers_mock, setup_logging_mock):
+ bind_signal_handlers_mock, setup_logging_mock,
+ time_sleep_mock, readAMHostPort_mock):
Controller_init_mock.return_value = None
isAlive_mock.return_value = False
parse_args_mock.return_value = (
TestMain.AgentOptions("agent", "host1:2181", "/registry/org-apache-slider/cl1", True, ""), [])
tmpdir = tempfile.gettempdir()
+ time_sleep_mock.return_value = None
+ readAMHostPort_mock.return_value = (None, None, None)
#testing call without command-line arguments
os.environ["AGENT_WORK_ROOT"] = os.path.join(tmpdir, "work")
os.environ["AGENT_LOG_ROOT"] = os.path.join(tmpdir, "log")
main.main()
self.assertTrue(AgentConfig_set_mock.call_count == 3)
+ self.assertTrue(readAMHostPort_mock.call_count == 10)
AgentConfig_set_mock.assert_any_call("server", "zk_quorum", "host1:2181")
AgentConfig_set_mock.assert_any_call("server", "zk_reg_path", "/registry/org-apache-slider/cl1")
+ def test_config1(self):
+ config = AgentConfig("", "")
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 5)
+ self.assertEqual(window, 5)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 0)
+ self.assertEqual(window, 0)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '33')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 0)
+ self.assertEqual(window, 0)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '-4,-6')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 0)
+ self.assertEqual(window, 0)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, 'wd,er')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 0)
+ self.assertEqual(window, 0)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '2,20')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 2)
+ self.assertEqual(window, 20)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, ' 2, 30')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 0)
+ self.assertEqual(window, 0)
+
+
if __name__ == "__main__":
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
unittest.main()
\ No newline at end of file
diff --git a/slider-agent/src/test/python/agent/TestPythonExecutor.py b/slider-agent/src/test/python/agent/TestPythonExecutor.py
index 1b12a0a..0a43639 100644
--- a/slider-agent/src/test/python/agent/TestPythonExecutor.py
+++ b/slider-agent/src/test/python/agent/TestPythonExecutor.py
@@ -17,6 +17,8 @@
See the License for the specific language governing permissions and
limitations under the License.
'''
+import platform
+IS_WINDOWS = platform.system() == "Windows"
import pprint
@@ -30,17 +32,24 @@
from PythonExecutor import PythonExecutor
from AgentConfig import AgentConfig
from mock.mock import MagicMock, patch
-
+from AgentToggleLogger import AgentToggleLogger
+import os
class TestPythonExecutor(TestCase):
+ def setUp(self):
+ self.agentToggleLogger = AgentToggleLogger("info")
@patch("shell.kill_process_with_children")
def test_watchdog_1(self, kill_process_with_children_mock):
+ # Test hangs on Windows TODO
+ if IS_WINDOWS:
+ return
+
"""
Tests whether watchdog works
"""
subproc_mock = self.Subprocess_mockup()
- executor = PythonExecutor("/tmp", AgentConfig("", ""))
+ executor = PythonExecutor("/tmp", AgentConfig("", ""), self.agentToggleLogger)
_, tmpoutfile = tempfile.mkstemp()
_, tmperrfile = tempfile.mkstemp()
_, tmpstrucout = tempfile.mkstemp()
@@ -65,11 +74,14 @@
def test_watchdog_2(self):
+ # Test hangs on Windows TODO
+ if IS_WINDOWS:
+ return
"""
Tries to catch false positive watchdog invocations
"""
subproc_mock = self.Subprocess_mockup()
- executor = PythonExecutor("/tmp", AgentConfig("", ""))
+ executor = PythonExecutor("/tmp", AgentConfig("", ""), self.agentToggleLogger)
_, tmpoutfile = tempfile.mkstemp()
_, tmperrfile = tempfile.mkstemp()
_, tmpstrucout = tempfile.mkstemp()
@@ -99,19 +111,24 @@
@patch("subprocess.Popen")
@patch("os.environ.copy")
def test_set_env_values(self, os_env_copy_mock, subprocess_mock, open_mock):
- actual_vars = {"someOther" : "value1"}
- executor = PythonExecutor("/tmp", AgentConfig("", ""))
- environment_vars = [("PYTHONPATH", "a:b")]
- os_env_copy_mock.return_value = actual_vars
- executor.run_file("script.pynot", ["a","b"], "", "", 10, "", "INFO", True, environment_vars)
- self.assertEquals(2, len(os_env_copy_mock.return_value))
+ if not IS_WINDOWS:
+ actual_vars = {"someOther" : "value1"}
+ executor = PythonExecutor("/tmp", AgentConfig("", ""), self.agentToggleLogger)
+ environment_vars = [("PYTHONPATH", "a:b")]
+ os_env_copy_mock.return_value = actual_vars
+ executor.run_file("script.pynot", ["a","b"], "", "", 10, "", "INFO", True, environment_vars)
+ self.assertEquals(2, len(os_env_copy_mock.return_value))
def test_execution_results(self):
+ self.assertEqual.__self__.maxDiff = None
subproc_mock = self.Subprocess_mockup()
- executor = PythonExecutor("/tmp", AgentConfig("", ""))
+ executor = PythonExecutor("/tmp", AgentConfig("", ""), self.agentToggleLogger)
_, tmpoutfile = tempfile.mkstemp()
_, tmperrfile = tempfile.mkstemp()
_, tmpstroutfile = tempfile.mkstemp()
+ if IS_WINDOWS:
+ if os.path.exists(tmpstroutfile):
+ tmpstroutfile = tmpstroutfile + "_t"
PYTHON_TIMEOUT_SECONDS = 5
def launch_python_subprocess_method(command, tmpout, tmperr, environment_vars):
@@ -124,13 +141,13 @@
executor.runShellKillPgrp = runShellKillPgrp_method
subproc_mock.returncode = 0
subproc_mock.should_finish_event.set()
- result = executor.run_file("file", ["arg1", "arg2"], tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstroutfile, "INFO")
+ result = executor.run_file("file", ["arg1", "arg2"], tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstroutfile, "INFO", True, None)
self.assertEquals(result, {'exitcode': 0, 'stderr': 'Dummy err', 'stdout': 'Dummy output',
'structuredOut': {}})
def test_is_successfull(self):
- executor = PythonExecutor("/tmp", AgentConfig("", ""))
+ executor = PythonExecutor("/tmp", AgentConfig("", ""), self.agentToggleLogger)
executor.python_process_has_been_killed = False
self.assertTrue(executor.isSuccessfull(0))
@@ -142,7 +159,7 @@
def test_python_command(self):
- executor = PythonExecutor("/tmp", AgentConfig("", ""))
+ executor = PythonExecutor("/tmp", AgentConfig("", ""), self.agentToggleLogger)
command = executor.python_command("script", ["script_param1"])
self.assertEqual(4, len(command))
self.assertTrue("python" in command[0].lower(), "Looking for python in %s" % (command[0].lower()))
diff --git a/slider-agent/src/test/python/agent/TestRegistration.py b/slider-agent/src/test/python/agent/TestRegistration.py
index f91fe29..735fcc3 100644
--- a/slider-agent/src/test/python/agent/TestRegistration.py
+++ b/slider-agent/src/test/python/agent/TestRegistration.py
@@ -28,6 +28,7 @@
from Register import Register
from Controller import State
from AgentConfig import AgentConfig
+import posixpath
class TestRegistration(TestCase):
@@ -37,37 +38,27 @@
config = AgentConfig(tmpdir, ver_dir)
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
- try:
- os.mkdir(ver_dir)
- except OSError as exception:
- if exception.errno != errno.EEXIST:
- raise
- pass
- ver_file = os.path.join(ver_dir, "version")
- with open(ver_file, "w") as text_file:
- text_file.write("1.3.0")
register = Register(config)
- data = register.build(State.INIT, State.INIT, {}, 1)
+ data = register.build(State.INIT, State.INIT, {}, {}, "tag", 1)
#print ("Register: " + pprint.pformat(data))
- self.assertEquals(data['hostname'] != "", True, "hostname should not be empty")
+ self.assertEquals(data['label'] != "", True, "hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp should not be empty")
- self.assertEquals(data['agentVersion'], '1.3.0', "agentVersion should not be empty")
+ self.assertEquals(data['agentVersion'], '1', "agentVersion should not be empty")
self.assertEquals(data['actualState'], State.INIT, "actualState should not be empty")
self.assertEquals(data['expectedState'], State.INIT, "expectedState should not be empty")
self.assertEquals(data['allocatedPorts'], {}, "allocatedPorts should be empty")
- self.assertEquals(len(data), 8)
+ self.assertEquals(data['logFolders'], {}, "allocated log should be empty")
+ self.assertEquals(data['tags'], "tag", "tags should be tag")
+ self.assertEquals(len(data), 10)
- self.assertEquals(os.path.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir"))
- self.assertEquals(os.path.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir"))
- self.assertEquals(os.path.join(ver_dir, "."), config.getResolvedPath("app_log_dir"))
- self.assertEquals(os.path.join(ver_dir, "."), config.getResolvedPath("log_dir"))
- self.assertEquals(os.path.join(ver_dir, "."), config.getResolvedPath("app_task_dir"))
-
- os.remove(ver_file)
- os.removedirs(ver_dir)
+ self.assertEquals(posixpath.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir"))
+ self.assertEquals(posixpath.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir"))
+ self.assertEquals(posixpath.join(ver_dir, "."), config.getResolvedPath("app_log_dir"))
+ self.assertEquals(posixpath.join(ver_dir, "."), config.getResolvedPath("log_dir"))
+ self.assertEquals(posixpath.join(ver_dir, "."), config.getResolvedPath("app_task_dir"))
if __name__ == "__main__":
unittest.main()
\ No newline at end of file
diff --git a/slider-agent/src/test/python/python-wrap b/slider-agent/src/test/python/python-wrap
index 40dc785..88a8c55 100755
--- a/slider-agent/src/test/python/python-wrap
+++ b/slider-agent/src/test/python/python-wrap
@@ -17,23 +17,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-export PYTHONPATH=/usr/lib/python2.6/site-packages/common_functions:$PYTHONPATH
-
# reset settings
unset PYTHON
+if [ -a /usr/bin/python2.6 ] && [ -z "$PYTHON" ]; then
+ PYTHON=/usr/bin/python2.6
+fi
+
# checking for preferable python versions
if [ -a /usr/bin/python2.7 ] && [ -z "$PYTHON" ]; then
PYTHON=/usr/bin/python2.7
fi
-if [ -a /usr/bin/python2.6 ] && [ -z "$PYTHON" ]; then
- PYTHON=/usr/bin/python2.6
+# if no preferable python versions found, try to use system one
+if [ -a /usr/bin/python ] && [ -z "$PYTHON" ]; then
+ PYTHON=/usr/bin/python
fi
# if no preferable python versions found, try to use system one
if [[ -z "$PYTHON" ]]; then
- PYTHON=/usr/bin/python
+ PYTHON=python
fi
# execute script
diff --git a/slider-agent/src/test/python/resource_management/TestContentSources.py b/slider-agent/src/test/python/resource_management/TestContentSources.py
index 2527f30..28d7435 100644
--- a/slider-agent/src/test/python/resource_management/TestContentSources.py
+++ b/slider-agent/src/test/python/resource_management/TestContentSources.py
@@ -30,6 +30,9 @@
from jinja2 import UndefinedError, TemplateNotFound
import urllib2
import os
+import platform
+
+IS_WINDOWS = platform.system() == "Windows"
@patch.object(System, "os_family", new = 'redhat')
@@ -46,8 +49,12 @@
file_mock.read.return_value = 'content'
open_mock.return_value = file_mock
+ filepath = "/absolute/path/file"
+ if IS_WINDOWS:
+ filepath = "\\absolute\\path\\file"
+
with Environment("/base") as env:
- static_file = StaticFile("/absolute/path/file")
+ static_file = StaticFile(filepath)
content = static_file.get_content()
self.assertEqual('content', content)
@@ -108,130 +115,6 @@
self.assertEqual(exists_mock.call_count, 1)
pass
- @patch.object(urllib2, "urlopen")
- @patch.object(os, "makedirs")
- @patch.object(os.path, "exists")
- def test_download_source_get_content_nocache(self, exists_mock, makedirs_mock, urlopen_mock):
- """
- Testing DownloadSource.get_content without cache
- """
- exists_mock.return_value = True
- web_file_mock = MagicMock()
- web_file_mock.read.return_value = 'web_content'
- urlopen_mock.return_value = web_file_mock
-
- with Environment("/base") as env:
- download_source = DownloadSource("http://download/source", cache=False)
- content = download_source.get_content()
-
- self.assertEqual('web_content', content)
- self.assertEqual(urlopen_mock.call_count, 1)
- urlopen_mock.assert_called_with('http://download/source')
- self.assertEqual(web_file_mock.read.call_count, 1)
-
- @patch.object(urllib2, "urlopen")
- @patch.object(os, "makedirs")
- @patch.object(os.path, "exists")
- def test_download_source_get_content_cache_new(self, exists_mock, makedirs_mock, urlopen_mock):
- """
- Testing DownloadSource.get_content with cache on non-cached resource
- """
- exists_mock.side_effect = [True, False]
- web_file_mock = MagicMock()
- web_file_mock.read.return_value = 'web_content'
- urlopen_mock.return_value = web_file_mock
-
- with Environment("/base") as env:
- download_source = DownloadSource("http://download/source", cache=True)
- content = download_source.get_content()
-
- self.assertEqual('web_content', content)
- self.assertEqual(urlopen_mock.call_count, 1)
- urlopen_mock.assert_called_with('http://download/source')
- self.assertEqual(web_file_mock.read.call_count, 1)
-
- @patch("__builtin__.open")
- @patch.object(os, "makedirs")
- @patch.object(os.path, "exists")
- def test_download_source_get_content_cache_existent(self, exists_mock, makedirs_mock, open_mock):
- """
- Testing DownloadSource.get_content with cache on cached resource
- """
- exists_mock.side_effect = [True, True, False]
-
- file_mock = MagicMock(name = 'file_mock')
- file_mock.__enter__.return_value = file_mock
- file_mock.read.return_value = 'cached_content'
- open_mock.return_value = file_mock
-
-
- with Environment("/base") as env:
- download_source = DownloadSource("http://download/source", cache=True)
- content = download_source.get_content()
-
- self.assertEqual('cached_content', content)
- self.assertEqual(open_mock.call_count, 1)
- open_mock.assert_called_with('/var/tmp/downloads/source')
- self.assertEqual(file_mock.read.call_count, 1)
-
- @patch.object(urllib2, "urlopen")
- @patch("__builtin__.open")
- @patch.object(os, "makedirs")
- @patch.object(os.path, "exists")
- def test_download_source_get_content_cache_existent_md5_match(self, exists_mock, makedirs_mock, open_mock,
- urlopen_mock):
- """
- Testing DownloadSource.get_content with cache on cached resource with md5 check
- """
- exists_mock.side_effect = [True, True, False]
-
- file_mock = MagicMock(name = 'file_mock')
- file_mock.__enter__.return_value = file_mock
- file_mock.read.return_value = 'cached_content'
- open_mock.return_value = file_mock
-
-
- with Environment("/base") as env:
- download_source = DownloadSource("http://download/source", cache=True)
- content = download_source.get_content()
-
- self.assertEqual('cached_content', content)
- self.assertEqual(open_mock.call_count, 1)
- open_mock.assert_called_with('/var/tmp/downloads/source')
- self.assertEqual(file_mock.read.call_count, 1)
- self.assertEqual(urlopen_mock.call_count, 0)
-
- @patch.object(urllib2, "urlopen")
- @patch("__builtin__.open")
- @patch.object(os, "makedirs")
- @patch.object(os.path, "exists")
- def test_download_source_get_content_cache_existent_md5_unmatch(self, exists_mock, makedirs_mock, open_mock,
- urlopen_mock):
- """
- Testing DownloadSource.get_content with cache on cached resource with md5 check
- """
- exists_mock.side_effect = [True, True, False]
- fake_md5 = "144c9defac04969c7bfad8efaa8ea194"
- file_mock = MagicMock(name = 'file_mock')
- file_mock.__enter__.return_value = file_mock
- file_mock.read.return_value = 'cached_content'
- open_mock.return_value = file_mock
- web_file_mock = MagicMock()
- web_file_mock.read.return_value = 'web_content'
- urlopen_mock.return_value = web_file_mock
-
- with Environment("/base") as env:
- download_source = DownloadSource("http://download/source", cache=True, md5sum=fake_md5)
- content = download_source.get_content()
-
- self.assertEqual('web_content', content)
- self.assertEqual(open_mock.call_count, 2)
- open_mock.assert_once_called('/var/tmp/downloads/source', 'w')
- open_mock.assert_once_called('/var/tmp/downloads/source')
- self.assertEqual(file_mock.read.call_count, 1)
- self.assertEqual(urlopen_mock.call_count, 1)
- urlopen_mock.assert_called_with('http://download/source')
-
@patch("__builtin__.open")
@patch.object(os.path, "getmtime")
@patch.object(os.path, "exists")
@@ -250,9 +133,15 @@
template = Template("test.j2")
self.assertEqual(open_mock.call_count, 1)
- open_mock.assert_called_with('/base/templates/test.j2', 'rb')
+ if IS_WINDOWS:
+ open_mock.assert_called_with('/base\\templates\\test.j2', 'rb')
+ else:
+ open_mock.assert_called_with('/base/templates/test.j2', 'rb')
self.assertEqual(getmtime_mock.call_count, 1)
- getmtime_mock.assert_called_with('/base/templates/test.j2')
+ if IS_WINDOWS:
+ getmtime_mock.assert_called_with('/base\\templates\\test.j2')
+ else:
+ getmtime_mock.assert_called_with('/base/templates/test.j2')
@patch.object(os.path, "exists")
def test_template_loader_fail(self, exists_mock):
@@ -340,4 +229,7 @@
with Environment("/base") as env:
template = InlineTemplate("{{test_arg1}} template content {{os.path.join(path[0],path[1])}}", [os], test_arg1 = "test", path = ["/one","two"])
content = template.get_content()
- self.assertEqual(u'test template content /one/two\n', content)
+ if IS_WINDOWS:
+ self.assertEqual(u'test template content /one\\two\n', content)
+ else:
+ self.assertEqual(u'test template content /one/two\n', content)
diff --git a/slider-agent/src/test/python/resource_management/TestCopyFromLocal.py b/slider-agent/src/test/python/resource_management/TestCopyFromLocal.py
deleted file mode 100644
index 7653b24..0000000
--- a/slider-agent/src/test/python/resource_management/TestCopyFromLocal.py
+++ /dev/null
@@ -1,65 +0,0 @@
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from unittest import TestCase
-from mock.mock import patch
-from resource_management import *
-
-@patch.object(System, "os_family", new = 'redhat')
-class TestCopyFromLocal(TestCase):
-
- @patch("resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider")
- def test_run_default_args(self, execute_hadoop_mock):
- with Environment() as env:
- CopyFromLocal('/user/testdir/*.files',
- owner='user1',
- dest_dir='/apps/test/',
- kinnit_if_needed='',
- hdfs_user='hdfs'
- )
- self.assertEqual(execute_hadoop_mock.call_count, 2)
- call_arg_list = execute_hadoop_mock.call_args_list
- self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
- call_arg_list[0][0][0].command)
- self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
- call_arg_list[0][0][0].arguments)
- self.assertEquals('fs -chown user1 /apps/test/*.files', call_arg_list[1][0][0].command)
- self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
-
-
- @patch("resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider")
- def test_run_with_chmod(self, execute_hadoop_mock):
- with Environment() as env:
- CopyFromLocal('/user/testdir/*.files',
- mode=0655,
- owner='user1',
- group='hdfs',
- dest_dir='/apps/test/',
- kinnit_if_needed='',
- hdfs_user='hdfs'
- )
- self.assertEqual(execute_hadoop_mock.call_count, 3)
- call_arg_list = execute_hadoop_mock.call_args_list
- self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
- call_arg_list[0][0][0].command)
- self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
- call_arg_list[0][0][0].arguments)
- self.assertEquals('fs -chown user1:hdfs /apps/test/*.files', call_arg_list[1][0][0].command)
- self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
-
-
diff --git a/slider-agent/src/test/python/resource_management/TestDirectoryResource.py b/slider-agent/src/test/python/resource_management/TestDirectoryResource.py
index d9a262c..e3282d0 100644
--- a/slider-agent/src/test/python/resource_management/TestDirectoryResource.py
+++ b/slider-agent/src/test/python/resource_management/TestDirectoryResource.py
@@ -33,11 +33,10 @@
@patch.object(os.path, "isdir")
@patch.object(os, "stat")
@patch.object(os,"chmod")
- @patch.object(os,"chown")
@patch("resource_management.core.providers.system._coerce_uid")
@patch("resource_management.core.providers.system._coerce_gid")
def test_create_directory_recursive(self, _coerce_gid_mock, _coerce_uid_mock,
- os_chown_mock, os_chmod_mock, os_stat_mock,
+ os_chmod_mock, os_stat_mock,
isdir_mock, os_makedirs_mock,
os_path_exists_mock):
os_path_exists_mock.return_value = False
@@ -57,20 +56,17 @@
os_makedirs_mock.assert_called_with('/a/b/c/d', 0777)
os_chmod_mock.assert_called_with('/a/b/c/d', 0777)
- os_chown_mock.assert_any_call('/a/b/c/d', 66, -1)
- os_chown_mock.assert_any_call('/a/b/c/d', -1, 77)
-
+
@patch.object(os.path, "exists")
@patch.object(os.path, "dirname")
@patch.object(os.path, "isdir")
@patch.object(os, "mkdir")
@patch.object(os, "stat")
@patch.object(os,"chmod")
- @patch.object(os,"chown")
@patch("resource_management.core.providers.system._coerce_uid")
@patch("resource_management.core.providers.system._coerce_gid")
def test_create_directory_not_recursive(self, _coerce_gid_mock, _coerce_uid_mock,
- os_chown_mock, os_chmod_mock, os_stat_mock,
+ os_chmod_mock, os_stat_mock,
mkdir_mock, isdir_mock, os_dirname_mock,
os_path_exists_mock):
os_path_exists_mock.return_value = False
@@ -90,9 +86,7 @@
mkdir_mock.assert_called_with('/a/b/c/d', 0777)
os_chmod_mock.assert_called_with('/a/b/c/d', 0777)
- os_chown_mock.assert_any_call('/a/b/c/d', 66, -1)
- os_chown_mock.assert_any_call('/a/b/c/d', -1, 77)
-
+
@patch.object(os.path, "exists")
@patch.object(os.path, "dirname")
@patch.object(os.path, "isdir")
diff --git a/slider-agent/src/test/python/resource_management/TestExecuteResource.py b/slider-agent/src/test/python/resource_management/TestExecuteResource.py
index f0a4539..0673b66 100644
--- a/slider-agent/src/test/python/resource_management/TestExecuteResource.py
+++ b/slider-agent/src/test/python/resource_management/TestExecuteResource.py
@@ -15,6 +15,9 @@
See the License for the specific language governing permissions and
limitations under the License.
'''
+import platform
+
+IS_WINDOWS = platform.system() == "Windows"
from unittest import TestCase
from mock.mock import patch, MagicMock, call
@@ -27,8 +30,10 @@
import logging
import os
from resource_management import Fail
-import grp
-import pwd
+
+if not IS_WINDOWS:
+ import grp
+ import pwd
@patch.object(System, "os_family", new='redhat')
@@ -62,6 +67,32 @@
self.assertTrue(popen_mock.called, 'subprocess.Popen should have been called!')
self.assertFalse(proc_communicate_mock.called, 'proc.communicate should not have been called!')
+ @patch('subprocess.Popen.communicate')
+ @patch('subprocess.Popen')
+ def test_attribute_wait_and_poll(self, popen_mock, proc_communicate_mock):
+ with Environment("/") as env:
+ try:
+ Execute('echo "1"',
+ wait_for_finish=False,
+ poll_after = 5)
+ self.assertTrue(False, "Should fail as process does not run for 5 seconds")
+ except Fail as e:
+ self.assertTrue("returned 1" in str(e))
+ pass
+
+ self.assertTrue(popen_mock.called, 'subprocess.Popen should have been called!')
+ self.assertFalse(proc_communicate_mock.called, 'proc.communicate should not have been called!')
+
+ if not IS_WINDOWS:
+ @patch('subprocess.Popen.communicate')
+ def test_attribute_wait_and_poll_and_success(self, proc_communicate_mock):
+ with Environment("/") as env:
+ Execute('sleep 6',
+ wait_for_finish=False,
+ poll_after = 2)
+
+ self.assertFalse(proc_communicate_mock.called, 'proc.communicate should not have been called!')
+
@patch.object(os.path, "exists")
@patch.object(subprocess, "Popen")
def test_attribute_creates(self, popen_mock, exists_mock):
@@ -90,7 +121,11 @@
execute_resource = Execute('echo "1"',
path=["/test/one", "test/two"]
)
- self.assertEqual(execute_resource.environment["PATH"], '/test/one:test/two')
+
+ if IS_WINDOWS:
+ self.assertEqual(execute_resource.environment["PATH"], '/test/one;test/two')
+ else:
+ self.assertEqual(execute_resource.environment["PATH"], '/test/one:test/two')
@patch('time.sleep')
@patch.object(subprocess, "Popen")
@@ -111,37 +146,39 @@
time_mock.assert_called_once_with(10)
- @patch.object(pwd, "getpwnam")
- def test_attribute_group(self, getpwnam_mock):
- def error(argument):
- self.assertEqual(argument, "test_user")
- raise KeyError("fail")
+ if not IS_WINDOWS:
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_group(self, getpwnam_mock):
+ def error(argument):
+ self.assertEqual(argument, "test_user")
+ raise KeyError("fail")
- getpwnam_mock.side_effect = error
- try:
- with Environment("/") as env:
- Execute('echo "1"',
- user="test_user",
- )
- except Fail as e:
- pass
+ getpwnam_mock.side_effect = error
+ try:
+ with Environment("/") as env:
+ Execute('echo "1"',
+ user="test_user",
+ )
+ except Fail as e:
+ pass
- @patch.object(grp, "getgrnam")
- @patch.object(pwd, "getpwnam")
- def test_attribute_group(self, getpwnam_mock, getgrnam_mock):
- def error(argument):
- self.assertEqual(argument, "test_group")
- raise KeyError("fail")
+ @patch.object(grp, "getgrnam")
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_group(self, getpwnam_mock, getgrnam_mock):
+ def error(argument):
+ self.assertEqual(argument, "test_group")
+ raise KeyError("fail")
- getpwnam_mock.side_effect = 1
- getgrnam_mock.side_effect = error
- try:
- with Environment("/") as env:
- Execute('echo "1"',
- group="test_group",
- )
- except Fail as e:
- pass
+ getpwnam_mock.side_effect = 1
+ getgrnam_mock.side_effect = error
+ try:
+ with Environment("/") as env:
+ Execute('echo "1"',
+ group="test_group",
+ )
+ except Fail as e:
+ pass
+
@patch.object(subprocess, "Popen")
def test_attribute_environment(self, popen_mock):
diff --git a/slider-agent/src/test/python/resource_management/TestFileResource.py b/slider-agent/src/test/python/resource_management/TestFileResource.py
index 7da0dbd..642412c 100644
--- a/slider-agent/src/test/python/resource_management/TestFileResource.py
+++ b/slider-agent/src/test/python/resource_management/TestFileResource.py
@@ -269,16 +269,12 @@
self.assertEqual(open_mock.call_count, 0)
- @patch("resource_management.core.providers.system._coerce_uid")
- @patch("resource_management.core.providers.system._coerce_gid")
- @patch.object(os, "chown")
@patch.object(os, "chmod")
@patch.object(os, "stat")
@patch("__builtin__.open")
@patch.object(os.path, "exists")
@patch.object(os.path, "isdir")
- def test_ensure_metadata(self, isdir_mock, exists_mock, open_mock, stat_mock, chmod_mock, chown_mock, gid_mock,
- uid_mock):
+ def test_ensure_metadata(self, isdir_mock, exists_mock, open_mock, stat_mock, chmod_mock):
"""
Tests if _ensure_metadata changes owner, usergroup and permissions of file to proper values
"""
@@ -292,8 +288,6 @@
self.st_gid = 1
stat_mock.return_value = stat()
- gid_mock.return_value = 0
- uid_mock.return_value = 0
with Environment('/') as env:
File('/directory/file',
@@ -308,15 +302,7 @@
open_mock.assert_called_with('/directory/file', 'wb')
self.assertEqual(open_mock.call_count, 1)
stat_mock.assert_called_with('/directory/file')
- self.assertEqual(chmod_mock.call_count, 1)
- self.assertEqual(chown_mock.call_count, 2)
- gid_mock.assert_called_once_with('hdfs')
- uid_mock.assert_called_once_with('root')
- chmod_mock.reset_mock()
- chown_mock.reset_mock()
- gid_mock.return_value = 1
- uid_mock.return_value = 1
with Environment('/') as env:
File('/directory/file',
@@ -326,7 +312,5 @@
owner='root',
group='hdfs'
)
-
+ self.assertTrue(chmod_mock.called)
- self.assertEqual(chmod_mock.call_count, 1)
- self.assertEqual(chown_mock.call_count, 0)
diff --git a/slider-agent/src/test/python/resource_management/TestLinkResource.py b/slider-agent/src/test/python/resource_management/TestLinkResource.py
deleted file mode 100644
index 87af645..0000000
--- a/slider-agent/src/test/python/resource_management/TestLinkResource.py
+++ /dev/null
@@ -1,148 +0,0 @@
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from unittest import TestCase
-from mock.mock import patch, MagicMock
-
-from resource_management.core import Environment, Fail
-from resource_management.core.system import System
-from resource_management.core.resources.system import Link
-
-import os
-
-@patch.object(System, "os_family", new = 'redhat')
-class TestLinkResource(TestCase):
-
- @patch.object(os.path, "realpath")
- @patch.object(os.path, "lexists")
- @patch.object(os.path, "islink")
- @patch.object(os, "unlink")
- @patch.object(os, "symlink")
- def test_action_create_relink(self, symlink_mock, unlink_mock,
- islink_mock, lexists_mock,
- realmock):
- lexists_mock.return_value = True
- realmock.return_value = "/old_to_link_path"
- islink_mock.return_value = True
- with Environment('/') as env:
- Link("/some_path",
- to = "/a/b/link_to_path"
- )
-
- unlink_mock.assert_called_with("/some_path")
- symlink_mock.assert_called_with("/a/b/link_to_path", "/some_path")
-
- @patch.object(os.path, "realpath")
- @patch.object(os.path, "lexists")
- @patch.object(os.path, "islink")
- def test_action_create_failed_due_to_file_exists(self, islink_mock,
- lexists_mock, realmock):
- lexists_mock.return_value = True
- realmock.return_value = "/old_to_link_path"
- islink_mock.return_value = False
- with Environment('/') as env:
- try:
- Link("/some_path",
- to = "/a/b/link_to_path"
- )
-
- self.fail("Must fail when directory or file with name /some_path exist")
- except Fail as e:
- self.assertEqual("LinkProvider[Link['/some_path']] trying to create a symlink with the same name as an existing file or directory",
- str(e))
-
- @patch.object(os.path, "lexists")
- @patch.object(os, "symlink")
- def test_action_create_symlink_clean_create(self, symlink_mock, lexists_mock):
- lexists_mock.return_value = False
-
- with Environment('/') as env:
- Link("/some_path",
- to = "/a/b/link_to_path"
- )
-
- symlink_mock.assert_called_with("/a/b/link_to_path", "/some_path")
-
- @patch.object(os.path, "isdir")
- @patch.object(os.path, "exists")
- @patch.object(os.path, "lexists")
- @patch.object(os, "link")
- def test_action_create_hardlink_clean_create(self, link_mock, lexists_mock,
- exists_mock, isdir_mock):
- lexists_mock.return_value = False
- exists_mock.return_value = True
- isdir_mock.return_value = False
-
- with Environment('/') as env:
- Link("/some_path",
- hard = True,
- to = "/a/b/link_to_path"
- )
-
- link_mock.assert_called_with("/a/b/link_to_path", "/some_path")
-
- @patch.object(os.path, "exists")
- @patch.object(os.path, "lexists")
- def test_action_create_hardlink_target_doesnt_exist(self, lexists_mock,
- exists_mock):
- lexists_mock.return_value = False
- exists_mock.return_value = False
-
- with Environment('/') as env:
- try:
- Link("/some_path",
- hard = True,
- to = "/a/b/link_to_path"
- )
- self.fail("Must fail when target directory do doenst exist")
- except Fail as e:
- self.assertEqual("Failed to apply Link['/some_path'], linking to nonexistent location /a/b/link_to_path",
- str(e))
-
- @patch.object(os.path, "isdir")
- @patch.object(os.path, "exists")
- @patch.object(os.path, "lexists")
- def test_action_create_hardlink_target_is_dir(self, lexists_mock,
- exists_mock, isdir_mock):
- lexists_mock.return_value = False
- exists_mock.return_value = True
- isdir_mock = True
-
- with Environment('/') as env:
- try:
- Link("/some_path",
- hard = True,
- to = "/a/b/link_to_path"
- )
- self.fail("Must fail when hardlinking to directory")
- except Fail as e:
- self.assertEqual("Failed to apply Link['/some_path'], cannot create hard link to a directory (/a/b/link_to_path)",
- str(e))
-
- @patch.object(os, "unlink")
- @patch.object(os.path, "exists")
- def test_action_delete(self, exists_mock, unlink_mock):
- exists_mock.return_value = True
-
- with Environment('/') as env:
- Link("/some_path",
- action = "delete"
- )
- unlink_mock.assert_called_with("/some_path")
-
-
diff --git a/slider-agent/src/test/python/resource_management/TestPackage.py b/slider-agent/src/test/python/resource_management/TestPackage.py
new file mode 100644
index 0000000..fe03ccf
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestPackage.py
@@ -0,0 +1,32 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+
+from resource_management.core import Environment, Fail
+from resource_management.core.system import System
+from resource_management.core.resources import Package
+
+from resource_management.core.providers import get_class_path, PROVIDERS
+
+class TestPackage(TestCase):
+
+ def test_default_provider(self):
+ self.assertEquals(PROVIDERS['default']['Tarball'], get_class_path('something unknown', 'Tarball'))
+
diff --git a/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py b/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py
index 6eb01cf..479c799 100644
--- a/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py
+++ b/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py
@@ -27,6 +27,7 @@
from resource_management.core import Environment
from resource_management.core.system import System
from resource_management.libraries import PropertiesFile
+import posixpath
@patch.object(System, "os_family", new='redhat')
class TestPropertiesFileResource(TestCase):
@@ -100,7 +101,7 @@
properties={},
)
- open_mock.assert_called_with('/dir/and/dir/file.txt', 'wb')
+ open_mock.assert_called_with(os.path.join('/dir/and/dir', 'file.txt'), 'wb')
result_file.__enter__().write.assert_called_with(u'# Generated by Apache Slider. Some other day\n \n \n')
self.assertEqual(open_mock.call_count, 1)
ensure_mock.assert_called()
@@ -213,7 +214,7 @@
)
result_file.read.assert_called()
- open_mock.assert_called_with('/dir1/new_file', 'wb')
+ open_mock.assert_called_with(os.path.join('/dir1','new_file'), 'wb')
result_file.__enter__().write.assert_called_with(u'# Generated by Apache Slider. 777\n \nproperty_1=value1\n \n')
self.assertEqual(open_mock.call_count, 2)
ensure_mock.assert_called()
diff --git a/slider-agent/src/test/python/unitTests.py b/slider-agent/src/test/python/unitTests.py
index e3f2d7c..b65c075 100644
--- a/slider-agent/src/test/python/unitTests.py
+++ b/slider-agent/src/test/python/unitTests.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-'''
+"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -16,14 +16,13 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-'''
+"""
import unittest
-import doctest
-from os.path import dirname, split, isdir
-import logging.handlers
+from os.path import isdir
import logging
-from random import shuffle
+import os
+import sys
LOG_FILE_NAME='tests.log'
SELECTED_PREFIX = "_"
@@ -31,7 +30,7 @@
ignoredDirs = ["mock"]
class TestAgent(unittest.TestSuite):
- def run(self, result):
+ def run(self, result, debug=False):
run = unittest.TestSuite.run
run(self, result)
return result
@@ -41,21 +40,22 @@
if isdir(path):
if path.endswith(os.sep):
path = os.path.dirname(path)
- parent_dir = os.path.dirname(path)
+ parent = os.path.dirname(path)
else:
- parent_dir = os.path.dirname(os.path.dirname(path))
+ parent = os.path.dirname(os.path.dirname(path))
- return parent_dir
+ return parent
def all_tests_suite():
- src_dir = os.getcwd()
+ root_dir = os.getcwd()
files_list = []
- for directory in os.listdir(src_dir):
+ for directory in os.listdir(root_dir):
if os.path.isdir(directory) and not directory in ignoredDirs:
- files_list += os.listdir(src_dir + os.sep + directory)
+ files_list += os.listdir(root_dir + os.sep + directory)
## temporarily deleting to add more predictability
## shuffle(files_list)
+ files_list.sort()
tests_list = []
logger.info('------------------------TESTS LIST:-------------------------------------')
@@ -70,8 +70,9 @@
else:
for file_name in files_list:
if file_name.endswith(PY_EXT) and not file_name == __file__:
- logger.info(file_name)
- tests_list.append(file_name.replace(PY_EXT, ''))
+ replaced = file_name.replace(PY_EXT, '')
+ logger.info(replaced)
+ tests_list.append(replaced)
logger.info('------------------------------------------------------------------------')
suite = unittest.TestLoader().loadTestsFromNames(tests_list)
@@ -98,9 +99,7 @@
logger.info('------------------------------------------------------------------------')
if __name__ == '__main__':
- import os
- import sys
- import io
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python')
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python' + os.sep + 'agent')
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index 928713e..cd4b6e6 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
</parent>
@@ -33,11 +33,15 @@
<rpm.bindir>${rpm.basedir}/bin</rpm.bindir>
<rpm.libdir>${rpm.basedir}/lib</rpm.libdir>
<rpm.agentdir>${rpm.basedir}/agent</rpm.agentdir>
- <rpm.username>mapred</rpm.username>
- <rpm.groupname>hadoop</rpm.groupname>
+ <rpm.username>root</rpm.username>
+ <rpm.groupname>root</rpm.groupname>
<src.confdir>src/conf-hdp</src.confdir>
<src.libdir>${project.build.directory}/lib</src.libdir>
<src.agent.ini.dir>${project.build.directory}/../../slider-agent/conf</src.agent.ini.dir>
+ <python.ver>python >= 2.6</python.ver>
+ <executable.python>${project.basedir}/../slider-agent/src/test/python/python-wrap</executable.python>
+ <python.path.l>${project.basedir}/src/main/scripts:${project.basedir}/../slider-agent/src/test/python/mock:${project.basedir}/src/test/python/scripts</python.path.l>
+ <skipTests>false</skipTests>
</properties>
<build>
@@ -63,6 +67,32 @@
</execution>
</executions>
</plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>${maven-exec-plugin.version}</version>
+ <executions>
+ <execution>
+ <configuration>
+ <executable>${executable.python}</executable>
+ <workingDirectory>src/test/python</workingDirectory>
+ <arguments>
+ <argument>unitTests.py</argument>
+ </arguments>
+ <environmentVariables>
+ <PYTHONPATH>${python.path.l}</PYTHONPATH>
+ </environmentVariables>
+ <skip>${skipTests}</skip>
+ </configuration>
+ <id>python-test</id>
+ <phase>test</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
<!-- pull in all dependencies -->
<plugin>
@@ -104,6 +134,26 @@
</artifactItems>
</configuration>
</execution>
+ <execution>
+ <!-- copy in the agent tar file -->
+ <id>copy-to-lib</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-agent</artifactId>
+ <type>tar.gz</type>
+ <overWrite>false</overWrite>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ <destFileName>slider-agent.tar.gz</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
</executions>
</plugin>
@@ -188,6 +238,27 @@
</reporting>
<profiles>
+ <profile>
+ <id>Windows</id>
+ <activation>
+ <os><family>windows</family></os>
+ </activation>
+ <properties>
+ <executable.python>python</executable.python>
+ <python.path.l>${project.basedir}\src\main\scripts;${project.basedir}\..\slider-agent\src\test\python\mock;${project.basedir}\src\test\python\scripts</python.path.l>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>Linux</id>
+ <activation>
+ <os><family>!windows</family></os>
+ </activation>
+ <properties>
+ <executable.python>${project.basedir}/../slider-agent/src/test/python/python-wrap</executable.python>
+ <python.path.l>${project.basedir}/src/main/scripts:${project.basedir}/../slider-agent/src/test/python/mock:${project.basedir}/src/test/python/scripts</python.path.l>
+ </properties>
+ </profile>
<profile>
<id>rpm</id>
<build>
@@ -231,8 +302,8 @@
<mappings>
<mapping>
<directory>${rpm.basedir}</directory>
- <!-- RW.R..R.. -->
- <filemode>644</filemode>
+ <!-- RWXR.XR.X -->
+ <filemode>755</filemode>
<username>${rpm.username}</username>
<groupname>${rpm.groupname}</groupname>
@@ -258,8 +329,8 @@
<!-- library -->
<mapping>
<directory>${rpm.libdir}</directory>
- <!-- RW.R..R.. -->
- <filemode>644</filemode>
+ <!-- RWXR.XR.X -->
+ <filemode>755</filemode>
<username>${rpm.username}</username>
<groupname>${rpm.groupname}</groupname>
@@ -287,10 +358,10 @@
<!-- agent -->
<mapping>
<directory>${rpm.agentdir}</directory>
- <configuration>true</configuration>
<filemode>0755</filemode>
<username>${rpm.username}</username>
<groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>false</directoryIncluded>
<sources>
<source>
<location>${project.build.directory}/agent</location>
@@ -308,6 +379,7 @@
<filemode>0755</filemode>
<username>${rpm.username}</username>
<groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>false</directoryIncluded>
<sources>
<source>
<location>${src.agent.ini.dir}</location>
@@ -318,6 +390,24 @@
</source>
</sources>
</mapping>
+
+ <!-- needed to apply attribute to directory -->
+ <mapping>
+ <directory>${rpm.agentdir}</directory>
+ <filemode>0755</filemode>
+ <username>${rpm.username}</username>
+ <groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>true</directoryIncluded>
+ </mapping>
+
+ <mapping>
+ <directory>${rpm.agentdir}/conf</directory>
+ <filemode>0755</filemode>
+ <username>${rpm.username}</username>
+ <groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>true</directoryIncluded>
+ </mapping>
+
</mappings>
<!--
Scripts. Very dangerous in RPMs unless you know exactly what you are doing.
diff --git a/slider-assembly/src/assembly/slider-bin.xml b/slider-assembly/src/assembly/slider-bin.xml
index 6f3b021..c5611f8 100644
--- a/slider-assembly/src/assembly/slider-bin.xml
+++ b/slider-assembly/src/assembly/slider-bin.xml
@@ -59,6 +59,14 @@
<fileSet>
<directory>${project.build.directory}/agent</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>slider-agent.tar.gz</include>
+ </includes>
+ </fileSet>
+
+ <fileSet>
+ <directory>${project.build.directory}/agent</directory>
<outputDirectory>agent</outputDirectory>
<includes>
<include>slider-agent.tar.gz</include>
diff --git a/slider-assembly/src/conf-hdp/log4j-server.properties b/slider-assembly/src/conf-hdp/log4j-server.properties
new file mode 100644
index 0000000..793454f
--- /dev/null
+++ b/slider-assembly/src/conf-hdp/log4j-server.properties
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the log4j configuration for Slider Application Master
+
+# Log rotation based on size (100KB) with a max of 10 backup files
+log4j.rootLogger=INFO, amlog
+log4j.threshhold=ALL
+log4j.appender.amlog=org.apache.log4j.RollingFileAppender
+log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
+log4j.appender.amlog.File=${LOG_DIR}/slider.log
+log4j.appender.amlog.MaxFileSize=100KB
+log4j.appender.amlog.MaxBackupIndex=10
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stderr
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+# for debugging Slider
+#log4j.logger.org.apache.slider=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
diff --git a/slider-assembly/src/conf-hdp/log4j.properties b/slider-assembly/src/conf-hdp/log4j.properties
index 3c0d08c..fc5cc29 100644
--- a/slider-assembly/src/conf-hdp/log4j.properties
+++ b/slider-assembly/src/conf-hdp/log4j.properties
@@ -15,9 +15,9 @@
# limitations under the License.
#
-# log4j configuration used during build and unit tests
+# log4j configuration used by Slider client (and during build and unit tests)
-log4j.rootLogger=INFO,stdout
+log4j.rootLogger=INFO,stderr
log4j.threshhold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
@@ -28,6 +28,14 @@
# debug edition is much more expensive
#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+# configure stderr
+# set the conversion pattern of stdout
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
@@ -36,7 +44,6 @@
# for debugging Slider
#log4j.logger.org.apache.slider=DEBUG
-#log4j.logger.org.apache.slider=DEBUG
# uncomment to debug service lifecycle issues
#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
@@ -51,8 +58,12 @@
#crank back on some noise
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
log4j.logger.org.apache.hadoop.hdfs=WARN
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
+
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
diff --git a/slider-assembly/src/conf-hdp/slider-client.xml b/slider-assembly/src/conf-hdp/slider-client.xml
index f844106..136abf0 100644
--- a/slider-assembly/src/conf-hdp/slider-client.xml
+++ b/slider-assembly/src/conf-hdp/slider-client.xml
@@ -19,59 +19,63 @@
<!--
Properties set here are picked up in the client.
- They are not passed to the AM -though the filesystem
- binding details (URL And principal) are added to the
- hbase-site.xml file when a cluster is created.
-->
<configuration>
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
-
+ <!--
+ The recommended approach is to configure slider-env.sh and set HADOOP_CONF_DIR.
+ Otherwise, appropriate configurations from hdfs-site, yarn-site, can be dropped in this file
+ for Slider client to work. The following list is not an exhaustive list but the minimal config
+ needed to interact with a non-secure cluster.
+ -->
- <property>
- <name>slider.yarn.queue</name>
- <value>default</value>
- <description>YARN queue for the Application Master</description>
- </property>
+ <!--
+ <property>
+ <name>slider.client.resource.origin</name>
+ <value>conf-hdp/slider-client.xml</value>
+ <description>This is just for diagnostics</description>
+ </property>
+ <property>
+ <name>slider.security.protocol.acl</name>
+ <value>*</value>
+ <description>When security is enabled, set appropriate acl. Default value means allow everyone.</description>
+ </property>
- <property>
- <name>yarn.application.classpath</name>
- <value>
- /etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
- </value>
- </property>
-
-<!--
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>PROVIDE_CLASS_PATH_FROM_YARN</value>
+ </property>
- <property>
- <name>yarn.resourcemanager.address</name>
- <value>master:8032</value>
- </property>
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>@RM-ADDR</value>
+ </property>
- <property>
- <name>fs.defaultFS</name>
- <value>hdfs://master:9090</value>
- </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>@RM-SCHEDULER-ADDR</value>
+ </property>
- <property>
- <name>yarn.resourcemanager.principal</name>
- <value>yarn/master@MINICLUSTER</value>
- </property>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>@FILESYSTEM</value>
+ </property>
- <property>
- <name>slider.security.enabled</name>
- <value>true</value>
- </property>
+ <property>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>@ZK-QUORUM</value>
+ </property>
- <property>
- <name>dfs.namenode.kerberos.principal</name>
- <value>hdfs/master@MINICLUSTER</value>
- </property>
--->
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>yarn/master@MINICLUSTER</value>
+ </property>
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>hdfs/master@MINICLUSTER</value>
+ </property>
+ -->
</configuration>
diff --git a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE b/slider-assembly/src/conf-hdp/slider-env.sh
similarity index 72%
rename from app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
rename to slider-assembly/src/conf-hdp/slider-env.sh
index dd934d5..f3ec550 100644
--- a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
+++ b/slider-assembly/src/conf-hdp/slider-env.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,4 +15,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-Replace with the actual storm package.
+
+# this is the shell script to start Slider deploying an application
+# Usage: slider <action> <commands>
+
+# The env variable SLIDER_JVM_OPTS can be used to override
+# the default JVM opts
+
+export JAVA_HOME=${JAVA_HOME}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
\ No newline at end of file
diff --git a/slider-assembly/src/conf/log4j-server.properties b/slider-assembly/src/conf/log4j-server.properties
new file mode 100644
index 0000000..793454f
--- /dev/null
+++ b/slider-assembly/src/conf/log4j-server.properties
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the log4j configuration for Slider Application Master
+
+# Log rotation based on size (100KB) with a max of 10 backup files
+log4j.rootLogger=INFO, amlog
+log4j.threshhold=ALL
+log4j.appender.amlog=org.apache.log4j.RollingFileAppender
+log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
+log4j.appender.amlog.File=${LOG_DIR}/slider.log
+log4j.appender.amlog.MaxFileSize=100KB
+log4j.appender.amlog.MaxBackupIndex=10
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stderr
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+# for debugging Slider
+#log4j.logger.org.apache.slider=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
diff --git a/slider-assembly/src/conf/log4j.properties b/slider-assembly/src/conf/log4j.properties
index 3c0d08c..fc5cc29 100644
--- a/slider-assembly/src/conf/log4j.properties
+++ b/slider-assembly/src/conf/log4j.properties
@@ -15,9 +15,9 @@
# limitations under the License.
#
-# log4j configuration used during build and unit tests
+# log4j configuration used by Slider client (and during build and unit tests)
-log4j.rootLogger=INFO,stdout
+log4j.rootLogger=INFO,stderr
log4j.threshhold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
@@ -28,6 +28,14 @@
# debug edition is much more expensive
#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+# configure stderr
+# set the conversion pattern of stdout
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
@@ -36,7 +44,6 @@
# for debugging Slider
#log4j.logger.org.apache.slider=DEBUG
-#log4j.logger.org.apache.slider=DEBUG
# uncomment to debug service lifecycle issues
#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
@@ -51,8 +58,12 @@
#crank back on some noise
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
log4j.logger.org.apache.hadoop.hdfs=WARN
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
+
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
diff --git a/slider-assembly/src/conf/slider-client.xml b/slider-assembly/src/conf/slider-client.xml
index bd17254..3a42bee 100644
--- a/slider-assembly/src/conf/slider-client.xml
+++ b/slider-assembly/src/conf/slider-client.xml
@@ -21,52 +21,66 @@
Properties set here are picked up in the client.
-->
<configuration>
- <property>
- <name>slider.client.resource.origin</name>
- <value>conf/slider-client.xml</value>
- <description>This is just for diagnostics</description>
- </property>
+ <!--
+ The recommended approach is to configure slider-env.sh and set HADOOP_CONF_DIR.
+ Otherwise, appropriate configurations from hdfs-site, yarn-site, can be dropped in this file
+ for Slider client to work. The following list is not an exhaustive list but the minimal config
+ needed to interact with a non-secure cluster.
+ -->
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
-
+ <!--
+ <property>
+ <name>slider.client.resource.origin</name>
+ <value>conf/slider-client.xml</value>
+ <description>This is just for diagnostics</description>
+ </property>
- <property>
- <name>slider.yarn.queue</name>
- <value>default</value>
- <description>YARN queue for the Application Master</description>
- </property>
-
-<!--
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
- <property>
- <name>yarn.resourcemanager.address</name>
- <value>master:8032</value>
- </property>
+ <property>
+ <name>slider.security.protocol.acl</name>
+ <value>*</value>
+ <description>When security is enabled, set appropriate acl. Default value means allow everyone.</description>
+ </property>
- <property>
- <name>fs.defaultFS</name>
- <value>hdfs://master:9090</value>
- </property>
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>PROVIDE_CLASS_PATH_FROM_YARN</value>
+ </property>
- <property>
- <name>yarn.resourcemanager.principal</name>
- <value>yarn/master@MINICLUSTER</value>
- </property>
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>@RM-ADDR</value>
+ </property>
- <property>
- <name>slider.security.enabled</name>
- <value>true</value>
- </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>@RM-SCHEDULER-ADDR</value>
+ </property>
- <property>
- <name>dfs.namenode.kerberos.principal</name>
- <value>hdfs/master@MINICLUSTER</value>
- </property>
--->
+ <property>
+ <name>fs.defaultFS</name>
+ <value>@FILESYSTEM</value>
+ </property>
+ <property>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>@ZK-QUORUM</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>yarn/master@MINICLUSTER</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>hdfs/master@MINICLUSTER</value>
+ </property>
+ -->
</configuration>
diff --git a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE b/slider-assembly/src/conf/slider-env.sh
similarity index 72%
copy from app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
copy to slider-assembly/src/conf/slider-env.sh
index dd934d5..f3ec550 100644
--- a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
+++ b/slider-assembly/src/conf/slider-env.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,4 +15,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-Replace with the actual storm package.
+
+# this is the shell script to start Slider deploying an application
+# Usage: slider <action> <commands>
+
+# The env variable SLIDER_JVM_OPTS can be used to override
+# the default JVM opts
+
+export JAVA_HOME=${JAVA_HOME}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
\ No newline at end of file
diff --git a/slider-assembly/src/main/bash/README.md b/slider-assembly/src/main/bash/README.md
index a4b7b08..d818aeb 100644
--- a/slider-assembly/src/main/bash/README.md
+++ b/slider-assembly/src/main/bash/README.md
@@ -79,7 +79,7 @@
---------
* slider_destroy will do the following
- 1. Freeze the slider application based on provided name
+ 1. Stop the slider application based on provided name
2. Destory the slider application based on provided name
* The following args are required
diff --git a/slider-assembly/src/main/bash/slider_destroy b/slider-assembly/src/main/bash/slider_destroy
index 9039751..7ca30b7 100755
--- a/slider-assembly/src/main/bash/slider_destroy
+++ b/slider-assembly/src/main/bash/slider_destroy
@@ -53,8 +53,8 @@
#
# Main
#
-echo -e "\n## Freezing app $app_name"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider freeze $app_name --manager $RM_ADDRESS || exit 1
+echo -e "\n## Stopping app $app_name"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider stop $app_name --manager $RM_ADDRESS || exit 1
echo "SUCCESS"
echo -e "\n## Destroying app $app_name"
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
index caf275b..97dabaf 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -16,59 +16,30 @@
# limitations under the License.
-# this is the shell script to start Slider deploying an application
+# this is the wrapper shell script to invoke the Slider.py script in the same folder
# Usage: slider <action> <commands>
-# The env variable SLIDER_JVM_OPTS can be used to override
-# the default JVM opts
+import os
+import sys
+import subprocess
-function usage
-{
- echo "Usage: slider <action> <arguments>"
- echo ""
-}
+pyExecPath = None
-# Slider works out its own location
-this="${BASH_SOURCE-$0}"
-bindir=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
-script="$(basename -- "$this")"
+if os.path.isfile('/usr/bin/python2.7') and (pyExecPath is None):
+ pyExecPath = '/usr/bin/python2.7'
+if os.path.isfile('/usr/bin/python2.6') and (pyExecPath is None):
+ pyExecPath = '/usr/bin/python2.6'
+if os.path.isfile('/usr/bin/python') and (pyExecPath is None):
+ pyExecPath = '/usr/bin/python'
+if pyExecPath is None:
+ pyExecPath = 'python'
-# lib directory is one up; it is expected to contain
-# slider.jar and any other dependencies that are not in the
-# standard Hadoop classpath
+ON_POSIX = 'posix' in sys.builtin_module_names
+currDir = os.path.dirname(os.path.realpath(__file__))
+args = sys.argv[1:]
+cmd = [pyExecPath + " " + currDir + "/slider.py"]
+cmd.extend(args)
+finalCmd = " ".join(cmd)
-slider_home="${bindir}/.."
-slider_home=`cd -P "${slider_home}" && pwd -P`
-
-libdir="${slider_home}/lib"
-libdir=`cd -P "${libdir}" && pwd -P`
-
-
-confdir="${slider_home}/conf"
-
-# normalize the conf dir so it can be passed down
-confdir=`cd -P "${confdir}" && pwd -P`
-confdir=${SLIDER_CONF_DIR:-$confdir}
-
-
-slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=${confdir}"
-slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts}
-
-# allow for an extra classpath
-slider_classpath_extra=${SLIDER_CLASSPATH_EXTRA:-""}
-
-slider_classpath="${libdir}/*:${confdir}:${slider_classpath_extra}"
-
-launcher=org.apache.slider.Slider
-
-
-echo "slider_home = \"${slider_home}\""
-echo "slider_jvm_opts = \"${slider_jvm_opts}\""
-echo "classpath = \"${slider_classpath}\""
-export CLASSPATH="${slider_classpath}"
-echo ""
-
-echo "command is java ${slider_jvm_opts} --classpath \"${slider_classpath}\" ${launcher} $@"
-echo ""
-echo ""
-exec java ${slider_jvm_opts} ${launcher} $@
+result = subprocess.call(finalCmd, shell=True)
+sys.exit(result)
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index e60bed3..26c9adc 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -18,24 +18,54 @@
import sys
import os
import subprocess
+import time
+import platform
+from threading import Thread
CONF = "conf"
-
+IS_WINDOWS = platform.system() == "Windows"
LIB = "lib"
+ENV_KEYS = ["JAVA_HOME", "HADOOP_CONF_DIR", "SLIDER_CONF_DIR", "SLIDER_JVM_OPTS", "SLIDER_CLASSPATH_EXTRA"]
SLIDER_CONF_DIR = "SLIDER_CONF_DIR"
SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS"
SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
+HADOOP_CONF_DIR = "HADOOP_CONF_DIR"
SLIDER_CLASSNAME = "org.apache.slider.Slider"
-DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s"
+SLIDER_CONFDIR_OPTS ="-Dslider.confdir=%s"
+SLIDER_LIBDIR_OPTS ="-Dslider.libdir=%s"
+DEFAULT_JVM_OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m"
+
+ON_POSIX = 'posix' in sys.builtin_module_names
+
+finished = False
+DEBUG = False
"""
Launches slider
+Nonblocking IO on windows is "tricky" ... see
+http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
+to explain the code here
+
"""
+def executeEnvSh(confDir):
+ envscript = '%s/slider-env.sh' % confDir
+ if not IS_WINDOWS and os.path.exists(envscript):
+ envCmd = 'source %s && env' % envscript
+ command = ['bash', '-c', envCmd]
+
+ proc = subprocess.Popen(command, stdout = subprocess.PIPE)
+
+ for line in proc.stdout:
+ (key, _, value) = line.strip().partition("=")
+ if key in ENV_KEYS:
+ os.environ[key] = value
+
+ proc.communicate()
def scriptDir():
@@ -58,13 +88,51 @@
override the relative path
"""
localconf = os.path.join(sliderdir, CONF)
- return os.environ.get(SLIDER_CONF_DIR,localconf)
+ return os.environ.get(SLIDER_CONF_DIR, localconf)
def dirMustExist(dirname):
if not os.path.exists(dirname):
raise Exception("Directory does not exist: %s " % dirname)
return dirname
+
+def debug(text):
+ if DEBUG: print '[DEBUG] ' + text
+
+
+def error(text):
+ print '[ERROR] ' + text
+ sys.stdout.flush()
+
+def info(text):
+ print text
+ sys.stdout.flush()
+
+
+def out(toStdErr, text) :
+ """
+ Write to one of the system output channels.
+ This action does not add newlines. If you want that: write them yourself
+ :param toStdErr: flag set if stderr is to be the dest
+ :param text: text to write.
+ :return:
+ """
+ if toStdErr:
+ sys.stderr.write(text)
+ else:
+ sys.stdout.write(text)
+
+def flush(toStdErr) :
+ """
+ Flush the output stream
+ :param toStdErr: flag set if stderr is to be the dest
+ :return:
+ """
+ if toStdErr:
+ sys.stderr.flush()
+ else:
+ sys.stdout.flush()
+
def read(pipe, line):
"""
read a char, append to the listing if there is a char that is not \n
@@ -85,40 +153,88 @@
return line, False
+def print_output(name, src, toStdErr):
+ """
+ Relay the output stream to stdout line by line
+ :param name:
+ :param src: source stream
+ :param toStdErr: flag set if stderr is to be the dest
+ :return:
+ """
+
+ debug ("starting printer for %s" % name )
+ line = ""
+ while not finished:
+ (line, done) = read(src, line)
+ if done:
+ out(toStdErr, line + "\n")
+ flush(toStdErr)
+ line = ""
+ out(toStdErr, line)
+ # closedown: read remainder of stream
+ c = src.read(1)
+ while c!="" :
+ c = c.decode('utf-8')
+ out(toStdErr, c)
+ if c == "\n":
+ flush(toStdErr)
+ c = src.read(1)
+ flush(toStdErr)
+ src.close()
+
+
def runProcess(commandline):
"""
Run a process
:param commandline: command line
:return:the return code
"""
- print "ready to exec : %s" % commandline
+ global finished
+ debug ("Executing : %s" % commandline)
exe = subprocess.Popen(commandline,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
- shell=False)
- stdout = exe.stdout
- stderr = exe.stderr
- outline = ""
- errline = ""
+ shell=False,
+ bufsize=1,
+ close_fds=ON_POSIX)
+
+ t = Thread(target=print_output, args=("stdout", exe.stdout, False))
+ t.daemon = True
+ t.start()
+ t2 = Thread(target=print_output, args=("stderr", exe.stderr, True))
+ t2.daemon = True
+ t2.start()
+
+ debug("Waiting for completion")
while exe.poll() is None:
# process is running; grab output and echo every line
- outline, done = read(stdout, outline)
- if done:
- print outline
- outline = ""
- errline, done = read(stderr, errline)
- if done:
- print errline
- errline = ""
-
- # get tail
- out, err = exe.communicate()
- print outline + out.decode()
- print errline + err.decode()
+ time.sleep(1)
+ debug("completed with exit code : %d" % exe.returncode)
+ finished = True
+ t.join()
+ t2.join()
return exe.returncode
+def is_exe(fpath):
+ return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+def which(program):
+
+ fpath, fname = os.path.split(program)
+ if fpath:
+ if is_exe(program):
+ return program
+ else:
+ for path in os.environ["PATH"].split(os.pathsep):
+ path = path.strip('"')
+ exe_file = os.path.join(path, program)
+ if is_exe(exe_file):
+ return exe_file
+
+ return None
+
def java(classname, args, classpath, jvm_opts_list):
"""
Execute a java process, hooking up stdout and stderr
@@ -131,7 +247,12 @@
"""
# split the JVM opts by space
# java = "/usr/bin/java"
- commandline = ["java"]
+ if os.environ["JAVA_HOME"] is not None and os.environ["JAVA_HOME"]:
+ prg = os.path.join(os.environ["JAVA_HOME"], "bin", "java")
+ else:
+ prg = which("java")
+
+ commandline = [prg]
commandline.extend(jvm_opts_list)
commandline.append("-classpath")
commandline.append(classpath)
@@ -140,42 +261,47 @@
return runProcess(commandline)
-def usage():
- print "Usage: slider <action> <arguments>"
- return 1
-
-
def main():
"""
Slider main method
:return: exit code of the process
"""
- if len(sys.argv)==1 :
- return usage()
- # print "stdout encoding: "+ sys.stdout.encoding
args = sys.argv[1:]
slider_home = sliderDir()
libdir = dirMustExist(libDir(slider_home))
confdir = dirMustExist(confDir(slider_home))
- default_jvm_opts = DEFAULT_JVM__OPTS % confdir
+ executeEnvSh(confdir)
+
+ #create sys property for conf dirs
+ jvm_opts_list = (SLIDER_CONFDIR_OPTS % confdir).split()
+
+ #extend with libdir
+ libdir_jvm_opts = (SLIDER_LIBDIR_OPTS % libdir)
+ jvm_opts_list.extend(libdir_jvm_opts.split())
+
+ #append user specified additional properties
+ default_jvm_opts = DEFAULT_JVM_OPTS
slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
- jvm_opts_split = slider_jvm_opts.split()
+ jvm_opts_list.extend(slider_jvm_opts.split())
+
slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
+ hadoop_conf_dir = os.environ.get(HADOOP_CONF_DIR, "")
p = os.pathsep # path separator
d = os.sep # dir separator
slider_classpath = libdir + d + "*" + p \
+ confdir + p \
- + slider_classpath_extra
-
+ + slider_classpath_extra + p \
+ + hadoop_conf_dir
- print "slider_home = \"%s\"" % slider_home
- print "slider_jvm_opts = \"%s\"" % slider_jvm_opts
- print "slider_classpath = \"%s\"" % slider_classpath
+
+ debug("slider_home = \"%s\"" % slider_home)
+ debug("slider_jvm_opts = \"%s\"" % slider_jvm_opts)
+ debug("slider_classpath = \"%s\"" % slider_classpath)
return java(SLIDER_CLASSNAME,
args,
slider_classpath,
- jvm_opts_split)
+ jvm_opts_list)
if __name__ == '__main__':
"""
@@ -184,7 +310,7 @@
try:
returncode = main()
except Exception as e:
- print "Exception: %s " % e.message
+ print "Exception: %s " % str(e)
returncode = -1
sys.exit(returncode)
diff --git a/slider-assembly/src/test/python/scripts/TestSlider.py b/slider-assembly/src/test/python/scripts/TestSlider.py
new file mode 100644
index 0000000..0ed5386
--- /dev/null
+++ b/slider-assembly/src/test/python/scripts/TestSlider.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import StringIO
+import sys
+
+from mock import MagicMock, patch, ANY
+import unittest
+import logging
+import slider
+import os
+import platform
+
+IS_WINDOWS = platform.system() == "Windows"
+
+logger = logging.getLogger()
+
+class TestSlider(unittest.TestCase):
+
+ @patch("os.environ.get")
+ @patch.object(slider, "confDir")
+ @patch.object(slider, "libDir")
+ @patch.object(slider, "executeEnvSh")
+ @patch("os.path.exists")
+ @patch.object(slider, "java")
+ def test_main(self, java_mock, exists_mock, executeEnvSh_mock, libDir_mock, confDir_mock, os_env_get_mock):
+ sys.argv = ["slider", "list"]
+ exists_mock.return_value = True
+ libDir_mock.return_value = "/dir/libdir"
+ confDir_mock.return_value = "/dir/confdir"
+ os_env_get_mock.return_value = "env_val"
+ slider.main()
+ self.assertTrue(java_mock.called)
+ if IS_WINDOWS:
+ java_mock.assert_called_with(
+ 'org.apache.slider.Slider',
+ ['list'],
+ '/dir/libdir\\*;/dir/confdir;env_val;env_val',
+ ['-Dslider.confdir=/dir/confdir', '-Dslider.libdir=/dir/libdir', 'env_val'])
+ else:
+ java_mock.assert_called_with(
+ 'org.apache.slider.Slider',
+ ['list'],
+ '/dir/libdir/*:/dir/confdir:env_val:env_val',
+ ['-Dslider.confdir=/dir/confdir', '-Dslider.libdir=/dir/libdir', 'env_val'])
+ pass
+
+
+if __name__ == "__main__":
+ logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
+ unittest.main()
\ No newline at end of file
diff --git a/slider-assembly/src/test/python/unitTests.py b/slider-assembly/src/test/python/unitTests.py
new file mode 100644
index 0000000..ec1a78c
--- /dev/null
+++ b/slider-assembly/src/test/python/unitTests.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import unittest
+from os.path import isdir
+import logging
+import os
+import sys
+
+LOG_FILE_NAME='tests.log'
+SELECTED_PREFIX = "_"
+PY_EXT='.py'
+ignoredDirs = ["mock"]
+
+class TestAgent(unittest.TestSuite):
+ def run(self, result, debug=False):
+ run = unittest.TestSuite.run
+ run(self, result)
+ return result
+
+
+def parent_dir(path):
+ if isdir(path):
+ if path.endswith(os.sep):
+ path = os.path.dirname(path)
+ parent = os.path.dirname(path)
+ else:
+ parent = os.path.dirname(os.path.dirname(path))
+
+ return parent
+
+
+def all_tests_suite():
+ root_dir = os.getcwd()
+ files_list = []
+ for directory in os.listdir(root_dir):
+ if os.path.isdir(directory) and not directory in ignoredDirs:
+ files_list += os.listdir(root_dir + os.sep + directory)
+ ## temporarily deleting to add more predictability
+ ## shuffle(files_list)
+ files_list.sort()
+ tests_list = []
+
+ logger.info('------------------------TESTS LIST:-------------------------------------')
+ # If test with special name exists, run only this test
+ selected_test = None
+ for file_name in files_list:
+ if file_name.endswith(PY_EXT) and not file_name == __file__ and file_name.startswith(SELECTED_PREFIX):
+ logger.info("Running only selected test " + str(file_name))
+ selected_test = file_name
+ if selected_test is not None:
+ tests_list.append(selected_test.replace(PY_EXT, ''))
+ else:
+ for file_name in files_list:
+ if file_name.endswith(PY_EXT) and not file_name == __file__:
+ replaced = file_name.replace(PY_EXT, '')
+ logger.info(replaced)
+ tests_list.append(replaced)
+ logger.info('------------------------------------------------------------------------')
+
+ suite = unittest.TestLoader().loadTestsFromNames(tests_list)
+ return TestAgent([suite])
+
+def main():
+
+ logger.info('------------------------------------------------------------------------')
+ logger.info('PYTHON SCRIPT TESTS')
+ logger.info('------------------------------------------------------------------------')
+ runner = unittest.TextTestRunner(verbosity=2, stream=sys.stdout)
+ suite = all_tests_suite()
+ status = runner.run(suite).wasSuccessful()
+
+ if not status:
+ logger.error('-----------------------------------------------------------------------')
+ logger.error('Python unit tests failed')
+ logger.error('Find detailed logs in ' + path)
+ logger.error('-----------------------------------------------------------------------')
+ exit(1)
+ else:
+ logger.info('------------------------------------------------------------------------')
+ logger.info('Python unit tests finished succesfully')
+ logger.info('------------------------------------------------------------------------')
+
+if __name__ == '__main__':
+
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python')
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python' + os.sep + 'agent')
+ logger = logging.getLogger()
+ logger.setLevel(logging.INFO)
+ formatter = logging.Formatter("[%(levelname)s] %(message)s")
+ src_dir = os.getcwd()
+ target_dir = parent_dir(parent_dir(parent_dir(src_dir))) + os.sep + 'target'
+ if not os.path.exists(target_dir):
+ os.mkdir(target_dir)
+ path = target_dir + os.sep + LOG_FILE_NAME
+ file=open(path, "w")
+ consoleLog = logging.StreamHandler(file)
+ consoleLog.setFormatter(formatter)
+ logger.addHandler(consoleLog)
+ main()
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index b228874..9c616c3 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
</parent>
<build>
@@ -116,6 +116,9 @@
<argLine>${test.argLine}</argLine>
<failIfNoTests>${test.failIfNoTests}</failIfNoTests>
<redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
+ <environmentVariables>
+ <PATH>${test.env.path}</PATH>
+ </environmentVariables>
<systemPropertyVariables>
<java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
<java.awt.headless>true</java.awt.headless>
@@ -188,43 +191,6 @@
<shortRevisionLength>10</shortRevisionLength>
</configuration>
</plugin>
-
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>src/test/python/agent.ini</exclude>
- <exclude>src/test/python/version</exclude>
- <exclude>src/main/resources/webapps/slideram/.keep</exclude>
- <exclude>src/main/resources/webapps/slideragent/.keep</exclude>
- <exclude>src/main/resources/webapps/static/yarn.dt.plugins.js</exclude>
- <!-- jQuery DataTables files (BSD license) -->
- <exclude>src/main/resources/webapps/static/dt-1.9.4/**</exclude>
- <!-- jQuery (MIT license) -->
- <exclude>src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js</exclude>
- <!-- jQuery UI (MIT license) -->
- <exclude>src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js</exclude>
- <exclude>src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css</exclude>
- <!-- jQuery jsTree (MIT license) -->
- <exclude>src/main/resources/webapps/static/jt/jquery.jstree.js</exclude>
- <!-- protobuf generated classes -->
- <exclude>src/main/java/org/apache/slider/api/proto/Messages.java</exclude>
- <exclude>src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java</exclude>
- </excludes>
- </configuration>
- </plugin>
-
</plugins>
</build>
@@ -306,6 +272,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
@@ -377,16 +349,6 @@
</dependency>
<dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-x-discovery</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-x-discovery-server</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
@@ -484,6 +446,41 @@
<artifactId>jetty-sslengine</artifactId>
</dependency>
+ <dependency>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-reflect</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-easymock</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
@@ -527,54 +524,45 @@
</build>
</profile>
- <!--
- a test run, currently hard-coded for stevel's secure
- VM cluster
- -->
+
<profile>
- <id>testrun</id>
+ <id>rat</id>
<build>
<plugins>
+
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>1.2.1</version>
- <configuration>
- <executable>java</executable>
- <arguments>
- <argument>-Xms512m</argument>
- <argument>-Xmx512m</argument>
- <argument>-classpath</argument>
- <classpath/>
- <argument>org.apache.slider.Slider</argument>
- <argument>list</argument>
- </arguments>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
-
- <profile>
- <id>list</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>1.2.1</version>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
<executions>
<execution>
+ <id>check-licenses</id>
<goals>
- <goal>java</goal>
+ <goal>check</goal>
</goals>
</execution>
</executions>
<configuration>
- <mainClass>org.apache.slider.Slider</mainClass>
- <arguments>
- <argument>list</argument>
- </arguments>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>src/test/python/agent.ini</exclude>
+ <exclude>src/test/python/version</exclude>
+ <exclude>src/main/resources/webapps/slideram/.keep</exclude>
+ <exclude>src/main/resources/webapps/slideragent/.keep</exclude>
+ <exclude>src/main/resources/webapps/static/yarn.dt.plugins.js</exclude>
+ <!-- jQuery DataTables files (BSD license) -->
+ <exclude>src/main/resources/webapps/static/dt-1.9.4/**</exclude>
+ <!-- jQuery (MIT license) -->
+ <exclude>src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js</exclude>
+ <!-- jQuery UI (MIT license) -->
+ <exclude>src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js</exclude>
+ <exclude>src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css</exclude>
+ <!-- jQuery jsTree (MIT license) -->
+ <exclude>src/main/resources/webapps/static/jt/jquery.jstree.js</exclude>
+ <!-- protobuf generated classes -->
+ <exclude>src/main/java/org/apache/slider/api/proto/Messages.java</exclude>
+ <exclude>src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java</exclude>
+ </excludes>
</configuration>
</plugin>
</plugins>
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index ad384e2..b360fbe 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -47,7 +47,11 @@
/**
* internal temp directory: {@value}
*/
- String INTERNAL_AM_TMP_DIR = "internal.tmp.dir";
+ String INTERNAL_AM_TMP_DIR = "internal.am.tmp.dir";
+ /**
+ * internal temp directory: {@value}
+ */
+ String INTERNAL_TMP_DIR = "internal.tmp.dir";
/**
* where a snapshot of the original conf dir is: {@value}
*/
@@ -75,10 +79,25 @@
*/
int DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY = 5000;
/**
+ * Time in seconds before a container is considered long-lived.
+ * Shortlived containers are interpreted as a problem with the role
+ * and/or the host: {@value}
+ */
+ String INTERNAL_CONTAINER_FAILURE_SHORTLIFE =
+ "internal.container.failure.shortlife";
+ /**
+ * Default short life threshold: {@value}
+ */
+ int DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE = 60;
+ /**
* Version of the app: {@value}
*/
String KEYTAB_LOCATION = "internal.keytab.location";
+ /**
+ * Queue used to deploy the app: {@value}
+ */
+ String INTERNAL_QUEUE = "internal.queue";
/**
* Flag to indicate whether or not the chaos monkey is enabled:
@@ -102,6 +121,14 @@
int DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0;
int DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0;
+ String CHAOS_MONKEY_DELAY = "internal.chaos.monkey.delay";
+ String CHAOS_MONKEY_DELAY_DAYS = CHAOS_MONKEY_DELAY + ".days";
+ String CHAOS_MONKEY_DELAY_HOURS = CHAOS_MONKEY_DELAY + ".hours";
+ String CHAOS_MONKEY_DELAY_MINUTES = CHAOS_MONKEY_DELAY + ".minutes";
+ String CHAOS_MONKEY_DELAY_SECONDS = CHAOS_MONKEY_DELAY + ".seconds";
+
+ int DEFAULT_CHAOS_MONKEY_STARTUP_DELAY = 0;
+
/**
* Prefix for all chaos monkey probabilities
*/
@@ -114,7 +141,8 @@
/**
* Probability of a monkey check killing the AM: {@value}
*/
- String CHAOS_MONKEY_PROBABILITY_AM_FAILURE = CHAOS_MONKEY_PROBABILITY +".amfailure";
+ String CHAOS_MONKEY_PROBABILITY_AM_FAILURE =
+ CHAOS_MONKEY_PROBABILITY + ".amfailure";
/**
* Default probability of a monkey check killing the AM: {@value}
@@ -122,6 +150,12 @@
int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 0;
/**
+ * Probability of a monkey check killing the AM: {@value}
+ */
+ String CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE =
+ CHAOS_MONKEY_PROBABILITY + ".amlaunchfailure";
+
+ /**
* Probability of a monkey check killing a container: {@value}
*/
@@ -134,4 +168,14 @@
int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 0;
+ /**
+ * 1% of chaos
+ */
+ int PROBABILITY_PERCENT_1 = 100;
+
+ /**
+ * 100% for chaos values
+ */
+ int PROBABILITY_PERCENT_100 = 100 * PROBABILITY_PERCENT_1;
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index 3d54140..538189a 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -65,10 +65,21 @@
* {@value}
*/
String YARN_CORES = "yarn.vcores";
-
+
/** {@value} */
int DEF_YARN_CORES = 1;
-
+
+
+ /**
+ * Label expression that this container must satisfy
+ * {@value}
+ */
+ String YARN_LABEL_EXPRESSION = "yarn.label.expression";
+
+ /** {@value} */
+ String DEF_YARN_LABEL_EXPRESSION = null;
+
+
/**
* Constant to indicate that the requirements of a YARN resource limit
* (cores, memory, ...) should be set to the maximum allowed by
@@ -92,19 +103,6 @@
/**
- * Time in seconds before a container is considered long-lived.
- * Shortlived containers are interpreted as a problem with the role
- * and/or the host: {@value}
- */
- String CONTAINER_FAILURE_SHORTLIFE =
- "container.failure.shortlife";
-
- /**
- * Default short life threshold: {@value}
- */
- int DEFAULT_CONTAINER_FAILURE_SHORTLIFE = 60;
-
- /**
* maximum number of failed containers (in a single role)
* before the cluster is deemed to have failed {@value}
*/
@@ -131,4 +129,9 @@
*/
int DEFAULT_CONTAINER_FAILURE_THRESHOLD = 5;
+ /**
+ * Log aggregation include, exclude patterns
+ */
+ String YARN_LOG_INCLUDE_PATTERNS = "yarn.log.include.patterns";
+ String YARN_LOG_EXCLUDE_PATTERNS = "yarn.log.exclude.patterns";
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java b/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
index 4bfcf41..4e46605 100644
--- a/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
@@ -67,7 +67,8 @@
String INFO_AM_RPC_PORT = "info.am.rpc.port";
String INFO_AM_WEB_PORT = "info.am.web.port";
String INFO_AM_WEB_URL = "info.am.web.url";
- String INFO_AM_AGENT_PORT = "info.am.agent.port";
- String INFO_AM_AGENT_URL = "info.am.agent.url";
- String INFO_AM_SECURED_AGENT_PORT = "info.am.agent.secured.port";
+ String INFO_AM_AGENT_STATUS_PORT = "info.am.agent.status.port";
+ String INFO_AM_AGENT_OPS_PORT = "info.am.agent.ops.port";
+ String INFO_AM_AGENT_OPS_URL = "info.am.agent.ops.url";
+ String INFO_AM_AGENT_STATUS_URL = "info.am.agent.status.url";
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
index 250e5ff..9174f88 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: SliderClusterMessages.proto
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java b/slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
index c00f99d..75bf05b 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
@@ -1,6 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: SliderClusterProtocol.proto
-
package org.apache.slider.api.proto;
public final class SliderClusterAPI {
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 93f6207..ee5be65 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -19,20 +19,46 @@
package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.*;
+
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.exceptions.NoRecordException;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
@@ -43,19 +69,28 @@
import org.apache.slider.common.Constants;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionDiagnosticArgs;
+import org.apache.slider.common.params.ActionExistsArgs;
+import org.apache.slider.common.params.ActionInstallKeytabArgs;
+import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionGetConfArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
+import org.apache.slider.common.params.ActionListArgs;
+import org.apache.slider.common.params.ActionLookupArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.params.ClientArgs;
+import org.apache.slider.common.params.CommonArgs;
import org.apache.slider.common.params.LaunchArgsAccessor;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.Duration;
@@ -75,6 +110,7 @@
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.exceptions.NotFoundException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
import org.apache.slider.core.exceptions.WaitTimeoutException;
@@ -84,16 +120,20 @@
import org.apache.slider.core.launch.JavaCommandLineBuilder;
import org.apache.slider.core.launch.LaunchedApplication;
import org.apache.slider.core.launch.RunningApplication;
+import org.apache.slider.core.launch.SerializedApplicationReport;
import org.apache.slider.core.main.RunService;
+import org.apache.slider.core.persist.ApplicationReportSerDeser;
import org.apache.slider.core.persist.ConfPersister;
import org.apache.slider.core.persist.LockAcquireFailedException;
-import org.apache.slider.core.registry.YARNRegistryClient;
+import org.apache.slider.core.registry.SliderRegistryUtils;
+import org.apache.slider.core.registry.YarnAppListClient;
import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.docstore.PublishedExportsOutputter;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import org.apache.slider.core.registry.retrieve.RegistryRetriever;
import org.apache.slider.core.zk.BlockingZKWatcher;
import org.apache.slider.core.zk.ZKIntegration;
@@ -104,39 +144,44 @@
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
-import org.apache.slider.server.services.curator.CuratorServiceInstance;
-import org.apache.slider.server.services.registry.SliderRegistryService;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
-
-import static org.apache.slider.common.params.SliderActions.*;
-
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.PrintStream;
import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
-import java.util.Properties;
+import java.util.Map.Entry;
import java.util.Set;
+import java.util.regex.Pattern;
+
+import static org.apache.slider.common.params.SliderActions.*;
/**
* Client service for Slider
*/
public class SliderClient extends AbstractSliderLaunchedService implements RunService,
- SliderExitCodes, SliderKeys, ErrorStrings {
+ SliderExitCodes, SliderKeys, ErrorStrings, SliderClientAPI {
private static final Logger log = LoggerFactory.getLogger(SliderClient.class);
private ClientArgs serviceArgs;
@@ -149,15 +194,19 @@
*/
private SliderClusterOperations sliderClusterOperations;
- private SliderFileSystem sliderFileSystem;
+ protected SliderFileSystem sliderFileSystem;
/**
* Yarn client service
*/
private SliderYarnClientImpl yarnClient;
- private YARNRegistryClient YARNRegistryClient;
+ private YarnAppListClient YarnAppListClient;
private AggregateConf launchedInstanceDefinition;
- private SliderRegistryService registry;
+
+ /**
+ * The YARN registry service
+ */
+ private RegistryOperations registryOperations;
/**
* Constructor
@@ -168,6 +217,16 @@
new YarnConfiguration();
}
+ /**
+ * This is called <i>Before serviceInit is called</i>
+ * @param config the initial configuration build up by the
+ * service launcher.
+ * @param args argument list list of arguments passed to the command line
+ * after any launcher-specific commands have been stripped.
+ * @return the post-binding configuration to pass to the <code>init()</code>
+ * operation.
+ * @throws Exception
+ */
@Override
public Configuration bindArgs(Configuration config, String... args) throws Exception {
config = super.bindArgs(config, args);
@@ -181,7 +240,7 @@
@Override
protected void serviceInit(Configuration conf) throws Exception {
Configuration clientConf = SliderUtils.loadClientConfigurationResource();
- ConfigHelper.mergeConfigurations(conf, clientConf, CLIENT_RESOURCE);
+ ConfigHelper.mergeConfigurations(conf, clientConf, CLIENT_RESOURCE, true);
serviceArgs.applyDefinitions(conf);
serviceArgs.applyFileSystemBinding(conf);
// init security with our conf
@@ -189,16 +248,11 @@
SliderUtils.forceLogin();
SliderUtils.initProcessSecurity(conf);
}
- //create the YARN client
- yarnClient = new SliderYarnClientImpl();
- addService(yarnClient);
-
+ AbstractActionArgs coreAction = serviceArgs.getCoreAction();
+ if (coreAction.getHadoopServicesRequired()) {
+ initHadoopBinding();
+ }
super.serviceInit(conf);
-
- //here the superclass is inited; getConfig returns a non-null value
- sliderFileSystem = new SliderFileSystem(getConfig());
- YARNRegistryClient =
- new YARNRegistryClient(yarnClient, getUsername(), getConfig());
}
/**
@@ -259,7 +313,7 @@
serviceArgs.getActionAMSuicideArgs());
break;
case ACTION_LIST:
- exitCode = actionList(clusterName);
+ exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
break;
case ACTION_REGISTRY:
exitCode = actionRegistry(
@@ -281,58 +335,85 @@
}
*/
+
+ /**
+ * Launched service execution. This runs {@link #exec()}
+ * then catches some exceptions and converts them to exit codes
+ * @return an exit code
+ * @throws Throwable
+ */
@Override
public int runService() throws Throwable {
+ try {
+ return exec();
+ } catch (FileNotFoundException nfe) {
+ throw new NotFoundException(nfe, nfe.toString());
+ } catch (PathNotFoundException nfe) {
+ throw new NotFoundException(nfe, nfe.toString());
+ }
+ }
+
+ /**
+ * Execute the command line
+ * @return an exit code
+ * @throws Throwable on a failure
+ */
+ public int exec() throws Throwable {
// choose the action
String action = serviceArgs.getAction();
+
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
// actions
- if (ACTION_BUILD.equals(action)) {
+ if (ACTION_INSTALL_PACKAGE.equals(action)) {
+ exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs());
+ } else if (ACTION_INSTALL_KEYTAB.equals(action)) {
+ exitCode = actionInstallKeytab(serviceArgs.getActionInstallKeytabArgs());
+ } else if (ACTION_BUILD.equals(action)) {
exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs());
} else if (ACTION_CREATE.equals(action)) {
exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs());
} else if (ACTION_FREEZE.equals(action)) {
exitCode = actionFreeze(clusterName,
- serviceArgs.getActionFreezeArgs());
+ serviceArgs.getActionFreezeArgs());
} else if (ACTION_THAW.equals(action)) {
exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
} else if (ACTION_DESTROY.equals(action)) {
exitCode = actionDestroy(clusterName);
+ } else if (ACTION_DIAGNOSTICS.equals(action)) {
+ exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
} else if (ACTION_EXISTS.equals(action)) {
exitCode = actionExists(clusterName,
- serviceArgs.getActionExistsArgs().live);
+ serviceArgs.getActionExistsArgs());
} else if (ACTION_FLEX.equals(action)) {
exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
- } else if (ACTION_GETCONF.equals(action)) {
- exitCode = actionGetConf(clusterName, serviceArgs.getActionGetConfArgs());
- } else if (ACTION_HELP.equals(action) ||
- ACTION_USAGE.equals(action)) {
+ } else if (ACTION_HELP.equals(action)) {
log.info(serviceArgs.usage());
-
} else if (ACTION_KILL_CONTAINER.equals(action)) {
exitCode = actionKillContainer(clusterName,
serviceArgs.getActionKillContainerArgs());
-
} else if (ACTION_AM_SUICIDE.equals(action)) {
exitCode = actionAmSuicide(clusterName,
serviceArgs.getActionAMSuicideArgs());
-
} else if (ACTION_LIST.equals(action)) {
- exitCode = actionList(clusterName);
+ exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
+ } else if (ACTION_LOOKUP.equals(action)) {
+ exitCode = actionLookup(serviceArgs.getActionLookupArgs());
} else if (ACTION_REGISTRY.equals(action)) {
- exitCode = actionRegistry(
- serviceArgs.getActionRegistryArgs());
+ exitCode = actionRegistry(serviceArgs.getActionRegistryArgs());
+ } else if (ACTION_RESOLVE.equals(action)) {
+ exitCode = actionResolve(serviceArgs.getActionResolveArgs());
} else if (ACTION_STATUS.equals(action)) {
exitCode = actionStatus(clusterName,
serviceArgs.getActionStatusArgs());
} else if (ACTION_UPDATE.equals(action)) {
exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
-
} else if (ACTION_VERSION.equals(action)) {
-
exitCode = actionVersion();
+ } else if (SliderUtils.isUnset(action)) {
+ throw new SliderException(EXIT_USAGE,
+ serviceArgs.usage());
} else {
throw new SliderException(EXIT_UNIMPLEMENTED,
"Unimplemented: " + action);
@@ -340,26 +421,47 @@
return exitCode;
}
+
+/**
+ * Perform everything needed to init the hadoop binding.
+ * This assumes that the service is already in inited or started state
+ * @throws IOException
+ * @throws SliderException
+ */
+ protected void initHadoopBinding() throws IOException, SliderException {
+ // validate the client
+ SliderUtils.validateSliderClientEnvironment(null);
+ //create the YARN client
+ yarnClient = new SliderYarnClientImpl();
+ yarnClient.init(getConfig());
+ if (getServiceState() == STATE.STARTED) {
+ yarnClient.start();
+ }
+ addService(yarnClient);
+ YarnAppListClient =
+ new YarnAppListClient(yarnClient, getUsername(), getConfig());
+ // create the filesystem
+ sliderFileSystem = new SliderFileSystem(getConfig());
+ }
+
/**
* Delete the zookeeper node associated with the calling user and the cluster
+ * TODO: YARN registry operations
**/
- protected boolean deleteZookeeperNode(String clusterName) throws YarnException, IOException {
+ @VisibleForTesting
+ public boolean deleteZookeeperNode(String clusterName) throws YarnException, IOException {
String user = getUsername();
String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
Exception e = null;
try {
Configuration config = getConfig();
- if (!SliderUtils.isHadoopClusterSecure(config)) {
- ZKIntegration client = getZkClient(clusterName, user);
- if (client != null) {
- if (client.exists(zkPath)) {
- log.info("Deleting zookeeper path {}", zkPath);
- }
- client.deleteRecursive(zkPath);
- return true;
+ ZKIntegration client = getZkClient(clusterName, user);
+ if (client != null) {
+ if (client.exists(zkPath)) {
+ log.info("Deleting zookeeper path {}", zkPath);
}
- } else {
- log.warn("Default zookeeper node is not available for secure cluster");
+ client.deleteRecursive(zkPath);
+ return true;
}
} catch (InterruptedException ignored) {
e = ignored;
@@ -369,7 +471,7 @@
e = ignored;
}
if (e != null) {
- log.warn("Unable to recursively delete zk node {}", zkPath);
+ log.debug("Unable to recursively delete zk node {}", zkPath);
log.debug("Reason: ", e);
}
@@ -379,28 +481,24 @@
/**
* Create the zookeeper node associated with the calling user and the cluster
*/
- protected String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException {
+ @VisibleForTesting
+ public String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException {
String user = getUsername();
String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
- if(nameOnly) {
+ if (nameOnly) {
return zkPath;
}
Configuration config = getConfig();
- if (!SliderUtils.isHadoopClusterSecure(config)) {
- ZKIntegration client = getZkClient(clusterName, user);
- if (client != null) {
- try {
- client.createPath(zkPath, "", ZooDefs.Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
- return zkPath;
-
- //JDK7
-// } catch (InterruptedException | KeeperException e) {
- } catch (InterruptedException e) {
- log.warn("Unable to create zk node {}", zkPath, e);
- } catch ( KeeperException e) {
- log.warn("Unable to create zk node {}", zkPath, e);
- }
+ ZKIntegration client = getZkClient(clusterName, user);
+ if (client != null) {
+ try {
+ client.createPath(zkPath, "", ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ return zkPath;
+ } catch (InterruptedException e) {
+ log.warn("Unable to create default zk node {}", zkPath, e);
+ } catch (KeeperException e) {
+ log.warn("Unable to create default zk node {}", zkPath, e);
}
}
@@ -427,36 +525,49 @@
return client;
}
- /**
- * Destroy a cluster. There's two race conditions here
- * #1 the cluster is started between verifying that there are no live
- * clusters of that name.
- */
+ @Override
public int actionDestroy(String clustername) throws YarnException,
IOException {
// verify that a live cluster isn't there
SliderUtils.validateClusterName(clustername);
//no=op, it is now mandatory.
verifyBindingsDefined();
- verifyNoLiveClusters(clustername);
+ verifyNoLiveClusters(clustername, "Destroy");
// create the directory path
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
// delete the directory;
- boolean exists = sliderFileSystem.getFileSystem().exists(clusterDirectory);
+ FileSystem fs = sliderFileSystem.getFileSystem();
+ boolean exists = fs.exists(clusterDirectory);
if (exists) {
- log.info("Application Instance {} found at {}: destroying", clustername, clusterDirectory);
+ log.debug("Application Instance {} found at {}: destroying", clustername, clusterDirectory);
+ boolean deleted =
+ fs.delete(clusterDirectory, true);
+ if (!deleted) {
+ log.warn("Filesystem returned false from delete() operation");
+ }
+
+ if(!deleteZookeeperNode(clustername)) {
+ log.warn("Unable to perform node cleanup in Zookeeper.");
+ }
+
+ if (fs.exists(clusterDirectory)) {
+ log.warn("Failed to delete {}", clusterDirectory);
+ }
+
} else {
- log.info("Application Instance {} already destroyed", clustername);
- }
- boolean deleted =
- sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
- if (!deleted) {
- log.warn("Filesystem returned false from delete() operation");
+ log.debug("Application Instance {} already destroyed", clustername);
}
- if(!deleteZookeeperNode(clustername)) {
- log.warn("Unable to perform node cleanup in Zookeeper.");
+ // rm the registry entry —do not let this block the destroy operations
+ String registryPath = SliderRegistryUtils.registryPathForInstance(
+ clustername);
+ try {
+ getRegistryOperations().delete(registryPath, true);
+ } catch (IOException e) {
+ log.warn("Error deleting registry entry {}: {} ", registryPath, e, e);
+ } catch (SliderException e) {
+ log.warn("Error binding to registry {} ", e, e);
}
List<ApplicationReport> instances = findAllLiveInstances(clustername);
@@ -473,26 +584,17 @@
return EXIT_SUCCESS;
}
- /**
- * AM to commit an asynchronous suicide
- */
+ @Override
public int actionAmSuicide(String clustername,
- ActionAMSuicideArgs args) throws
- YarnException,
- IOException {
+ ActionAMSuicideArgs args) throws YarnException, IOException {
SliderClusterOperations clusterOperations =
createClusterOperations(clustername);
clusterOperations.amSuicide(args.message, args.exitcode, args.waittime);
return EXIT_SUCCESS;
}
- /**
- * Get the provider for this cluster
- * @param provider the name of the provider
- * @return the provider instance
- * @throws SliderException problems building the provider
- */
- private AbstractClientProvider createClientProvider(String provider)
+ @Override
+ public AbstractClientProvider createClientProvider(String provider)
throws SliderException {
SliderProviderFactory factory =
SliderProviderFactory.createSliderProviderFactory(provider);
@@ -512,20 +614,48 @@
IOException {
actionBuild(clustername, createArgs);
+ Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
+ AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
+ clustername, clusterDirectory);
+ try {
+ checkForCredentials(getConfig(), instanceDefinition.getAppConf());
+ } catch (IOException e) {
+ sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
+ throw e;
+ }
return startCluster(clustername, createArgs);
}
- /**
- * Build up the cluster specification/directory
- *
- * @param clustername cluster name
- * @param buildInfo the arguments needed to build the cluster
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- * @throws BadCommandArgumentsException bad arguments.
- */
+ private void checkForCredentials(Configuration conf,
+ ConfTree tree) throws IOException {
+ if (tree.credentials == null || tree.credentials.size()==0) {
+ log.info("No credentials requested");
+ return;
+ }
+
+ for (Entry<String, List<String>> cred : tree.credentials.entrySet()) {
+ String provider = cred.getKey();
+ List<String> aliases = cred.getValue();
+ if (aliases == null || aliases.size()==0) {
+ continue;
+ }
+ Configuration c = new Configuration(conf);
+ c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
+ CredentialProvider credentialProvider =
+ CredentialProviderFactory.getProviders(c).get(0);
+ Set<String> existingAliases = new HashSet<String>(credentialProvider.getAliases());
+ for (String alias : aliases) {
+ if (!existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
+ throw new IOException("Specified credentials have not been " +
+ "initialized in provider " + provider + ": " + alias);
+ }
+ }
+ }
+ }
+
+ @Override
public int actionBuild(String clustername,
- AbstractClusterBuildingActionArgs buildInfo) throws
+ AbstractClusterBuildingActionArgs buildInfo) throws
YarnException,
IOException {
@@ -533,15 +663,91 @@
return EXIT_SUCCESS;
}
- /**
- * Update the cluster specification
- *
- * @param clustername cluster name
- * @param buildInfo the arguments needed to update the cluster
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- */
- public int actionUpdate(String clustername, AbstractClusterBuildingActionArgs buildInfo) throws
+ @Override
+ public int actionInstallKeytab(ActionInstallKeytabArgs installKeytabInfo)
+ throws YarnException, IOException {
+
+ Path srcFile = null;
+ if (StringUtils.isEmpty(installKeytabInfo.folder)) {
+ throw new BadCommandArgumentsException(
+ "A valid destination keytab sub-folder name is required (e.g. 'security').\n"
+ + CommonArgs.usage(serviceArgs, ACTION_INSTALL_KEYTAB));
+ }
+
+ if (StringUtils.isEmpty(installKeytabInfo.keytabUri)) {
+ throw new BadCommandArgumentsException("A valid local keytab location is required.");
+ } else {
+ File keytabFile = new File(installKeytabInfo.keytabUri);
+ if (!keytabFile.exists() || keytabFile.isDirectory()) {
+ throw new BadCommandArgumentsException("Unable to access supplied keytab file at " +
+ keytabFile.getAbsolutePath());
+ } else {
+ srcFile = new Path(keytabFile.toURI());
+ }
+ }
+
+ Path pkgPath = sliderFileSystem.buildKeytabInstallationDirPath(installKeytabInfo.folder);
+ sliderFileSystem.getFileSystem().mkdirs(pkgPath);
+ sliderFileSystem.getFileSystem().setPermission(pkgPath, new FsPermission(
+ FsAction.ALL, FsAction.NONE, FsAction.NONE));
+
+ Path fileInFs = new Path(pkgPath, srcFile.getName());
+ log.info("Installing keytab {} at {} and overwrite is {}.", srcFile, fileInFs, installKeytabInfo.overwrite);
+ if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installKeytabInfo.overwrite) {
+ throw new BadCommandArgumentsException("Keytab exists at " +
+ fileInFs.toUri().toString() +
+ ". Use --overwrite to overwrite.");
+ }
+
+ sliderFileSystem.getFileSystem().copyFromLocalFile(false, installKeytabInfo.overwrite, srcFile, fileInFs);
+ sliderFileSystem.getFileSystem().setPermission(fileInFs, new FsPermission(
+ FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
+
+ return EXIT_SUCCESS;
+ }
+
+ @Override
+ public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws
+ YarnException,
+ IOException {
+
+ Path srcFile = null;
+ if (StringUtils.isEmpty(installPkgInfo.name)) {
+ throw new BadCommandArgumentsException(
+ "A valid application type name is required (e.g. HBASE).\n"
+ + CommonArgs.usage(serviceArgs, ACTION_INSTALL_PACKAGE));
+ }
+
+ if (StringUtils.isEmpty(installPkgInfo.packageURI)) {
+ throw new BadCommandArgumentsException("A valid application package location required.");
+ } else {
+ File pkgFile = new File(installPkgInfo.packageURI);
+ if (!pkgFile.exists() || pkgFile.isDirectory()) {
+ throw new BadCommandArgumentsException("Unable to access supplied pkg file at " +
+ pkgFile.getAbsolutePath());
+ } else {
+ srcFile = new Path(pkgFile.toURI());
+ }
+ }
+
+ Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name);
+ sliderFileSystem.getFileSystem().mkdirs(pkgPath);
+
+ Path fileInFs = new Path(pkgPath, srcFile.getName());
+ log.info("Installing package {} at {} and overwrite is {}.", srcFile, fileInFs, installPkgInfo.replacePkg);
+ if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installPkgInfo.replacePkg) {
+ throw new BadCommandArgumentsException("Pkg exists at " +
+ fileInFs.toUri().toString() +
+ ". Use --replacepkg to overwrite.");
+ }
+
+ sliderFileSystem.getFileSystem().copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
+ return EXIT_SUCCESS;
+ }
+
+ @Override
+ public int actionUpdate(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo) throws
YarnException, IOException {
buildInstanceDefinition(clustername, buildInfo, true, true);
return EXIT_SUCCESS;
@@ -564,7 +770,9 @@
// verify that a live cluster isn't there
SliderUtils.validateClusterName(clustername);
verifyBindingsDefined();
- if (!liveClusterAllowed) verifyNoLiveClusters(clustername);
+ if (!liveClusterAllowed) {
+ verifyNoLiveClusters(clustername, "Create");
+ }
Configuration conf = getConfig();
String registryQuorum = lookupZKQuorum();
@@ -660,8 +868,9 @@
builder.init(providerName, instanceDefinition);
builder.propagateFilename();
builder.propagatePrincipals();
- builder.setImageDetails(buildInfo.getImage(), buildInfo.getAppHomeDir());
-
+ builder.setImageDetailsIfAvailable(buildInfo.getImage(),
+ buildInfo.getAppHomeDir());
+ builder.setQueue(buildInfo.queue);
String quorum = buildInfo.getZKhosts();
if (SliderUtils.isUnset(quorum)) {
@@ -683,14 +892,14 @@
String createDefaultZkNode = appConf.getGlobalOptions().getOption(AgentKeys.CREATE_DEF_ZK_NODE, "false");
if (createDefaultZkNode.equals("true")) {
String defaultZKPath = createZookeeperNode(clustername, false);
- log.info("ZK node created for application instance: {}.", defaultZKPath);
+ log.debug("ZK node created for application instance: {}", defaultZKPath);
if (defaultZKPath != null) {
zkPaths.setAppPath(defaultZKPath);
}
} else {
// create AppPath if default is being used
String defaultZKPath = createZookeeperNode(clustername, true);
- log.info("ZK node assigned to application instance: {}.", defaultZKPath);
+ log.debug("ZK node assigned to application instance: {}", defaultZKPath);
zkPaths.setAppPath(defaultZKPath);
}
}
@@ -702,41 +911,77 @@
appConf.set(AgentKeys.PACKAGE_PATH, buildInfo.packageURI);
}
- // provider to validate what there is
+ propagatePythonExecutable(conf, instanceDefinition);
+
+ // make any substitutions needed at this stage
+ replaceTokens(appConf.getConfTree(), getUsername(), clustername);
+
+ // providers to validate what there is
+ AggregateConf instanceDescription = builder.getInstanceDescription();
+ validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem);
+ validateInstanceDefinition(provider, instanceDescription, sliderFileSystem);
try {
- sliderAM.validateInstanceDefinition(builder.getInstanceDescription());
- provider.validateInstanceDefinition(builder.getInstanceDescription());
- } catch (SliderException e) {
- //problem, reject it
- log.info("Error {} validating application instance definition ", e.toString());
- log.debug("Error validating application instance definition ", e);
- log.info(instanceDefinition.toString());
- throw e;
- }
- try {
- builder.persist(appconfdir, overwrite);
+ persistInstanceDefinition(overwrite, appconfdir, builder);
} catch (LockAcquireFailedException e) {
log.warn("Failed to get a Lock on {} : {}", builder, e);
throw new BadClusterStateException("Failed to save " + clustername
+ ": " + e);
}
}
-
+
+ protected void persistInstanceDefinition(boolean overwrite,
+ Path appconfdir,
+ InstanceBuilder builder)
+ throws IOException, SliderException, LockAcquireFailedException {
+ builder.persist(appconfdir, overwrite);
+ }
+
+ @VisibleForTesting
+ public static void replaceTokens(ConfTree conf,
+ String userName, String clusterName) throws IOException {
+ Map<String,String> newglobal = new HashMap<String,String>();
+ for (Entry<String,String> entry : conf.global.entrySet()) {
+ newglobal.put(entry.getKey(), replaceTokens(entry.getValue(),
+ userName, clusterName));
+ }
+ conf.global.putAll(newglobal);
+
+ Map<String,List<String>> newcred = new HashMap<String,List<String>>();
+ for (Entry<String,List<String>> entry : conf.credentials.entrySet()) {
+ List<String> resultList = new ArrayList<String>();
+ for (String v : entry.getValue()) {
+ resultList.add(replaceTokens(v, userName, clusterName));
+ }
+ newcred.put(replaceTokens(entry.getKey(), userName, clusterName),
+ resultList);
+ }
+ conf.credentials.clear();
+ conf.credentials.putAll(newcred);
+ }
+
+ private static String replaceTokens(String s, String userName,
+ String clusterName) throws IOException {
+ return s.replaceAll(Pattern.quote("${USER}"), userName)
+ .replaceAll(Pattern.quote("${USER_NAME}"), userName)
+ .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName);
+ }
+
public FsPermission getClusterDirectoryPermissions(Configuration conf) {
String clusterDirPermsOct =
conf.get(CLUSTER_DIRECTORY_PERMISSIONS,
- DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
+ DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
return new FsPermission(clusterDirPermsOct);
}
/**
- * Verify that the Resource MAnager is configured, if not fail
+ * Verify that the Resource Manager is configured (on a non-HA cluster).
* with a useful error message
* @throws BadCommandArgumentsException the exception raised on an invalid config
*/
public void verifyBindingsDefined() throws BadCommandArgumentsException {
InetSocketAddress rmAddr = SliderUtils.getRmAddress(getConfig());
- if (!SliderUtils.isAddressDefined(rmAddr)) {
+ if (!getConfig().getBoolean(YarnConfiguration.RM_HA_ENABLED, false)
+ && !SliderUtils.isAddressDefined(rmAddr)) {
throw new BadCommandArgumentsException(
"No valid Resource Manager address provided in the argument "
+ Arguments.ARG_MANAGER
@@ -744,7 +989,6 @@
+ YarnConfiguration.RM_ADDRESS
+ " value :" + rmAddr);
}
-
}
/**
@@ -772,7 +1016,22 @@
serviceArgs.isDebug());
applicationId = launchedApplication.getApplicationId();
- return waitForAppAccepted(launchedApplication, launchArgs.getWaittime());
+ if (launchArgs.getOutputFile() != null) {
+ // output file has been requested. Get the app report and serialize it
+ ApplicationReport report =
+ launchedApplication.getApplicationReport();
+ SerializedApplicationReport sar = new SerializedApplicationReport(report);
+ sar.submitTime = System.currentTimeMillis();
+ ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser();
+ serDeser.save(sar, launchArgs.getOutputFile());
+ }
+ int waittime = launchArgs.getWaittime();
+ if (waittime > 0) {
+ return waitForAppRunning(launchedApplication, waittime, waittime);
+ } else {
+ // no waiting
+ return EXIT_SUCCESS;
+ }
}
/**
@@ -793,6 +1052,7 @@
AggregateConf definition =
InstanceIO.loadInstanceDefinitionUnresolved(sliderFileSystem,
clusterDirectory);
+ definition.setName(name);
return definition;
} catch (FileNotFoundException e) {
throw UnknownApplicationInstanceException.unknownInstance(name, e);
@@ -825,10 +1085,10 @@
/**
*
- * @param clustername
- * @param clusterDirectory
- * @param instanceDefinition
- * @param debugAM
+ * @param clustername name of the cluster
+ * @param clusterDirectory cluster dir
+ * @param instanceDefinition the instance definition
+ * @param debugAM enable debug AM options
* @return the launched application
* @throws YarnException
* @throws IOException
@@ -842,7 +1102,7 @@
deployedClusterName = clustername;
SliderUtils.validateClusterName(clustername);
- verifyNoLiveClusters(clustername);
+ verifyNoLiveClusters(clustername, "Launch");
Configuration config = getConfig();
lookupZKQuorum();
boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config);
@@ -873,15 +1133,12 @@
InternalKeys.INTERNAL_PROVIDER_NAME));
// make sure the conf dir is valid;
- // now build up the image path
- // TODO: consider supporting apps that don't have an image path
- Path imagePath =
- SliderUtils.extractImagePath(sliderFileSystem, internalOptions);
if (log.isDebugEnabled()) {
log.debug(instanceDefinition.toString());
}
MapOperations sliderAMResourceComponent =
resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM);
+ MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions();
// add the tags if available
Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
@@ -893,6 +1150,7 @@
yarnClient,
clusterSecure,
sliderAMResourceComponent,
+ resourceGlobalOptions,
applicationTags);
ApplicationId appId = amLauncher.getApplicationId();
@@ -915,7 +1173,9 @@
// local files or archives as needed
// In this scenario, the jar file for the application master is part of the local resources
Map<String, LocalResource> localResources = amLauncher.getLocalResources();
- // conf directory setup
+
+ // look for the configuration directory named on the command line
+ boolean hasServerLog4jProperties = false;
Path remoteConfPath = null;
String relativeConfDir = null;
String confdirProp =
@@ -929,11 +1189,14 @@
confDir);
}
Path localConfDirPath = SliderUtils.createLocalPath(confDir);
- log.debug("Copying AM configuration data from {}", localConfDirPath);
- remoteConfPath = new Path(clusterDirectory,
- SliderKeys.SUBMITTED_CONF_DIR);
- SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath,
- null);
+ remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR);
+ log.debug("Slider configuration directory is {}; remote to be {}",
+ localConfDirPath, remoteConfPath);
+ SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null);
+
+ File log4jserver =
+ new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
+ hasServerLog4jProperties = log4jserver.isFile();
}
// the assumption here is that minimr cluster => this is a test run
// and the classpath can look after itself
@@ -957,6 +1220,15 @@
// will be valid.
propagatePrincipals(config, instanceDefinition);
+ // validate security data
+
+/*
+ // turned off until tested
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config,
+ instanceDefinition, clustername);
+
+*/
Configuration clientConfExtras = new Configuration(false);
// then build up the generated path.
FsPermission clusterPerms = getClusterDirectoryPermissions(config);
@@ -964,8 +1236,6 @@
clusterPerms);
- // add AM and provider specific artifacts to the resource map
- Map<String, LocalResource> providerResources;
// standard AM resources
sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem,
config,
@@ -1013,12 +1283,13 @@
);
- // now add the image if it was set
+ // TODO: consider supporting apps that don't have an image path
+ Path imagePath =
+ SliderUtils.extractImagePath(sliderFileSystem, internalOptions);
if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) {
log.debug("Registered image path {}", imagePath);
}
-
// build the environment
amLauncher.putEnv(
SliderUtils.buildEnvMap(sliderAMResourceComponent));
@@ -1054,6 +1325,13 @@
sliderAM.addJVMOptions(instanceDefinition, commandLine);
// enable asserts if the text option is set
commandLine.enableJavaAssertions();
+
+ // if the conf dir has a log4j-server.properties, switch to that
+ if (hasServerLog4jProperties) {
+ commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME);
+ commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ }
+
// add the AM sevice entry point
commandLine.add(SliderAppMaster.SERVICE_CLASSNAME);
@@ -1075,11 +1353,15 @@
if (serviceArgs.getFilesystemBinding() != null) {
commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding());
}
-
+
+ /**
+ * pass the registry binding
+ */
addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
DEFAULT_REGISTRY_PATH);
- addMandatoryConfOptionToCLI(commandLine, config, REGISTRY_ZK_QUORUM);
-
+ addMandatoryConfOptionToCLI(commandLine, config,
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+
if (clusterSecure) {
// if the cluster is secure, make sure that
// the relevant security settings go over
@@ -1088,7 +1370,7 @@
*/
addConfOptionToCLI(commandLine,
config,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
@@ -1114,50 +1396,66 @@
// Set the queue to which this application is to be submitted in the RM
// Queue for App master
String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE);
+ String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE);
+ if(!SliderUtils.isUnset(suppliedQueue)) {
+ amQueue = suppliedQueue;
+ log.info("Using queue {} for the application instance.", amQueue);
+ }
- amLauncher.setQueue(amQueue);
-
- // Submit the application to the applications manager
- // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
- // Ignore the response as either a valid response object is returned on success
- // or an exception thrown to denote some form of a failure
-
+ if (amQueue != null) {
+ amLauncher.setQueue(amQueue);
+ }
// submit the application
LaunchedApplication launchedApplication = amLauncher.submitApplication();
return launchedApplication;
}
-
-
+
+ private void propagatePythonExecutable(Configuration config,
+ AggregateConf instanceDefinition) {
+ String pythonExec = config.get(
+ SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH);
+ if (pythonExec != null) {
+ instanceDefinition.getAppConfOperations().getGlobalOptions().putIfUnset(
+ SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH,
+ pythonExec);
+ }
+ }
+
+
/**
- * Wait for the launched app to be accepted
- * @param waittime time in millis
- * @return exit code
+ * Wait for the launched app to be accepted in the time
+ * and, optionally running.
+ * <p>
+ * If the application
+ *
+ * @param launchedApplication application
+ * @param acceptWaitMillis time in millis to wait for accept
+ * @param runWaitMillis time in millis to wait for the app to be running.
+ * May be null, in which case no wait takes place
+ * @return exit code: success
* @throws YarnException
* @throws IOException
*/
- public int waitForAppAccepted(LaunchedApplication launchedApplication,
- int waittime) throws
- YarnException,
- IOException {
+ public int waitForAppRunning(LaunchedApplication launchedApplication,
+ int acceptWaitMillis, int runWaitMillis) throws YarnException, IOException {
assert launchedApplication != null;
int exitCode;
// wait for the submit state to be reached
ApplicationReport report = launchedApplication.monitorAppToState(
YarnApplicationState.ACCEPTED,
- new Duration(Constants.ACCEPT_TIME));
-
+ new Duration(acceptWaitMillis));
// may have failed, so check that
if (SliderUtils.hasAppFinished(report)) {
exitCode = buildExitCode(report);
} else {
// exit unless there is a wait
- exitCode = EXIT_SUCCESS;
- if (waittime != 0) {
+
+ if (runWaitMillis != 0) {
// waiting for state to change
- Duration duration = new Duration(waittime * 1000);
+ Duration duration = new Duration(runWaitMillis * 1000);
duration.start();
report = launchedApplication.monitorAppToState(
YarnApplicationState.RUNNING, duration);
@@ -1165,10 +1463,10 @@
report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
exitCode = EXIT_SUCCESS;
} else {
-
- launchedApplication.kill("");
exitCode = buildExitCode(report);
}
+ } else {
+ exitCode = EXIT_SUCCESS;
}
}
return exitCode;
@@ -1182,10 +1480,11 @@
*/
private void propagatePrincipals(Configuration config,
AggregateConf clusterSpec) {
- String dfsPrincipal = config.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ String dfsPrincipal = config.get(
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY;
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset(
siteDfsPrincipal,
dfsPrincipal);
@@ -1197,20 +1496,46 @@
Configuration conf,
String key) {
String val = conf.get(key);
+ return defineIfSet(cmdLine, key, val);
+ }
+
+ private String addConfOptionToCLI(CommandLineBuilder cmdLine,
+ Configuration conf,
+ String key,
+ String defVal) {
+ String val = conf.get(key, defVal);
+ define(cmdLine, key, val);
+ return val;
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI
+ * @param cmdLine command line
+ * @param key key
+ * @param val value
+ */
+ private void define(CommandLineBuilder cmdLine, String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ Preconditions.checkArgument(val != null, "null value");
+ cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
+ * is not null
+ * @param cmdLine command line
+ * @param key key
+ * @param val value
+ */
+ private boolean defineIfSet(CommandLineBuilder cmdLine, String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
if (val != null) {
- cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
+ define(cmdLine, key, val);
return true;
} else {
return false;
}
}
-
- private void addConfOptionToCLI(CommandLineBuilder cmdLine,
- Configuration conf,
- String key, String defVal) {
- String val = conf.get(key, defVal);
- cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
- }
private void addMandatoryConfOptionToCLI(CommandLineBuilder cmdLine,
Configuration conf,
@@ -1235,23 +1560,27 @@
/**
* verify that a live cluster isn't there
* @param clustername cluster name
+ * @param action
* @throws SliderException with exit code EXIT_CLUSTER_LIVE
* if a cluster of that name is either live or starting up.
*/
- public void verifyNoLiveClusters(String clustername) throws
+ public void verifyNoLiveClusters(String clustername, String action) throws
IOException,
YarnException {
List<ApplicationReport> existing = findAllLiveInstances(clustername);
if (!existing.isEmpty()) {
throw new SliderException(EXIT_APPLICATION_IN_USE,
- clustername + ": " + E_CLUSTER_RUNNING + " :" +
+ action +" failed for "
+ + clustername
+ + ": "
+ + E_CLUSTER_RUNNING + " :" +
existing.get(0));
}
}
public String getUsername() throws IOException {
- return UserGroupInformation.getCurrentUser().getShortUserName();
+ return RegistryUtils.currentUser();
}
/**
@@ -1273,7 +1602,7 @@
*/
private boolean getUsingMiniMRCluster() {
return getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER,
- false);
+ false);
}
/**
@@ -1298,16 +1627,17 @@
}
/**
- * Build an exit code for an application Id and its report.
- * If the report parameter is null, the app is killed
- * @param report report
+ * Build an exit code for an application from its report.
+ * If the report parameter is null, its interpreted as a timeout
+ * @param report report application report
* @return the exit code
+ * @throws IOException
+ * @throws YarnException
*/
private int buildExitCode(ApplicationReport report) throws
IOException,
YarnException {
if (null == report) {
- forceKillApplication("Reached client specified timeout for application");
return EXIT_TIMED_OUT;
}
@@ -1334,6 +1664,7 @@
log.info("Application Failed. YarnState={}, DSFinalStatus={}", state,
dsStatus);
return EXIT_YARN_SERVICE_FAILED;
+
default:
//not in any of these states
return EXIT_SUCCESS;
@@ -1361,23 +1692,14 @@
return launchedApplication.monitorAppToState(desiredState, duration);
}
- /**
- * Get the report of a this application
- * @return the app report or null if it could not be found.
- * @throws IOException
- * @throws YarnException
- */
+ @Override
public ApplicationReport getApplicationReport() throws
IOException,
YarnException {
return getApplicationReport(applicationId);
}
- /**
- * Kill the submitted application by sending a call to the ASM
- * @throws YarnException
- * @throws IOException
- */
+ @Override
public boolean forceKillApplication(String reason)
throws YarnException, IOException {
if (applicationId != null) {
@@ -1388,61 +1710,190 @@
}
/**
- * List Slider instances belonging to a specific user
- * @param user user: "" means all users
+ * List Slider instances belonging to a specific user. This will include
+ * failed and killed instances; there may be duplicates
+ * @param user user: "" means all users, null means "default"
* @return a possibly empty list of Slider AMs
*/
@VisibleForTesting
public List<ApplicationReport> listSliderInstances(String user)
throws YarnException, IOException {
- return YARNRegistryClient.listInstances();
+ return YarnAppListClient.listInstances(user);
}
/**
- * Implement the list action: list all nodes
- * @return exit code of 0 if a list was created
+ * A basic list action to list live instances
+ * @param clustername cluster name
+ * @return success if the listing was considered successful
+ * @throws IOException
+ * @throws YarnException
*/
- @VisibleForTesting
public int actionList(String clustername) throws IOException, YarnException {
+ ActionListArgs args = new ActionListArgs();
+ args.live = true;
+ return actionList(clustername, args);
+ }
+
+ /**
+ * Implement the list action.
+ * @param clustername List out specific instance name
+ * @param args Action list arguments
+ * @return 0 if one or more entries were listed
+ * @throws IOException
+ * @throws YarnException
+ * @throws UnknownApplicationInstanceException if a specific instance
+ * was named but it was not found
+ */
+ @Override
+ @VisibleForTesting
+ public int actionList(String clustername, ActionListArgs args)
+ throws IOException, YarnException {
verifyBindingsDefined();
- String user = UserGroupInformation.getCurrentUser().getUserName();
- List<ApplicationReport> instances = listSliderInstances(user);
+ boolean live = args.live;
+ String state = args.state;
+ boolean verbose = args.verbose;
- if (isUnset(clustername)) {
- log.info("Instances for {}: {}",
- (user != null ? user : "all users"),
- instances.size());
- for (ApplicationReport report : instances) {
- logAppReport(report);
- }
- return EXIT_SUCCESS;
+ if (live && !state.isEmpty()) {
+ throw new BadCommandArgumentsException(
+ Arguments.ARG_LIVE + " and " + Arguments.ARG_STATE + " are exclusive");
+ }
+ // flag to indicate only services in a specific state are to be listed
+ boolean listOnlyInState = live || !state.isEmpty();
+
+ YarnApplicationState min, max;
+ if (live) {
+ min = YarnApplicationState.NEW;
+ max = YarnApplicationState.RUNNING;
+ } else if (!state.isEmpty()) {
+ YarnApplicationState stateVal = extractYarnApplicationState(state);
+ min = max = stateVal;
} else {
- SliderUtils.validateClusterName(clustername);
- log.debug("Listing cluster named {}", clustername);
- ApplicationReport report =
- findClusterInInstanceList(instances, clustername);
- if (report != null) {
- logAppReport(report);
- return EXIT_SUCCESS;
- } else {
+ min = YarnApplicationState.NEW;
+ max = YarnApplicationState.KILLED;
+ }
+ // get the complete list of persistent instances
+ Map<String, Path> persistentInstances = sliderFileSystem.listPersistentInstances();
+
+ if (persistentInstances.isEmpty() && isUnset(clustername)) {
+ // an empty listing is a success if no cluster was named
+ log.debug("No application instances found");
+ return EXIT_SUCCESS;
+ }
+
+ // and those the RM knows about
+ List<ApplicationReport> instances = listSliderInstances(null);
+ SliderUtils.sortApplicationsByMostRecent(instances);
+ Map<String, ApplicationReport> reportMap =
+ SliderUtils.buildApplicationReportMap(instances, min, max);
+ log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
+ persistentInstances.size(),
+ instances.size(),
+ min, max,
+ reportMap.size() );
+
+ if (isSet(clustername)) {
+ // only one instance is expected
+ // resolve the persistent value
+ Path persistent = persistentInstances.get(clustername);
+ if (persistent == null) {
throw unknownClusterException(clustername);
}
+ // create a new map with only that instance in it.
+ // this restricts the output of results to this instance
+ persistentInstances = new HashMap<String, Path>();
+ persistentInstances.put(clustername, persistent);
}
+
+ // at this point there is either the entire list or a stripped down instance
+ int listed = 0;
+
+ for (String name : persistentInstances.keySet()) {
+ ApplicationReport report = reportMap.get(name);
+ if (!listOnlyInState || report != null) {
+ // list the details if all were requested, or the filtering contained
+ // a report
+ listed++;
+ String details = instanceDetailsToString(name, report, verbose);
+ print(details);
+ }
+ }
+
+ return listed > 0 ? EXIT_SUCCESS: EXIT_FALSE;
}
/**
- * Log the application report at INFO
- * @param report report to log
+ * Convert the instance details of an application to a string
+ * @param name instance name
+ * @param report the application report
+ * @param verbose verbose output
+ * @return a string
*/
- public void logAppReport(ApplicationReport report) {
- log.info(SliderUtils.appReportToString(report, "\n"));
+ String instanceDetailsToString(String name,
+ ApplicationReport report,
+ boolean verbose) {
+ // format strings
+ String staticf = "%-30s";
+ String reportedf = staticf + " %10s %-40s";
+ String livef = reportedf + " %s";
+ StringBuilder builder = new StringBuilder(200);
+ if (report == null) {
+ builder.append(String.format(staticf, name));
+ } else {
+ // there's a report to look at
+ String appId = report.getApplicationId().toString();
+ String state = report.getYarnApplicationState().toString();
+ if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
+ // running: there's a URL
+ builder.append(String.format(livef, name, state, appId ,report.getTrackingUrl()));
+ } else {
+ builder.append(String.format(reportedf, name, state, appId));
+ }
+ if (verbose) {
+ builder.append('\n');
+ builder.append(SliderUtils.appReportToString(report, "\n "));
+ }
+ }
+
+ builder.append('\n');
+ return builder.toString();
+ }
+
+ /**
+ * Extract the state of a Yarn application --state argument
+ * @param state state argument
+ * @return the application state
+ * @throws BadCommandArgumentsException if the argument did not match
+ * any known state
+ */
+ private YarnApplicationState extractYarnApplicationState(String state) throws
+ BadCommandArgumentsException {
+ YarnApplicationState stateVal;
+ try {
+ stateVal = YarnApplicationState.valueOf(state.toUpperCase(Locale.ENGLISH));
+ } catch (IllegalArgumentException e) {
+ throw new BadCommandArgumentsException("Unknown state: " + state);
+
+ }
+ return stateVal;
+ }
+
+ /**
+ * Is an application active: accepted or running
+ * @param report the application report
+ * @return true if it is running or scheduled to run.
+ */
+ public boolean isApplicationActive(ApplicationReport report) {
+ return report.getYarnApplicationState() == YarnApplicationState.RUNNING
+ || report.getYarnApplicationState() == YarnApplicationState.ACCEPTED;
}
/**
* Implement the islive action: probe for a cluster of the given name existing
* @return exit code
*/
+
+ @Override
@VisibleForTesting
public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException {
verifyBindingsDefined();
@@ -1464,64 +1915,79 @@
return flex(name, roleInstances);
}
- /**
- * Test for a cluster existing probe for a cluster of the given name existing
- * in the filesystem. If the live param is set, it must be a live cluster
- * @return exit code
- */
- @VisibleForTesting
- public int actionExists(String name, boolean live) throws YarnException, IOException {
+ @Override
+ public int actionExists(String name, boolean checkLive) throws YarnException, IOException {
+ ActionExistsArgs args = new ActionExistsArgs();
+ args.live = checkLive;
+ return actionExists(name, args);
+ }
+
+ public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException {
verifyBindingsDefined();
SliderUtils.validateClusterName(name);
- log.debug("actionExists({}, {})", name, live);
+ boolean checkLive = args.live;
+ log.debug("actionExists({}, {}, {})", name, checkLive, args.state);
//initial probe for a cluster in the filesystem
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name);
if (!sliderFileSystem.getFileSystem().exists(clusterDirectory)) {
throw unknownClusterException(name);
}
-
- //test for liveness if desired
-
- if (live) {
- ApplicationReport instance = findInstance(name);
- if (instance == null) {
- log.info("cluster {} not running", name);
- return EXIT_FALSE;
- } else {
- // the app exists, but it may be in a terminated state
- SliderUtils.OnDemandReportStringifier report =
- new SliderUtils.OnDemandReportStringifier(instance);
- YarnApplicationState state =
- instance.getYarnApplicationState();
- if (state.ordinal() >= YarnApplicationState.FINISHED.ordinal()) {
- //cluster in the list of apps but not running
- log.info("Cluster {} found but is in state {}", name, state);
- log.debug("State {}", report);
- return EXIT_FALSE;
- }
- log.info("Cluster {} is running:\n{}", name, report);
- }
- } else {
- log.info("Cluster {} exists but is not running", name);
-
+ String state = args.state;
+ if (!checkLive && SliderUtils.isUnset(state)) {
+ log.info("Application {} exists", name);
+ return EXIT_SUCCESS;
}
- return EXIT_SUCCESS;
+
+ //test for liveness/state
+ boolean inDesiredState = false;
+ ApplicationReport instance;
+ instance = findInstance(name);
+ if (instance == null) {
+ log.info("Application {} not running", name);
+ return EXIT_FALSE;
+ }
+ if (checkLive) {
+ // the app exists, check that it is not in any terminated state
+ YarnApplicationState appstate = instance.getYarnApplicationState();
+ log.debug(" current app state = {}", appstate);
+ inDesiredState =
+ appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
+ } else {
+ // scan for instance in single --state state
+ List<ApplicationReport> userInstances = yarnClient.listInstances("");
+ state = state.toUpperCase(Locale.ENGLISH);
+ YarnApplicationState desiredState = extractYarnApplicationState(state);
+ ApplicationReport foundInstance =
+ yarnClient.findAppInInstanceList(userInstances, name, desiredState);
+ if (foundInstance != null) {
+ // found in selected state: success
+ inDesiredState = true;
+ // mark this as the instance to report
+ instance = foundInstance;
+ }
+ }
+
+ SliderUtils.OnDemandReportStringifier report =
+ new SliderUtils.OnDemandReportStringifier(instance);
+ if (!inDesiredState) {
+ //cluster in the list of apps but not running
+ log.info("Application {} found but is in wrong state {}", name,
+ instance.getYarnApplicationState());
+ log.debug("State {}", report);
+ return EXIT_FALSE;
+ } else {
+ log.debug("Application instance is in desired state");
+ log.info("Application {} is {}\n{}", name,
+ instance.getYarnApplicationState(), report);
+ return EXIT_SUCCESS;
+ }
}
- /**
- * Kill a specific container of the cluster
- * @param name cluster name
- * @param args arguments
- * @return exit code
- * @throws YarnException
- * @throws IOException
- */
+ @Override
public int actionKillContainer(String name,
- ActionKillContainerArgs args) throws
- YarnException,
- IOException {
+ ActionKillContainerArgs args) throws YarnException, IOException {
String id = args.id;
if (SliderUtils.isUnset(id)) {
throw new BadCommandArgumentsException("Missing container id");
@@ -1538,14 +2004,7 @@
return EXIT_SUCCESS;
}
- /**
- * Echo operation (not currently wired up to command line)
- * @param name cluster name
- * @param args arguments
- * @return the echoed text
- * @throws YarnException
- * @throws IOException
- */
+ @Override
public String actionEcho(String name, ActionEchoArgs args) throws
YarnException,
IOException {
@@ -1562,8 +2021,8 @@
* Get at the service registry operations
* @return registry client -valid after the service is inited.
*/
- public YARNRegistryClient getYARNRegistryClient() {
- return YARNRegistryClient;
+ public YarnAppListClient getYarnAppListClient() {
+ return YarnAppListClient;
}
/**
@@ -1573,18 +2032,16 @@
* @throws YarnException YARN issues
* @throws IOException IO problems
*/
- private ApplicationReport findInstance(String appname) throws
- YarnException,
- IOException {
- return YARNRegistryClient.findInstance(appname);
+ private ApplicationReport findInstance(String appname)
+ throws YarnException, IOException {
+ return YarnAppListClient.findInstance(appname);
}
- private RunningApplication findApplication(String appname) throws
- YarnException,
- IOException {
+ private RunningApplication findApplication(String appname)
+ throws YarnException, IOException {
ApplicationReport applicationReport = findInstance(appname);
- return applicationReport != null ? new RunningApplication(yarnClient, applicationReport): null;
-
+ return applicationReport != null ?
+ new RunningApplication(yarnClient, applicationReport): null;
}
/**
@@ -1596,13 +2053,7 @@
private List<ApplicationReport> findAllLiveInstances(String appname)
throws YarnException, IOException {
- return YARNRegistryClient.findAllLiveInstances(appname);
- }
-
-
- public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
- String appname) {
- return yarnClient.findClusterInInstanceList(instances, appname);
+ return YarnAppListClient.findAllLiveInstances(appname);
}
/**
@@ -1612,9 +2063,8 @@
* @throws YarnException
* @throws IOException
*/
- private SliderClusterProtocol connect(ApplicationReport app) throws
- YarnException,
- IOException {
+ private SliderClusterProtocol connect(ApplicationReport app)
+ throws YarnException, IOException {
try {
return RpcBinder.getProxy(getConfig(),
@@ -1629,15 +2079,7 @@
}
}
- /**
- * Status operation
- *
- * @param clustername cluster name
- * @param statusArgs status arguments
- * @return 0 -for success, else an exception is thrown
- * @throws YarnException
- * @throws IOException
- */
+ @Override
@VisibleForTesting
public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws
YarnException,
@@ -1655,26 +2097,15 @@
return EXIT_SUCCESS;
}
- /**
- * Version Details
- * @return exit code
- */
+ @Override
public int actionVersion() {
SliderVersionInfo.loadAndPrintVersionInfo(log);
return EXIT_SUCCESS;
}
- /**
- * Freeze the cluster
- *
- * @param clustername cluster name
- * @param freezeArgs arguments to the freeze
- * @return EXIT_SUCCESS if the cluster was not running by the end of the operation
- */
+ @Override
public int actionFreeze(String clustername,
- ActionFreezeArgs freezeArgs) throws
- YarnException,
- IOException {
+ ActionFreezeArgs freezeArgs) throws YarnException, IOException {
verifyBindingsDefined();
SliderUtils.validateClusterName(clustername);
int waittime = freezeArgs.getWaittime();
@@ -1691,10 +2122,10 @@
if (app == null) {
// exit early
log.info("Cluster {} not running", clustername);
- // not an error to freeze a frozen cluster
+ // not an error to stop a stopped cluster
return EXIT_SUCCESS;
}
- log.debug("App to freeze was found: {}:\n{}", clustername,
+ log.debug("App to stop was found: {}:\n{}", clustername,
new SliderUtils.OnDemandReportStringifier(app));
if (app.getYarnApplicationState().ordinal() >=
YarnApplicationState.FINISHED.ordinal()) {
@@ -1702,14 +2133,22 @@
app.getYarnApplicationState());
return EXIT_SUCCESS;
}
+
+ // IPC request for a managed shutdown is only possible if the app is running.
+ // so we need to force kill if the app is accepted or submitted
+ if (!forcekill
+ && app.getYarnApplicationState().ordinal() < YarnApplicationState.RUNNING.ordinal()) {
+ log.info("Cluster {} is in a pre-running state {}. Force killing it", clustername,
+ app.getYarnApplicationState());
+ forcekill = true;
+ }
+
LaunchedApplication application = new LaunchedApplication(yarnClient, app);
applicationId = application.getApplicationId();
-
if (forcekill) {
- //escalating to forced kill
- application.kill("Forced freeze of " + clustername +
- ": " + text);
+ // escalating to forced kill
+ application.kill("Forced stop of " + clustername + ": " + text);
} else {
try {
SliderClusterProtocol appMaster = connect(app);
@@ -1746,120 +2185,22 @@
// JDK7 } catch (YarnException | IOException e) {
} catch (YarnException e) {
- log.warn("Exception while waiting for the cluster {} to shut down: {}",
+ log.warn("Exception while waiting for the application {} to shut down: {}",
clustername, e);
} catch ( IOException e) {
- log.warn("Exception while waiting for the cluster {} to shut down: {}",
+ log.warn("Exception while waiting for the application {} to shut down: {}",
clustername, e);
}
return EXIT_SUCCESS;
}
- /*
- * Creates a site conf with entries from clientProperties of ClusterStatus
- * @param desc ClusterDescription, can be null
- * @param clustername, can be null
- * @return site conf
- */
- public Configuration getSiteConf(ClusterDescription desc, String clustername)
- throws YarnException, IOException {
- if (desc == null) {
- desc = getClusterDescription();
- }
- if (clustername == null) {
- clustername = getDeployedClusterName();
- }
- String description = "Slider Application Instance " + clustername;
-
- Configuration siteConf = new Configuration(false);
- for (String key : desc.clientProperties.keySet()) {
- siteConf.set(key, desc.clientProperties.get(key), description);
- }
- return siteConf;
- }
-
-
- /**
- * get the cluster configuration
- * @param clustername cluster name
- * @return the cluster name
- */
-
- @SuppressWarnings(
- {"UseOfSystemOutOrSystemErr", "IOResourceOpenedButNotSafelyClosed"})
- public int actionGetConf(String clustername, ActionGetConfArgs confArgs) throws
- YarnException,
- IOException {
- File outfile = null;
-
- if (confArgs.getOutput() != null) {
- outfile = new File(confArgs.getOutput());
- }
-
- String format = confArgs.getFormat();
- verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
- ClusterDescription status = getClusterDescription(clustername);
- Writer writer;
- boolean toPrint;
- if (outfile != null) {
- writer = new FileWriter(outfile);
- toPrint = false;
- } else {
- writer = new StringWriter();
- toPrint = true;
- }
- try {
- String description = "Slider Application Instance " + clustername;
-// JDK7
-/*
- switch (format) {
- case Arguments.FORMAT_XML:
- Configuration siteConf = getSiteConf(status, clustername);
- siteConf.writeXml(writer);
- break;
- case Arguments.FORMAT_PROPERTIES:
- Properties props = new Properties();
- props.putAll(status.clientProperties);
- props.store(writer, description);
- break;
- default:
- throw new BadCommandArgumentsException("Unknown format: " + format);
- }
-*/
- if (Arguments.FORMAT_XML.equals(format)) {
- Configuration siteConf = getSiteConf(status, clustername);
- siteConf.writeXml(writer);
- } else if (Arguments.FORMAT_PROPERTIES.equals(format)) {
- Properties props = new Properties();
- props.putAll(status.clientProperties);
- props.store(writer, description);
- } else {
- throw new BadCommandArgumentsException("Unknown format: " + format);
- }
- } finally {
- // data is written.
- // close the file
- writer.close();
- }
- // then, if this is not a file write, print it
- if (toPrint) {
- // not logged
- System.err.println(writer.toString());
- }
- return EXIT_SUCCESS;
- }
-
- /**
- * Restore a cluster
- */
+ @Override
public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException {
SliderUtils.validateClusterName(clustername);
- // see if it is actually running and bail out;
verifyBindingsDefined();
- verifyNoLiveClusters(clustername);
-
+ // see if it is actually running and bail out;
+ verifyNoLiveClusters(clustername, "Start");
//start the cluster
return startCluster(clustername, thaw);
@@ -1873,10 +2214,8 @@
* @throws YarnException
* @throws IOException
*/
- public int flex(String clustername,
- Map<String, Integer> roleInstances) throws
- YarnException,
- IOException {
+ public int flex(String clustername, Map<String, Integer> roleInstances)
+ throws YarnException, IOException {
verifyBindingsDefined();
SliderUtils.validateClusterName(clustername);
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
@@ -1889,30 +2228,30 @@
for (Map.Entry<String, Integer> entry : roleInstances.entrySet()) {
String role = entry.getKey();
int count = entry.getValue();
- if (count < 0) {
- throw new BadCommandArgumentsException("Requested number of " + role
- + " instances is out of range");
- }
resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES,
Integer.toString(count));
-
log.debug("Flexed cluster specification ( {} -> {}) : \n{}",
role,
count,
resources);
}
+ SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
+ AbstractClientProvider provider = createClientProvider(
+ instanceDefinition.getInternalOperations().getGlobalOptions().getMandatoryOption(
+ InternalKeys.INTERNAL_PROVIDER_NAME));
+ // slider provider to validate what there is
+ validateInstanceDefinition(sliderAM, instanceDefinition, sliderFileSystem);
+ validateInstanceDefinition(provider, instanceDefinition, sliderFileSystem);
+
int exitCode = EXIT_FALSE;
// save the specification
try {
- InstanceIO.updateInstanceDefinition(sliderFileSystem, clusterDirectory,instanceDefinition);
+ InstanceIO.updateInstanceDefinition(sliderFileSystem, clusterDirectory, instanceDefinition);
} catch (LockAcquireFailedException e) {
// lock failure
log.debug("Failed to lock dir {}", clusterDirectory, e);
- log.warn("Failed to save new resource definition to {} : {}", clusterDirectory,
- e.toString());
-
-
+ log.warn("Failed to save new resource definition to {} : {}", clusterDirectory, e);
}
// now see if it is actually running and tell it about the update if it is
@@ -1921,18 +2260,34 @@
log.info("Flexing running cluster");
SliderClusterProtocol appMaster = connect(instance);
SliderClusterOperations clusterOps = new SliderClusterOperations(appMaster);
- if (clusterOps.flex(instanceDefinition.getResources())) {
- log.info("Cluster size updated");
- exitCode = EXIT_SUCCESS;
- } else {
- log.info("Requested size is the same as current size: no change");
- }
+ clusterOps.flex(instanceDefinition.getResources());
+ log.info("application instance size updated");
+ exitCode = EXIT_SUCCESS;
} else {
log.info("No running instance to update");
}
return exitCode;
}
+ /**
+ * Validate an instance definition against a provider.
+ * @param provider the provider performing the validation
+ * @param instanceDefinition the instance definition
+ * @throws SliderException if invalid.
+ */
+ protected void validateInstanceDefinition(AbstractClientProvider provider,
+ AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException {
+ try {
+ provider.validateInstanceDefinition(instanceDefinition, fs);
+ } catch (SliderException e) {
+ //problem, reject it
+ log.info("Error {} validating application instance definition ", e.getMessage());
+ log.debug("Error validating application instance definition ", e);
+ log.info(instanceDefinition.toString());
+ throw e;
+ }
+ }
+
/**
* Load the persistent cluster description
@@ -2141,7 +2496,6 @@
public ApplicationReport getApplicationReport(ApplicationId appId)
throws YarnException, IOException {
return new LaunchedApplication(appId, yarnClient).getApplicationReport();
-
}
/**
@@ -2154,16 +2508,87 @@
}
- /**
- * Registry operation
- *
- * @param registryArgs registry Arguments
- * @return 0 for success, -1 for some issues that aren't errors, just failures
- * to retrieve information (e.g. no configurations for that entry)
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
- @VisibleForTesting
+ @Override
+ public int actionResolve(ActionResolveArgs args)
+ throws YarnException, IOException {
+ // as this is an API entry point, validate
+ // the arguments
+ args.validate();
+ RegistryOperations operations = getRegistryOperations();
+ String path = SliderRegistryUtils.resolvePath(args.path);
+ ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal();
+ try {
+ if (args.list) {
+ File destDir = args.destdir;
+ if (destDir != null) {
+ destDir.mkdirs();
+ }
+
+ Map<String, ServiceRecord> recordMap;
+ Map<String, RegistryPathStatus> znodes;
+ try {
+ znodes = statChildren(registryOperations, path);
+ recordMap = extractServiceRecords(registryOperations,
+ path,
+ znodes.values());
+ } catch (PathNotFoundException e) {
+ // treat the root directory as if if is always there
+
+ if ("/".equals(path)) {
+ znodes = new HashMap<String, RegistryPathStatus>(0);
+ recordMap = new HashMap<String, ServiceRecord>(0);
+ } else {
+ throw e;
+ }
+ }
+ // subtract all records from the znodes map to get pure directories
+ log.info("Entries: {}", znodes.size());
+
+ for (String name : znodes.keySet()) {
+ println(" " + name);
+ }
+ println("");
+
+ log.info("Service records: {}", recordMap.size());
+ for (Entry<String, ServiceRecord> recordEntry : recordMap.entrySet()) {
+ String name = recordEntry.getKey();
+ ServiceRecord instance = recordEntry.getValue();
+ String json = serviceRecordMarshal.toJson(instance);
+ if (destDir == null) {
+ println(name);
+ println(json);
+ } else {
+ String filename = RegistryPathUtils.lastPathEntry(name) + ".json";
+ File jsonFile = new File(destDir, filename);
+ SliderUtils.write(jsonFile,
+ serviceRecordMarshal.toBytes(instance),
+ true);
+ }
+ }
+ } else {
+ // resolve single entry
+ ServiceRecord instance = resolve(path);
+ File outFile = args.out;
+ if (args.destdir != null) {
+ outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path));
+ }
+ if (outFile != null) {
+ SliderUtils.write(outFile, serviceRecordMarshal.toBytes(instance), true);
+ } else {
+ println(serviceRecordMarshal.toJson(instance));
+ }
+ }
+// TODO JDK7
+ } catch (PathNotFoundException e) {
+ // no record at this path
+ throw new NotFoundException(e, path);
+ } catch (NoRecordException e) {
+ throw new NotFoundException(e, path);
+ }
+ return EXIT_SUCCESS;
+ }
+
+ @Override
public int actionRegistry(ActionRegistryArgs registryArgs) throws
YarnException,
IOException {
@@ -2175,21 +2600,33 @@
actionRegistryList(registryArgs);
} else if (registryArgs.listConf) {
// list the configurations
- actionRegistryListConfigs(registryArgs);
+ actionRegistryListConfigsYarn(registryArgs);
+ } else if (registryArgs.listExports) {
+ // list the exports
+ actionRegistryListExports(registryArgs);
} else if (SliderUtils.isSet(registryArgs.getConf)) {
// get a configuration
PublishedConfiguration publishedConfiguration =
actionRegistryGetConfig(registryArgs);
outputConfig(publishedConfiguration, registryArgs);
+ } else if (SliderUtils.isSet(registryArgs.getExport)) {
+ // get a export group
+ PublishedExports publishedExports =
+ actionRegistryGetExport(registryArgs);
+ outputExport(publishedExports, registryArgs);
} else {
// it's an unknown command
- throw new BadCommandArgumentsException(
- "Bad command arguments for " + ACTION_REGISTRY + " " +
- registryArgs);
+ log.info(CommonArgs.usage(serviceArgs, ACTION_DIAGNOSTICS));
+ return EXIT_USAGE;
}
+// JDK7
} catch (FileNotFoundException e) {
- log.info("{}", e.toString());
- log.debug("{}",e, e);
+ log.info("{}", e);
+ log.debug("{}", e, e);
+ return EXIT_NOT_FOUND;
+ } catch (PathNotFoundException e) {
+ log.info("{}", e);
+ log.debug("{}", e, e);
return EXIT_NOT_FOUND;
}
return EXIT_SUCCESS;
@@ -2204,73 +2641,459 @@
* @throws IOException Network or other problems
*/
@VisibleForTesting
- public List<ServiceInstanceData> actionRegistryList(
+ public Collection<ServiceRecord> actionRegistryList(
ActionRegistryArgs registryArgs)
throws YarnException, IOException {
- SliderRegistryService registryService = getRegistry();
String serviceType = registryArgs.serviceType;
String name = registryArgs.name;
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
- registryService.findInstances(serviceType, name);
- int size = instances.size();
- if (size == 0) {
- throw new FileNotFoundException("No entries for servicetype "
- + serviceType
- + " name " + name);
+ RegistryOperations operations = getRegistryOperations();
+ Collection<ServiceRecord> serviceRecords;
+ if (StringUtils.isEmpty(name)) {
+ String path =
+ serviceclassPath(
+ currentUser(),
+ serviceType);
+
+ try {
+ Map<String, ServiceRecord> recordMap =
+ listServiceRecords(operations, path);
+ if (recordMap.isEmpty()) {
+ throw new UnknownApplicationInstanceException(
+ "No applications registered under " + path);
+ }
+ serviceRecords = recordMap.values();
+ } catch (PathNotFoundException e) {
+ throw new NotFoundException(path, e);
+ }
+ } else {
+ ServiceRecord instance = lookupServiceRecord(registryArgs);
+ serviceRecords = new ArrayList<ServiceRecord>(1);
+ serviceRecords.add(instance);
}
- List<ServiceInstanceData> sids = new ArrayList<ServiceInstanceData>(size);
- for (CuratorServiceInstance<ServiceInstanceData> instance : instances) {
- ServiceInstanceData payload = instance.payload;
- logInstance(payload, registryArgs.verbose);
- sids.add(payload);
+
+ for (ServiceRecord serviceRecord : serviceRecords) {
+ logInstance(serviceRecord, registryArgs.verbose);
}
- return sids;
+ return serviceRecords;
}
- private void logInstance(ServiceInstanceData instance,
- boolean verbose) {
- if (!verbose) {
- log.info("{}", instance.id);
- } else {
- log.info("{}: ", instance.id);
- logEndpoints(instance);
+ @Override
+ public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
+ try {
+ if (diagnosticArgs.client) {
+ actionDiagnosticClient(diagnosticArgs);
+ } else if (diagnosticArgs.application) {
+ actionDiagnosticApplication(diagnosticArgs);
+ } else if (diagnosticArgs.yarn) {
+ actionDiagnosticYarn(diagnosticArgs);
+ } else if (diagnosticArgs.credentials) {
+ actionDiagnosticCredentials();
+ } else if (diagnosticArgs.all) {
+ actionDiagnosticAll(diagnosticArgs);
+ } else if (diagnosticArgs.level) {
+ actionDiagnosticIntelligent(diagnosticArgs);
+ } else {
+ // it's an unknown option
+ log.info(CommonArgs.usage(serviceArgs, ACTION_DIAGNOSTICS));
+ return EXIT_USAGE;
+ }
+ } catch (Exception e) {
+ log.error(e.toString());
+ return EXIT_FALSE;
+ }
+ return EXIT_SUCCESS;
+ }
+
+ private void actionDiagnosticIntelligent(ActionDiagnosticArgs diagnosticArgs)
+ throws YarnException, IOException, URISyntaxException {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.name;
+ if(SliderUtils.isUnset(clusterName)){
+ throw new BadCommandArgumentsException("application name must be provided with --name option");
+ }
+
+ try {
+ SliderUtils.validateClientConfigFile();
+ log.info("Slider-client.xml is accessible");
+ } catch (IOException e) {
+ // we are catching exceptions here because those are indication of
+ // validation result, and we need to print them here
+ log.error(
+ "validation of slider-client.xml fails because: " + e.toString(), e);
+ return;
+ }
+ SliderClusterOperations clusterOperations = createClusterOperations(clusterName);
+ // cluster not found exceptions will be thrown upstream
+ ClusterDescription clusterDescription = clusterOperations
+ .getClusterDescription();
+ log.info("Slider AppMaster is accessible");
+
+ if (clusterDescription.state == ClusterDescription.STATE_LIVE) {
+ AggregateConf instanceDefinition = clusterOperations
+ .getInstanceDefinition();
+ String imagePath = instanceDefinition.getInternalOperations().get(
+ InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ // if null, that means slider uploaded the agent tarball for the user
+ // and we need to use where slider has put
+ if (imagePath == null) {
+ ApplicationReport appReport = findInstance(clusterName);
+ Path path1 = sliderFileSystem.getTempPathForCluster(clusterName);
+
+ Path subPath = new Path(path1, appReport.getApplicationId().toString()
+ + "/agent");
+ imagePath = subPath.toString();
+ }
+ try {
+ SliderUtils.validateHDFSFile(sliderFileSystem, imagePath + "/" + AGENT_TAR);
+ log.info("Slider agent package is properly installed");
+ } catch (FileNotFoundException e) {
+ log.error("can not find agent package: " + e.toString());
+ return;
+ } catch (IOException e) {
+ log.error("can not open agent package: " + e.toString());
+ return;
+ }
+ String pkgTarballPath = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ try {
+ SliderUtils.validateHDFSFile(sliderFileSystem, pkgTarballPath);
+ log.info("Application package is properly installed");
+ } catch (FileNotFoundException e) {
+ log.error("can not find application package: {}", e);
+ return;
+ } catch (IOException e) {
+ log.error("can not open application package: {} ", e);
+ return;
+ }
}
}
-
- private void logEndpoints(ServiceInstanceData instance) {
- Map<String, RegisteredEndpoint> endpoints =
- instance.listEndpoints(true);
- for (Map.Entry<String, RegisteredEndpoint> entry : endpoints.entrySet()) {
- String name = entry.getKey();
- RegisteredEndpoint endpoint = entry.getValue();
- log.info(" {}", endpoint);
+
+ private void actionDiagnosticAll(ActionDiagnosticArgs diagnosticArgs)
+ throws IOException, YarnException {
+ // assign application name from param to each sub diagnostic function
+ actionDiagnosticClient(diagnosticArgs);
+ actionDiagnosticApplication(diagnosticArgs);
+ actionDiagnosticSlider(diagnosticArgs);
+ actionDiagnosticYarn(diagnosticArgs);
+ actionDiagnosticCredentials();
+ }
+
+ private void actionDiagnosticCredentials() throws BadConfigException,
+ IOException {
+ if (SliderUtils.isHadoopClusterSecure(SliderUtils
+ .loadClientConfigurationResource())) {
+ String credentialCacheFileDescription = null;
+ try {
+ credentialCacheFileDescription = SliderUtils.checkCredentialCacheFile();
+ } catch (BadConfigException e) {
+ log.error("The credential config is not valid: " + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Unable to read the credential file: " + e.toString());
+ throw e;
+ }
+ log.info("Credential cache file for the current user: "
+ + credentialCacheFileDescription);
+ } else {
+ log.info("the cluster is not in secure mode");
+ }
+ }
+
+ private void actionDiagnosticYarn(ActionDiagnosticArgs diagnosticArgs)
+ throws IOException, YarnException {
+ JSONObject converter = null;
+ log.info("the node in the YARN cluster has below state: ");
+ List<NodeReport> yarnClusterInfo;
+ try {
+ yarnClusterInfo = yarnClient.getNodeReports(NodeState.RUNNING);
+ } catch (YarnException e1) {
+ log.error("Exception happened when fetching node report from the YARN cluster: "
+ + e1.toString());
+ throw e1;
+ } catch (IOException e1) {
+ log.error("Network problem happened when fetching node report YARN cluster: "
+ + e1.toString());
+ throw e1;
+ }
+ for (NodeReport nodeReport : yarnClusterInfo) {
+ log.info(nodeReport.toString());
+ }
+
+ if (diagnosticArgs.verbose) {
+ Writer configWriter = new StringWriter();
+ try {
+ Configuration.dumpConfiguration(yarnClient.getConfig(), configWriter);
+ } catch (IOException e1) {
+ log.error("Network problem happened when retrieving YARN config from YARN: "
+ + e1.toString());
+ throw e1;
+ }
+ try {
+ converter = new JSONObject(configWriter.toString());
+ log.info("the configuration of the YARN cluster is: "
+ + converter.toString(2));
+
+ } catch (JSONException e) {
+ log.error("JSONException happened during parsing response from YARN: "
+ + e.toString());
+ }
+ }
+ }
+
+ private void actionDiagnosticSlider(ActionDiagnosticArgs diagnosticArgs)
+ throws YarnException, IOException {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.name;
+ if(SliderUtils.isUnset(clusterName)){
+ throw new BadCommandArgumentsException("application name must be provided with --name option");
+ }
+ SliderClusterOperations clusterOperations;
+ AggregateConf instanceDefinition = null;
+ try {
+ clusterOperations = createClusterOperations(clusterName);
+ instanceDefinition = clusterOperations.getInstanceDefinition();
+ } catch (YarnException e) {
+ log.error("Exception happened when retrieving instance definition from YARN: "
+ + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Network problem happened when retrieving instance definition from YARN: "
+ + e.toString());
+ throw e;
+ }
+ String imagePath = instanceDefinition.getInternalOperations().get(
+ InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ // if null, it will be uploaded by Slider and thus at slider's path
+ if (imagePath == null) {
+ ApplicationReport appReport = findInstance(clusterName);
+ Path path1 = sliderFileSystem.getTempPathForCluster(clusterName);
+ Path subPath = new Path(path1, appReport.getApplicationId().toString()
+ + "/agent");
+ imagePath = subPath.toString();
+ }
+ log.info("The path of slider agent tarball on HDFS is: " + imagePath);
+ }
+
+ private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs)
+ throws YarnException, IOException {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.name;
+ if(SliderUtils.isUnset(clusterName)){
+ throw new BadCommandArgumentsException("application name must be provided with --name option");
+ }
+ SliderClusterOperations clusterOperations;
+ AggregateConf instanceDefinition = null;
+ try {
+ clusterOperations = createClusterOperations(clusterName);
+ instanceDefinition = clusterOperations.getInstanceDefinition();
+ } catch (YarnException e) {
+ log.error("Exception happened when retrieving instance definition from YARN: "
+ + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Network problem happened when retrieving instance definition from YARN: "
+ + e.toString());
+ throw e;
+ }
+ String clusterDir = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.APP_ROOT);
+ String pkgTarball = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.APP_DEF);
+ String runAsUser = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.RUNAS_USER);
+
+ log.info("The location of the cluster instance directory in HDFS is: "
+ + clusterDir);
+ log.info("The name of the application package tarball on HDFS is: "
+ + pkgTarball);
+ log.info("The runas user of the application in the cluster is: "
+ + runAsUser);
+
+ if (diagnosticArgs.verbose) {
+ log.info("App config of the application: "
+ + instanceDefinition.getAppConf().toJson());
+ log.info("Resource config of the application: "
+ + instanceDefinition.getResources().toJson());
+ }
+ }
+
+ private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs)
+ throws SliderException, IOException {
+ try {
+ String currentCommandPath = SliderUtils.getCurrentCommandPath();
+ SliderVersionInfo.loadAndPrintVersionInfo(log);
+ String clientConfigPath = SliderUtils.getClientConfigPath();
+ String jdkInfo = SliderUtils.getJDKInfo();
+ println("The slider command path: %s", currentCommandPath);
+ println("The slider-client.xml used by current running command path: %s",
+ clientConfigPath);
+ println(jdkInfo);
+
+ // security info
+ Configuration config = getConfig();
+ if (SliderUtils.isHadoopClusterSecure(config)) {
+ println("Hadoop Cluster is secure");
+ println("Login user is %s", UserGroupInformation.getLoginUser());
+ println("Current user is %s", UserGroupInformation.getCurrentUser());
+
+ } else {
+ println("Hadoop Cluster is insecure");
+ }
+
+ // verbose?
+ if (diagnosticArgs.verbose) {
+ // do the environment
+ Map<String, String> env = System.getenv();
+ Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet());
+ StringBuilder builder = new StringBuilder("Environment variables:\n");
+ for (String key : envList) {
+ builder.append(key).append("=").append(env.get(key)).append("\n");
+ }
+ println(builder.toString());
+
+ // Java properties
+ builder = new StringBuilder("JVM Properties\n");
+ Map<String, String> props =
+ SliderUtils.sortedMap(SliderUtils.toMap(System.getProperties()));
+ for (Entry<String, String> entry : props.entrySet()) {
+ builder.append(entry.getKey()).append("=")
+ .append(entry.getValue()).append("\n");
+ }
+
+ println(builder.toString());
+
+ // then the config
+ println("Slider client configuration:\n"
+ + ConfigHelper.dumpConfigToString(config));
+
+ }
+
+ SliderUtils.validateSliderClientEnvironment(log);
+ } catch (SliderException e) {
+ log.error(e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error(e.toString());
+ throw e;
+ }
+
+ }
+
+ /**
+ * Log a service record instance
+ * @param instance record
+ * @param verbose verbose logging of all external endpoints
+ */
+ private void logInstance(ServiceRecord instance,
+ boolean verbose) {
+ if (!verbose) {
+ log.info("{}", instance.get(YarnRegistryAttributes.YARN_ID, ""));
+ } else {
+ log.info("{}: ", instance.get(YarnRegistryAttributes.YARN_ID, ""));
+ logEndpoints(instance);
}
}
/**
+ * Log the external endpoints of a service record
+ * @param instance service record instance
+ */
+ private void logEndpoints(ServiceRecord instance) {
+ List<Endpoint> endpoints = instance.external;
+ for (Endpoint endpoint : endpoints) {
+ log.info(endpoint.toString());
+ }
+ }
+
+ /**
* list configs available for an instance
*
* @param registryArgs registry Arguments
* @throws YarnException YARN problems
* @throws IOException Network or other problems
*/
- public void actionRegistryListConfigs(ActionRegistryArgs registryArgs)
+ public void actionRegistryListConfigsYarn(ActionRegistryArgs registryArgs)
throws YarnException, IOException {
- ServiceInstanceData instance = lookupInstance(registryArgs);
+
+ ServiceRecord instance = lookupServiceRecord(registryArgs);
RegistryRetriever retriever = new RegistryRetriever(instance);
PublishedConfigSet configurations =
retriever.getConfigurations(!registryArgs.internal);
-
- for (String configName : configurations.keys()) {
- if (!registryArgs.verbose) {
- log.info("{}", configName);
+ PrintStream out = null;
+ try {
+ if (registryArgs.out != null) {
+ out = new PrintStream(new FileOutputStream(registryArgs.out));
} else {
- PublishedConfiguration published =
- configurations.get(configName);
- log.info("{} : {}",
- configName,
- published.description);
+ out = System.out;
+ }
+ for (String configName : configurations.keys()) {
+ if (!registryArgs.verbose) {
+ out.println(configName);
+ } else {
+ PublishedConfiguration published =
+ configurations.get(configName);
+ out.printf("%s: %s\n",
+ configName,
+ published.description);
+ }
+ }
+ } finally {
+ if (registryArgs.out != null && out != null) {
+ out.flush();
+ out.close();
+ }
+ }
+ }
+
+ /**
+ * list exports available for an instance
+ *
+ * @param registryArgs registry Arguments
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ public void actionRegistryListExports(ActionRegistryArgs registryArgs)
+ throws YarnException, IOException {
+ ServiceRecord instance = lookupServiceRecord(registryArgs);
+
+ RegistryRetriever retriever = new RegistryRetriever(instance);
+ PublishedExportsSet exports =
+ retriever.getExports(!registryArgs.internal);
+ PrintStream out = null;
+ boolean streaming = false;
+ try {
+ if (registryArgs.out != null) {
+ out = new PrintStream(new FileOutputStream(registryArgs.out));
+ streaming = true;
+ log.debug("Saving output to {}", registryArgs.out);
+ } else {
+ out = System.out;
+ }
+ log.debug("Number of exports: {}", exports.keys().size());
+ for (String exportName : exports.keys()) {
+ if (streaming) {
+ log.debug(exportName);
+ }
+ if (!registryArgs.verbose) {
+ out.println(exportName);
+ } else {
+ PublishedExports published = exports.get(exportName);
+ out.printf("%s: %s\n",
+ exportName,
+ published.description);
+ }
+ }
+ } finally {
+ if (streaming) {
+ out.flush();
+ out.close();
}
}
}
@@ -2286,7 +3109,7 @@
@VisibleForTesting
public PublishedConfiguration actionRegistryGetConfig(ActionRegistryArgs registryArgs)
throws YarnException, IOException {
- ServiceInstanceData instance = lookupInstance(registryArgs);
+ ServiceRecord instance = lookupServiceRecord(registryArgs);
RegistryRetriever retriever = new RegistryRetriever(instance);
boolean external = !registryArgs.internal;
@@ -2300,9 +3123,36 @@
}
/**
- * write out the config
- * @param published
- * @param registryArgs
+ * get a specific export group
+ *
+ * @param registryArgs registry Arguments
+ *
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ * @throws FileNotFoundException if the config is not found
+ */
+ @VisibleForTesting
+ public PublishedExports actionRegistryGetExport(ActionRegistryArgs registryArgs)
+ throws YarnException, IOException {
+ ServiceRecord instance = lookupServiceRecord(registryArgs);
+
+ RegistryRetriever retriever = new RegistryRetriever(instance);
+ boolean external = !registryArgs.internal;
+ PublishedExportsSet exports =
+ retriever.getExports(external);
+
+ PublishedExports published = retriever.retrieveExports(exports,
+ registryArgs.getExport,
+ external);
+ return published;
+ }
+
+ /**
+ * write out the config. If a destination is provided and that dir is a
+ * directory, the entry is written to it with the name provided + extension,
+ * else it is printed to standard out.
+ * @param published published config
+ * @param registryArgs registry Arguments
* @throws BadCommandArgumentsException
* @throws IOException
*/
@@ -2321,10 +3171,48 @@
PublishedConfigurationOutputter outputter =
PublishedConfigurationOutputter.createOutputter(configFormat,
published);
- boolean print = registryArgs.dest == null;
+ boolean print = registryArgs.out == null;
+ if (!print) {
+ File outputPath = registryArgs.out;
+ if (outputPath.isDirectory()) {
+ // creating it under a directory
+ outputPath = new File(outputPath, entry + "." + format);
+ }
+ log.debug("Destination path: {}", outputPath);
+ outputter.save(outputPath);
+ } else {
+ print(outputter.asString());
+ }
+
+ }
+
+ /**
+ * write out the config
+ * @param published
+ * @param registryArgs
+ * @throws BadCommandArgumentsException
+ * @throws IOException
+ */
+ private void outputExport(PublishedExports published,
+ ActionRegistryArgs registryArgs) throws
+ BadCommandArgumentsException,
+ IOException {
+ // decide whether or not to print
+ String entry = registryArgs.getExport;
+ String format = ConfigFormat.JSON.toString();
+ ConfigFormat configFormat = ConfigFormat.resolve(format);
+ if (configFormat == null || configFormat != ConfigFormat.JSON) {
+ throw new BadCommandArgumentsException(
+ "Unknown/Unsupported format %s . Only JSON is supported.", format);
+ }
+
+ PublishedExportsOutputter outputter =
+ PublishedExportsOutputter.createOutputter(configFormat,
+ published);
+ boolean print = registryArgs.out == null;
if (!print) {
File destFile;
- destFile = registryArgs.dest;
+ destFile = registryArgs.out;
if (destFile.isDirectory()) {
// creating it under a directory
destFile = new File(destFile, entry + "." + format);
@@ -2334,64 +3222,83 @@
} else {
print(outputter.asString());
}
-
}
/**
* Look up an instance
* @return instance data
- * @throws UnknownApplicationInstanceException no match
* @throws SliderException other failures
* @throws IOException IO problems or wrapped exceptions
*/
- private ServiceInstanceData lookupInstance(ActionRegistryArgs registryArgs) throws
- UnknownApplicationInstanceException,
+ private ServiceRecord lookupServiceRecord(ActionRegistryArgs registryArgs) throws
SliderException,
IOException {
- return lookupInstance(registryArgs.name, registryArgs.serviceType);
+ String user;
+ if (StringUtils.isNotEmpty(registryArgs.user)) {
+ user = RegistryPathUtils.encodeForRegistry(registryArgs.user);
+ } else {
+ user = currentUser();
+ }
+
+ String path = servicePath(user, registryArgs.serviceType,
+ registryArgs.name);
+ return resolve(path);
}
/**
- * Look up an instance
- * @param id instance ID
+ * Look up a service record of the current user
* @param serviceType service type
+ * @param id instance ID
* @return instance data
- * @throws UnknownApplicationInstanceException no match
+ * @throws UnknownApplicationInstanceException no path or service record
+ * at the end of the path
* @throws SliderException other failures
* @throws IOException IO problems or wrapped exceptions
*/
- private ServiceInstanceData lookupInstance(String id,
- String serviceType) throws
- IOException {
- try {
- CuratorServiceInstance<ServiceInstanceData> csi =
- getRegistry().queryForInstance(serviceType, id);
- if (csi == null) {
- throw new FileNotFoundException(
- String.format("instance %s of type %s not found",
- id, serviceType));
- }
- return csi.getPayload();
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- /**
- * List instances in the registry
- * @return
- * @throws IOException
- * @throws YarnException
- */
- public List<CuratorServiceInstance<ServiceInstanceData>> listRegistryInstances()
- throws IOException, YarnException {
- maybeStartRegistry();
- return registry.listInstances(SliderKeys.APP_TYPE);
+ public ServiceRecord lookupServiceRecord(String serviceType, String id)
+ throws IOException, SliderException {
+ String path = servicePath(currentUser(), serviceType, id);
+ return resolve(path);
}
/**
+ *
+ * Look up an instance
+ * @param path path
+ * @return instance data
+ * @throws NotFoundException no path/no service record
+ * at the end of the path
+ * @throws SliderException other failures
+ * @throws IOException IO problems or wrapped exceptions
+ */
+ public ServiceRecord resolve(String path)
+ throws IOException, SliderException {
+ try {
+ return getRegistryOperations().resolve(path);
+ } catch (PathNotFoundException e) {
+ throw new NotFoundException(e.getPath().toString(), e);
+ } catch (NoRecordException e) {
+ throw new NotFoundException(e.getPath().toString(), e);
+ }
+ }
+
+ /**
+ * List instances in the registry for the current user
+ * @return a list of slider registry instances
+ * @throws IOException Any IO problem ... including no path in the registry
+ * to slider service classes for this user
+ * @throws SliderException other failures
+ */
+
+ public Map<String, ServiceRecord> listRegistryInstances()
+ throws IOException, SliderException {
+ Map<String, ServiceRecord> recordMap = listServiceRecords(
+ getRegistryOperations(),
+ serviceclassPath(currentUser(), SliderKeys.APP_TYPE));
+ return recordMap;
+ }
+
+ /**
* List instances in the registry
* @return the instance IDs
* @throws IOException
@@ -2401,8 +3308,10 @@
IOException,
YarnException {
try {
- maybeStartRegistry();
- return registry.instanceIDs(SliderKeys.APP_TYPE);
+ Map<String, ServiceRecord> recordMap = listServiceRecords(
+ getRegistryOperations(),
+ serviceclassPath(currentUser(), SliderKeys.APP_TYPE));
+ return new ArrayList<String>(recordMap.keySet());
/// JDK7 } catch (YarnException | IOException e) {
} catch (IOException e) {
throw e;
@@ -2419,29 +3328,19 @@
* @throws SliderException
* @throws IOException
*/
- private synchronized SliderRegistryService maybeStartRegistry() throws
- SliderException,
- IOException {
+ private synchronized RegistryOperations maybeStartYarnRegistry()
+ throws SliderException, IOException {
- if (registry == null) {
- registry = startRegistrationService();
+ if (registryOperations == null) {
+ registryOperations = startRegistryOperationsService();
}
- return registry;
+ return registryOperations;
}
- /**
- * Get the registry binding. As this may start the registry, it can take time
- * and fail
- * @return registry the registry service
- * @throws SliderException slider-specific failures
- * @throws IOException other failures
- */
- @VisibleForTesting
-
- public SliderRegistryService getRegistry() throws
- SliderException,
- IOException {
- return maybeStartRegistry();
+ @Override
+ public RegistryOperations getRegistryOperations()
+ throws SliderException, IOException {
+ return maybeStartYarnRegistry();
}
/**
@@ -2452,4 +3351,57 @@
private static void print(CharSequence src) {
System.out.append(src);
}
+
+ /**
+ * Output to standard out/stderr with a newline after
+ * @param message message
+ */
+ private static void println(String message) {
+ print(message);
+ print("\n");
+ }
+ /**
+ * Output to standard out/stderr with a newline after, formatted
+ * @param message message
+ * @param args arguments for string formatting
+ */
+ private static void println(String message, Object ... args) {
+ print(String.format(message, args));
+ print("\n");
+ }
+
+ /**
+ * Implement the lookup action.
+ * @param args Action arguments
+ * @return 0 if the entry was found
+ * @throws IOException
+ * @throws YarnException
+ * @throws UnknownApplicationInstanceException if a specific instance
+ * was named but it was not found
+ */
+ @VisibleForTesting
+ public int actionLookup(ActionLookupArgs args)
+ throws IOException, YarnException {
+ verifyBindingsDefined();
+ try {
+ ApplicationId id = ConverterUtils.toApplicationId(args.id);
+ ApplicationReport report = yarnClient.getApplicationReport(id);
+ SerializedApplicationReport sar = new SerializedApplicationReport(report);
+ ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser();
+ if (args.outputFile != null) {
+ serDeser.save(sar, args.outputFile);
+ } else {
+ println(serDeser.toJson(sar));
+ }
+ } catch (IllegalArgumentException e) {
+ throw new BadCommandArgumentsException(e, "%s : %s", args, e);
+ } catch (ApplicationAttemptNotFoundException notFound) {
+ throw new NotFoundException(notFound, notFound.toString());
+ } catch (ApplicationNotFoundException notFound) {
+ throw new NotFoundException(notFound, notFound.toString());
+ }
+ return EXIT_SUCCESS;
+ }
}
+
+
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
new file mode 100644
index 0000000..61948e3
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionAMSuicideArgs;
+import org.apache.slider.common.params.ActionDiagnosticArgs;
+import org.apache.slider.common.params.ActionEchoArgs;
+import org.apache.slider.common.params.ActionFlexArgs;
+import org.apache.slider.common.params.ActionFreezeArgs;
+import org.apache.slider.common.params.ActionInstallKeytabArgs;
+import org.apache.slider.common.params.ActionInstallPackageArgs;
+import org.apache.slider.common.params.ActionKillContainerArgs;
+import org.apache.slider.common.params.ActionListArgs;
+import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionResolveArgs;
+import org.apache.slider.common.params.ActionStatusArgs;
+import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.providers.AbstractClientProvider;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Interface of those method calls in the slider API that are intended
+ * for direct public invocation.
+ * <p>
+ * Stability: evolving
+ */
+public interface SliderClientAPI extends Service {
+ /**
+ * Destroy a cluster. There's two race conditions here
+ * #1 the cluster is started between verifying that there are no live
+ * clusters of that name.
+ */
+ int actionDestroy(String clustername) throws YarnException,
+ IOException;
+
+ /**
+ * AM to commit an asynchronous suicide
+ */
+ int actionAmSuicide(String clustername,
+ ActionAMSuicideArgs args) throws YarnException, IOException;
+
+ /**
+ * Get the provider for this cluster
+ * @param provider the name of the provider
+ * @return the provider instance
+ * @throws SliderException problems building the provider
+ */
+ AbstractClientProvider createClientProvider(String provider)
+ throws SliderException;
+
+ /**
+ * Build up the cluster specification/directory
+ *
+ * @param clustername cluster name
+ * @param buildInfo the arguments needed to build the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ int actionBuild(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo) throws YarnException, IOException;
+
+ /**
+ * Upload keytab to a designated sub-directory of the user home directory
+ *
+ * @param installKeytabInfo the arguments needed to upload the keytab
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ int actionInstallKeytab(ActionInstallKeytabArgs installKeytabInfo)
+ throws YarnException, IOException;
+
+ /**
+ * Upload application package to user home directory
+ *
+ * @param installPkgInfo the arguments needed to upload the package
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ int actionInstallPkg(ActionInstallPackageArgs installPkgInfo)
+ throws YarnException, IOException;
+
+ /**
+ * Update the cluster specification
+ *
+ * @param clustername cluster name
+ * @param buildInfo the arguments needed to update the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ */
+ int actionUpdate(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo)
+ throws YarnException, IOException;
+ /**
+ * Get the report of a this application
+ * @return the app report or null if it could not be found.
+ * @throws IOException
+ * @throws YarnException
+ */
+ ApplicationReport getApplicationReport()
+ throws IOException, YarnException;
+
+ /**
+ * Kill the submitted application via YARN
+ * @throws YarnException
+ * @throws IOException
+ */
+ boolean forceKillApplication(String reason)
+ throws YarnException, IOException;
+
+ /**
+ * Implement the list action: list all nodes
+ * @return exit code of 0 if a list was created
+ */
+ int actionList(String clustername, ActionListArgs args) throws IOException, YarnException;
+
+ /**
+ * Implement the islive action: probe for a cluster of the given name existing
+ * @return exit code
+ */
+ int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException;
+
+ /**
+ * Test for a cluster existing probe for a cluster of the given name existing
+ * in the filesystem. If the live param is set, it must be a live cluster
+ * @return exit code
+ */
+ int actionExists(String name, boolean checkLive) throws YarnException, IOException;
+
+ /**
+ * Kill a specific container of the cluster
+ * @param name cluster name
+ * @param args arguments
+ * @return exit code
+ * @throws YarnException
+ * @throws IOException
+ */
+ int actionKillContainer(String name, ActionKillContainerArgs args)
+ throws YarnException, IOException;
+
+ /**
+ * Echo operation (not currently wired up to command line)
+ * @param name cluster name
+ * @param args arguments
+ * @return the echoed text
+ * @throws YarnException
+ * @throws IOException
+ */
+ String actionEcho(String name, ActionEchoArgs args)
+ throws YarnException, IOException;
+
+ /**
+ * Status operation
+ *
+ * @param clustername cluster name
+ * @param statusArgs status arguments
+ * @return 0 -for success, else an exception is thrown
+ * @throws YarnException
+ * @throws IOException
+ */
+ int actionStatus(String clustername, ActionStatusArgs statusArgs)
+ throws YarnException, IOException;
+
+ /**
+ * Version Details
+ * @return exit code
+ */
+ int actionVersion();
+
+ /**
+ * Stop the cluster
+ *
+ * @param clustername cluster name
+ * @param freezeArgs arguments to the stop
+ * @return EXIT_SUCCESS if the cluster was not running by the end of the operation
+ */
+ int actionFreeze(String clustername, ActionFreezeArgs freezeArgs)
+ throws YarnException, IOException;
+
+ /**
+ * Restore a cluster
+ */
+ int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException;
+
+ /**
+ * Registry operation
+ *
+ * @param args registry Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just failures
+ * to retrieve information (e.g. no configurations for that entry)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ int actionResolve(ActionResolveArgs args)
+ throws YarnException, IOException;
+
+ /**
+ * Registry operation
+ *
+ * @param registryArgs registry Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just failures
+ * to retrieve information (e.g. no configurations for that entry)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ int actionRegistry(ActionRegistryArgs registryArgs)
+ throws YarnException, IOException;
+
+ /**
+ * diagnostic operation
+ *
+ * @param clusterName
+ * application name
+ * @param diagosticArgs
+ * diagnostic Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just
+ * failures to retrieve information (e.g. no application name
+ * specified)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs);
+
+ /**
+ * Get the registry binding. As this may start the registry, it can take time
+ * and fail
+ * @return the registry
+ */
+ RegistryOperations getRegistryOperations()
+ throws SliderException, IOException;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java b/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
index 9e1f568..30f17b6 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
@@ -56,7 +56,6 @@
/**
* Get a node from the AM
- * @param appMaster AM
* @param uuid uuid of node
* @return deserialized node
* @throws IOException IO problems
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 3151a09..a2a7fe7 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -250,22 +250,59 @@
return results;
}
+ /**
+ * Find a cluster in the instance list; biased towards live instances
+ * @param instances list of instances
+ * @param appname application name
+ * @return the first found instance, else a failed/finished instance, or null
+ * if there are none of those
+ */
public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
String appname) {
+ // sort by most recent
+ SliderUtils.sortApplicationsByMostRecent(instances);
ApplicationReport found = null;
- ApplicationReport foundAndLive = null;
for (ApplicationReport app : instances) {
if (app.getName().equals(appname)) {
- found = app;
if (isApplicationLive(app)) {
- foundAndLive = app;
+ return app;
+ }
+ // set the found value if not set
+ found = found != null ? found : app;
+ }
+ }
+ return found;
+ }
+
+ /**
+ * Find an app in the instance list in the desired state
+ * @param instances instance list
+ * @param appname application name
+ * @param desiredState yarn state desired
+ * @return the match or null for none
+ */
+ public ApplicationReport findAppInInstanceList(List<ApplicationReport> instances,
+ String appname,
+ YarnApplicationState desiredState) {
+ ApplicationReport found = null;
+ ApplicationReport foundAndLive = null;
+ log.debug("Searching {} records for instance name {} in state '{}'",
+ instances.size(), appname, desiredState);
+ for (ApplicationReport app : instances) {
+ if (app.getName().equals(appname)) {
+
+ YarnApplicationState appstate =
+ app.getYarnApplicationState();
+ log.debug("app ID {} is in state {}", app.getApplicationId(), appstate);
+ if (appstate.equals(desiredState)) {
+ log.debug("match");
+ return app;
}
}
}
- if (foundAndLive != null) {
- found = foundAndLive;
- }
- return found;
+ // nothing found in desired state
+ log.debug("No match");
+ return null;
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/Constants.java b/slider-core/src/main/java/org/apache/slider/common/Constants.java
index 2fe0250..868ea57 100644
--- a/slider-core/src/main/java/org/apache/slider/common/Constants.java
+++ b/slider-core/src/main/java/org/apache/slider/common/Constants.java
@@ -19,7 +19,6 @@
package org.apache.slider.common;
public class Constants {
- public static final int ACCEPT_TIME = 60000;
public static final int CONNECT_TIMEOUT = 10000;
public static final int RPC_TIMEOUT = 15000;
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java b/slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
index b115d98..5758f79 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
@@ -26,72 +26,63 @@
* starting point for exit codes; not an exception itself
*/
int _EXIT_CODE_BASE = 64;
-
- /**
- * internal error: {@value}
- */
- int EXIT_INTERNAL_ERROR = _EXIT_CODE_BASE;
-
- /**
- * Unimplemented feature: {@value}
- */
- int EXIT_UNIMPLEMENTED = 65;
/**
* service entered the failed state: {@value}
*/
- int EXIT_YARN_SERVICE_FAILED = 66;
+ int EXIT_YARN_SERVICE_FAILED = 65;
/**
* service was killed: {@value}
*/
- int EXIT_YARN_SERVICE_KILLED = 67;
+ int EXIT_YARN_SERVICE_KILLED = 66;
/**
* timeout on monitoring client: {@value}
*/
- int EXIT_TIMED_OUT = 68;
+ int EXIT_TIMED_OUT = 67;
/**
* service finished with an error: {@value}
*/
- int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 69;
+ int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68;
/**
* the application instance is unknown: {@value}
*/
- int EXIT_UNKNOWN_INSTANCE = 70;
+ int EXIT_UNKNOWN_INSTANCE = 69;
/**
* the application instance is in the wrong state for that operation: {@value}
*/
- int EXIT_BAD_STATE = 71;
+ int EXIT_BAD_STATE = 70;
/**
* A spawned master process failed
*/
- int EXIT_PROCESS_FAILED = 72;
+ int EXIT_PROCESS_FAILED = 71;
/**
* The instance failed -too many containers were
* failing or some other threshold was reached
*/
- int EXIT_DEPLOYMENT_FAILED = 73;
+ int EXIT_DEPLOYMENT_FAILED = 72;
/**
* The application is live -and the requested operation
* does not work if the cluster is running
*/
- int EXIT_APPLICATION_IN_USE = 74;
+ int EXIT_APPLICATION_IN_USE = 73;
/**
* There already is an application instance of that name
* when an attempt is made to create a new instance
*/
- int EXIT_INSTANCE_EXISTS = 75;
+ int EXIT_INSTANCE_EXISTS = 75;
/**
- * The resource was not found
+ * Exit code when the configurations in valid/incomplete: {@value}
*/
- int EXIT_NOT_FOUND = 77;
+ int EXIT_BAD_CONFIGURATION = 77;
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 38f55c2..b9dc8e1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -86,6 +86,7 @@
String HISTORY_DIR_NAME = "history";
String HISTORY_FILENAME_SUFFIX = "json";
String HISTORY_FILENAME_PREFIX = "rolehistory-";
+ String KEYTAB_DIR = "keytabs";
/**
* Filename pattern is required to save in strict temporal order.
@@ -112,6 +113,8 @@
String CLUSTER_DIRECTORY = "cluster";
+ String PACKAGE_DIRECTORY = "package";
+
/**
* JVM property to define the slider configuration directory;
* this is set by the slider script: {@value}
@@ -119,11 +122,37 @@
String PROPERTY_CONF_DIR = "slider.confdir";
/**
+ * JVM property to define the slider lib directory;
+ * this is set by the slider script: {@value}
+ */
+ String PROPERTY_LIB_DIR = "slider.libdir";
+
+ /**
* name of generated dir for this conf: {@value}
*/
String SUBMITTED_CONF_DIR = "confdir";
/**
+ * Slider AM log4j file name : {@value}
+ */
+ String LOG4J_SERVER_PROP_FILENAME = "log4j-server.properties";
+
+ /**
+ * Standard log4j file name : {@value}
+ */
+ String LOG4J_PROP_FILENAME = "log4j.properties";
+
+ /**
+ * Log4j sysprop to name the resource :{@value}
+ */
+ String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
+
+ /**
+ * sysprop for Slider AM log4j directory :{@value}
+ */
+ String SYSPROP_LOG_DIR = "LOG_DIR";
+
+ /**
* name of the Slider client resource
* loaded when the service is loaded.
*/
@@ -142,6 +171,7 @@
String SLIDER_JAR = "slider.jar";
String JCOMMANDER_JAR = "jcommander.jar";
String GSON_JAR = "gson.jar";
+ String AGENT_TAR = "slider-agent.tar.gz";
String DEFAULT_JVM_HEAP = "256M";
int DEFAULT_YARN_MEMORY = 256;
@@ -164,12 +194,30 @@
String KEY_FILE_NAME = "ca.key";
String KEYSTORE_FILE_NAME = "keystore.p12";
String CRT_PASS_FILE_NAME = "pass.txt";
- String PASSPHRASE = "DEV";
String PASS_LEN = "50";
- String KEYSTORE_LOCATION = "ssl.server.keystore.location";
/**
* Python specific
*/
String PYTHONPATH = "PYTHONPATH";
+
+
+ /**
+ * Name of the AM filter to use: {@value}
+ */
+ String AM_FILTER_NAME =
+ "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
+
+ String KEY_ALLOWED_PORT_RANGE = "site.global.slider.allowed.ports";
+ /**
+ * Allowed port range
+ */
+ String KEY_AM_ALLOWED_PORT_RANGE = "slider.am.allowed.port.range";
+
+ /**
+ * env var for custom JVM options.
+ */
+ String SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS";
+
+ String SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA";
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
index e31cfb6..eb55c71 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
@@ -41,7 +41,7 @@
String KEY_TEST_TIMEOUT = "slider.test.timeout.seconds";
- int DEFAULT_TEST_TIMEOUT_SECONDS = 10 * 60;
+ int DEFAULT_TEST_TIMEOUT_SECONDS = 30 * 60;
String KEY_TEST_HBASE_LAUNCH_TIME = "slider.test.hbase.launch.wait.seconds";
@@ -53,9 +53,16 @@
String KEY_ACCUMULO_LAUNCH_TIME =
"slider.test.accumulo.launch.wait.seconds";
-
int DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS = 60 * 3;
+
+ String KEY_ACCUMULO_GO_LIVE_TIME =
+ "slider.test.accumulo.live.wait.seconds";
+ int DEFAULT_ACCUMULO_LIVE_TIME_SECONDS = 90;
+
String KEY_TEST_AGENT_ENABLED = "slider.test.agent.enabled";
+ String KEY_AGENTTESTS_QUEUE_LABELED_DEFINED = "slider.test.agent.labeled.queue.enabled";
+ String KEY_AGENTTESTS_LABELS_RED_BLUE_DEFINED = "slider.test.agent.labels.defined";
+ String KEY_AGENTTESTS_AM_FAILURES_ENABLED = "slider.test.agent.am.failures.enabled";
int DEFAULT_AGENT_LAUNCH_TIME_SECONDS = 60 * 3;
@@ -76,4 +83,9 @@
* security related keys
*/
String TEST_SECURITY_DIR = "/tmp/work/security";
+
+ /**
+ * Local path to AM keytab: {@value}
+ */
+ String KEY_TEST_AM_KEYTAB = "slider.test.am.keytab.local";
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index 1bbe9ae..a967ebf 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -19,6 +19,7 @@
package org.apache.slider.common;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
/**
@@ -75,7 +76,7 @@
* Name of the property for ACLs for Slider AM.
* {@value}
*/
- String KEY_PROTOCOL_ACL = "security.slider.protocol.acl";
+ String KEY_PROTOCOL_ACL = "slider.security.protocol.acl";
/**
* Limit on restarts for the AM
@@ -84,10 +85,10 @@
String KEY_AM_RESTART_LIMIT = "slider.yarn.restart.limit";
/**
- * queue name
+ * queue name, by default let YARN pick the queue
*/
String KEY_YARN_QUEUE = "slider.yarn.queue";
- String DEFAULT_YARN_QUEUE = YarnConfiguration.DEFAULT_QUEUE_NAME;
+ String DEFAULT_YARN_QUEUE = null;
/**
* default priority
@@ -127,7 +128,7 @@
/**
* Default value for the registry: {@value}
*/
- String DEFAULT_REGISTRY_PATH = "/registry";
+ String DEFAULT_REGISTRY_PATH = RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT;
String REGISTRY_ZK_QUORUM = "slider.zookeeper.quorum";
@@ -142,4 +143,22 @@
"ipc.client.fallback-to-simple-auth-allowed";
String HADOOP_HTTP_FILTER_INITIALIZERS =
"hadoop.http.filter.initializers";
+ String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location";
+ String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
+ String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
+ String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
+ String KEY_SECURITY_ENABLED = "site.global.security_enabled";
+
+ /**
+ * Set to disable server-side checks for python, openssl &c.
+ * This should only be set for testing
+ */
+ String KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED =
+ "slider.am.dependency.checks.disabled";
+
+ /**
+ * The path to the python executable utilized to launch the agent.
+ */
+ String PYTHON_EXECUTABLE_PATH = "agent.python.exec.path";
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index f4a4569..22473d5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.UsageException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,13 +119,14 @@
return getMinParams();
}
- public void validate() throws BadCommandArgumentsException {
+ public void validate() throws BadCommandArgumentsException, UsageException {
int minArgs = getMinParams();
int actionArgSize = parameters.size();
if (minArgs > actionArgSize) {
throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName());
+ ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
+ " Expected minimum " + minArgs + " but got " + actionArgSize);
}
int maxArgs = getMaxParams();
if (maxArgs == -1) {
@@ -148,4 +150,15 @@
public String toString() {
return super.toString() + ": " + getActionName();
}
+
+ /**
+ * Override point:
+ * Flag to indicate that core hadoop API services are needed (HDFS, YARN, etc)
+ * —and that validation of the client state should take place.
+ *
+ * @return a flag to indicate that the core hadoop services will be needed.
+ */
+ public boolean getHadoopServicesRequired() {
+ return true;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index fa71677..56e01c3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -93,6 +93,10 @@
description = "Template application configuration")
public File template;
+ @Parameter(names = {ARG_QUEUE},
+ description = "Queue to submit the application")
+ public String queue;
+
@ParametersDelegate
public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
index cfcfb9d..e70f30a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
@@ -21,6 +21,8 @@
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
+import java.io.File;
+
@Parameters(commandNames = {SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
@@ -36,6 +38,11 @@
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
@Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
+
+ @Override
public String getRmAddress() {
return launchArgs.getRmAddress();
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
index d4acea6..1203d28 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
@@ -24,6 +24,7 @@
commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
public class ActionDestroyArgs extends AbstractActionArgs {
+
@Override
public String getActionName() {
return SliderActions.ACTION_DESTROY;
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
new file mode 100644
index 0000000..4b03336
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(
+ commandNames = {SliderActions.ACTION_DIAGNOSTICS},
+ commandDescription = SliderActions.DESCRIBE_ACTION_DIAGNOSTIC)
+public class ActionDiagnosticArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_DIAGNOSTICS;
+ }
+
+ @Parameter(names = {ARG_NAME},
+ description = "the name of the running application")
+ public String name;
+
+ @Parameter(names = {ARG_CLIENT},
+ description = "print configuration of the slider client")
+ public boolean client = false;
+
+ @Parameter(names = {ARG_APPLICATION},
+ description = "print configuration of the running application")
+ public boolean application;
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "print out information in details")
+ public boolean verbose = false;
+
+ @Parameter(names = {ARG_YARN},
+ description = "print configuration of the YARN cluster")
+ public boolean yarn = false;
+
+ @Parameter(names = {ARG_CREDENTIALS},
+ description = "print credentials of the current user")
+ public boolean credentials = false;
+
+ @Parameter(names = {ARG_ALL},
+ description = "print all of the information above")
+ public boolean all;
+
+ @Parameter(names = {ARG_LEVEL},
+ description = "diagnose each slider configuration one by one")
+ public boolean level;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
index 2d6bef2..dd1c04b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
@@ -21,6 +21,8 @@
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
+import java.io.File;
+
@Parameters(commandNames = {SliderActions.ACTION_EXISTS},
commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
@@ -30,7 +32,16 @@
public String getActionName() {
return SliderActions.ACTION_EXISTS;
}
+
@Parameter(names = {ARG_LIVE},
- description = "verify that the cluster is running")
+ description = "verify that the application is running")
public boolean live;
+
+ @Parameter(names = {ARG_STATE},
+ description = "verify that the application is in the specific YARN state")
+ public String state = "";
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any application report")
+ public File out;
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
index 04f6305..e3085d9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
@@ -32,7 +32,7 @@
return SliderActions.ACTION_FREEZE;
}
- public static final String FREEZE_COMMAND_ISSUED = "freeze command issued";
+ public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
@ParametersDelegate
public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionGetConfArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionGetConfArgs.java
deleted file mode 100644
index b636a5e..0000000
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionGetConfArgs.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = {SliderActions.ACTION_GETCONF},
- commandDescription = SliderActions.DESCRIBE_ACTION_GETCONF)
-
-public class ActionGetConfArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_GETCONF;
- }
-
- //--format
- @Parameter(names = ARG_FORMAT,
- description = "Format for a response: [text|xml|json|properties]")
- public String format = FORMAT_XML;
-
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output file for the configuration data")
- private String output;
-
-
- public String getFormat() {
- return format;
- }
-
- public String getOutput() {
- return output;
- }
-}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
index 4d4098f..62773c4 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
@@ -20,9 +20,11 @@
import com.beust.jcommander.Parameters;
-@Parameters(commandNames = {SliderActions.ACTION_HELP, SliderActions.ACTION_USAGE},
- commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
+/**
+ * The Help command
+ */
+@Parameters(commandNames = {SliderActions.ACTION_HELP},
+ commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
public class ActionHelpArgs extends AbstractActionArgs {
@Override
public String getActionName() {
@@ -38,4 +40,12 @@
return 0;
}
+ /**
+ * This action does not need hadoop services
+ * @return false
+ */
+ @Override
+ public boolean getHadoopServicesRequired() {
+ return false;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java
new file mode 100644
index 0000000..4cfb889
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_INSTALL_KEYTAB},
+ commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_KEYTAB)
+
+public class ActionInstallKeytabArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_INSTALL_KEYTAB;
+ }
+
+ @Parameter(names = {ARG_KEYTAB},
+ description = "Path to keytab on local disk")
+ public String keytabUri;
+
+ @Parameter(names = {ARG_FOLDER},
+ description = "The name of the folder in which to store the keytab")
+ public String folder;
+
+ @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
+ public boolean overwrite = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 3;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java
new file mode 100644
index 0000000..646e795
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+@Parameters(commandNames = {SliderActions.ACTION_INSTALL_PACKAGE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_PACKAGE)
+
+public class ActionInstallPackageArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_INSTALL_PACKAGE;
+ }
+
+ @Parameter(names = {ARG_PACKAGE},
+ description = "Path to app package on local disk")
+ public String packageURI;
+
+ @Parameter(names = {ARG_NAME},
+ description = "The type of the package")
+ public String name;
+
+ @Parameter(names = {ARG_REPLACE_PKG}, description = "Overwrite existing package")
+ public boolean replacePkg = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
index 8f42b69..30cc93e 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
@@ -18,6 +18,7 @@
package org.apache.slider.common.params;
+import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_LIST},
@@ -29,6 +30,18 @@
return SliderActions.ACTION_LIST;
}
+ @Parameter(names = {ARG_LIVE},
+ description = "List only live application instances")
+ public boolean live;
+
+ @Parameter(names = {ARG_STATE},
+ description = "list only applications in the specific YARN state")
+ public String state = "";
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "print out information in details")
+ public boolean verbose = false;
+
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
new file mode 100644
index 0000000..3b69e74
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.commons.lang.StringUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.UsageException;
+
+import java.io.File;
+
+@Parameters(commandNames = {SliderActions.ACTION_LOOKUP},
+ commandDescription = SliderActions.DESCRIBE_ACTION_LOOKUP)
+
+public class ActionLookupArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_LOOKUP;
+ }
+
+ public int getMinParams() {
+ return 0;
+ }
+ public int getMaxParams() {
+ return 0;
+ }
+
+ @Parameter(names = {ARG_ID},
+ description = "ID of the container")
+ public String id;
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any application report")
+ public File outputFile;
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (StringUtils.isEmpty(id)) {
+ throw new BadCommandArgumentsException("Missing mandatory argument "
+ + ARG_ID);
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(SliderActions.ACTION_LOOKUP);
+ if (id!=null) {
+ sb.append(" ");
+ sb.append(ARG_ID).append(" ").append(id);
+ }
+ if (outputFile != null) {
+ sb.append(" ");
+ sb.append(ARG_OUTPUT).append(" ").append(outputFile.getAbsolutePath());
+ }
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
index 9d311c2..da1b0e5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
@@ -22,14 +22,13 @@
import com.beust.jcommander.Parameters;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.UsageException;
import org.apache.slider.core.registry.docstore.ConfigFormat;
import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY;
import static org.apache.slider.common.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
import java.io.File;
-
/**
* Registry actions
*
@@ -43,6 +42,25 @@
public class ActionRegistryArgs extends AbstractActionArgs {
+ public static final String USAGE =
+ "Usage: " + SliderActions.ACTION_REGISTRY
+ + " ("
+ + Arguments.ARG_LIST + "|"
+ + Arguments.ARG_LISTCONF + "|"
+ + Arguments.ARG_LISTEXP + "|"
+ + Arguments.ARG_LISTFILES + "|"
+ + Arguments.ARG_GETCONF + "|"
+ + Arguments.ARG_GETEXP + "> "
+ + Arguments.ARG_NAME + " <name> "
+ + " )"
+ + "[" + Arguments.ARG_VERBOSE + "] "
+ + "[" + Arguments.ARG_USER + "] "
+ + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
+ + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
+ + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
+ + System.getProperty("line.separator")
+ + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
+ ;
public ActionRegistryArgs() {
}
@@ -55,7 +73,6 @@
return ACTION_REGISTRY;
}
-
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
@@ -77,8 +94,15 @@
description = "get configuration")
public String getConf;
+ @Parameter(names = {ARG_LISTEXP},
+ description = "list exports")
+ public boolean listExports;
- @Parameter(names = {ARG_LISTFILES},
+ @Parameter(names = {ARG_GETEXP},
+ description = "get export")
+ public String getExport;
+
+ @Parameter(names = {ARG_LISTFILES},
description = "list files")
public String listFiles;
@@ -86,16 +110,14 @@
description = "get files")
public String getFiles;
-
//--format
@Parameter(names = ARG_FORMAT,
- description = "Format for a response: [xml|json|properties]")
+ description = "Format for a response: <xml|json|properties>")
public String format = ConfigFormat.XML.toString() ;
-
- @Parameter(names = {ARG_DEST},
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
description = "Output destination")
- public File dest;
+ public File out;
@Parameter(names = {ARG_NAME},
description = "name of an instance")
@@ -112,30 +134,34 @@
@Parameter(names = {ARG_INTERNAL},
description = "fetch internal registry entries")
public boolean internal;
-
+
+ @Parameter(names = {ARG_USER},
+ description = "the name of the user whose application is being resolved")
+ public String user;
+
/**
* validate health of all the different operations
* @throws BadCommandArgumentsException
*/
@Override
- public void validate() throws BadCommandArgumentsException {
+ public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
//verify that at most one of the operations is set
- int gets = s(getConf) + s(getFiles);
- int lists = s(list) + s(listConf) + s(listFiles);
+ int gets = s(getConf) + s(getFiles) + s(getExport);
+ int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
int set = lists + gets;
if (set > 1) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_TOO_MANY_ARGUMENTS);
+ throw new UsageException(USAGE);
}
- if (dest != null && (lists > 0 || set == 0)) {
- throw new BadCommandArgumentsException("Argument " + ARG_DEST
- + " is only supported on 'get' operations");
+
+ if (out != null && ( set == 0)) {
+ throw new UsageException("output path"
+ + " is only supported on 'get' operations: ");
}
if (!list && !is(name)) {
- throw new BadCommandArgumentsException("Argument " + ARG_NAME
- +" missing");
+ throw new UsageException("Argument " + ARG_NAME
+ +" missing: ");
}
}
@@ -182,8 +208,8 @@
sb.append(ifdef(ARG_VERBOSE, verbose));
sb.append(ifdef(ARG_INTERNAL, internal));
- if (dest != null) {
- sb.append(ifdef(ARG_DEST, dest.toString()));
+ if (out != null) {
+ sb.append(ifdef(ARG_OUTPUT, out.toString()));
}
sb.append(ifdef(ARG_FORMAT, format));
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
new file mode 100644
index 0000000..2ee075a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.commons.lang.StringUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.UsageException;
+
+import java.io.File;
+
+import static org.apache.slider.common.params.SliderActions.ACTION_RESOLVE;
+import static org.apache.slider.common.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+
+/**
+ * Resolve registry entries
+ *
+ * --path {path}
+ * --out {destfile}
+ * --verbose
+ * --list
+ */
+@Parameters(commandNames = {ACTION_RESOLVE},
+ commandDescription = DESCRIBE_ACTION_REGISTRY)
+public class ActionResolveArgs extends AbstractActionArgs {
+
+ public static final String USAGE =
+ "Usage: " + SliderActions.ACTION_RESOLVE
+ + " "
+ + ARG_PATH + " <path> "
+ + "[" + ARG_LIST + "] "
+ + "[" + ARG_OUTPUT + " <filename> ] "
+ + "[" + ARG_DESTDIR + " <directory> ] "
+ ;
+ public ActionResolveArgs() {
+ }
+
+ @Override
+ public String getActionName() {
+ return ACTION_RESOLVE;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Parameter(names = {ARG_LIST},
+ description = "list services")
+ public boolean list;
+
+ @Parameter(names = {ARG_PATH},
+ description = "resolve a path")
+ public String path;
+
+ @Parameter(names = {ARG_DESTDIR},
+ description = "destination directory for operations")
+ public File destdir;
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "dest file")
+ public File out;
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(ACTION_RESOLVE).append(" ");
+ sb.append(ARG_PATH).append(" ").append(path).append(" ");
+ if (list) {
+ sb.append(ARG_LIST).append(" ");
+ }
+ if (destdir != null) {
+ sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
+ }
+ if (out != null) {
+ sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (StringUtils.isEmpty(path)) {
+ throw new BadCommandArgumentsException("Missing mandatory argument "
+ + ARG_PATH);
+ }
+ if (list && out != null) {
+ throw new BadCommandArgumentsException("Argument "
+ + ARG_OUTPUT +
+ " not supported for " + ARG_LIST);
+ }
+ if (out != null && destdir != null) {
+ throw new BadCommandArgumentsException(
+ ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
+ );
+ }
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public boolean isList() {
+ return list;
+ }
+
+ public void setList(boolean list) {
+ this.list = list;
+ }
+
+ public File getDestdir() {
+ return destdir;
+ }
+
+ public void setDestdir(File destdir) {
+ this.destdir = destdir;
+ }
+
+ public File getOut() {
+ return out;
+ }
+
+ public void setOut(File out) {
+ this.out = out;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
index 8408385..b43a14e 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
@@ -21,6 +21,8 @@
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
+import java.io.File;
+
@Parameters(commandNames = {SliderActions.ACTION_THAW},
commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
public class ActionThawArgs extends AbstractActionArgs implements
@@ -50,4 +52,10 @@
public void setWaittime(int waittime) {
launchArgs.setWaittime(waittime);
}
+
+
+ @Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
index c36ef62..b9d212b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
@@ -20,9 +20,11 @@
import com.beust.jcommander.Parameters;
+/**
+ * The version command
+ */
@Parameters(commandNames = {SliderActions.ACTION_VERSION},
commandDescription = SliderActions.DESCRIBE_ACTION_VERSION)
-
public class ActionVersionArgs extends AbstractActionArgs {
@Override
public String getActionName() {
@@ -33,4 +35,12 @@
return 0;
}
+ /**
+ * This action does not need hadoop services
+ * @return false
+ */
+ @Override
+ public boolean getHadoopServicesRequired() {
+ return false;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
index 83754b3..aeb2979 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
@@ -83,7 +83,7 @@
for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
String key = entry.getKey();
String val = entry.getValue();
- log.debug("configuration[{}]=\"{}\"", key, val);
+ log.debug("configuration[{}]<=\"{}\"", key, val);
conf.set(key, val, "command line");
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 764ffca..42efb33 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -21,48 +21,65 @@
/**
* Here are all the arguments that may be parsed by the client or server
* command lines.
+ *
+ * Important: Please keep the main list in alphabetical order
+ * so it is easier to see what arguments are there
*/
public interface Arguments {
-
+ String ARG_ALL = "--all";
+ String ARG_APPLICATION = "--application";
String ARG_APP_HOME = "--apphome";
+ String ARG_BASE_PATH = "--basepath";
+ String ARG_CLIENT = "--client";
String ARG_CONFDIR = "--appconf";
String ARG_COMPONENT = "--component";
String ARG_COMPONENT_SHORT = "--comp";
String ARG_COMP_OPT= "--compopt";
String ARG_COMP_OPT_SHORT = "--co";
-
+ String ARG_CREDENTIALS = "--credentials";
+
String ARG_DEBUG = "--debug";
- String ARG_DEST = "--dest";
String ARG_DEFINE = "-D";
+ String ARG_DEST = "--dest";
+ String ARG_DESTDIR = "--destdir";
+ String ARG_DESTFILE = "--destfile";
String ARG_EXITCODE = "--exitcode";
/**
filesystem-uri: {@value}
*/
String ARG_FILESYSTEM = "--fs";
String ARG_FILESYSTEM_LONG = "--filesystem";
- String ARG_BASE_PATH = "--basepath";
- String ARG_FORMAT = "--format";
+ String ARG_FOLDER = "--folder";
String ARG_FORCE = "--force";
+ String ARG_FORMAT = "--format";
String ARG_GETCONF = "--getconf";
+ String ARG_GETEXP = "--getexp";
String ARG_GETFILES = "--getfiles";
String ARG_HELP = "--help";
String ARG_ID = "--id";
String ARG_IMAGE = "--image";
String ARG_INTERNAL = "--internal";
+ String ARG_KEYTAB = "--keytab";
+ String ARG_LEVEL = "--level";
String ARG_LIST = "--list";
- String ARG_LISTFILES = "--listfiles";
String ARG_LISTCONF = "--listconf";
+ String ARG_LISTEXP = "--listexp";
+ String ARG_LISTFILES = "--listfiles";
String ARG_LIVE = "--live";
String ARG_MANAGER = "--manager";
String ARG_MANAGER_SHORT = "--m";
String ARG_MESSAGE = "--message";
+ String ARG_NAME = "--name";
String ARG_OPTION = "--option";
String ARG_OPTION_SHORT = "-O";
- String ARG_NAME = "--name";
String ARG_OUTPUT = "--out";
String ARG_OUTPUT_SHORT = "-o";
+ String ARG_OVERWRITE = "--overwrite";
String ARG_PACKAGE = "--package";
+ String ARG_PATH = "--path";
String ARG_PROVIDER = "--provider";
+ String ARG_QUEUE = "--queue";
+ String ARG_REPLACE_PKG = "--replacepkg";
String ARG_RESOURCES = "--resources";
String ARG_RES_COMP_OPT = "--rescompopt";
String ARG_RES_COMP_OPT_SHORT = "--rco";
@@ -70,13 +87,17 @@
String ARG_RESOURCE_OPT = "--resopt";
String ARG_RESOURCE_OPT_SHORT = "-ro";
String ARG_SERVICETYPE = "--servicetype";
+ String ARG_SLIDER = "--slider";
+ String ARG_STATE = "--state";
String ARG_SYSPROP = "-S";
String ARG_TEMPLATE = "--template";
+ String ARG_USER = "--user";
String ARG_VERBOSE = "--verbose";
String ARG_WAIT = "--wait";
+ String ARG_YARN = "--yarn";
+ String ARG_ZKHOSTS = "--zkhosts";
String ARG_ZKPATH = "--zkpath";
String ARG_ZKPORT = "--zkport";
- String ARG_ZKHOSTS = "--zkhosts";
/**
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index ca854f1..a3b7997 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.SliderException;
@@ -42,25 +43,31 @@
* This is not bonded to jcommander, it is set up
* after the construction to point to the relevant
* entry
+ *
+ * KEEP IN ALPHABETICAL ORDER
*/
private AbstractClusterBuildingActionArgs buildingActionArgs;
private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs();
private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
- private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
+ private final ActionDiagnosticArgs actionDiagnosticArgs = new ActionDiagnosticArgs();
private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
- private final ActionGetConfArgs actionGetConfArgs = new ActionGetConfArgs();
+ private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
+ private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs();
+ private final ActionInstallKeytabArgs actionInstallKeytabArgs = new ActionInstallKeytabArgs();
private final ActionKillContainerArgs actionKillContainerArgs =
new ActionKillContainerArgs();
private final ActionListArgs actionListArgs = new ActionListArgs();
+ private final ActionLookupArgs actionLookupArgs = new ActionLookupArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
+ private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
+ private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
- private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
public ClientArgs(String[] args) {
@@ -75,23 +82,27 @@
protected void addActionArguments() {
addActions(
- actionAMSuicideArgs,
- actionBuildArgs,
- actionCreateArgs,
- actionUpdateArgs,
- actionDestroyArgs,
- actionExistsArgs,
- actionFlexArgs,
- actionFreezeArgs,
- actionGetConfArgs,
- actionKillContainerArgs,
- actionListArgs,
- actionRegistryArgs,
- actionStatusArgs,
- actionThawArgs,
- actionHelpArgs,
- actionVersionArgs
- );
+ actionAMSuicideArgs,
+ actionBuildArgs,
+ actionCreateArgs,
+ actionUpdateArgs,
+ actionDestroyArgs,
+ actionDiagnosticArgs,
+ actionExistsArgs,
+ actionFlexArgs,
+ actionFreezeArgs,
+ actionHelpArgs,
+ actionInstallPackageArgs,
+ actionInstallKeytabArgs,
+ actionKillContainerArgs,
+ actionListArgs,
+ actionLookupArgs,
+ actionRegistryArgs,
+ actionResolveArgs,
+ actionStatusArgs,
+ actionThawArgs,
+ actionVersionArgs
+ );
}
@Override
@@ -110,6 +121,10 @@
}
}
+ public ActionDiagnosticArgs getActionDiagnosticArgs() {
+ return actionDiagnosticArgs;
+ }
+
public AbstractClusterBuildingActionArgs getBuildingActionArgs() {
return buildingActionArgs;
}
@@ -122,6 +137,12 @@
return actionBuildArgs;
}
+ public ActionInstallPackageArgs getActionInstallPackageArgs() {
+ return actionInstallPackageArgs; }
+
+ public ActionInstallKeytabArgs getActionInstallKeytabArgs() {
+ return actionInstallKeytabArgs; }
+
public ActionUpdateArgs getActionUpdateArgs() {
return actionUpdateArgs;
}
@@ -146,10 +167,6 @@
return actionFreezeArgs;
}
- public ActionGetConfArgs getActionGetConfArgs() {
- return actionGetConfArgs;
- }
-
public ActionKillContainerArgs getActionKillContainerArgs() {
return actionKillContainerArgs;
}
@@ -158,10 +175,18 @@
return actionListArgs;
}
+ public ActionLookupArgs getActionLookupArgs() {
+ return actionLookupArgs;
+ }
+
public ActionRegistryArgs getActionRegistryArgs() {
return actionRegistryArgs;
}
+ public ActionResolveArgs getActionResolveArgs() {
+ return actionResolveArgs;
+ }
+
public ActionStatusArgs getActionStatusArgs() {
return actionStatusArgs;
}
@@ -174,7 +199,7 @@
* Look at the chosen action and bind it as the core action for the operation.
* In theory this could be done by introspecting on the list of actions and
* choosing it without the switch statement. In practise this switch, while
- * verbose, is easier to debug.
+ * verbose, is easier to debug. And in JDK7, much simpler.
* @throws SliderException bad argument or similar
*/
@Override
@@ -189,9 +214,6 @@
//its a builder, so set those actions too
buildingActionArgs = actionCreateArgs;
- } else if (SliderActions.ACTION_UPDATE.equals(action)) {
- bindCoreAction(actionUpdateArgs);
-
} else if (SliderActions.ACTION_FREEZE.equals(action)) {
bindCoreAction(actionFreezeArgs);
@@ -204,36 +226,50 @@
} else if (SliderActions.ACTION_DESTROY.equals(action)) {
bindCoreAction(actionDestroyArgs);
+ } else if (SliderActions.ACTION_DIAGNOSTICS.equals(action)) {
+ bindCoreAction(actionDiagnosticArgs);
+
} else if (SliderActions.ACTION_EXISTS.equals(action)) {
bindCoreAction(actionExistsArgs);
} else if (SliderActions.ACTION_FLEX.equals(action)) {
bindCoreAction(actionFlexArgs);
- } else if (SliderActions.ACTION_GETCONF.equals(action)) {
- bindCoreAction(actionGetConfArgs);
-
- } else if (SliderActions.ACTION_HELP.equals(action) ||
- SliderActions.ACTION_USAGE.equals(action)) {
+ } else if (SliderActions.ACTION_HELP.equals(action)) {
bindCoreAction(actionHelpArgs);
+ } else if (SliderActions.ACTION_INSTALL_PACKAGE.equals(action)) {
+ bindCoreAction(actionInstallPackageArgs);
+
+ } else if (SliderActions.ACTION_INSTALL_KEYTAB.equals(action)) {
+ bindCoreAction(actionInstallKeytabArgs);
+
} else if (SliderActions.ACTION_KILL_CONTAINER.equals(action)) {
bindCoreAction(actionKillContainerArgs);
} else if (SliderActions.ACTION_LIST.equals(action)) {
bindCoreAction(actionListArgs);
+ } else if (SliderActions.ACTION_LOOKUP.equals(action)) {
+ bindCoreAction(actionLookupArgs);
+
} else if (SliderActions.ACTION_REGISTRY.equals(action)) {
bindCoreAction(actionRegistryArgs);
+ } else if (SliderActions.ACTION_RESOLVE.equals(action)) {
+ bindCoreAction(actionResolveArgs);
+
} else if (SliderActions.ACTION_STATUS.equals(action)) {
bindCoreAction(actionStatusArgs);
+ } else if (SliderActions.ACTION_UPDATE.equals(action)) {
+ bindCoreAction(actionUpdateArgs);
+
} else if (SliderActions.ACTION_VERSION.equals(action)) {
bindCoreAction(actionVersionArgs);
- } else if (action == null || action.isEmpty()) {
- throw new BadCommandArgumentsException(ErrorStrings.ERROR_NO_ACTION);
+ } else if (SliderUtils.isUnset(action)) {
+ bindCoreAction(actionHelpArgs);
} else {
throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
index 5d94182..57d5fb4 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
@@ -20,13 +20,16 @@
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterDescription;
import com.beust.jcommander.ParameterException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.UsageException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,6 +51,9 @@
protected static final Logger log = LoggerFactory.getLogger(CommonArgs.class);
+ private static final int DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME = 30;
+
+
@Parameter(names = ARG_HELP, help = true)
public boolean help;
@@ -96,14 +102,49 @@
commander = new JCommander(this);
}
-
public String usage() {
- StringBuilder builder = new StringBuilder("\n");
- commander.usage(builder, " ");
- builder.append("\nactions: ");
- return builder.toString();
+ return usage(this, null);
}
+ public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
+ String result = null;
+ StringBuilder helperMessage = new StringBuilder();
+ if (commandOfInterest == null) {
+ // JCommander.usage is too verbose for a command with many options like
+ // slider no short version of that is found Instead, we compose our msg by
+ helperMessage.append("\nUsage: slider COMMAND [options]\n");
+ helperMessage.append("where COMMAND is one of\n");
+ for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
+ helperMessage.append(String.format("\t%-"
+ + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", jcommand,
+ serviceArgs.commander.getCommandDescription(jcommand) + "\n"));
+ }
+ helperMessage
+ .append("Most commands print help when invoked without parameters");
+ result = helperMessage.toString();
+ } else {
+ helperMessage.append("\nUsage: slider " + commandOfInterest);
+ helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <application>" : "");
+ helperMessage.append("\n");
+ for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
+ .get(commandOfInterest).getParameters()) {
+ String optional = paramDesc.getParameter().required() ? " (required)"
+ : " (optional)";
+ String paramName = paramDesc.getParameterized().getType() == Boolean.TYPE ? paramDesc
+ .getLongestName() : paramDesc.getLongestName() + " <"
+ + paramDesc.getParameterized().getName() + ">";
+ helperMessage.append(String.format("\t%-"
+ + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", paramName,
+ paramDesc.getDescription() + optional + "\n"));
+ result = helperMessage.toString();
+ }
+ }
+ return result;
+ }
+
+ public static String usage(CommonArgs serviceArgs) {
+ return usage(serviceArgs, null);
+ }
/**
* Parse routine -includes registering the action-specific argument classes
@@ -193,14 +234,25 @@
/**
* Validate the arguments against the action requested
*/
- public void validate() throws BadCommandArgumentsException {
+ public void validate() throws BadCommandArgumentsException, UsageException {
if (coreAction == null) {
- throw new BadCommandArgumentsException(ErrorStrings.ERROR_NO_ACTION
- + usage());
+ throw new UsageException(ErrorStrings.ERROR_NO_ACTION + usage());
}
log.debug("action={}", getAction());
- //let the action validate itself
- coreAction.validate();
+ // let the action validate itself
+ try {
+ coreAction.validate();
+ } catch (BadCommandArgumentsException e) {
+ StringBuilder badArgMsgBuilder = new StringBuilder();
+ badArgMsgBuilder.append(e.toString() + "\n");
+ badArgMsgBuilder.append(usage(this, coreAction.getActionName()));
+ throw new BadCommandArgumentsException(badArgMsgBuilder.toString());
+ } catch (UsageException e) {
+ StringBuilder badArgMsgBuilder = new StringBuilder();
+ badArgMsgBuilder.append(e.toString() + "\n");
+ badArgMsgBuilder.append(usage(this, coreAction.getActionName()));
+ throw new UsageException(badArgMsgBuilder.toString());
+ }
}
/**
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
index f4ff4ce..7524053 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
@@ -18,9 +18,13 @@
package org.apache.slider.common.params;
+import java.io.File;
+
/**
- * Launch args for create and thaw and anything else that can start something
+ * Launch args for create and start and anything else that can start something
*/
public interface LaunchArgsAccessor extends WaitTimeAccessor {
String getRmAddress();
+
+ File getOutputFile();
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
index f0068e2..bc7e94c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
@@ -20,6 +20,8 @@
import com.beust.jcommander.Parameter;
+import java.io.File;
+
/**
* Any launch-time args
*/
@@ -37,4 +39,13 @@
public String getRmAddress() {
return rmAddress;
}
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any application report")
+ public File outputFile;
+
+ @Override
+ public File getOutputFile() {
+ return outputFile;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
index 0d084da..197c22b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
@@ -22,6 +22,8 @@
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
+import java.io.File;
+
@Parameters(commandNames = {SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
@@ -63,5 +65,10 @@
public void setWaittime(int waittime) {
launchArgs.setWaittime(waittime);
}
+
+ @Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index 8e50a83..4b92a32 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -32,18 +32,21 @@
String ACTION_ECHO = "echo";
String ACTION_EXISTS = "exists";
String ACTION_FLEX = "flex";
- String ACTION_FREEZE = "freeze";
- String ACTION_GETCONF = "getconf";
+ String ACTION_FREEZE = "stop";
String ACTION_HELP = "help";
String ACTION_KILL_CONTAINER = "kill-container";
String ACTION_LIST = "list";
+ String ACTION_LOOKUP = "lookup";
String ACTION_PREFLIGHT = "preflight";
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
+ String ACTION_RESOLVE = "resolve";
String ACTION_STATUS = "status";
- String ACTION_THAW = "thaw";
- String ACTION_USAGE = "usage";
+ String ACTION_THAW = "start";
String ACTION_VERSION = "version";
+ String ACTION_DIAGNOSTICS = "diagnostics";
+ String ACTION_INSTALL_PACKAGE = "install-package";
+ String ACTION_INSTALL_KEYTAB = "install-keytab";
String DESCRIBE_ACTION_AM_SUICIDE =
"Tell the Slider Application Master to simulate a process failure by terminating itself";
String DESCRIBE_ACTION_BUILD =
@@ -58,7 +61,7 @@
"Probe for an application running";
String DESCRIBE_ACTION_FLEX = "Flex a Slider application";
String DESCRIBE_ACTION_FREEZE =
- "Freeze/suspend a running application";
+ "Stop a running application";
String DESCRIBE_ACTION_GETCONF =
"Get the configuration of an application";
String DESCRIBE_ACTION_KILL_CONTAINER =
@@ -66,14 +69,23 @@
String DESCRIBE_ACTION_HELP = "Print help information";
String DESCRIBE_ACTION_LIST =
"List running Slider applications";
+ String DESCRIBE_ACTION_LOOKUP =
+ "look up a YARN application";
String DESCRIBE_ACTION_MONITOR =
"Monitor a running application";
String DESCRIBE_ACTION_REGISTRY =
"Query the registry of a YARN application";
+ String DESCRIBE_ACTION_RESOLVE =
+ "Resolve or list records in the YARN registry";
String DESCRIBE_ACTION_STATUS =
"Get the status of an application";
String DESCRIBE_ACTION_THAW =
- "Thaw a frozen application";
+ "Start a stopped application";
String DESCRIBE_ACTION_VERSION =
"Print the Slider version information";
+ String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install the application package in the home directory under sub-folder packages";
+ String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab file in the sub-folder 'keytabs' of the user's Slider base directory";
+ String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
+
}
+
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index b7e1323..af5ceee 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.exceptions.BadConfigException;
@@ -48,6 +49,7 @@
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import java.util.TreeSet;
/**
@@ -65,22 +67,21 @@
* @param conf config
* @return the sorted keyset
*/
- public static TreeSet<String> dumpConf(Configuration conf) {
- TreeSet<String> keys = sortedConfigKeys(conf);
+ public static Set<String> dumpConf(Configuration conf) {
+ Set<String> keys = sortedConfigKeys(conf);
for (String key : keys) {
log.info("{}={}", key, conf.get(key));
}
return keys;
}
-
/**
* Take a configuration and return a sorted set
* @param conf config
* @return the sorted keyset
*/
- public static TreeSet<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
+ public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
TreeSet<String> sorted = new TreeSet<String>();
for (Map.Entry<String, String> entry : conf) {
sorted.add(entry.getKey());
@@ -338,7 +339,7 @@
try {
conf.addResource(file.toURI().toURL());
} catch (MalformedURLException e) {
- //should never happen...
+ // should never happen...
throw new IOException(
"File " + file.toURI() + " doesn't have a valid URL");
}
@@ -346,6 +347,34 @@
}
/**
+ * Add a configuration from a file to an existing configuration
+ * @param conf existing configuration
+ * @param file file to load
+ * @param overwrite flag to indicate new values should overwrite the predecessor
+ * @return the merged configuration
+ * @throws IOException
+ */
+ public static Configuration addConfigurationFile(Configuration conf,
+ File file, boolean overwrite)
+ throws IOException {
+ Configuration c2 = loadConfFromFile(file, false);
+ mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite);
+ return conf;
+ }
+
+ /**
+ * Add the system env variables with the given prefix (by convention, env.)
+ * @param conf existing configuration
+ * @param prefix prefix
+ */
+ public static void addEnvironmentVariables(Configuration conf, String prefix) {
+ Map<String, String> env = System.getenv();
+ for (Map.Entry<String, String> entry : env.entrySet()) {
+ conf.set(prefix + entry.getKey(),entry.getValue(), "env");
+ }
+ }
+
+ /**
* looks for the config under $confdir/$templateFilename; if not there
* loads it from /conf/templateFile.
* The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
@@ -383,8 +412,8 @@
*/
public static Configuration loadTemplateConfiguration(FileSystem fs,
Path templatePath,
- String fallbackResource) throws
- IOException {
+ String fallbackResource)
+ throws IOException {
Configuration conf;
String origin;
if (fs.exists(templatePath)) {
@@ -415,7 +444,7 @@
* @return listing in key=value style
*/
public static String dumpConfigToString(Configuration conf) {
- TreeSet<String> sorted = sortedConfigKeys(conf);
+ Set<String> sorted = sortedConfigKeys(conf);
StringBuilder builder = new StringBuilder();
for (String key : sorted) {
@@ -434,13 +463,18 @@
* @param merge one to merge. This MUST be a non-default-load config to avoid
* merge origin confusion
* @param origin description of the origin for the put operation
+ * @param overwrite flag to indicate new values should overwrite the predecessor
* @return the base with the merged values
*/
public static Configuration mergeConfigurations(Configuration base,
- Iterable<Map.Entry<String, String>> merge,
- String origin) {
+ Iterable<Map.Entry<String, String>> merge,
+ String origin,
+ boolean overwrite) {
for (Map.Entry<String, String> entry : merge) {
- base.set(entry.getKey(), entry.getValue(), origin);
+ String key = entry.getKey();
+ if (overwrite || base.get(key) == null) {
+ base.set(key, entry.getValue(), origin);
+ }
}
return base;
}
@@ -489,8 +523,8 @@
* @return the loaded configuration
* @throws FileNotFoundException if the resource is missing
*/
- public static Configuration loadMandatoryResource(String resource) throws
- FileNotFoundException {
+ public static Configuration loadMandatoryResource(String resource)
+ throws FileNotFoundException {
Configuration conf = new Configuration(false);
URL resURL = ConfigHelper.class.getClassLoader()
.getResource(resource);
@@ -561,4 +595,17 @@
}
return result;
}
+
+ /**
+ * Register anything we consider deprecated
+ */
+ public static void registerDeprecatedConfigItems() {
+ Configuration.addDeprecation(
+ SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+ Configuration.addDeprecation(
+ SliderXmlConfKeys.REGISTRY_PATH,
+ RegistryConstants.KEY_REGISTRY_ZK_ROOT);
+
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index def252a..fed0bc2 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -122,6 +122,49 @@
}
/**
+ * Build up the path string for package install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for persistent app package
+ */
+ public Path buildPackageDirPath(String packageName) {
+ Preconditions.checkNotNull(packageName);
+ Path path = getBaseApplicationPath();
+ return new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName);
+ }
+
+ /**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab
+ */
+ public Path buildKeytabInstallationDirPath(String keytabFolder) {
+ Preconditions.checkNotNull(keytabFolder);
+ Path path = getBaseApplicationPath();
+ return new Path(path, SliderKeys.KEYTAB_DIR + "/" + keytabFolder);
+ }
+
+ /**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab installation location
+ */
+ public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) {
+ Path homePath = getHomeDirectory();
+ Path baseKeytabDir;
+ if (keytabDir != null) {
+ baseKeytabDir = new Path(homePath, keytabDir);
+ } else {
+ baseKeytabDir = new Path(buildClusterDirPath(clusterName),
+ SliderKeys.KEYTAB_DIR);
+ }
+ return keytabName == null ? baseKeytabDir :
+ new Path(baseKeytabDir, keytabName);
+ }
+
+ /**
* Create the Slider cluster path for a named cluster and all its subdirs
* This is a directory; a mkdirs() operation is executed
* to ensure that it is there.
@@ -131,11 +174,10 @@
* @throws java.io.IOException trouble
* @throws SliderException slider-specific exceptions
*/
- public Path createClusterDirectories(String clustername, Configuration conf) throws
- IOException,
- SliderException {
-
-
+ public Path createClusterDirectories(String clustername, Configuration conf)
+ throws IOException, SliderException {
+
+
Path clusterDirectory = buildClusterDirPath(clustername);
InstancePaths instancePaths = new InstancePaths(clusterDirectory);
createClusterDirectories(instancePaths);
@@ -177,8 +219,8 @@
*
* @param dir directory
* @param clusterPerms cluster permissions
- * @throws IOException IO problem
- * @throws org.apache.slider.core.exceptions.BadClusterStateException any cluster state problem
+ * @throws IOException IO problem
+ * @throws BadClusterStateException any cluster state problem
*/
public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
IOException,
@@ -513,6 +555,40 @@
return builder.toString();
}
+ /**
+ * List all application instances persisted for this user, giving the
+ * patha. The instance name is the last element in the path
+ * @return a possibly empty map of application instance names to paths
+ */
+ public Map<String, Path> listPersistentInstances() throws IOException {
+ FileSystem fs = getFileSystem();
+ Path path = new Path(getBaseApplicationPath(), SliderKeys.CLUSTER_DIRECTORY);
+ log.debug("Looking for all persisted application at {}", path.toString());
+ if (!fs.exists(path)) {
+ // special case: no instances have ever been created
+ return new HashMap<String, Path>(0);
+ }
+ FileStatus[] statuses = fs.listStatus(path);
+ Map<String, Path> instances = new HashMap<String, Path>(statuses.length);
+
+ // enum the child entries
+ for (FileStatus status : statuses) {
+ if (status.isDirectory()) {
+ // for directories, look for an internal.json underneath
+ Path child = status.getPath();
+ Path internalJson = new Path(child, Filenames.INTERNAL);
+ if (fs.exists(internalJson)) {
+ // success => this is an instance
+ instances.put(child.getName(), child);
+ } else {
+ log.info("Malformed cluster found at {}. It does not appear to be a valid persisted instance.",
+ child.toString());
+ }
+ }
+ }
+ return instances;
+ }
+
public void touch(Path path, boolean overwrite) throws IOException {
FSDataOutputStream out = fileSystem.create(path, overwrite);
out.close();
@@ -545,7 +621,7 @@
*
* @param clustername name of the cluster
* @return the path to the spec.
- * @throws IOException IO problems
+ * @throws IOException IO problems
* @throws SliderException if the path isn't there
*/
public Path locateInstanceDefinition(String clustername) throws IOException,
@@ -564,23 +640,15 @@
* @throws IOException IO problems
* @throws SliderException if the cluster specification is not present
*/
- public void verifyClusterSpecExists(String clustername,
- Path clusterSpecPath) throws
- IOException,
+ public void verifyClusterSpecExists(String clustername, Path clusterSpecPath)
+ throws IOException,
SliderException {
if (!fileSystem.isFile(clusterSpecPath)) {
log.debug("Missing specification file {}", clusterSpecPath);
- throw UnknownApplicationInstanceException.unknownInstance(clustername
- +
- "\n (definition not found at "
- +
- clusterSpecPath);
+ throw UnknownApplicationInstanceException.unknownInstance(
+ clustername + "\n (definition not found at " + clusterSpecPath);
}
}
-
- public Path fileToPath(File file) {
- return new Path(file.getAbsoluteFile().toURI());
- }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
new file mode 100644
index 0000000..0f4cfbc
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.common.tools;
+
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ *
+ */
+public class PortScanner {
+ private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$");
+ private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$");
+
+ private List<Integer> remainingPortsToCheck;
+
+ public PortScanner() {
+ }
+
+ public void setPortRange(String input) {
+ // first split based on commas
+ Set<Integer> inputPorts= new TreeSet<Integer>();
+ String[] ranges = input.split(",");
+ for ( String range : ranges ) {
+ Matcher m = SINGLE_NUMBER.matcher(range.trim());
+ if (m.find()) {
+ inputPorts.add(Integer.parseInt(m.group()));
+ } else {
+ m = NUMBER_RANGE.matcher(range.trim());
+ if (m.find()) {
+ String[] boundaryValues = m.group(0).split("-");
+ int start = Integer.parseInt(boundaryValues[0].trim());
+ int end = Integer.parseInt(boundaryValues[1].trim());
+ for (int i = start; i < end + 1; i++) {
+ inputPorts.add(i);
+ }
+ }
+ }
+ }
+ this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts);
+ }
+
+ public List<Integer> getRemainingPortsToCheck() {
+ return remainingPortsToCheck;
+ }
+
+ public int getAvailablePort () throws SliderException{
+ boolean found = false;
+ int availablePort = -1;
+ Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
+ while (portsToCheck.hasNext() && !found) {
+ int portToCheck = portsToCheck.next();
+ found = SliderUtils.isPortAvailable(portToCheck);
+ if (found) {
+ availablePort = portToCheck;
+ portsToCheck.remove();
+ }
+ }
+
+ if (availablePort < 0) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "No available ports found in configured range {}",
+ remainingPortsToCheck);
+ }
+
+ return availablePort;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 188b7d9..f4cea00 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -19,6 +19,7 @@
package org.apache.slider.common.tools;
import com.google.common.base.Preconditions;
+
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.io.output.ByteArrayOutputStream;
@@ -33,16 +34,20 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.Slider;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.common.SliderKeys;
@@ -52,10 +57,11 @@
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.MissingArgException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ClasspathConstructor;
+import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.server.services.utility.PatternValidator;
+import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.zookeeper.server.util.KerberosUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,18 +69,28 @@
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
+import java.io.Serializable;
import java.io.StringWriter;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
@@ -83,9 +99,12 @@
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.TimeZone;
import java.util.Timer;
import java.util.TimerTask;
+import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -100,12 +119,25 @@
* turned on (prevents re-entrancy)
*/
private static final AtomicBoolean processSecurityAlreadyInitialized =
- new AtomicBoolean(false);
+ new AtomicBoolean(false);
public static final String JAVA_SECURITY_KRB5_REALM =
"java.security.krb5.realm";
public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
-
+ /**
+ * Winutils
+ */
+ public static final String WINUTILS = "WINUTILS.EXE";
+ /**
+ * name of openssl program
+ */
+ public static final String OPENSSL = "openssl";
+
+ /**
+ * name of python program
+ */
+ public static final String PYTHON = "python";
+
private SliderUtils() {
}
@@ -128,7 +160,8 @@
* @param msg the message to be shown in exception
*/
@SuppressWarnings("ResultOfMethodCallIgnored")
- private static void validateNumber(String num, String msg) throws BadConfigException {
+ private static void validateNumber(String num, String msg) throws
+ BadConfigException {
try {
Integer.parseInt(num);
} catch (NumberFormatException nfe) {
@@ -142,15 +175,16 @@
* @param heapsize
* @return heapsize in MB
*/
- public static String translateTrailingHeapUnit(String heapsize) throws BadConfigException {
+ public static String translateTrailingHeapUnit(String heapsize) throws
+ BadConfigException {
String errMsg = "Bad heapsize: ";
if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
- String num = heapsize.substring(0, heapsize.length()-1);
+ String num = heapsize.substring(0, heapsize.length() - 1);
validateNumber(num, errMsg);
return num;
}
if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
- String num = heapsize.substring(0, heapsize.length()-1)+"000";
+ String num = heapsize.substring(0, heapsize.length() - 1) + "000";
validateNumber(num, errMsg);
return num;
}
@@ -219,7 +253,7 @@
ClassLoader loader = my_class.getClassLoader();
if (loader == null) {
throw new IOException(
- "Class " + my_class + " does not have a classloader!");
+ "Class " + my_class + " does not have a classloader!");
}
String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
Enumeration<URL> urlEnumeration = loader.getResources(class_file);
@@ -252,16 +286,16 @@
}
public static void checkPort(String hostname, int port, int connectTimeout)
- throws IOException {
+ throws IOException {
InetSocketAddress addr = new InetSocketAddress(hostname, port);
checkPort(hostname, addr, connectTimeout);
}
@SuppressWarnings("SocketOpenedButNotSafelyClosed")
public static void checkPort(String name,
- InetSocketAddress address,
- int connectTimeout)
- throws IOException {
+ InetSocketAddress address,
+ int connectTimeout)
+ throws IOException {
Socket socket = null;
try {
socket = new Socket();
@@ -271,14 +305,14 @@
+ " at " + address
+ " after " + connectTimeout + "millisconds"
+ ": " + e,
- e);
+ e);
} finally {
IOUtils.closeSocket(socket);
}
}
public static void checkURL(String name, String url, int timeout) throws
- IOException {
+ IOException {
InetSocketAddress address = NetUtils.createSocketAddr(url);
checkPort(name, address, timeout);
}
@@ -291,22 +325,22 @@
* @return the file
*/
public static File requiredFile(String filename, String role) throws
- IOException {
+ IOException {
if (filename.isEmpty()) {
throw new ExitUtil.ExitException(-1, role + " file not defined");
}
File file = new File(filename);
if (!file.exists()) {
throw new ExitUtil.ExitException(-1,
- role + " file not found: " +
- file.getCanonicalPath());
+ role + " file not found: " +
+ file.getCanonicalPath());
}
return file;
}
private static final PatternValidator clusternamePattern
= new PatternValidator("[a-z][a-z0-9_-]*");
-
+
/**
* Normalize a cluster name then verify that it is valid
* @param name proposed cluster name
@@ -315,12 +349,13 @@
public static boolean isClusternameValid(String name) {
return name != null && clusternamePattern.matches(name);
}
+
public static boolean oldIsClusternameValid(String name) {
if (name == null || name.isEmpty()) {
return false;
}
int first = name.charAt(0);
- if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
+ if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
return false;
}
@@ -351,11 +386,11 @@
* @return # of files copies
*/
public static int copyDirectory(Configuration conf,
- Path srcDirPath,
- Path destDirPath,
- FsPermission permission) throws
- IOException,
- BadClusterStateException {
+ Path srcDirPath,
+ Path destDirPath,
+ FsPermission permission) throws
+ IOException,
+ BadClusterStateException {
FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
//list all paths in the src.
@@ -363,7 +398,8 @@
throw new FileNotFoundException("Source dir not found " + srcDirPath);
}
if (!srcFS.isDirectory(srcDirPath)) {
- throw new FileNotFoundException("Source dir not a directory " + srcDirPath);
+ throw new FileNotFoundException(
+ "Source dir not a directory " + srcDirPath);
}
GlobFilter dotFilter = new GlobFilter("[!.]*");
FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
@@ -375,7 +411,8 @@
permission = FsPermission.getDirDefault();
}
if (!destFS.exists(destDirPath)) {
- new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath, permission);
+ new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
+ permission);
}
Path[] sourcePaths = new Path[srcFileCount];
for (int i = 0; i < srcFileCount; i++) {
@@ -391,8 +428,8 @@
sourcePaths[i] = srcFile;
}
log.debug("Copying {} files from {} to dest {}", srcFileCount,
- srcDirPath,
- destDirPath);
+ srcDirPath,
+ destDirPath);
FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
return srcFileCount;
}
@@ -427,7 +464,8 @@
//if the fallback option is NOT set, enable it.
//if it is explicitly set to anything -leave alone
- if (conf.get(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH) == null) {
+ if (conf.get(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH) ==
+ null) {
conf.set(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH, "true");
}
return conf;
@@ -466,7 +504,9 @@
* @param trailing add a trailing entry or not
* @return the joined entries
*/
- public static String join(Collection collection, String separator, boolean trailing) {
+ public static String join(Collection collection,
+ String separator,
+ boolean trailing) {
StringBuilder b = new StringBuilder();
// fast return on empty collection
if (collection.isEmpty()) {
@@ -478,9 +518,8 @@
}
int length = separator.length();
String s = b.toString();
- return (trailing || s.isEmpty())?
- s
- : (b.substring(0, b.length() - length));
+ return (trailing || s.isEmpty()) ?
+ s : (b.substring(0, b.length() - length));
}
/**
@@ -492,9 +531,10 @@
*/
public static String join(String[] collection, String separator) {
return join(collection, separator, true);
-
-
+
+
}
+
/**
* Join an array of strings with a separator that appears after every
* instance in the list -optionally at the end
@@ -504,7 +544,7 @@
* @return the joined entries
*/
public static String join(String[] collection, String separator,
- boolean trailing) {
+ boolean trailing) {
return join(Arrays.asList(collection), separator, trailing);
}
@@ -516,7 +556,7 @@
* @return the list
*/
public static String joinWithInnerSeparator(String separator,
- Object... collection) {
+ Object... collection) {
StringBuilder b = new StringBuilder();
boolean first = true;
@@ -532,39 +572,164 @@
return b.toString();
}
- public static String mandatoryEnvVariable(String key) {
+ /**
+ * Resolve a mandatory environment variable
+ * @param key env var
+ * @return the resolved value
+ * @throws BadClusterStateException
+ */
+ public static String mandatoryEnvVariable(String key) throws
+ BadClusterStateException {
String v = System.getenv(key);
if (v == null) {
- throw new MissingArgException("Missing Environment variable " + key);
+ throw new BadClusterStateException("Missing Environment variable " + key);
}
return v;
}
- public static String appReportToString(ApplicationReport r, String separator) {
+ public static String appReportToString(ApplicationReport r,
+ String separator) {
StringBuilder builder = new StringBuilder(512);
- builder.append("application ").append(
- r.getName()).append("/").append(r.getApplicationType()).append(separator);
+ builder.append("application ")
+ .append(
+ r.getName())
+ .append("/")
+ .append(r.getApplicationType())
+ .append(separator);
Set<String> tags = r.getApplicationTags();
if (!tags.isEmpty()) {
for (String tag : tags) {
builder.append(tag).append(separator);
}
}
+ DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
+ dateFormat.setTimeZone(TimeZone.getDefault());
builder.append("state: ").append(r.getYarnApplicationState());
- builder.append(separator).append("URL: ").append(r.getTrackingUrl());
- builder.append(separator).append("Started ").append(new Date(r.getStartTime()).toGMTString());
- long finishTime = r.getFinishTime();
- if (finishTime>0) {
- builder.append(separator).append("Finished ").append(new Date(finishTime).toGMTString());
+ String trackingUrl = r.getTrackingUrl();
+ if (isSet(trackingUrl)) {
+ builder.append(separator).append("URL: ").append(trackingUrl);
}
- builder.append(separator).append("RPC :").append(r.getHost()).append(':').append(r.getRpcPort());
+ builder.append(separator)
+ .append("Started: ")
+ .append(dateFormat.format(new Date(r.getStartTime())));
+ long finishTime = r.getFinishTime();
+ if (finishTime > 0) {
+ builder.append(separator)
+ .append("Finished: ")
+ .append(dateFormat.format(new Date(finishTime)));
+ }
+ String rpcHost = r.getHost();
+ if (!isSet(rpcHost)) {
+ builder.append(separator)
+ .append("RPC :")
+ .append(rpcHost)
+ .append(':')
+ .append(r.getRpcPort());
+ }
String diagnostics = r.getDiagnostics();
- if (!diagnostics.isEmpty()) {
+ if (!isSet(diagnostics)) {
builder.append(separator).append("Diagnostics :").append(diagnostics);
}
return builder.toString();
}
+
+ /**
+ * Sorts the given list of application reports, most recently started
+ * or finished instance first.
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
+ Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
+ }
+
+ /**
+ * Sorts the given list of application reports
+ * Finished instances are ordered by finished time and running/accepted instances are
+ * ordered by start time
+ * Finally Instance are order by finished instances coming after running instances
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationReport(List<ApplicationReport> instances) {
+ if (instances.size() <= 1) {
+ return;
+ }
+ List<ApplicationReport> nonLiveInstance =
+ new ArrayList<ApplicationReport>(instances.size());
+ List<ApplicationReport> liveInstance =
+ new ArrayList<ApplicationReport>(instances.size());
+
+ for (ApplicationReport report : instances) {
+ if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
+ ||
+ report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
+ liveInstance.add(report);
+ } else {
+ nonLiveInstance.add(report);
+ }
+ }
+
+ if (liveInstance.size() > 1) {
+ Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
+ }
+ if (nonLiveInstance.size() > 1) {
+ Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
+ }
+ instances.clear();
+ instances.addAll(liveInstance);
+ instances.addAll(nonLiveInstance);
+ }
+
+ /**
+ * Built a (sorted) map of application reports, mapped to the instance name
+ * The list is sorted, and the addition process does not add a report
+ * if there is already one that exists. If the list handed in is sorted,
+ * those that are listed first form the entries returned
+ * @param instances list of intances
+ * @param minState minimum YARN state to be included
+ * @param maxState maximum YARN state to be included
+ * @return all reports in the list whose state >= minimum and <= maximum
+ */
+ public static Map<String, ApplicationReport> buildApplicationReportMap(
+ List<ApplicationReport> instances,
+ YarnApplicationState minState, YarnApplicationState maxState) {
+ TreeMap<String, ApplicationReport> map = new TreeMap<String, ApplicationReport>();
+ for (ApplicationReport report : instances) {
+ YarnApplicationState state = report.getYarnApplicationState();
+ if (state.ordinal() >= minState.ordinal() &&
+ state.ordinal() <= maxState.ordinal() &&
+ map.get(report.getName()) == null) {
+ map.put(report.getName(), report);
+ }
+ }
+ return map;
+ }
+
+ /**
+ * Take a map and produce a sorted equivalent
+ * @param source source map
+ * @return a map whose iterator returns the string-sorted ordering of entries
+ */
+ public static Map<String, String> sortedMap(Map<String, String> source) {
+ Map<String, String> out = new TreeMap<String, String>(source);
+ return out;
+ }
+
+ /**
+ * Convert a properties instance to a string map.
+ * @param properties source property object
+ * @return a string map
+ */
+ public static Map<String, String> toMap(Properties properties) {
+ Map<String, String> out = new HashMap<String, String>(properties.size());
+ for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+ out.put(entry.getKey().toString(), entry.getValue().toString());
+ }
+ return out;
+ }
+
/**
* Merge in one map to another -all entries in the second map are
* merged into the first -overwriting any duplicate keys.
@@ -572,8 +737,8 @@
* @param second the map that is merged in
* @return the first map
*/
- public static Map<String, String> mergeMap(Map<String, String> first,
- Map<String, String> second) {
+ public static Map<String, String> mergeMap(Map<String, String> first,
+ Map<String, String> second) {
first.putAll(second);
return first;
}
@@ -586,8 +751,8 @@
* @return dest -with the entries merged in
*/
public static Map<String, String> mergeEntries(Map<String, String> dest,
- Iterable<Map.Entry<String, String>> entries) {
- for (Map.Entry<String, String> entry: entries) {
+ Iterable<Map.Entry<String, String>> entries) {
+ for (Map.Entry<String, String> entry : entries) {
dest.put(entry.getKey(), entry.getValue());
}
return dest;
@@ -601,8 +766,8 @@
* @param <T2> value type
* @return 'first' merged with the second
*/
- public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
- Map<T1, T2> second) {
+ public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
+ Map<T1, T2> second) {
first.putAll(second);
return first;
}
@@ -616,7 +781,7 @@
* @return 'first' merged with the second
*/
public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
- Map<T1, T2> second) {
+ Map<T1, T2> second) {
Preconditions.checkArgument(first != null, "Null 'first' value");
Preconditions.checkArgument(second != null, "Null 'second' value");
for (Map.Entry<T1, T2> entry : second.entrySet()) {
@@ -634,8 +799,8 @@
* @return a string representation of the map
*/
public static String stringifyMap(Map<String, String> map) {
- StringBuilder builder =new StringBuilder();
- for (Map.Entry<String, String> entry: map.entrySet()) {
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
builder.append(entry.getKey())
.append("=\"")
.append(entry.getValue())
@@ -656,11 +821,11 @@
* @throws BadConfigException if the value could not be parsed
*/
public static int getIntValue(Map<String, String> roleMap,
- String key,
- int defVal,
- int min,
- int max
- ) throws BadConfigException {
+ String key,
+ int defVal,
+ int min,
+ int max
+ ) throws BadConfigException {
String valS = roleMap.get(key);
return parseAndValidate(key, valS, defVal, min, max);
@@ -676,10 +841,10 @@
* @throws BadConfigException if the value could not be parsed
*/
public static int parseAndValidate(String errorKey,
- String valS,
- int defVal,
- int min, int max) throws
- BadConfigException {
+ String valS,
+ int defVal,
+ int min, int max) throws
+ BadConfigException {
if (valS == null) {
valS = Integer.toString(defVal);
}
@@ -706,14 +871,14 @@
public static InetSocketAddress getRmAddress(Configuration conf) {
return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
}
public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
}
/**
@@ -756,11 +921,11 @@
return "null container";
}
return String.format(Locale.ENGLISH,
- "ContainerID=%s nodeID=%s http=%s priority=%s",
- container.getId(),
- container.getNodeId(),
- container.getNodeHttpAddress(),
- container.getPriority());
+ "ContainerID=%s nodeID=%s http=%s priority=%s",
+ container.getId(),
+ container.getNodeId(),
+ container.getNodeHttpAddress(),
+ container.getPriority());
}
/**
@@ -841,12 +1006,12 @@
public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
Map<String, String> env = new HashMap<String, String>();
if (roleOpts != null) {
- for (Map.Entry<String, String> entry: roleOpts.entrySet()) {
+ for (Map.Entry<String, String> entry : roleOpts.entrySet()) {
String key = entry.getKey();
if (key.startsWith(RoleKeys.ENV_PREFIX)) {
String envName = key.substring(RoleKeys.ENV_PREFIX.length());
if (!envName.isEmpty()) {
- env.put(envName,entry.getValue());
+ env.put(envName, entry.getValue());
}
}
}
@@ -860,8 +1025,8 @@
* @param commandOptions command opts
*/
public static void applyCommandLineRoleOptsToRoleMap(Map<String, Map<String, String>> clusterRoleMap,
- Map<String, Map<String, String>> commandOptions) {
- for (Map.Entry<String, Map<String, String>> entry: commandOptions.entrySet()) {
+ Map<String, Map<String, String>> commandOptions) {
+ for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
String key = entry.getKey();
Map<String, String> optionMap = entry.getValue();
Map<String, String> existingMap = clusterRoleMap.get(key);
@@ -869,7 +1034,7 @@
existingMap = new HashMap<String, String>();
}
log.debug("Overwriting role options with command line values {}",
- stringifyMap(optionMap));
+ stringifyMap(optionMap));
mergeMap(existingMap, optionMap);
//set or overwrite the role
clusterRoleMap.put(key, existingMap);
@@ -882,10 +1047,10 @@
* @throws BadCommandArgumentsException if it is invalid
*/
public static void validateClusterName(String clustername) throws
- BadCommandArgumentsException {
+ BadCommandArgumentsException {
if (!isClusternameValid(clustername)) {
throw new BadCommandArgumentsException(
- "Illegal cluster name: " + clustername);
+ "Illegal cluster name: " + clustername);
}
}
@@ -897,12 +1062,12 @@
* @throws BadConfigException if the key is not set
*/
public static void verifyPrincipalSet(Configuration conf,
- String principal) throws
- BadConfigException {
+ String principal) throws
+ BadConfigException {
String principalName = conf.get(principal);
if (principalName == null) {
throw new BadConfigException("Unset Kerberos principal : %s",
- principal);
+ principal);
}
log.debug("Kerberos princial {}={}", principal, principalName);
}
@@ -925,8 +1090,8 @@
* @throws BadConfigException the configuration/process is invalid
*/
public static boolean maybeInitSecurity(Configuration conf) throws
- IOException,
- BadConfigException {
+ IOException,
+ BadConfigException {
boolean clusterSecure = isHadoopClusterSecure(conf);
if (clusterSecure) {
log.debug("Enabling security");
@@ -943,8 +1108,8 @@
* @throws BadConfigException the configuration and system state are inconsistent
*/
public static boolean initProcessSecurity(Configuration conf) throws
- IOException,
- BadConfigException {
+ IOException,
+ BadConfigException {
if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
//security is already inited
@@ -956,9 +1121,9 @@
//this gets UGI to reset its previous world view (i.e simple auth)
//security
log.debug("java.security.krb5.realm={}",
- System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
+ System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
log.debug("java.security.krb5.kdc={}",
- System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
+ System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
log.debug("hadoop.security.authentication={}",
conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
log.debug("hadoop.security.authorization={}",
@@ -971,20 +1136,23 @@
log.debug("Login user is {}", UserGroupInformation.getLoginUser());
if (!UserGroupInformation.isSecurityEnabled()) {
throw new BadConfigException("Although secure mode is enabled," +
- "the application has already set up its user as an insecure entity %s",
- authUser);
+ "the application has already set up its user as an insecure entity %s",
+ authUser);
}
if (authUser.getAuthenticationMethod() ==
UserGroupInformation.AuthenticationMethod.SIMPLE) {
throw new BadConfigException("Auth User is not Kerberized %s" +
- " -security has already been set up with the wrong authentication method",
- authUser);
+ " -security has already been set up with the wrong authentication method. "
+ +
+ "This can occur if a file system has already been created prior to the loading of "
+ + "the security configuration.",
+ authUser);
}
SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
SliderUtils.verifyPrincipalSet(conf,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
return true;
}
@@ -1014,24 +1182,61 @@
* @throws IOException trouble copying to HDFS
*/
public static LocalResource putJar(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Class clazz,
- Path tempPath,
- String libdir,
- String jarName
- )
- throws IOException, SliderException {
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
LocalResource res = sliderFileSystem.submitJarWithClass(
- clazz,
- tempPath,
- libdir,
- jarName);
- providerResources.put(libdir + "/"+ jarName, res);
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/" + jarName, res);
return res;
}
- public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
- Map<String, Map<String, String>> dest = new HashMap<String, Map<String, String>>();
+ /**
+ * Submit a JAR containing and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param libDir lib directory
+ * @param srcPath copy jars from
+ * @throws IOException, SliderException trouble copying to HDFS
+ */
+ public static void putAllJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String srcPath
+ )
+ throws IOException, SliderException {
+ log.info("Loading all dependencies from {}", srcPath);
+ if (SliderUtils.isSet(srcPath)) {
+ File srcFolder = new File(srcPath);
+ FilenameFilter jarFilter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ String lowercaseName = name.toLowerCase();
+ if (lowercaseName.endsWith(".jar")) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ };
+ File[] listOfJars = srcFolder.listFiles(jarFilter);
+ for (File jarFile : listOfJars) {
+ LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
+ providerResources.put(libDir + "/" + jarFile.getName(), res);
+ }
+ }
+ }
+
+ public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
+ Map<String, Map<String, String>> dest =
+ new HashMap<String, Map<String, String>>();
for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
dest.put(entry.getKey(), stringMapClone(entry.getValue()));
}
@@ -1039,7 +1244,7 @@
}
public static Map<String, String> stringMapClone(Map<String, String> src) {
- Map<String, String> dest = new HashMap<String, String>();
+ Map<String, String> dest = new HashMap<String, String>();
return mergeEntries(dest, src.entrySet());
}
@@ -1143,24 +1348,24 @@
* @return a classpath
*/
public static ClasspathConstructor buildClasspath(String sliderConfDir,
- String libdir,
- Configuration config,
- boolean usingMiniMRCluster) {
+ String libdir,
+ Configuration config,
+ boolean usingMiniMRCluster) {
ClasspathConstructor classpath = new ClasspathConstructor();
-
+
// add the runtime classpath needed for tests to work
if (usingMiniMRCluster) {
// for mini cluster we pass down the java CP properties
// and nothing else
classpath.appendAll(classpath.localJVMClasspath());
} else {
- classpath.addLibDir(libdir);
if (sliderConfDir != null) {
classpath.addClassDirectory(sliderConfDir);
}
+ classpath.addLibDir(libdir);
classpath.addRemoteClasspathEnvVar();
- classpath.appendAll(classpath.yarnApplicationClasspath(config));
+ classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$());
}
return classpath;
}
@@ -1172,17 +1377,18 @@
* @param errorlog log for output on an error
* @throws FileNotFoundException if it is not a directory
*/
- public static void verifyIsDir(File dir, Logger errorlog) throws FileNotFoundException {
+ public static void verifyIsDir(File dir, Logger errorlog) throws
+ FileNotFoundException {
if (!dir.exists()) {
errorlog.warn("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
+ listDir(dir.getParentFile()));
throw new FileNotFoundException(dir.toString());
}
if (!dir.isDirectory()) {
errorlog.info("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
+ listDir(dir.getParentFile()));
throw new FileNotFoundException(
- "Not a directory: " + dir);
+ "Not a directory: " + dir);
}
}
@@ -1192,10 +1398,11 @@
* @param errorlog log for output on an error
* @throws FileNotFoundException
*/
- public static void verifyFileExists(File file, Logger errorlog) throws FileNotFoundException {
+ public static void verifyFileExists(File file, Logger errorlog) throws
+ FileNotFoundException {
if (!file.exists()) {
errorlog.warn("contents of {}: {}", file,
- listDir(file.getParentFile()));
+ listDir(file.getParentFile()));
throw new FileNotFoundException(file.toString());
}
if (!file.isFile()) {
@@ -1211,15 +1418,15 @@
* @throws BadConfigException if the key is missing
*/
public static String verifyOptionSet(Configuration configuration, String key,
- boolean allowEmpty) throws BadConfigException {
+ boolean allowEmpty) throws BadConfigException {
String val = configuration.get(key);
if (val == null) {
throw new BadConfigException(
- "Required configuration option \"%s\" not defined ", key);
+ "Required configuration option \"%s\" not defined ", key);
}
if (!allowEmpty && val.isEmpty()) {
throw new BadConfigException(
- "Configuration option \"%s\" must not be empty", key);
+ "Configuration option \"%s\" must not be empty", key);
}
return val;
}
@@ -1232,24 +1439,25 @@
* @return the file referenced
* @throws BadConfigException on a failure
*/
- public static File verifyKeytabExists(Configuration siteConf, String prop) throws
- BadConfigException {
+ public static File verifyKeytabExists(Configuration siteConf,
+ String prop) throws
+ BadConfigException {
String keytab = siteConf.get(prop);
if (keytab == null) {
throw new BadConfigException("Missing keytab property %s",
- prop);
+ prop);
}
File keytabFile = new File(keytab);
if (!keytabFile.exists()) {
throw new BadConfigException("Missing keytab file %s defined in %s",
- keytabFile,
- prop);
+ keytabFile,
+ prop);
}
if (keytabFile.length() == 0 || !keytabFile.isFile()) {
throw new BadConfigException("Invalid keytab file %s defined in %s",
- keytabFile,
- prop);
+ keytabFile,
+ prop);
}
return keytabFile;
}
@@ -1275,12 +1483,12 @@
Properties props = SliderVersionInfo.loadVersionProperties();
info.put(prefix + "." + SliderVersionInfo.APP_BUILD_INFO, props.getProperty(
- SliderVersionInfo.APP_BUILD_INFO));
+ SliderVersionInfo.APP_BUILD_INFO));
info.put(prefix + "." + SliderVersionInfo.HADOOP_BUILD_INFO,
- props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO));
+ props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO));
info.put(prefix + "." + SliderVersionInfo.HADOOP_DEPLOYED_INFO,
- VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
+ VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
}
/**
@@ -1292,30 +1500,37 @@
* @param time timestamp
*/
public static void setInfoTime(Map info,
- String keyHumanTime,
- String keyMachineTime,
- long time) {
+ String keyHumanTime,
+ String keyMachineTime,
+ long time) {
info.put(keyHumanTime, SliderUtils.toGMTString(time));
info.put(keyMachineTime, Long.toString(time));
}
- public static Path extractImagePath(CoreFileSystem fs, MapOperations internalOptions) throws
+ public static Path extractImagePath(CoreFileSystem fs,
+ MapOperations internalOptions) throws
SliderException, IOException {
Path imagePath;
String imagePathOption =
internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- String appHomeOption = internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
+ String appHomeOption =
+ internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
if (!isUnset(imagePathOption)) {
+ if (!isUnset(appHomeOption)) {
+ throw new BadClusterStateException(
+ ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED);
+ }
imagePath = fs.createPathThatMustExist(imagePathOption);
} else {
imagePath = null;
if (isUnset(appHomeOption)) {
- throw new BadClusterStateException(ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED);
+ throw new BadClusterStateException(
+ ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED);
}
}
return imagePath;
}
-
+
/**
* trigger a JVM halt with no clean shutdown at all
* @param status status code for exit
@@ -1370,7 +1585,7 @@
* @param paths subpaths
* @return base+"/"+paths[0]+"/"+paths[1]...
*/
- public static String appendToURL(String base, String...paths) {
+ public static String appendToURL(String base, String... paths) {
String result = base;
for (String path : paths) {
result = appendToURL(result, path);
@@ -1387,19 +1602,19 @@
* @return
*/
public static String truncate(String toTruncate, int maxSize) {
- if(toTruncate == null || maxSize < 1
- || toTruncate.length() <= maxSize) {
+ if (toTruncate == null || maxSize < 1
+ || toTruncate.length() <= maxSize) {
return toTruncate;
}
String pad = "...";
- if(maxSize < 10) {
+ if (maxSize < 10) {
pad = "";
}
return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
}
-
-
+
+
/**
* Callable for async/scheduled halt
*/
@@ -1433,11 +1648,15 @@
*/
public static int compareTo(long left, long right) {
long diff = left - right;
- if (diff < 0) return -1;
- if (diff > 0) return 1;
+ if (diff < 0) {
+ return -1;
+ }
+ if (diff > 0) {
+ return 1;
+ }
return 0;
}
-
+
/**
* This wrapps ApplicationReports and generates a string version
* iff the toString() operator is invoked
@@ -1456,42 +1675,494 @@
}
public static InputStream getApplicationResourceInputStream(FileSystem fs,
- Path appPath,
- String entry)
+ Path appPath,
+ String entry)
throws IOException {
InputStream is = null;
- FSDataInputStream appStream = fs.open(appPath);
- ZipArchiveInputStream zis = new ZipArchiveInputStream(appStream);
- ZipArchiveEntry zipEntry;
- boolean done = false;
- while (!done && (zipEntry = zis.getNextZipEntry()) != null) {
- if (entry.equals(zipEntry.getName())) {
- int size = (int) zipEntry.getSize();
- if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(), zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zis.read(content, offset, size - offset);
- }
- is = new ByteArrayInputStream(content);
- } else {
- log.debug("Size unknown. Reading {}", zipEntry.getName());
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- while (true) {
- int byteRead = zis.read();
- if (byteRead == -1) {
- break;
+ FSDataInputStream appStream = null;
+ try {
+ appStream = fs.open(appPath);
+ ZipArchiveInputStream zis = new ZipArchiveInputStream(appStream);
+ ZipArchiveEntry zipEntry;
+ boolean done = false;
+ while (!done && (zipEntry = zis.getNextZipEntry()) != null) {
+ if (entry.equals(zipEntry.getName())) {
+ int size = (int) zipEntry.getSize();
+ if (size != -1) {
+ log.info("Reading {} of size {}", zipEntry.getName(),
+ zipEntry.getSize());
+ byte[] content = new byte[size];
+ int offset = 0;
+ while (offset < size) {
+ offset += zis.read(content, offset, size - offset);
}
- baos.write(byteRead);
+ is = new ByteArrayInputStream(content);
+ } else {
+ log.debug("Size unknown. Reading {}", zipEntry.getName());
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ while (true) {
+ int byteRead = zis.read();
+ if (byteRead == -1) {
+ break;
+ }
+ baos.write(byteRead);
+ }
+ is = new ByteArrayInputStream(baos.toByteArray());
}
- is = new ByteArrayInputStream(baos.toByteArray());
+ done = true;
}
- done = true;
}
+ } finally {
+ IOUtils.closeStream(appStream);
}
return is;
}
+ /**
+ * Check for any needed libraries being present. On Unix none are needed;
+ * on windows they must be present
+ * @return true if all is well
+ */
+ public static String checkForRequiredNativeLibraries() {
+
+ if (!Shell.WINDOWS) {
+ return "";
+ }
+ StringBuilder errorText = new StringBuilder("");
+ if (!NativeIO.isAvailable()) {
+ errorText.append("No native IO library. ");
+ }
+ try {
+ String path = Shell.getQualifiedBinPath("winutils.exe");
+ log.debug("winutils is at {}", path);
+ } catch (IOException e) {
+ errorText.append("No WINUTILS.EXE. ");
+ log.warn("No winutils: {}", e, e);
+ }
+ try {
+ File target = new File("target");
+ FileUtil.canRead(target);
+ } catch (UnsatisfiedLinkError e) {
+ log.warn("Failing to link to native IO methods: {}", e, e);
+ errorText.append("No native IO methods");
+ }
+ return errorText.toString();
+ }
+
+ /**
+ * Strictly verify that windows utils is present.
+ * Checks go as far as opening the file and looking for
+ * the headers.
+ * @throws IOException on any problem reading the file
+ * @throws FileNotFoundException if the file is not considered valid
+ */
+ public static void maybeVerifyWinUtilsValid() throws
+ IOException,
+ SliderException {
+ String errorText = SliderUtils.checkForRequiredNativeLibraries();
+ if (!errorText.isEmpty()) {
+ throw new BadClusterStateException(errorText);
+ }
+ }
+
+ public static void verifyIsFile(String program, File exe) throws
+ FileNotFoundException {
+ if (!exe.isFile()) {
+ throw new FileNotFoundException(program
+ + " at " + exe
+ + " is not a file");
+
+ }
+ }
+
+ public static void verifyFileSize(String program,
+ File exe,
+ int minFileSize) throws FileNotFoundException {
+ if (exe.length() < minFileSize) {
+ throw new FileNotFoundException(program
+ + " at " + exe
+ + " is too short to be an executable");
+ }
+ }
+
+ /**
+ * Look for the windows executable and check it has the right headers.
+ * <code>File.canRead()</code> doesn't work on windows, so the reading
+ * is mandatory.
+ *
+ * @param program program name for errors
+ * @param exe executable
+ * @throws IOException IOE
+ */
+ public static void verifyWindowsExe(String program, File exe)
+ throws IOException {
+ verifyIsFile(program, exe);
+
+ verifyFileSize(program, exe, 0x100);
+
+ // now read two bytes and verify the header.
+
+ FileReader reader = null;
+ try {
+ int[] header = new int[2];
+ reader = new FileReader(exe);
+ header[0] = reader.read();
+ header[1] = reader.read();
+ if ((header[0] != 'M' || header[1] != 'Z')) {
+ throw new FileNotFoundException(program
+ + " at " + exe
+ + " is not a windows executable file");
+ }
+ } finally {
+ IOUtils.closeStream(reader);
+ }
+ }
+
+ /**
+ * Verify that a Unix exe works
+ * @param program program name for errors
+ * @param exe executable
+ * @throws IOException IOE
+
+ */
+ public static void verifyUnixExe(String program, File exe)
+ throws IOException {
+ verifyIsFile(program, exe);
+
+ // read flag
+ if (!exe.canRead()) {
+ throw new IOException("Cannot read " + program + " at " + exe);
+ }
+ // exe flag
+ if (!exe.canExecute()) {
+ throw new IOException("Cannot execute " + program + " at " + exe);
+ }
+ }
+
+ /**
+ * Validate an executable
+ * @param program program name for errors
+ * @param exe program to look at
+ * @throws IOException
+ */
+ public static void validateExe(String program, File exe) throws IOException {
+ if (!Shell.WINDOWS) {
+ verifyWindowsExe(program, exe);
+ } else {
+ verifyUnixExe(program, exe);
+ }
+ }
+
+ /**
+ * Write bytes to a file
+ * @param outfile output file
+ * @param data data to write
+ * @param createParent flag to indicate that the parent dir should
+ * be created
+ * @throws IOException on any IO problem
+ */
+ public static void write(File outfile, byte[] data, boolean createParent)
+ throws IOException {
+ File parentDir = outfile.getParentFile();
+ if (createParent) {
+ parentDir.mkdirs();
+ }
+ SliderUtils.verifyIsDir(parentDir, log);
+ FileOutputStream out = new FileOutputStream(outfile);
+ try {
+ out.write(data);
+ } finally {
+ IOUtils.closeStream(out);
+ }
+
+ }
+
+ /**
+ * Execute a command for a test operation
+ * @param name name in error
+ * @param status status code expected
+ * @param timeoutMillis timeout in millis for process to finish
+ * @param logger
+ * @param outputString optional string to grep for (must not span a line)
+ * @param commands commands @return the process
+ * @throws IOException on any failure.
+ */
+ public static ForkedProcessService execCommand(String name,
+ int status,
+ long timeoutMillis,
+ Logger logger,
+ String outputString,
+ String... commands) throws IOException, SliderException {
+ Preconditions.checkArgument(isSet(name), "no name");
+ Preconditions.checkArgument(commands.length > 0, "no commands");
+ Preconditions.checkArgument(isSet(commands[0]), "empty command");
+
+ ForkedProcessService process;
+
+
+ process = new ForkedProcessService(
+ name,
+ new HashMap<String, String>(),
+ Arrays.asList(commands));
+ process.setProcessLog(logger);
+ process.init(new Configuration());
+ String errorText = null;
+ process.start();
+ try {
+ if (!process.waitForServiceToStop(timeoutMillis)) {
+ throw new TimeoutException(
+ "Process did not stop in " + timeoutMillis + "mS");
+ }
+ int exitCode = process.getExitCode();
+ List<String> recentOutput = process.getRecentOutput();
+ if (status != exitCode) {
+ // error condition
+ errorText = "Expected exit code={" + status + "}, "
+ + "actual exit code={" + exitCode + "}";
+ } else {
+ if (isSet(outputString)) {
+ boolean found = false;
+ for (String line : recentOutput) {
+ if (line.contains(outputString)) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ errorText = "Did not find \"" + outputString + "\""
+ + " in output";
+ }
+ }
+ }
+ if (errorText == null) {
+ return process;
+ }
+
+ } catch (TimeoutException e) {
+ errorText = e.toString();
+ }
+ // error text: non null ==> operation failed
+ log.warn(errorText);
+ List<String> recentOutput = process.getRecentOutput();
+ for (String line : recentOutput) {
+ log.info(line);
+ }
+ throw new SliderException(LauncherExitCodes.EXIT_OTHER_FAILURE,
+ "Process %s failed: %s", name, errorText);
+
+ }
+
+
+ /**
+ * Validate the slider client-side execution environment.
+ * This looks for everything felt to be critical for execution, including
+ * native binaries and other essential dependencies.
+ * @param logger logger to log to on normal execution
+ * @throws IOException on IO failures
+ * @throws SliderException on validation failures
+ */
+ public static void validateSliderClientEnvironment(Logger logger) throws
+ IOException,
+ SliderException {
+ maybeVerifyWinUtilsValid();
+ }
+
+ /**
+ * Validate the slider server-side execution environment.
+ * This looks for everything felt to be critical for execution, including
+ * native binaries and other essential dependencies.
+ * @param logger logger to log to on normal execution
+ * @param dependencyChecks flag to indicate checks for agent dependencies
+ * @throws IOException on IO failures
+ * @throws SliderException on validation failures
+ */
+ public static void validateSliderServerEnvironment(Logger logger,
+ boolean dependencyChecks) throws
+ IOException,
+ SliderException {
+ maybeVerifyWinUtilsValid();
+ if (dependencyChecks) {
+ validatePythonEnv(logger);
+ validateOpenSSLEnv(logger);
+ }
+ }
+
+ public static void validateOpenSSLEnv(Logger logger) throws
+ IOException,
+ SliderException {
+ execCommand(OPENSSL, 0, 5000, logger, "OpenSSL", OPENSSL, "version");
+ }
+
+ public static void validatePythonEnv(Logger logger) throws
+ IOException,
+ SliderException {
+ execCommand(PYTHON, 0, 5000, logger, "Python", PYTHON, "-V");
+ }
+
+ /**
+ * return the path to the currently running slider command
+ *
+ * @throws NullPointerException
+ * - If the pathname argument is null
+ * @throws SecurityException
+ * - if a security manager exists and its checkPermission method
+ * doesn't allow getting the ProtectionDomain
+ */
+ public static String getCurrentCommandPath() {
+ File f = new File(Slider.class.getProtectionDomain().getCodeSource()
+ .getLocation().getPath());
+ return f.getAbsolutePath();
+ }
+
+ /**
+ * return the path to the slider-client.xml used by the current running
+ * slider command
+ *
+ * @throws SecurityException
+ * - if a security manager exists and its checkPermission method
+ * denies access to the class loader for the class
+ */
+ public static String getClientConfigPath() {
+ URL path = ConfigHelper.class.getClassLoader().getResource(
+ SliderKeys.CLIENT_RESOURCE);
+ return path.toString();
+ }
+
+ /**
+ * validate if slider-client.xml under the path can be opened
+ *
+ * @throws IOException
+ * : the file can't be found or open
+ */
+ public static void validateClientConfigFile() throws IOException {
+ URL resURL = SliderVersionInfo.class.getClassLoader().getResource(
+ SliderKeys.CLIENT_RESOURCE);
+ if (resURL == null) {
+ throw new IOException(
+ "slider-client.xml doesn't exist on the path: "
+ + getClientConfigPath());
+ }
+
+ try {
+ InputStream inStream = resURL.openStream();
+ if (inStream == null) {
+ throw new IOException("slider-client.xml can't be opened");
+ }
+ } catch (IOException e) {
+ throw new IOException("slider-client.xml can't be opened: "
+ + e.toString());
+ }
+ }
+
+ /**
+ * validate if a file on HDFS can be open
+ *
+ * @throws IOException the file can't be found or opened
+ * @throws URISyntaxException
+ */
+ public static void validateHDFSFile(SliderFileSystem sliderFileSystem,
+ String pathStr)
+ throws IOException, URISyntaxException {
+ URI pathURI = new URI(pathStr);
+ InputStream inputStream =
+ sliderFileSystem.getFileSystem().open(new Path(pathURI));
+ if (inputStream == null) {
+ throw new IOException("HDFS file " + pathStr + " can't be opened");
+ }
+ }
+
+ /**
+ * return the version and path of the JDK invoking the current running
+ * slider command
+ *
+ * @throws SecurityException
+ * - if a security manager exists and its checkPropertyAccess
+ * method doesn't allow access to the specified system property.
+ */
+ public static String getJDKInfo() {
+ String version = System.getProperty("java.version");
+ String javaHome = System.getProperty("java.home");
+ return
+ "The version of the JDK invoking the current running slider command: "
+ + version + "; The path to it is: " + javaHome;
+ }
+
+ /**
+ * return a description of whether the current user has created credential
+ * cache files from kerberos servers
+ *
+ * @throws IOException
+ * @throws BadConfigException
+ * @throws SecurityException
+ * - if a security manager exists and its checkPropertyAccess
+ * method doesn't allow access to the specified system property.
+ */
+ public static String checkCredentialCacheFile() throws IOException,
+ BadConfigException {
+ String result = null;
+ if (!Shell.WINDOWS) {
+ result = Shell.execCommand("klist");
+ }
+ return result;
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first
+ * Specifically: the one whose start time value is greater.
+ */
+ private static class MostRecentlyStartedAppFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getStartTime();
+ long y = r2.getStartTime();
+ return compareTwoLongsReverse(x, y);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first.
+ * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
+ */
+ private static class MostRecentlyStartedOrFinishedFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long started1 = r1.getStartTime();
+ long started2 = r2.getStartTime();
+ long finished1 = r1.getFinishTime();
+ long finished2 = r2.getFinishTime();
+ long lastEvent1 = Math.max(started1, finished1);
+ long lastEvent2 = Math.max(started2, finished2);
+ return compareTwoLongsReverse(lastEvent1, lastEvent2);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recently finished app comes first
+ * Specifically: the one whose finish time value is greater.
+ */
+ private static class MostRecentAppFinishFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getFinishTime();
+ long y = r2.getFinishTime();
+ return compareTwoLongsReverse(x, y);
+ }
+ }
+
+ /**
+ * Compare two long values for sorting. As the return value for
+ * comparators must be int, the simple value of <code>x-y</code>
+ * is inapplicable
+ * @param x x value
+ * @param y y value
+ * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
+ */
+ public static int compareTwoLongsReverse(long x, long y) {
+ return (x < y) ? +1 : ((x == y) ? 0 : -1);
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 937b777..cee2663 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -49,7 +49,9 @@
import java.io.IOException;
import java.util.Map;
+import static org.apache.slider.api.InternalKeys.INTERNAL_QUEUE;
import static org.apache.slider.api.OptionKeys.INTERNAL_AM_TMP_DIR;
+import static org.apache.slider.api.OptionKeys.INTERNAL_TMP_DIR;
import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_HOME;
import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH;
import static org.apache.slider.api.OptionKeys.INTERNAL_DATA_DIR_PATH;
@@ -132,6 +134,8 @@
internalOps.set(INTERNAL_AM_TMP_DIR,
instancePaths.tmpPathAM.toUri());
+ internalOps.set(INTERNAL_TMP_DIR,
+ instancePaths.tmpPath.toUri());
internalOps.set(INTERNAL_SNAPSHOT_CONF_PATH,
instancePaths.snapshotConfPath.toUri());
internalOps.set(INTERNAL_GENERATED_CONF_PATH,
@@ -146,45 +150,58 @@
}
/**
- * Set up the image/app home path
- * @param appImage path in the DFS to the tar file
- * @param appHomeDir other strategy: home dir
- * @throws BadConfigException if both or neither are found (its an xor)
+ * Set the queue used to start the application
+ * @param queue
+ * @throws BadConfigException
*/
- public void setImageDetails(
- Path appImage,
- String appHomeDir) throws BadConfigException {
+ public void setQueue(String queue) throws BadConfigException {
+ if(queue != null) {
+ if(SliderUtils.isUnset(queue)) {
+ throw new BadConfigException("Queue value cannot be empty.");
+ }
+
+ instanceDescription.getInternalOperations().set(INTERNAL_QUEUE, queue);
+ }
+ }
+
+ /**
+ * Set up the image/app home path
+ * @param appImage path in the DFS to the tar file
+ * @param appHomeDir other strategy: home dir
+ * @throws BadConfigException if both are found
+ */
+ public void setImageDetailsIfAvailable(
+ Path appImage,
+ String appHomeDir) throws BadConfigException {
boolean appHomeUnset = SliderUtils.isUnset(appHomeDir);
// App home or image
if (appImage != null) {
if (!appHomeUnset) {
// both args have been set
throw new BadConfigException(
- ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED);
+ ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED);
}
instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_IMAGE_PATH,
- appImage.toUri());
+ appImage.toUri());
} else {
// the alternative is app home, which now MUST be set
- if (appHomeUnset) {
- // both args have been set
- throw new BadConfigException(ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED);
-
+ if (!appHomeUnset) {
+ instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_HOME,
+ appHomeDir);
}
- instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_HOME,
- appHomeDir);
-
}
}
+
/**
* Propagate any critical principals from the current site config down to the HBase one.
*/
public void propagatePrincipals() {
- String dfsPrincipal = conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ String dfsPrincipal = conf.get(
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY;
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
instanceDescription.getAppConfOperations().set(siteDfsPrincipal, dfsPrincipal);
}
}
@@ -236,7 +253,9 @@
IOException,
SliderException,
LockAcquireFailedException {
- if (!overwrite) coreFS.createClusterDirectories(instancePaths);
+ if (!overwrite) {
+ coreFS.createClusterDirectories(instancePaths);
+ }
ConfPersister persister =
new ConfPersister(coreFS, getInstanceDir());
ConfDirSnapshotAction action = null;
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
index f923ef3..25bb4ab 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
@@ -38,15 +38,15 @@
* Load in an instance definition -but do not resolve it
* @param sliderFileSystem filesystem
* @param clusterDirectory CD
- * @return
+ * @return the unresolved aggregate configuration
* @throws IOException
* @throws SliderException
+ * @throws BadClusterStateException if a lock could not be acquired
*/
public static AggregateConf loadInstanceDefinitionUnresolved(
CoreFileSystem sliderFileSystem,
- Path clusterDirectory) throws
- IOException,
- SliderException {
+ Path clusterDirectory)
+ throws IOException, SliderException {
AggregateConf instanceDefinition = new AggregateConf();
ConfPersister persister =
new ConfPersister(sliderFileSystem, clusterDirectory);
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
index 02669cd..103d44b 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
@@ -18,11 +18,15 @@
package org.apache.slider.core.conf;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.exceptions.BadConfigException;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
+import java.io.IOException;
+
/**
* Aggregate Configuration.
*
@@ -41,6 +45,8 @@
private ConfTreeOperations appConfOperations;
private ConfTreeOperations internalOperations;
+ private String passphrase;
+
public AggregateConf() {
this(new ConfTree(), new ConfTree(), new ConfTree());
}
@@ -58,6 +64,24 @@
setInternal(internal);
}
+ /**
+ * Take a snapshot of the configuration
+ * @param instanceDefinition source
+ * @throws IOException marshalling/copying problems
+ */
+ public AggregateConf(AggregateConf instanceDefinition) throws IOException {
+ ConfTreeOperations resourcesSnapshot =
+ ConfTreeOperations.fromInstance(instanceDefinition.getResources());
+ ConfTreeOperations appConfSnapshot =
+ ConfTreeOperations.fromInstance(instanceDefinition.getAppConf());
+ ConfTreeOperations internalsSnapshot =
+ ConfTreeOperations.fromInstance(instanceDefinition.getInternal());
+ //build a new aggregate from the snapshots
+ setResources(resourcesSnapshot.confTree);
+ setAppConf(appConfSnapshot.confTree);
+ setInternal(internalsSnapshot.confTree);
+ }
+
public void setResources(ConfTree resources) {
this.resources = resources;
resourceOperations = new ConfTreeOperations(resources);
@@ -134,6 +158,16 @@
appConfOperations.resolve();
}
+ @JsonIgnore
+ public String getPassphrase() {
+ if (passphrase == null) {
+ passphrase = RandomStringUtils.randomAlphanumeric(
+ Integer.valueOf(SliderKeys.PASS_LEN));
+ }
+
+ return passphrase;
+ }
+
/**
* string operation includes all the inner conftrees
* @return a string description
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
index 5517771..c1ce0e8 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
@@ -29,6 +29,7 @@
import java.io.IOException;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
/**
@@ -66,6 +67,12 @@
/**
+ * Credentials
+ */
+ public Map<String, List<String>> credentials =
+ new HashMap<String, List<String>>(INITAL_MAP_CAPACITY);
+
+ /**
* Role options,
* role -> option -> value
*/
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index bb17547..e946e43 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -275,6 +275,7 @@
getGlobalOptions().mergeWithoutOverwrite(that.global);
confTree.metadata.putAll(that.metadata);
+ confTree.credentials.putAll(that.credentials);
for (Map.Entry<String, Map<String, String>> entry : that.components.entrySet()) {
MapOperations comp = getOrAddComponent(entry.getKey());
@@ -290,7 +291,8 @@
getGlobalOptions().putAll(that.global);
confTree.metadata.putAll(that.metadata);
-
+ confTree.credentials.putAll(that.credentials);
+
for (Map.Entry<String, Map<String, String>> entry : that.components.entrySet()) {
MapOperations comp = getOrAddComponent(entry.getKey());
comp.putAll(entry.getValue());
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index 4b1b44f..5f7b5f0 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
@@ -38,6 +38,10 @@
public class MapOperations implements Map<String, String> {
private static final Logger log =
LoggerFactory.getLogger(MapOperations.class);
+ public static final String DAYS = ".days";
+ public static final String HOURS = ".hours";
+ public static final String MINUTES = ".minutes";
+ public static final String SECONDS = ".seconds";
/**
* Global options
@@ -53,16 +57,26 @@
/**
* Create an instance
- * @param name
- * @param options
+ * @param name name
+ * @param options source of options
*/
public MapOperations(String name, Map<String, String> options) {
- assert options != null : "null map";
+ Preconditions.checkArgument(options != null, "null map");
this.options = options;
this.name = name;
}
/**
+ * Create an instance from an iterative map entry
+ * @param entry entry to work with
+ */
+ public MapOperations(Map.Entry<String, Map<String, String>> entry) {
+ Preconditions.checkArgument(entry != null, "null entry");
+ this.name = entry.getKey();
+ this.options = entry.getValue();
+ }
+
+ /**
* Get an option value
*
* @param key key
@@ -102,7 +116,11 @@
log.debug("Missing key {} from config containing {}",
key, this);
}
- throw new BadConfigException("Missing option " + key);
+ String text = "Missing option " + key;
+ if (SliderUtils.isSet(name)) {
+ text += " from set " + name;
+ }
+ throw new BadConfigException(text);
}
return val;
}
@@ -263,12 +281,12 @@
/**
* Get the time range of a set of keys
- * @param basekey
+ * @param basekey base key to which suffix gets applied
* @param defDays
* @param defHours
* @param defMins
* @param defSecs
- * @return
+ * @return the aggregate time range in seconds
*/
public long getTimeRange(String basekey,
int defDays,
@@ -276,11 +294,11 @@
int defMins,
int defSecs) {
Preconditions.checkArgument(basekey != null);
- int days = getOptionInt(basekey + ".days", defDays);
- int hours = getOptionInt(basekey + ".hours", defHours);
+ int days = getOptionInt(basekey + DAYS, defDays);
+ int hours = getOptionInt(basekey + HOURS, defHours);
- int minutes = getOptionInt(basekey + ".minutes", defMins);
- int seconds = getOptionInt(basekey + ".seconds", defSecs);
+ int minutes = getOptionInt(basekey + MINUTES, defMins);
+ int seconds = getOptionInt(basekey + SECONDS, defSecs);
// range check
Preconditions.checkState(days >= 0 && hours >= 0 && minutes >= 0
&& seconds >= 0,
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java b/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
index 5494174..aad2757 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
@@ -27,7 +27,7 @@
void validatePropertyNamePrefix(String key) throws BadConfigException {
if (key.startsWith("yarn.")) {
throw new BadConfigException(
- "argument %s does not have 'yarn.' prefix", key);
+ "argument %s has 'yarn.' prefix - this is not allowed in templates", key);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java
index fdedd20..e73ce57 100644
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java
@@ -20,7 +20,7 @@
/**
- * YARN cluster itself is in a bad state
+ * The system is in a bad state
*/
public class BadClusterStateException extends SliderException {
public BadClusterStateException(String message,
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
index 894f19b..8b04969 100644
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
@@ -20,7 +20,7 @@
public interface ErrorStrings {
String E_UNSTABLE_CLUSTER = "Unstable Application Instance :";
- String E_CLUSTER_RUNNING = "Application Instance already running";
+ String E_CLUSTER_RUNNING = "Application Instance running";
String E_ALREADY_EXISTS = "already exists";
String PRINTF_E_INSTANCE_ALREADY_EXISTS = "Application Instance \"%s\" already exists and is defined in %s";
String PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS = "Application Instance dir already exists: %s";
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
index 98dc028..ed5ceb2 100644
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
@@ -20,6 +20,7 @@
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
+import org.apache.hadoop.fs.PathAccessDeniedException;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -39,17 +40,22 @@
public static IOException convertJerseyException(String targetURL,
UniformInterfaceException exception) {
+ IOException ioe = null;
ClientResponse response = exception.getResponse();
if (response != null) {
int status = response.getStatus();
+ if (status == 401) {
+ ioe = new PathAccessDeniedException(targetURL);
+ }
if (status >= 400 && status < 500) {
- FileNotFoundException fnfe =
- new FileNotFoundException(targetURL);
- fnfe.initCause(exception);
- return fnfe;
+ ioe = new FileNotFoundException(targetURL);
}
}
- return new IOException("Failed to GET " + targetURL + ": " + exception, exception);
+ if (ioe == null) {
+ ioe = new IOException("Failed to GET " + targetURL + ": " + exception);
+ }
+ ioe.initCause(exception);
+ return ioe;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java
similarity index 66%
copy from slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
copy to slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java
index ee24dc1..40cb94d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java
@@ -16,11 +16,20 @@
* limitations under the License.
*/
-package org.apache.slider.server.services.registry;
+package org.apache.slider.core.exceptions;
+
/**
- * These constants are unique to the slider registry service itself
+ * Whatever was being resolved: it was not found
*/
-public class RegistryServiceConstants {
- public static final int INSTANCE_REFRESH_MS = 1000;
+public class NotFoundException extends SliderException {
+ public NotFoundException(String message,
+ Object... args) {
+ super(EXIT_NOT_FOUND, message, args);
+ }
+
+ public NotFoundException(Throwable throwable,
+ String message, Object... args) {
+ super(EXIT_NOT_FOUND, throwable, message, args);
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
index 18e3157..7f3134a 100644
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
@@ -41,9 +41,9 @@
/**
* Format the exception as you create it
- * @param code
- * @param message
- * @param args
+ * @param code exit code
+ * @param message exception message -sprintf formatted
+ * @param args arguments for the formatting
*/
public SliderException(int code, String message, Object... args) {
super(code, String.format(message, args));
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
index 7f59e41..bb9f430 100644
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
@@ -18,21 +18,24 @@
package org.apache.slider.core.exceptions;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+
/**
* An Exception to be thrown for an explicit "shut down the cluster" operation
* raised by the application state or other parts of the AM
*/
public class TriggerClusterTeardownException extends SliderException {
+ private final FinalApplicationStatus finalApplicationStatus;
+
public TriggerClusterTeardownException(int code,
- String message,
- Object... args) {
+ FinalApplicationStatus finalApplicationStatus, String message,
+ Object... args) {
super(code, message, args);
+ this.finalApplicationStatus = finalApplicationStatus;
}
- public TriggerClusterTeardownException(int code,
- Throwable throwable,
- String message, Object... args) {
- super(code, throwable, message, args);
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ return finalApplicationStatus;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/MissingArgException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java
similarity index 70%
rename from slider-core/src/main/java/org/apache/slider/core/exceptions/MissingArgException.java
rename to slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java
index 0faffa9..8684294 100644
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/MissingArgException.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java
@@ -18,9 +18,17 @@
package org.apache.slider.core.exceptions;
+/**
+ * Used to raise a usage exception ... this has the exit code
+ * {@link #EXIT_USAGE}
+ */
+public class UsageException extends SliderException {
+ public UsageException(String s, Object... args) {
+ super(EXIT_USAGE, s, args);
+ }
-public class MissingArgException extends RuntimeException {
- public MissingArgException(String s) {
- super(s);
+ public UsageException(Throwable throwable, String message,
+ Object... args) {
+ super(EXIT_USAGE, throwable, message, args);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 644f627..94f8f24 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -19,14 +19,18 @@
package org.apache.slider.core.launch;
import com.google.common.base.Preconditions;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
import org.apache.slider.api.ResourceKeys;
@@ -51,6 +55,7 @@
public abstract class AbstractLauncher extends Configured {
private static final Logger log =
LoggerFactory.getLogger(AbstractLauncher.class);
+ public static final String CLASSPATH = "CLASSPATH";
/**
* Filesystem to use for the launch
*/
@@ -68,7 +73,8 @@
private final Map<String, ByteBuffer> serviceData =
new HashMap<String, ByteBuffer>();
// security
- Credentials credentials = new Credentials();
+ protected final Credentials credentials = new Credentials();
+ protected LogAggregationContext logAggregationContext;
protected AbstractLauncher(Configuration conf,
@@ -97,10 +103,18 @@
return env;
}
+ /**
+ * Get the launch commands.
+ * @return the live list of commands
+ */
public List<String> getCommands() {
return commands;
}
+ /**
+ * Get the map of local resources.
+ * @return the live map of local resources.
+ */
public Map<String, LocalResource> getLocalResources() {
return localResources;
}
@@ -122,6 +136,13 @@
return serviceData;
}
+ /**
+ * Accessor to the credentials
+ * @return the credentials associated with this launcher
+ */
+ public Credentials getCredentials() {
+ return credentials;
+ }
/**
* Add a command line. It is converted to a single command before being
@@ -138,7 +159,7 @@
/**
* Add a list of commands. Each element in the list becomes a single command
- * @param commandList
+ * @param commandList list of commands
*/
public void addCommands(List<String> commandList) {
commands.addAll(commandList);
@@ -157,25 +178,42 @@
* @return the container to launch
*/
public ContainerLaunchContext completeContainerLaunch() throws IOException {
- dumpLocalResources();
+
String cmdStr = SliderUtils.join(commands, " ", false);
log.debug("Completed setting up container command {}", cmdStr);
containerLaunchContext.setCommands(commands);
- //fix the env variables
+ //env variables
+ if (log.isDebugEnabled()) {
+ log.debug("Environment variables");
+ for (Map.Entry<String, String> envPair : envVars.entrySet()) {
+ log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
+ }
+ }
containerLaunchContext.setEnvironment(env);
+
//service data
+ if (log.isDebugEnabled()) {
+ log.debug("Service Data size");
+ for (Map.Entry<String, ByteBuffer> entry : serviceData.entrySet()) {
+ log.debug("\"{}\"=> {} bytes of data", entry.getKey(),
+ entry.getValue().array().length);
+ }
+ }
containerLaunchContext.setServiceData(serviceData);
+
+ // resources
+ dumpLocalResources();
containerLaunchContext.setLocalResources(localResources);
-
+ //tokens
+ log.debug("{} tokens", credentials.numberOfTokens());
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
ByteBuffer tokenBuffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
containerLaunchContext.setTokens(tokenBuffer);
-
return containerLaunchContext;
}
@@ -220,7 +258,6 @@
public void extractResourceRequirements(Resource resource,
Map<String, String> map) {
-
if (map != null) {
MapOperations options = new MapOperations("", map);
resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY,
@@ -230,13 +267,57 @@
}
}
+ public void extractLogAggregationContext(Map<String, String> map) {
+ if (map != null) {
+ String logPatternSepStr = "\\|";
+ String logPatternJoinStr = "|";
+ MapOperations options = new MapOperations("", map);
+
+ List<String> logIncludePatterns = new ArrayList<String>();
+ String includePatternExpression = options.getOption(
+ ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "").trim();
+ if (!includePatternExpression.isEmpty()) {
+ String[] includePatterns = includePatternExpression
+ .split(logPatternSepStr);
+ for (String includePattern : includePatterns) {
+ String trimmedIncludePattern = includePattern.trim();
+ if (!trimmedIncludePattern.isEmpty()) {
+ logIncludePatterns.add(trimmedIncludePattern);
+ }
+ }
+ }
+ String logIncludePattern = StringUtils.join(logIncludePatterns,
+ logPatternJoinStr);
+ log.info("Log include patterns: {}", logIncludePattern);
+
+ List<String> logExcludePatterns = new ArrayList<String>();
+ String excludePatternExpression = options.getOption(
+ ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "").trim();
+ if (!excludePatternExpression.isEmpty()) {
+ String[] excludePatterns = excludePatternExpression
+ .split(logPatternSepStr);
+ for (String excludePattern : excludePatterns) {
+ String trimmedExcludePattern = excludePattern.trim();
+ if (!trimmedExcludePattern.isEmpty()) {
+ logExcludePatterns.add(trimmedExcludePattern);
+ }
+ }
+ }
+ String logExcludePattern = StringUtils.join(logExcludePatterns,
+ logPatternJoinStr);
+ log.info("Log exclude patterns: {}", logExcludePattern);
+
+ logAggregationContext = LogAggregationContext.newInstance(
+ logIncludePattern, logExcludePattern);
+ }
+ }
/**
* Utility method to set up the classpath
* @param classpath classpath to use
*/
public void setClasspath(ClasspathConstructor classpath) {
- setEnv("CLASSPATH", classpath.buildClasspath());
+ setEnv(CLASSPATH, classpath.buildClasspath());
}
public void setEnv(String var, String value) {
Preconditions.checkArgument(var != null, "null variable name");
@@ -299,5 +380,18 @@
addLocalResources(confResources);
}
+ /**
+ * Return the label expression and if not set null
+ * @param map
+ * @return
+ */
+ public String extractLabelExpression(Map<String, String> map) {
+ if (map != null) {
+ MapOperations options = new MapOperations("", map);
+ return options.getOption(ResourceKeys.YARN_LABEL_EXPRESSION, null);
+ }
+ return null;
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index bd8a0a5..7023c80 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -35,6 +36,7 @@
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.InetAddress;
import java.util.Map;
import java.util.Set;
@@ -44,18 +46,19 @@
private static final Logger log =
LoggerFactory.getLogger(AppMasterLauncher.class);
- protected final YarnClientApplication application;
- private final String name;
- private final String type;
- private final ApplicationSubmissionContext submissionContext;
- private final ApplicationId appId;
- private final boolean secureCluster;
+ public final YarnClientApplication application;
+ public final String name;
+ public final String type;
+ public final ApplicationSubmissionContext submissionContext;
+ public final ApplicationId appId;
+ public final boolean secureCluster;
private int maxAppAttempts = 0;
private boolean keepContainersOverRestarts = true;
private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME;
private int priority = 1;
private final Resource resource = Records.newRecord(Resource.class);
private final SliderYarnClientImpl yarnClient;
+ private Long submitTime;
/**
* Build the AM Launcher
@@ -63,10 +66,15 @@
* @param type applicatin type
* @param conf hadoop config
* @param fs filesystem binding
- * @param application precreated YARN client app instance
+ * @param yarnClient yarn client
* @param secureCluster is the cluster secure?
- * @param options map of options. All values are extracted in this constructor only
* -the map is not retained.
+ * @param secureCluster flag to indicate secure cluster
+ * @param options map of options. All values are extracted in this constructor only
+ * @param resourceGlobalOptions global options
+ * @param applicationTags any app tags
+ * @throws IOException
+ * @throws YarnException
*/
public AppMasterLauncher(String name,
String type,
@@ -75,6 +83,7 @@
SliderYarnClientImpl yarnClient,
boolean secureCluster,
Map<String, String> options,
+ Map<String, String> resourceGlobalOptions,
Set<String> applicationTags
) throws IOException, YarnException {
super(conf, fs);
@@ -93,8 +102,9 @@
if (!applicationTags.isEmpty()) {
submissionContext.setApplicationTags(applicationTags);
}
+ submissionContext.setNodeLabelExpression(extractLabelExpression(options));
extractResourceRequirements(resource, options);
-
+ extractLogAggregationContext(resourceGlobalOptions);
}
public void setMaxAppAttempts(int maxAppAttempts) {
@@ -168,6 +178,7 @@
//container requirements
submissionContext.setResource(resource);
+ submissionContext.setLogAggregationContext(logAggregationContext);
if (keepContainersOverRestarts) {
log.debug("Requesting cluster stays running over AM failure");
@@ -187,7 +198,6 @@
completeContainerLaunch();
submissionContext.setAMContainerSpec(containerLaunchContext);
return submissionContext;
-
}
/**
@@ -196,7 +206,9 @@
*/
private void addSecurityTokens() throws IOException {
- String tokenRenewer = getConf().get(YarnConfiguration.RM_PRINCIPAL);
+ String tokenRenewer = SecurityUtil.getServerPrincipal(
+ getConf().get(YarnConfiguration.RM_PRINCIPAL),
+ InetAddress.getLocalHost().getCanonicalHostName());
if (SliderUtils.isUnset(tokenRenewer)) {
throw new IOException(
"Can't get Master Kerberos principal for the RM to use as renewer: "
@@ -209,13 +221,35 @@
fs.addDelegationTokens(tokenRenewer, credentials);
}
-
+ /**
+ * Submit the application.
+ * @return a launched application representing the submitted application
+ * @throws IOException
+ * @throws YarnException
+ */
public LaunchedApplication submitApplication() throws IOException, YarnException {
completeAppMasterLaunch();
log.info("Submitting application to Resource Manager");
ApplicationId applicationId =
yarnClient.submitApplication(submissionContext);
+ // implicit success; record the time
+ submitTime = System.currentTimeMillis();
return new LaunchedApplication(applicationId, yarnClient);
}
-
+
+ /**
+ * Build a serializable application report. This is a very minimal
+ * report that contains the application Id, name and type —the information
+ * available
+ * @return a data structure which can be persisted
+ */
+ public SerializedApplicationReport createSerializedApplicationReport() {
+ SerializedApplicationReport sar = new SerializedApplicationReport();
+ sar.applicationId = appId.toString();
+ sar.name = name;
+ sar.applicationType = type;
+ sar.queue = queue;
+ sar.submitTime = submitTime;
+ return sar;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
index 8e49435..f474ffd 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
@@ -113,7 +113,7 @@
}
public void addClassDirectory(String pathToDir) {
- append(buildLibDir(appendDirectoryTerminator(pathToDir)));
+ append(appendDirectoryTerminator(pathToDir));
}
public void insertClassDirectory(String pathToDir) {
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
index 0367e06..0b3fa10 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
@@ -32,7 +32,6 @@
add(getJavaBinary());
}
-
/**
* Get the java binary. This is called in the constructor so don't try and
* do anything other than return a constant.
@@ -53,6 +52,9 @@
}
}
+ /**
+ * Turn Java assertions on
+ */
public void enableJavaAssertions() {
add("-ea");
add("-esa");
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java b/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java
index e5a025c..632e3fd 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java
@@ -95,6 +95,12 @@
return yarnClient.killRunningApplication(applicationId, reason);
}
+ /**
+ * Get the application report of this application
+ * @return an application report
+ * @throws YarnException
+ * @throws IOException
+ */
public ApplicationReport getApplicationReport()
throws YarnException, IOException {
return yarnClient.getApplicationReport(applicationId);
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java b/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
new file mode 100644
index 0000000..dfa037d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.slider.core.persist.ApplicationReportSerDeser;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+
+/**
+ * Serialized form of an application report which can be persisted
+ * and then parsed. It can not be converted back into a
+ * real YARN application report
+ *
+ * Useful for testing
+ */
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+
+public class SerializedApplicationReport {
+
+ public String applicationId;
+ public String applicationAttemptId;
+ public String name;
+ public String applicationType;
+ public String user;
+ public String queue;
+ public String host;
+ public Integer rpcPort;
+ public String state;
+ public String diagnostics;
+ public String url;
+ /**
+ * This value is non-null only when a report is generated from a submission context.
+ * The YARN {@link ApplicationReport} structure does not propagate this value
+ * from the RM.
+ */
+ public Long submitTime;
+ public Long startTime;
+ public Long finishTime;
+ public String finalStatus;
+ public String origTrackingUrl;
+ public Float progress;
+
+ public SerializedApplicationReport() {
+ }
+
+ public SerializedApplicationReport(ApplicationReport report) {
+ this.applicationId = report.getApplicationId().toString();
+ this.applicationAttemptId = report.getCurrentApplicationAttemptId().toString();
+ this.name = report.getName();
+ this.applicationType = report.getApplicationType();
+ this.user = report.getUser();
+ this.queue = report.getQueue();
+ this.host = report.getHost();
+ this.rpcPort = report.getRpcPort();
+ this.state = report.getYarnApplicationState().toString();
+ this.diagnostics = report.getDiagnostics();
+ this.startTime = report.getStartTime();
+ this.finishTime = report.getFinishTime();
+ this.finalStatus = report.getFinalApplicationStatus().toString();
+ this.progress = report.getProgress();
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return ApplicationReportSerDeser.toString(this);
+ } catch (IOException e) {
+ return super.toString();
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java b/slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java
index 6fdebcd..3ffacd0 100644
--- a/slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java
+++ b/slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java
@@ -18,14 +18,27 @@
package org.apache.slider.core.main;
-
/*
* Common Exit codes
- * Exit codes from 32 up are relative to a base value that
- * we put a fair way up from the the base numbers, so that
- * applications can have their own set of failures
+ * <p>
+ * Exit codes from 64 up are application specific.
+ * <p>
+ * Many of the exit codes are designed to resemble HTTP error codes,
+ * squashed into a single byte. e.g 44 , "not found" is the equivalent
+ * of 404
+ * <pre>
+ * 0-10: general command issues
+ * 30-39: equivalent to the 3XX responses, where those responses are
+ * considered errors by the application.
+ * 40-49: request-related errors
+ * 50-59: server-side problems. These may be triggered by the request.
+ * 64- : application specific error codes
+ * </pre>
+ *
+ *
*/
public interface LauncherExitCodes {
+
/**
* 0: success
*/
@@ -48,40 +61,138 @@
int EXIT_TASK_LAUNCH_FAILURE = 2;
/**
- * Exit code when an exception was thrown from the service: {@value}
+ * Exit code when a control-C, kill -3, signal was picked up: {@value}
*/
- int EXIT_EXCEPTION_THROWN = 32;
+ int EXIT_INTERRUPTED = 3;
/**
* Exit code when a usage message was printed: {@value}
*/
- int EXIT_USAGE = 33;
+ int EXIT_USAGE = 4;
/**
* Exit code when something happened but we can't be specific: {@value}
*/
- int EXIT_OTHER_FAILURE = 34;
+ int EXIT_OTHER_FAILURE = 5;
/**
- * Exit code when a control-C, kill -3, signal was picked up: {@value}
+ * Exit code on connectivity problems: {@value}
*/
+ int EXIT_MOVED = 31;
+
+ /**
+ * found: {@value}.
+ * <p>
+ * This is low value as in HTTP it is normally a success/redirect;
+ * whereas on the command line 0 is the sole success code.
+ * <p>
+ * <code>302 Found</code>
+ */
+ int EXIT_FOUND = 32;
- int EXIT_INTERRUPTED = 35;
+ /**
+ * Exit code on a request where the destination has not changed
+ * and (somehow) the command specified that this is an error.
+ * That is, this exit code is somehow different from a "success"
+ * : {@value}
+ * <p>
+ * <code>304 Not Modified </code>
+ */
+ int EXIT_NOT_MODIFIED = 34;
/**
* Exit code when the command line doesn't parse: {@value}, or
* when it is otherwise invalid.
+ * <p>
+ * <code>400 BAD REQUEST</code>
*/
- int EXIT_COMMAND_ARGUMENT_ERROR = 36;
+ int EXIT_COMMAND_ARGUMENT_ERROR = 40;
/**
- * Exit code when the configurations in valid/incomplete: {@value}
+ * The request requires user authentication: {@value}
+ * <p>
+ * <code>401 Unauthorized</code>
*/
- int EXIT_BAD_CONFIGURATION = 37;
+ int EXIT_UNAUTHORIZE = 41;
+
+ /**
+ * Forbidden action: {@value}
+ * <p>
+ * <code>403: Forbidden</code>
+ */
+ int EXIT_FORBIDDEN = 43;
+
+ /**
+ * Something was not found: {@value}
+ * <p>
+ * <code>404: NOT FOUND</code>
+ */
+ int EXIT_NOT_FOUND = 44;
/**
- * Exit code when the configurations in valid/incomplete: {@value}
+ * The operation is not allowed: {@value}
+ * <p>
+ * <code>405: NOT ALLOWED</code>
*/
- int EXIT_CONNECTIVITY_PROBLEM = 38;
+ int EXIT_OPERATION_NOT_ALLOWED = 45;
+
+ /**
+ * The command is somehow not acceptable: {@value}
+ * <p>
+ * <code>406: NOT ACCEPTABLE</code>
+ */
+ int EXIT_NOT_ACCEPTABLE = 46;
+
+ /**
+ * Exit code on connectivity problems: {@value}
+ * <p>
+ * <code>408: Request Timeout</code>
+ */
+ int EXIT_CONNECTIVITY_PROBLEM = 48;
+
+ /**
+ * The request could not be completed due to a conflict with the current
+ * state of the resource. {@value}
+ * <p>
+ * <code>409: conflict</code>
+ */
+ int EXIT_CONFLICT = 49;
+
+ /**
+ * internal error: {@value}
+ * <p>
+ * <code>500 Internal Server Error</code>
+ */
+ int EXIT_INTERNAL_ERROR = 50;
+
+ /**
+ * Unimplemented feature: {@value}
+ * <p>
+ * <code>501: Not Implemented</code>
+ */
+ int EXIT_UNIMPLEMENTED = 51;
+
+ /**
+ * Service Unavailable; it may be available later: {@value}
+ * <p>
+ * <code>503 Service Unavailable</code>
+ */
+ int EXIT_SERVICE_UNAVAILABLE = 53;
+
+ /**
+ * The application does not support, or refuses to support this version: {@value}.
+ * If raised, this is expected to be raised server-side and likely due
+ * to client/server version incompatibilities.
+ * <p>
+ * <code> 505: Version Not Supported</code>
+ */
+ int EXIT_UNSUPPORTED_VERSION = 55;
+
+ /**
+ * Exit code when an exception was thrown from the service: {@value}
+ * <p>
+ * <code>5XX</code>
+ */
+ int EXIT_EXCEPTION_THROWN = 56;
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
index df12849..498a8ad 100644
--- a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
@@ -80,11 +80,13 @@
public static final String ARG_CONF = "--conf";
public static final String USAGE_MESSAGE =
- "Usage: " + NAME + " classname ["+ARG_CONF + "<conf file>] <service arguments> | ";
+ "Usage: " + NAME + " classname [" + ARG_CONF +
+ "<conf file>] <service arguments> | ";
static final int SHUTDOWN_TIME_ON_INTERRUPT = 30 * 1000;
private volatile S service;
private int serviceExitCode;
+ @SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
private final List<IrqHandler> interruptHandlers = new ArrayList<IrqHandler>(1);
private Configuration configuration;
private String serviceClassName;
@@ -220,7 +222,7 @@
Object instance = serviceClass.getConstructor().newInstance();
if (!(instance instanceof Service)) {
//not a service
- throw new ExitUtil.ExitException(EXIT_BAD_CONFIGURATION,
+ throw new ExitUtil.ExitException(EXIT_COMMAND_ARGUMENT_ERROR,
"Not a Service class: " + serviceClassName);
}
@@ -399,6 +401,8 @@
Configuration conf = new Configuration();
String[] processedArgs = extractConfigurationArgs(conf, args);
ExitUtil.ExitException ee = launchServiceRobustly(conf, processedArgs);
+ System.out.flush();
+ System.err.flush();
exit(ee);
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java
new file mode 100644
index 0000000..a8c72ce
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+import org.apache.slider.core.launch.SerializedApplicationReport;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.JsonMappingException;
+
+import java.io.IOException;
+
+/**
+ * Persistence of {@link SerializedApplicationReport}
+ *
+ */
+public class ApplicationReportSerDeser
+ extends JsonSerDeser<SerializedApplicationReport> {
+ public ApplicationReportSerDeser() {
+ super(SerializedApplicationReport.class);
+ }
+
+
+ private static final ApplicationReportSerDeser
+ staticinstance = new ApplicationReportSerDeser();
+
+ /**
+ * Convert an instance to a JSON string -sync access to a shared ser/deser
+ * object instance
+ * @param instance object to convert
+ * @return a JSON string description
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public static String toString(SerializedApplicationReport instance)
+ throws IOException, JsonGenerationException, JsonMappingException {
+ synchronized (staticinstance) {
+ return staticinstance.toJson(instance);
+ }
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index ab71683..4d7d310 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -33,12 +33,13 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
/**
* Support for marshalling objects to and from JSON.
@@ -51,14 +52,14 @@
private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class);
private static final String UTF_8 = "UTF-8";
- private final Class classType;
+ private final Class<T> classType;
private final ObjectMapper mapper;
/**
* Create an instance bound to a specific type
- * @param classType
+ * @param classType class type
*/
- public JsonSerDeser(Class classType) {
+ public JsonSerDeser(Class<T> classType) {
this.classType = classType;
this.mapper = new ObjectMapper();
mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
@@ -74,7 +75,7 @@
public T fromJson(String json)
throws IOException, JsonParseException, JsonMappingException {
try {
- return (T) (mapper.readValue(json, classType));
+ return mapper.readValue(json, classType);
} catch (IOException e) {
log.error("Exception while parsing json : " + e + "\n" + json, e);
throw e;
@@ -91,7 +92,7 @@
public T fromFile(File jsonFile)
throws IOException, JsonParseException, JsonMappingException {
try {
- return (T) (mapper.readValue(jsonFile, classType));
+ return mapper.readValue(jsonFile, classType);
} catch (IOException e) {
log.error("Exception while parsing json file {}: {}", jsonFile, e);
throw e;
@@ -126,6 +127,7 @@
* @throws IOException IO problems
* @throws JsonMappingException failure to map from the JSON to this class
*/
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
public synchronized T fromResource(String resource)
throws IOException, JsonParseException, JsonMappingException {
InputStream resStream = null;
@@ -144,6 +146,23 @@
}
/**
+ * Convert from an input stream, closing the stream afterwards.
+ * @param stream
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ */
+ public T fromStream(InputStream stream) throws IOException {
+ try {
+ return (T) (mapper.readValue(stream, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json input stream: {}", e);
+ throw e;
+ } finally {
+ IOUtils.closeStream(stream);
+ }
+ }
+
+ /**
* clone by converting to JSON and back again.
* This is much less efficient than any Java clone process.
* @param instance instance to duplicate
@@ -189,9 +208,10 @@
/**
- * Save a cluster description to a hadoop filesystem
+ * Save to a hadoop filesystem
* @param fs filesystem
* @param path path
+ * @param instance instance to save
* @param overwrite should any existing file be overwritten
* @throws IOException IO exception
*/
@@ -203,26 +223,38 @@
}
/**
+ * Save an instance to a file
+ * @param instance instance to save
+ * @param file file
+ * @throws IOException
+ */
+ public void save(T instance, File file) throws
+ IOException {
+ writeJsonAsBytes(instance, new FileOutputStream(file));
+ }
+
+ /**
* Write the json as bytes -then close the file
* @param dataOutputStream an outout stream that will always be closed
* @throws IOException on any failure
*/
private void writeJsonAsBytes(T instance,
- DataOutputStream dataOutputStream) throws
- IOException {
+ OutputStream dataOutputStream) throws IOException {
try {
String json = toJson(instance);
byte[] b = json.getBytes(UTF_8);
dataOutputStream.write(b);
- } finally {
+ dataOutputStream.flush();
dataOutputStream.close();
+ } finally {
+ IOUtils.closeStream(dataOutputStream);
}
}
/**
* Convert an object to a JSON string
- * @param o object to convert
+ * @param instance instance to convert
* @return a JSON string description
* @throws JsonParseException parse problems
* @throws JsonMappingException O/J mapping problems
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java b/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
new file mode 100644
index 0000000..37b36ea
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.slider.common.SliderKeys;
+
+/**
+ * Miscellaneous methods to assist slider registry work
+ *
+ */
+public class SliderRegistryUtils {
+
+
+ /**
+ * Get the registry path for an instance under the user's home node
+ * @param instanceName application instance
+ * @return a path to the registry location for this application instance.
+ */
+ public static String registryPathForInstance(String instanceName) {
+ return RegistryUtils.servicePath(
+ RegistryUtils.currentUser(), SliderKeys.APP_TYPE, instanceName
+ );
+ }
+
+ /**
+ * Process a path expanding it if needed.
+ * Validation is delegated to later as the core registry will need
+ * to do that anyway
+ * @param path path
+ * @return a path maybe with some expansion
+ */
+ public static String resolvePath(String path) {
+ Preconditions.checkArgument(path!=null, "null path");
+ Preconditions.checkArgument(!path.isEmpty(), "empty path");
+ String newpath = path;
+ if (path.startsWith("~/")) {
+ // add user expansion
+ newpath = RegistryUtils.homePathForCurrentUser() + path.substring(1);
+ } else if (path.equals("~")) {
+ newpath = RegistryUtils.homePathForCurrentUser();
+ }
+ return newpath;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/YARNRegistryClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
similarity index 65%
rename from slider-core/src/main/java/org/apache/slider/core/registry/YARNRegistryClient.java
rename to slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
index 31147ef..6f50fca 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/YARNRegistryClient.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -18,6 +18,7 @@
package org.apache.slider.core.registry;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -27,33 +28,39 @@
import java.util.List;
/**
- * Client code for interacting with a registry of service instances.
+ * Client code for interacting with a list of service instances.
* The initial logic just enumerates service instances in the YARN RM
*/
-public class YARNRegistryClient {
+public class YarnAppListClient {
final SliderYarnClientImpl yarnClient;
final String username;
final Configuration conf;
+ public YarnAppListClient(SliderYarnClientImpl yarnClient,
+ String username,
+ Configuration conf) {
- public YARNRegistryClient(SliderYarnClientImpl yarnClient,
- String username,
- Configuration conf) {
+ Preconditions.checkArgument(yarnClient != null,
+ "yarn client is null: is app inited?");
+ Preconditions.checkArgument(username != null,
+ "username is null");
+ Preconditions.checkArgument(conf != null,
+ "conf parameter is null");
this.yarnClient = yarnClient;
this.username = username;
this.conf = conf;
}
/**
- * find all live instances of a specific app -if there is >1 in the cluster,
- * this returns them all. State should be running or less
+ * find all live instances of a specific app -if there is more than one
+ * in the cluster, this returns them all. State should be running or earlier
+ * in the lifecycle
* @param appname application name
* @return the list of all matching application instances
*/
public List<ApplicationReport> findAllLiveInstances(String appname)
throws YarnException, IOException {
-
return yarnClient.findAllLiveInstances(username, appname);
}
@@ -68,17 +75,29 @@
public ApplicationReport findInstance(String appname) throws
YarnException,
IOException {
- List<ApplicationReport> instances = listInstances();
+ List<ApplicationReport> instances = listInstances(null);
return yarnClient.findClusterInInstanceList(instances, appname);
}
/**
+ * List instances belonging to the specific user
+ * @return a possibly empty list of AMs
+ */
+ public List<ApplicationReport> listInstances()
+ throws YarnException, IOException {
+ return listInstances(null);
+ }
+
+ /**
* List instances belonging to a specific user
* @return a possibly empty list of AMs
+ * @param user user if not the default. null means default, "" means all users,
+ * otherwise it is the name of a user
*/
- public List<ApplicationReport> listInstances()
- throws YarnException, IOException {
- return yarnClient.listInstances(username);
+ public List<ApplicationReport> listInstances(String user)
+ throws YarnException, IOException {
+ String listUser = user == null ? username : user;
+ return yarnClient.listInstances(listUser);
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
new file mode 100644
index 0000000..4bcf6c1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ExportEntry {
+
+ /**
+ * The value of the export
+ */
+ private String value;
+ /**
+ * The container id of the container that is responsible for the export
+ */
+ private String containerId;
+ /**
+ * Tag associated with the container - its usually an identifier different than container id
+ * that allows a soft serial id to all containers of a component - e.g. 1, 2, 3, ...
+ */
+ private String tag;
+ /**
+ * An export can be at the level of a component or an application
+ */
+ private String level;
+ /**
+ * The time when the export was updated
+ */
+ private String updatedTime;
+ /**
+ * The time when the export expires
+ */
+ private String validUntil;
+
+ public ExportEntry() {
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ public String getContainerId() {
+ return containerId;
+ }
+
+ public void setContainerId(String containerId) {
+ this.containerId = containerId;
+ }
+
+ public String getTag() {
+ return tag;
+ }
+
+ public void setTag(String tag) {
+ this.tag = tag;
+ }
+
+ public String getLevel() {
+ return level;
+ }
+
+ public void setLevel(String level) {
+ this.level = level;
+ }
+ public String getUpdatedTime() {
+ return updatedTime;
+ }
+
+ public void setUpdatedTime(String updatedTime) {
+ this.updatedTime = updatedTime;
+ }
+
+ public String getValidUntil() {
+ return validUntil;
+ }
+
+ public void setValidUntil(String validUntil) {
+ this.validUntil = validUntil;
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder("ExportEntry{").
+ append("value='").append(value).append("',").
+ append("containerId='").append(containerId).append("',").
+ append("tag='").append(tag).append("',").
+ append("level='").append(level).append("'").
+ append("updatedTime='").append(updatedTime).append("'").
+ append("validUntil='").append(validUntil).append("'").
+ append(" }").toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
index f76b93b..28f9d3d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -23,6 +23,7 @@
import org.apache.slider.core.exceptions.BadConfigException;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.io.IOException;
@@ -150,11 +151,12 @@
/**
* Return the values as json string
- * @return
- * @throws IOException
+ * @return the JSON representation
+ * @throws IOException marshalling failure
*/
public String asJson() throws IOException {
ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
String json = mapper.writeValueAsString(entries);
return json;
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
new file mode 100644
index 0000000..1919bfa
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedExports {
+
+ public String description;
+ public long updated;
+ public String updatedTime;
+ public Map<String, List<ExportEntry>> entries = new HashMap<String, List<ExportEntry>>();
+
+ public PublishedExports() {
+ }
+
+ /**
+ * build an empty published configuration
+ *
+ * @param description configuration description
+ */
+ public PublishedExports(String description) {
+ this.description = description;
+ }
+
+ /**
+ * Build a configuration from the entries
+ *
+ * @param description configuration description
+ * @param entries entries to put
+ */
+ public PublishedExports(String description,
+ Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
+ this.description = description;
+ putValues(entries);
+ }
+
+ /**
+ * Is the configuration empty. This means either that it has not been given any values, or it is stripped down copy
+ * set down over the wire.
+ *
+ * @return
+ */
+ public boolean isEmpty() {
+ return entries.isEmpty();
+ }
+
+ public long getUpdated() {
+ return updated;
+ }
+
+ public void setUpdated(long updated) {
+ this.updated = updated;
+ this.updatedTime = new Date(updated).toString();
+ }
+
+ /**
+ * Set the values from an iterable (this includes a Hadoop Configuration and Java properties object). Any existing
+ * value set is discarded
+ *
+ * @param entries entries to put
+ */
+ public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
+ this.entries = new HashMap<String, List<ExportEntry>>();
+ for (Map.Entry<String, List<ExportEntry>> entry : entries) {
+ this.entries.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Return the values as json string
+ *
+ * @return
+ *
+ * @throws IOException
+ */
+ public String asJson() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ String json = mapper.writeValueAsString(entries);
+ return json;
+ }
+
+ /**
+ * This makes a copy without the nested content -so is suitable for returning as part of the list of a parent's
+ * values
+ *
+ * @return the copy
+ */
+ public PublishedExports shallowCopy() {
+ PublishedExports that = new PublishedExports();
+ that.description = this.description;
+ that.updated = this.updated;
+ that.updatedTime = this.updatedTime;
+ return that;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("PublishedConfiguration{");
+ sb.append("description='").append(description).append('\'');
+ sb.append(" entries = ").append(entries.size());
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
new file mode 100644
index 0000000..b21e717
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/** Output a published configuration */
+public abstract class PublishedExportsOutputter {
+
+ protected final PublishedExports exports;
+
+ protected PublishedExportsOutputter(PublishedExports exports) {
+ this.exports = exports;
+ }
+
+ /**
+ * Create an outputter for the chosen format
+ *
+ * @param format format enumeration
+ * @param exports owning config
+ * @return the outputter
+ */
+
+ public static PublishedExportsOutputter createOutputter(ConfigFormat format,
+ PublishedExports exports) {
+ Preconditions.checkNotNull(exports);
+ switch (format) {
+ case JSON:
+ return new JsonOutputter(exports);
+ default:
+ throw new RuntimeException("Unsupported format :" + format);
+ }
+ }
+
+ public void save(File dest) throws IOException {
+ FileOutputStream out = null;
+ try {
+ out = new FileOutputStream(dest);
+ save(out);
+ out.close();
+ } finally {
+ org.apache.hadoop.io.IOUtils.closeStream(out);
+ }
+ }
+
+ /**
+ * Save the content. The default saves the asString() value to the output stream
+ *
+ * @param out output stream
+ * @throws IOException
+ */
+ public void save(OutputStream out) throws IOException {
+ IOUtils.write(asString(), out, Charsets.UTF_8);
+ }
+
+ /**
+ * Convert to a string
+ *
+ * @return
+ * @throws IOException
+ */
+ public abstract String asString() throws IOException;
+
+ public static class JsonOutputter extends PublishedExportsOutputter {
+
+ public JsonOutputter(PublishedExports exports) {
+ super(exports);
+ }
+
+ @Override
+ public void save(File dest) throws IOException {
+ FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return exports.asJson();
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
new file mode 100644
index 0000000..cdd35de
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+import org.apache.slider.server.services.utility.PatternValidator;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Represents a set of configurations for an application, component, etc.
+ * Json serialisable; accessors are synchronized
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedExportsSet {
+
+ private static final PatternValidator validator = new PatternValidator(
+ RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
+
+ public Map<String, PublishedExports> exports =
+ new HashMap<String, PublishedExports>();
+
+ public PublishedExportsSet() {
+ }
+
+ /**
+ * Put a name -it will be converted to lower case before insertion.
+ * Any existing entry will be overwritten (that includes an entry
+ * with a different case in the original name)
+ * @param name name of entry
+ * @param export published export
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public void put(String name, PublishedExports export) {
+ String name1 = name.toLowerCase(Locale.ENGLISH);
+ validateName(name1);
+ exports.put(name1, export);
+ }
+
+ /**
+ * Validate the name -restricting it to the set defined in
+ * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
+ * @param name name to validate
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public static void validateName(String name) {
+ validator.validate(name);
+
+ }
+
+ public PublishedExports get(String name) {
+ return exports.get(name);
+ }
+
+ public boolean contains(String name) {
+ return exports.containsKey(name);
+ }
+
+ public int size() {
+ return exports.size();
+ }
+
+ public Set<String> keys() {
+ TreeSet<String> keys = new TreeSet<String>();
+ keys.addAll(exports.keySet());
+ return keys;
+ }
+
+ public PublishedExportsSet shallowCopy() {
+ PublishedExportsSet that = new PublishedExportsSet();
+ for (Map.Entry<String, PublishedExports> entry :
+ exports.entrySet()) {
+ that.put(entry.getKey(), entry.getValue().shallowCopy());
+ }
+ return that;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
index 38fb4a5..0a3476c 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
@@ -24,17 +24,31 @@
public class CustomRegistryConstants {
public static final String MANAGEMENT_REST_API =
- "org.apache.slider.management";
+ "classpath:org.apache.slider.management";
+
public static final String REGISTRY_REST_API =
- "org.apache.slider.registry";
+ "classpath:org.apache.slider.registry";
public static final String PUBLISHER_REST_API =
- "org.apache.slider.publisher";
-
- public static final String AGENT_REST_API =
- "org.apache.slider.agents";
+ "classpath:org.apache.slider.publisher";
+
+ public static final String PUBLISHER_CONFIGURATIONS_API =
+ "classpath:org.apache.slider.publisher.configurations";
+
+ public static final String PUBLISHER_EXPORTS_API =
+ "classpath:org.apache.slider.publisher.exports";
+
+ public static final String PUBLISHER_DOCUMENTS_API =
+ "classpath:org.apache.slider.publisher.documents";
+
+ public static final String AGENT_SECURE_REST_API =
+ "classpath:org.apache.slider.agents.secure";
+
+ public static final String AGENT_ONEWAY_REST_API =
+ "classpath:org.apache.slider.agents.oneway";
public static final String AM_IPC_PROTOCOL =
- "org.apache.slider.appmaster";
+ "classpath:org.apache.slider.appmaster";
+ public static final String WEB_UI = "http://";
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java
deleted file mode 100644
index cde282b..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- * A registry document
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class RegisteredDocument {
- public String contentType;
- public String url;
- public String description;
-}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
deleted file mode 100644
index f3477d0..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class RegisteredEndpoint {
-
- // standard types
-
- /**
- * URL: {@value}
- */
- public static final String TYPE_URL = "url";
-
- /**
- * hostname: {@value}
- */
- public static final String TYPE_HOSTNAME = "hostname";
-
- /**
- * "hostname:port" pair: {@value}
- */
- public static final String TYPE_INETADDRESS = "inetaddress";
-
- /**
- * simple path string: {@value}
- */
- public static final String TYPE_PATH = "path";
-
- // standard protocols
-
- /**
- * Generic TCP protocol: {@value}
- */
-
- public static final String PROTOCOL_TCP = "tcp";
-
- /**
- * Generic TCP protocol: {@value}
- */
-
- public static final String PROTOCOL_UDP = "udp";
-
- /**
- * HTTP: {@value}
- */
-
- public static final String PROTOCOL_HTTP = "http";
-
- /**
- * HTTPS: {@value}
- */
-
- public static final String PROTOCOL_HTTPS = "http";
-
- /**
- * Classic "Writable" Hadoop IPC: {@value}
- */
- public static final String PROTOCOL_HADOOP_RPC = "org.apache.hadoop.ipc.RPC";
-
- /**
- * Protocol buffer based Hadoop IPC: {@value}
- */
- public static final String PROTOCOL_HADOOP_PROTOBUF = "org.apache.hadoop.ipc.Protobuf";
-
- /**
- * The address -format is driven by the type entry
- */
- public String address;
-
- /**
- * Protocol
- */
- public String protocol = "";
-
- public String type = "";
-
- /**
- * Human readable type
- */
- public String description = "";
-
- public RegisteredEndpoint() {
- }
-
- public RegisteredEndpoint(String address,
- String protocol,
- String type,
- String description) {
- this.address = address;
- this.protocol = protocol;
- this.type = type;
- this.description = description;
- }
-
- /**
- * Build an endpoint instance from a URI, extracting
- * the protocol from it
- * @param uri URI to set the value to
- * @param description description
- */
- public RegisteredEndpoint(URI uri,
- String description) {
-
- this.address = uri.toString();
- this.protocol = uri.getScheme();
- this.type = TYPE_URL;
- this.description = description;
- }
- /**
- * Build an endpoint instance from a URI, extracting
- * the protocol from it
- * @param uri URI to set the value to
- * @param description description
- */
- public RegisteredEndpoint(InetSocketAddress address,
- String protocol,
- String description) {
-
- this.address = address.toString();
- this.protocol = protocol;
- this.type = TYPE_INETADDRESS;
- this.description = description;
- }
-
- /**
- * Build an endpoint instance from a URL, extracting
- * the protocol from it
- * @param url URL to set the value to
- * @param description description
- */
- public RegisteredEndpoint(URL url,
- String description) throws URISyntaxException {
- this(url.toURI(), description);
- }
-
- /**
- * Get the value as a URL
- * @return URL of the value -if the value type is URL
- * @throws SliderException if the value is of the wrong type, or unparsable
- */
- public URL asURL() throws SliderException {
- verifyEndpointType(TYPE_URL);
- try {
- return new URL(address);
- } catch (MalformedURLException e) {
- throw new SliderException(-1, e,
- "could not create a URL from %s : %s", address, e.toString());
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder();
- if (TYPE_URL.equals(type)) {
- sb.append(address);
- } else {
- sb.append("protocol='").append(protocol).append('\'');
- sb.append(" address='").append(address).append('\'');
- sb.append(" type='").append(type).append('\'');
- }
- sb.append(" -- \"").append(description).append('"');
- return sb.toString();
- }
-
-
- /**
- * Verify that an endpoint is of the desired type
- * @param desiredType desired type
- * @throws SliderException if it is not
- */
- public void verifyEndpointType(String desiredType) throws SliderException {
- if (!type.equals(desiredType)) {
- throw new SliderException(-1, "Body of endpoint is of type %s and not %s",
- type, desiredType);
- }
- }
-}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java
deleted file mode 100644
index 441a7de..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-/**
- * Fieldnames in registry entries
- */
-public class RegistryFields {
-
- public static final String ENDPOINTS = "endpoints";
- public static final String INTERNAL_VIEW = "internalView";
- public static final String EXTERNAL_VIEW = "externalView";
-
- public static final String PROTOCOL = "protocol";
- public static final String VALUE = "value";
- public static final String DESCRIPTION = "description";
-
-}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java
deleted file mode 100644
index bd5cf27..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-import java.util.Locale;
-
-/**
- * Logic for greating names from registry entries; lets the policy be changed
- * later
- */
-public class RegistryNaming {
-
- public static String SLIDER_INSTANCE_NAME_FORMAT = "%2$s";
-
- public static String createRegistryServiceType(String instanceName,
- String userName,
- String serviceName) {
- return serviceName;
- }
-
- public static String createRegistryName(String instanceName,
- String userName,
- String serviceName,
- int appId) {
- return String.format(Locale.ENGLISH,
- SLIDER_INSTANCE_NAME_FORMAT,
- userName,
- instanceName,
- serviceName,
- appId);
-
- }
-
-
-}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
deleted file mode 100644
index c3c7e63..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Service instance data to serialize with JSON.
- *
- * The equality and hash codes are derived from the
- * service type and ID, which aren't final so that JSON marshalling
- * works. Do not change these fields if an instance is stored
- * in a map
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class ServiceInstanceData implements Serializable {
-
- public String serviceType;
- public String id;
- public String description;
- public String yarnApplicationId;
- public long registrationTimeUTC;
-
- /**
- * Anything can go into the internal view, it's specific
- * to the provider
- */
- public RegistryView internalView = new RegistryView();
-
- /**
- * External view of the service: public information
- */
- public RegistryView externalView = new RegistryView();
-
- public ServiceInstanceData() {
- }
-
- public ServiceInstanceData(String id, String serviceType) {
- this.serviceType = serviceType;
- this.id = id;
- }
-
- /**
- * Instances are equal if they look after the same service type
- * and name
- * @param o other
- * @return true if id and type match
- */
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ServiceInstanceData that = (ServiceInstanceData) o;
-
- if (!id.equals(that.id)) {
- return false;
- }
- if (!serviceType.equals(that.serviceType)) {
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = serviceType.hashCode();
- result = 31 * result + id.hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("ServiceInstanceData{");
- sb.append("id='").append(id).append('\'');
- sb.append(", serviceType='").append(serviceType).append('\'');
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * get the internal or external registry
- * @param external flag to indicate the external endpoints
- * @return a view -which may be null
- */
- public RegistryView getRegistryView(boolean external) {
- return external ? externalView : internalView;
- }
-
- /**
- * List the internal or external endpoints. This returns
- * an empty list if there are none registered
- * @param external flag to indicate the external endpoints
- * @return a map of published endpoints
- */
- public Map<String, RegisteredEndpoint> listEndpoints(boolean external) {
- RegistryView view = getRegistryView(external);
- if (view == null) {
- return new HashMap<String, RegisteredEndpoint>(0);
- }
- Map<String, RegisteredEndpoint> endpoints = view.endpoints;
- if (endpoints != null) {
- return endpoints;
- } else {
- return new HashMap<String, RegisteredEndpoint>(0);
- }
- }
-
-}
-
-
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index 4fe2e8f..1194270 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -25,18 +25,36 @@
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.RegistryIOException;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.ExceptionConverter;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.info.RegistryView;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.security.GeneralSecurityException;
+import java.util.List;
/**
* Registry retriever.
@@ -46,7 +64,10 @@
public class RegistryRetriever {
private static final Logger log = LoggerFactory.getLogger(RegistryRetriever.class);
- private final ServiceInstanceData instance;
+ private final String externalConfigurationURL;
+ private final String internalConfigurationURL;
+ private final String externalExportsURL;
+ private final String internalExportsURL;
private static final Client jerseyClient;
static {
@@ -54,26 +75,120 @@
clientConfig.getFeatures().put(
JSONConfiguration.FEATURE_POJO_MAPPING,
Boolean.TRUE);
- jerseyClient = Client.create(clientConfig);
+ clientConfig.getProperties().put(
+ URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true);
+ URLConnectionClientHandler handler = getUrlConnectionClientHandler();
+ jerseyClient = new Client(handler, clientConfig);
jerseyClient.setFollowRedirects(true);
}
+ private static URLConnectionClientHandler getUrlConnectionClientHandler() {
+ return new URLConnectionClientHandler(new HttpURLConnectionFactory() {
+ @Override
+ public HttpURLConnection getHttpURLConnection(URL url)
+ throws IOException {
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ if (connection.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP) {
+ // is a redirect - are we changing schemes?
+ String redirectLocation = connection.getHeaderField(HttpHeaders.LOCATION);
+ String originalScheme = url.getProtocol();
+ String redirectScheme = URI.create(redirectLocation).getScheme();
+ if (!originalScheme.equals(redirectScheme)) {
+ // need to fake it out by doing redirect ourselves
+ log.info("Protocol change during redirect. Redirecting {} to URL {}",
+ url, redirectLocation);
+ URL redirectURL = new URL(redirectLocation);
+ connection = (HttpURLConnection) redirectURL.openConnection();
+ }
+ }
+ if (connection instanceof HttpsURLConnection) {
+ log.debug("Attempting to configure HTTPS connection using client "
+ + "configuration");
+ final SSLFactory factory;
+ final SSLSocketFactory sf;
+ final HostnameVerifier hv;
- public RegistryRetriever(ServiceInstanceData instance) {
- this.instance = instance;
+ try {
+ HttpsURLConnection c = (HttpsURLConnection) connection;
+ factory = new SSLFactory(SSLFactory.Mode.CLIENT, new Configuration());
+ factory.init();
+ sf = factory.createSSLSocketFactory();
+ hv = factory.getHostnameVerifier();
+ c.setSSLSocketFactory(sf);
+ c.setHostnameVerifier(hv);
+ } catch (Exception e) {
+ log.info("Unable to configure HTTPS connection from "
+ + "configuration. Leveraging JDK properties.");
+ }
+
+ }
+ return connection;
+ }
+ });
+ }
+
+ public RegistryRetriever(String externalConfigurationURL, String internalConfigurationURL,
+ String externalExportsURL, String internalExportsURL) {
+ this.externalConfigurationURL = externalConfigurationURL;
+ this.internalConfigurationURL = internalConfigurationURL;
+ this.externalExportsURL = externalExportsURL;
+ this.internalExportsURL = internalExportsURL;
}
/**
- * Get the appropriate view for the flag
- * @param external
- * @return
+ * Retrieve from a service by locating the
+ * exported {@link CustomRegistryConstants#PUBLISHER_CONFIGURATIONS_API}
+ * and working off it.
+ * @param record service record
+ * @throws RegistryIOException the address type of the endpoint does
+ * not match that expected (i.e. not a list of URLs), missing endpoint...
*/
- private RegistryView getRegistryView(boolean external) {
- return external ? instance.externalView : instance.internalView;
- }
+ public RegistryRetriever(ServiceRecord record) throws RegistryIOException {
+ Endpoint internal = record.getInternalEndpoint(
+ CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
+ String url = null;
+ if (internal != null) {
+ List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType(
+ internal);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
+ }
+ internalConfigurationURL = url;
+ Endpoint external = record.getExternalEndpoint(
+ CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
+ url = null;
+ if (external != null) {
+ List<String> addresses =
+ RegistryTypeUtils.retrieveAddressesUriType(external);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
+ }
+ externalConfigurationURL = url;
- private String destination(boolean external) {
- return external ? "external" : "internal";
+ internal = record.getInternalEndpoint(
+ CustomRegistryConstants.PUBLISHER_EXPORTS_API);
+ url = null;
+ if (internal != null) {
+ List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType(
+ internal);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
+ }
+ internalExportsURL = url;
+ external = record.getExternalEndpoint(
+ CustomRegistryConstants.PUBLISHER_EXPORTS_API);
+ url = null;
+ if (external != null) {
+ List<String> addresses =
+ RegistryTypeUtils.retrieveAddressesUriType(external);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
+ }
+ externalExportsURL = url;
}
/**
@@ -82,8 +197,8 @@
* @return true if there is a URL to the configurations defined
*/
public boolean hasConfigurations(boolean external) {
- String confURL = getRegistryView(external).configurationsURL;
- return !Strings.isStringEmpty(confURL);
+ return !Strings.isStringEmpty(
+ external ? externalConfigurationURL : internalConfigurationURL);
}
/**
@@ -94,11 +209,7 @@
public PublishedConfigSet getConfigurations(boolean external) throws
FileNotFoundException, IOException {
- String confURL = getRegistryView(external).configurationsURL;
- if (Strings.isStringEmpty(confURL)) {
- throw new FileNotFoundException("No configuration URL at "
- + destination(external) + " view");
- }
+ String confURL = getConfigurationURL(external);
try {
WebResource webResource = jsonResource(confURL);
log.debug("GET {}", confURL);
@@ -109,6 +220,41 @@
}
}
+ protected String getConfigurationURL(boolean external) throws FileNotFoundException {
+ String confURL = external ? externalConfigurationURL: internalConfigurationURL;
+ if (Strings.isStringEmpty(confURL)) {
+ throw new FileNotFoundException("No configuration URL");
+ }
+ return confURL;
+ }
+
+ protected String getExportURL(boolean external) throws FileNotFoundException {
+ String confURL = external ? externalExportsURL: internalExportsURL;
+ if (Strings.isStringEmpty(confURL)) {
+ throw new FileNotFoundException("No configuration URL");
+ }
+ return confURL;
+ }
+
+ /**
+ * Get the configurations of the registry
+ * @param external flag to indicate that it is the external entries to fetch
+ * @return the configuration sets
+ */
+ public PublishedExportsSet getExports(boolean external) throws
+ FileNotFoundException, IOException {
+
+ String exportsUrl = getExportURL(external);
+ try {
+ WebResource webResource = jsonResource(exportsUrl);
+ log.debug("GET {}", exportsUrl);
+ PublishedExportsSet exportSet = webResource.get(PublishedExportsSet.class);
+ return exportSet;
+ } catch (UniformInterfaceException e) {
+ throw ExceptionConverter.convertJerseyException(exportsUrl, e);
+ }
+ }
+
private WebResource resource(String url) {
WebResource resource = jerseyClient.resource(url);
return resource;
@@ -122,7 +268,7 @@
/**
* Get a complete configuration, with all values
- * @param configSet
+ * @param configSet config set to ask for
* @param name name of the configuration
* @param external flag to indicate that it is an external configuration
* @return the retrieved config
@@ -131,10 +277,10 @@
public PublishedConfiguration retrieveConfiguration(PublishedConfigSet configSet,
String name,
boolean external) throws IOException {
+ String confURL = getConfigurationURL(external);
if (!configSet.contains(name)) {
throw new FileNotFoundException("Unknown configuration " + name);
}
- String confURL = getRegistryView(external).configurationsURL;
confURL = SliderUtils.appendToURL(confURL, name);
try {
WebResource webResource = jsonResource(confURL);
@@ -145,10 +291,38 @@
throw ExceptionConverter.convertJerseyException(confURL, e);
}
}
-
+
+ /**
+ * Get a complete export, with all values
+ * @param exportSet
+ * @param name name of the configuration
+ * @param external flag to indicate that it is an external configuration
+ * @return the retrieved config
+ * @throws IOException IO problems
+ */
+ public PublishedExports retrieveExports(PublishedExportsSet exportSet,
+ String name,
+ boolean external) throws IOException {
+ if (!exportSet.contains(name)) {
+ throw new FileNotFoundException("Unknown export " + name);
+ }
+ String exportsURL = getExportURL(external);
+ exportsURL = SliderUtils.appendToURL(exportsURL, name);
+ try {
+ WebResource webResource = jsonResource(exportsURL);
+ PublishedExports publishedExports =
+ webResource.get(PublishedExports.class);
+ return publishedExports;
+ } catch (UniformInterfaceException e) {
+ throw ExceptionConverter.convertJerseyException(exportsURL, e);
+ }
+ }
+
@Override
public String toString() {
- return super.toString() + " - " + instance;
+ return super.toString()
+ + ": internal URL: \"" + internalConfigurationURL
+ + "\"; external \"" + externalConfigurationURL +"\"";
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index 0d96559..0c28229 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -29,13 +29,14 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-public class ZKIntegration implements Watcher {
+public class ZKIntegration implements Watcher, Closeable {
/**
* Base path for services
@@ -115,7 +116,20 @@
createClusterPath,
watchEventHandler);
}
-
+
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (zookeeper != null) {
+ try {
+ zookeeper.close();
+ } catch (InterruptedException ignored) {
+
+ }
+ zookeeper = null;
+ }
+ }
+
public String getConnectionString() {
return zkConnection;
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
index 61b1ff0..cc1b2c9 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
@@ -27,6 +27,7 @@
import java.util.List;
public class ZookeeperUtils {
+ public static final int DEFAULT_PORT = 2181;
public static String buildConnectionString(String zkHosts, int port) {
String zkPort = Integer.toString(port);
@@ -73,7 +74,7 @@
List<HostAndPort> list = new ArrayList<HostAndPort>(len);
if (strings != null) {
for (String s : strings) {
- list.add(HostAndPort.fromString(s.trim()));
+ list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
}
}
return list;
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index f8008a4..f89f842 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -62,9 +62,11 @@
public abstract List<ProviderRole> getRoles();
/**
- * Validate the instance definition.
+ * Verify that an instance definition is considered valid by the provider
+ * @param instanceDefinition instance definition
+ * @throws SliderException if the configuration is not valid
*/
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
SliderException {
List<ProviderRole> roles = getRoles();
@@ -202,10 +204,9 @@
AggregateConf instanceDefinition,
Path clusterDirPath,
Path generatedConfDirPath,
- boolean secure) throws
- SliderException,
- IOException {
- validateInstanceDefinition(instanceDefinition);
+ boolean secure)
+ throws SliderException, IOException {
+ validateInstanceDefinition(instanceDefinition, sliderFileSystem);
}
/**
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 3fbd3cf..c628d8a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -22,7 +22,13 @@
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.types.AddressTypes;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -32,22 +38,21 @@
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.main.ExitCodeProvider;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
-import org.apache.slider.server.services.registry.RegistryViewForProviders;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.slider.server.services.workflow.ServiceParent;
import org.apache.slider.server.services.workflow.WorkflowSequenceService;
+import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
@@ -70,13 +75,13 @@
LoggerFactory.getLogger(AbstractProviderService.class);
protected StateAccessForProviders amState;
protected AgentRestOperations restOps;
- protected RegistryViewForProviders registry;
- protected ServiceInstanceData registryInstanceData;
protected URL amWebAPI;
+ protected YarnRegistryViewForProviders yarnRegistry;
protected QueueAccess queueAccess;
public AbstractProviderService(String name) {
super(name);
+ setStopIfNoChildServicesAtStartup(false);
}
@Override
@@ -96,17 +101,25 @@
this.amState = amState;
}
+
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg,
QueueAccess queueAccess,
List<Container> liveContainers) {
this.amState = stateAccessor;
- this.registry = reg;
this.queueAccess = queueAccess;
}
@Override
+ public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
+ this.yarnRegistry = yarnRegistry;
+ }
+
+ public YarnRegistryViewForProviders getYarnRegistry() {
+ return yarnRegistry;
+ }
+
+ @Override
public AgentRestOperations getAgentRestOperations() {
return restOps;
}
@@ -184,7 +197,25 @@
}
return false;
}
-
+
+ /**
+ * override point to allow a process to start executing in this container
+ * @param instanceDefinition cluster description
+ * @param confDir configuration directory
+ * @param env environment
+ * @param execInProgress the callback for the exec events
+ * @return false
+ * @throws IOException
+ * @throws SliderException
+ */
+ @Override
+ public boolean exec(AggregateConf instanceDefinition,
+ File confDir,
+ Map<String, String> env,
+ ProviderCompleted execInProgress) throws IOException, SliderException {
+ return false;
+ }
+
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
@Override // ExitCodeProvider
public int getExitCode() {
@@ -302,7 +333,7 @@
public Map<String, String> buildMonitorDetails(ClusterDescription clusterDesc) {
Map<String, String> details = new LinkedHashMap<String, String>();
- // add in all the
+ // add in all the endpoints
buildEndpointDetails(details);
return details;
@@ -316,28 +347,35 @@
@Override
public void buildEndpointDetails(Map<String, String> details) {
- ServiceInstanceData self = registry.getSelfRegistration();
- buildEndpointDetails(details, self);
- }
+ ServiceRecord self = yarnRegistry.getSelfRegistration();
- public static void buildEndpointDetails(Map<String, String> details,
- ServiceInstanceData self) {
- Map<String, RegisteredEndpoint> endpoints =
- self.getRegistryView(true).endpoints;
- for (Map.Entry<String, RegisteredEndpoint> endpoint : endpoints.entrySet()) {
- RegisteredEndpoint val = endpoint.getValue();
- if (val.type.equals(RegisteredEndpoint.TYPE_URL)) {
- details.put(val.description, val.address);
+ List<Endpoint> externals = self.external;
+ for (Endpoint endpoint : externals) {
+ String addressType = endpoint.addressType;
+ if (AddressTypes.ADDRESS_URI.equals(addressType)) {
+ try {
+ List<URL> urls = RegistryTypeUtils.retrieveAddressURLs(endpoint);
+ if (!urls.isEmpty()) {
+ details.put(endpoint.api, urls.get(0).toString());
+ }
+ } catch (InvalidRecordException ignored) {
+ // Ignored
+ } catch (MalformedURLException ignored) {
+ // ignored
+ }
+
}
+
}
}
- @Override
- public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData registryInstanceData) throws IOException {
- this.amWebAPI = unsecureWebAPI;
- this.registryInstanceData = registryInstanceData;
+ @Override
+ public void applyInitialRegistryDefinitions(URL amWebURI,
+ URL agentOpsURI,
+ URL agentStatusURI,
+ ServiceRecord serviceRecord)
+ throws IOException {
+ this.amWebAPI = amWebURI;
}
/**
@@ -361,6 +399,13 @@
// no-op
}
+ @Override
+ public int cancelContainerRequests(Priority priority1,
+ Priority priority2,
+ int count) {
+ return 0;
+ }
+
/**
* No-op implementation of this method.
*/
diff --git a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
index 444c041..dc6c910 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
@@ -23,9 +23,29 @@
*/
public class PlacementPolicy {
+ /**
+ * Default values
+ */
public static final int DEFAULT = 0;
- public static final int EXCLUDE_FROM_FLEXING = 1;
+
+ /**
+ * Strict placement: when asking for an instance for which there is
+ * history, mandate that it is strict
+ */
+ public static final int STRICT = 1;
+
+ /**
+ * No data locality; do not bother trying to ask for any location
+ */
public static final int NO_DATA_LOCALITY = 2;
+ /**
+ * Anti-affinity is mandatory.
+ */
public static final int ANTI_AFFINITY_REQUIRED = 4;
+
+ /**
+ * Exclude from flexing; used internally to mark AMs.
+ */
+ public static final int EXCLUDE_FROM_FLEXING = 16;
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
index f1883fc..9767430 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.exceptions.SliderException;
import java.util.List;
@@ -28,9 +29,15 @@
String getName();
List<ProviderRole> getRoles();
-
+
Configuration getConf();
+ /**
+ * Verify that an instance definition is considered valid by the provider
+ * @param instanceDefinition instance definition
+ * @throws SliderException if the configuration is not valid
+ */
void validateInstanceDefinition(AggregateConf instanceDefinition) throws
SliderException;
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 0f5b4fb..b080ecb 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
@@ -31,13 +32,12 @@
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.main.ExitCodeProvider;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
+import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
-import org.apache.slider.server.services.registry.RegistryViewForProviders;
+import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import java.io.File;
import java.io.IOException;
@@ -158,11 +158,16 @@
Map<String, String> buildMonitorDetails(ClusterDescription clusterSpec);
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg,
QueueAccess queueAccess,
List<Container> liveContainers);
/**
+ * Bind to the YARN registry
+ * @param yarnRegistry YARN registry
+ */
+ void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry);
+
+ /**
* Returns the agent rest operations interface.
* @return the interface if available, null otherwise.
*/
@@ -176,13 +181,15 @@
/**
* Prior to going live -register the initial service registry data
- * @param unsecureWebAPI
- * @param secureWebAPI
- * @param registryInstanceData
+ * @param amWebURI
+ * @param agentOpsURI
+ * @param agentStatusURI
+ * @param serviceRecord
*/
- void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData registryInstanceData)
+ void applyInitialRegistryDefinitions(URL amWebURI,
+ URL agentOpsURI,
+ URL agentStatusURI,
+ ServiceRecord serviceRecord)
throws IOException;
/**
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index 4b8724a..e26105e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -106,6 +106,30 @@
}
}
}
+
+ /**
+ * Add/overwrite the agent tarball (overwritten every time application is restarted)
+ * @param provider
+ * @param tarName
+ * @param sliderFileSystem
+ * @param agentDir
+ * @return
+ * @throws IOException
+ */
+ public static boolean addAgentTar(Object provider,
+ String tarName,
+ SliderFileSystem sliderFileSystem,
+ Path agentDir) throws
+ IOException {
+ File localFile = SliderUtils.findContainingJar(provider.getClass());
+ if(localFile != null) {
+ String parentDir = localFile.getParent();
+ Path agentTarPath = new Path(parentDir, tarName);
+ sliderFileSystem.getFileSystem().copyFromLocalFile(false, true, agentTarPath, agentDir);
+ return true;
+ }
+ return false;
+ }
/**
* Add a set of dependencies to the provider resources being built up,
@@ -151,6 +175,34 @@
}
/**
+ * Loads all dependency jars from the default path
+ * @param providerResources map of provider resources to add these entries to
+ * @param sliderFileSystem target filesystem
+ * @param tempPath path in the cluster FS for temp files
+ * @param libDir relative directory to place resources
+ * @param libLocalSrcDir explicitly supplied local libs dir
+ * @throws IOException
+ * @throws SliderException
+ */
+ public static void addAllDependencyJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String libLocalSrcDir
+ ) throws
+ IOException,
+ SliderException {
+ String libSrcToUse = libLocalSrcDir;
+ if (SliderUtils.isSet(libLocalSrcDir)) {
+ File file = new File(libLocalSrcDir);
+ if (!file.exists() || !file.isDirectory()) {
+ throw new BadCommandArgumentsException("Supplied lib src dir %s is not valid", libLocalSrcDir);
+ }
+ }
+ SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath, libDir, libSrcToUse);
+ }
+
+ /**
* build the log directory
* @return the log dir
*/
@@ -180,11 +232,13 @@
/**
* Validate the node count and heap size values of a node class
- *
+ * <p>
+ * If max <= 0: min <= count
+ * If max > 0: min <= count <= max
* @param name node class name
* @param count requested node count
* @param min requested heap size
- * @param max
+ * @param max maximum value.
* @throws BadCommandArgumentsException if the values are out of range
*/
public void validateNodeCount(String name,
@@ -252,7 +306,7 @@
/**
* Propagate an option from the cluster specification option map
* to the site XML map, using the site key for the name
- * @param clusterSpec cluster specification
+ * @param global global config spec
* @param optionKey key in the option map
* @param sitexml map for XML file to build up
* @param siteKey key to assign the value to in the site XML
@@ -268,14 +322,13 @@
/**
* Build the image dir. This path is relative and only valid at the far end
- * @param clusterSpec cluster spec
+ * @param instanceDefinition instance definition
* @param bindir bin subdir
* @param script script in bin subdir
* @return the path to the script
* @throws FileNotFoundException if a file is not found, or it is not a directory*
*/
- public String buildPathToHomeDir(AggregateConf instanceDefinition
- ,
+ public String buildPathToHomeDir(AggregateConf instanceDefinition,
String bindir,
String script) throws
FileNotFoundException,
@@ -327,7 +380,7 @@
/**
* Build the image dir. This path is relative and only valid at the far end
- * @param internal internal options
+ * @param instance instance options
* @param bindir bin subdir
* @param script script in bin subdir
* @return the path to the script
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 3a1ee76..0ef8a33 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -36,6 +36,7 @@
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
+import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,6 +45,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
@@ -109,25 +111,55 @@
}
@Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
SliderException {
- super.validateInstanceDefinition(instanceDefinition);
+ super.validateInstanceDefinition(instanceDefinition, fs);
log.debug("Validating conf {}", instanceDefinition);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
- ConfTreeOperations appConf =
- instanceDefinition.getAppConfOperations();
providerUtils.validateNodeCount(instanceDefinition, ROLE_NODE,
0, -1);
+ try {
+ // Validate the app definition
+ instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ } catch (BadConfigException bce) {
+ throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
+ }
+ String appDef = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ log.info("Validating app definition {}", appDef);
+ String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length());
+ if (!"zip".equals(extension.toLowerCase(Locale.ENGLISH))) {
+ throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef);
+ }
+
Set<String> names = resources.getComponentNames();
names.remove(SliderKeys.COMPONENT_AM);
Map<Integer, String> priorityMap = new HashMap<Integer, String>();
+
+ Metainfo metaInfo = null;
+ if (fs != null) {
+ try {
+ metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef);
+ } catch (IOException ioe) {
+ // Ignore missing metainfo file for now
+ log.info("Missing metainfo.xml {}", ioe.getMessage());
+ }
+ }
+
for (String name : names) {
MapOperations component = resources.getMandatoryComponent(name);
- // Validate count against the metainfo.xml
+ if (metaInfo != null) {
+ Component componentDef = metaInfo.getApplicationComponent(name);
+ if (componentDef == null) {
+ throw new BadConfigException(
+ "Component %s is not a member of application.", name);
+ }
+ }
int priority =
component.getMandatoryOptionInt(ResourceKeys.COMPONENT_PRIORITY);
@@ -150,46 +182,59 @@
priorityMap.put(priority, name);
}
- try {
- // Validate the app definition
- instanceDefinition.getAppConfOperations().
- getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
- } catch (BadConfigException bce) {
- throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
- }
- String appDef = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
- log.info("Validating app definition {}", appDef);
- String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length());
- if (!"zip".equalsIgnoreCase(extension)) {
- throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef);
- }
+ // fileSystem may be null for tests
+ if (metaInfo != null) {
+ for (String name : names) {
+ Component componentDef = metaInfo.getApplicationComponent(name);
+ if (componentDef == null) {
+ throw new BadConfigException(
+ "Component %s is not a member of application.", name);
+ }
- String appHome = instanceDefinition.getAppConfOperations().
- getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
- String agentImage = instanceDefinition.getInternalOperations().
- get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
-
- if (SliderUtils.isUnset(appHome) && SliderUtils.isUnset(agentImage)) {
- throw new BadConfigException("Either agent package path " +
- AgentKeys.PACKAGE_PATH + " or image root " +
- InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH
- + " must be provided.");
+ MapOperations componentConfig = resources.getMandatoryComponent(name);
+ int count =
+ componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
+ int definedMinCount = componentDef.getMinInstanceCountInt();
+ int definedMaxCount = componentDef.getMaxInstanceCountInt();
+ if (count < definedMinCount || count > definedMaxCount) {
+ throw new BadConfigException("Component %s, %s value %d out of range. "
+ + "Expected minimum is %d and maximum is %d",
+ name,
+ ResourceKeys.COMPONENT_INSTANCES,
+ count,
+ definedMinCount,
+ definedMaxCount);
+ }
+ }
}
}
+
@Override
public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
- Configuration serviceConf,
- AbstractLauncher launcher,
- AggregateConf instanceDescription,
- Path snapshotConfDirPath,
- Path generatedConfDirPath,
- Configuration clientConfExtras,
- String libdir,
- Path tempPath, boolean miniClusterTestRun) throws
+ Configuration serviceConf,
+ AbstractLauncher launcher,
+ AggregateConf instanceDefinition,
+ Path snapshotConfDirPath,
+ Path generatedConfDirPath,
+ Configuration clientConfExtras,
+ String libdir,
+ Path tempPath,
+ boolean miniClusterTestRun) throws
IOException,
SliderException {
+ String agentImage = instanceDefinition.getInternalOperations().
+ get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ if (SliderUtils.isUnset(agentImage)) {
+ Path agentPath = new Path(tempPath.getParent(), AgentKeys.PROVIDER_AGENT);
+ log.info("Automatically uploading the agent tarball at {}", agentPath);
+ fileSystem.getFileSystem().mkdirs(agentPath);
+ if(ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) {
+ instanceDefinition.getInternalOperations().set(
+ InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH,
+ new Path(agentPath, AGENT_TAR).toUri());
+ }
+ }
}
@Override
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 419fa1a..e682b13 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -24,7 +24,7 @@
public interface AgentKeys {
String PROVIDER_AGENT = "agent";
- String ROLE_NODE = "node";
+ String ROLE_NODE = "echo";
/**
* {@value}
*/
@@ -60,6 +60,11 @@
* Execution home for the agent.
*/
String APP_HOME = "app.home";
+ String APP_ROOT = "site.global.app_root";
+ /**
+ * Runas user of the application
+ */
+ String RUNAS_USER = "site.global.app_user";
/**
* Name of the service.
*/
@@ -85,11 +90,16 @@
String JAVA_HOME = "java_home";
String PACKAGE_LIST = "package_list";
+ String SYSTEM_CONFIGS = "system_configs";
String WAIT_HEARTBEAT = "wait.heartbeat";
String PYTHON_EXE = "python";
String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
+ String AGENT_OUT_FILE = "slider-agent.out";
+ String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth";
+ String CERT_FILE_LOCALIZATION_PATH = "certs/ca.crt";
+ String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index c7a82d3..4f981c7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -19,9 +19,17 @@
package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.curator.utils.ZKPaths;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ProtocolTypes;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
@@ -33,8 +41,11 @@
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.StatusKeys;
+import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
@@ -46,23 +57,26 @@
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.ExportEntry;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.docstore.PublishedExports;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
+import org.apache.slider.providers.agent.application.metadata.CommandScript;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.ComponentExport;
+import org.apache.slider.providers.agent.application.metadata.ConfigFile;
+import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
import org.apache.slider.providers.agent.application.metadata.Export;
import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
+import org.apache.slider.providers.agent.application.metadata.PropertyInfo;
import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
import org.apache.slider.server.appmaster.state.ContainerPriority;
@@ -79,6 +93,7 @@
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand;
+import org.apache.slider.server.services.security.CertificateManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,6 +106,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -100,14 +116,15 @@
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
-/** This class implements the server-side logic for application deployment
- * through Slider application package
- **/
+/**
+ * This class implements the server-side logic for application deployment through Slider application package
+ */
public class AgentProviderService extends AbstractProviderService implements
ProviderCore,
AgentKeys,
@@ -121,17 +138,24 @@
private static final String CONTAINER_ID = "container_id";
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
+ private static final String HOST_FOLDER_FORMAT = "%s:%s";
+ private static final String CONTAINER_LOGS_TAG = "container_log_dirs";
+ private static final String CONTAINER_PWDS_TAG = "container_work_dirs";
+ private static final String COMPONENT_TAG = "component";
+ private static final String APPLICATION_TAG = "application";
private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
private static final String SHARED_PORT_TAG = "SHARED";
- private static final String DO_NOT_PROPAGATE_TAG = "{DO_NOT_PROPAGATE}";
- private static final int MAX_LOG_ENTRIES = 20;
+ private static final String PER_CONTAINER_TAG = "{PER_CONTAINER}";
+ private static final int MAX_LOG_ENTRIES = 40;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
+ private final ComponentTagProvider tags = new ComponentTagProvider();
private int heartbeatMonitorInterval = 0;
private AgentClientProvider clientProvider;
private AtomicInteger taskId = new AtomicInteger(0);
private volatile Metainfo metainfo = null;
+ private Map<String, DefaultConfig> defaultConfigs = null;
private ComponentCommandOrder commandOrder = null;
private HeartbeatMonitor monitor;
private Boolean canAnyMasterPublish = null;
@@ -142,16 +166,25 @@
new ConcurrentHashMap<String, ComponentInstanceState>();
private final Map<String, Map<String, String>> componentInstanceData =
new ConcurrentHashMap<String, Map<String, String>>();
- private final Map<String, Map<String, String>> exportGroups =
- new ConcurrentHashMap<String, Map<String, String>>();
+ private final Map<String, Map<String, List<ExportEntry>>> exportGroups =
+ new ConcurrentHashMap<String, Map<String, List<ExportEntry>>>();
private final Map<String, Map<String, String>> allocatedPorts =
new ConcurrentHashMap<String, Map<String, String>>();
- private final Map<String, String> workFolders =
- Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) {
+
+ private final Map<String, ExportEntry> logFolderExports =
+ Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) {
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > MAX_LOG_ENTRIES;
}
});
+ private final Map<String, ExportEntry> workFolderExports =
+ Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) {
+ protected boolean removeEldestEntry(Map.Entry eldest) {
+ return size() > MAX_LOG_ENTRIES;
+ }
+ });
+ private final Map<String, Set<String>> containerExportsMap =
+ new HashMap<String, Set<String>>();
/**
* Create an instance of AgentProviderService
@@ -184,12 +217,40 @@
public void validateInstanceDefinition(AggregateConf instanceDefinition)
throws
SliderException {
- clientProvider.validateInstanceDefinition(instanceDefinition);
+ clientProvider.validateInstanceDefinition(instanceDefinition, null);
+
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+
+ Set<String> names = resources.getComponentNames();
+ names.remove(SliderKeys.COMPONENT_AM);
+ for (String name : names) {
+ Component componentDef = getMetainfo().getApplicationComponent(name);
+ if (componentDef == null) {
+ throw new BadConfigException(
+ "Component %s is not a member of application.", name);
+ }
+
+ MapOperations componentConfig = resources.getMandatoryComponent(name);
+ int count =
+ componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
+ int definedMinCount = componentDef.getMinInstanceCountInt();
+ int definedMaxCount = componentDef.getMaxInstanceCountInt();
+ if (count < definedMinCount || count > definedMaxCount) {
+ throw new BadConfigException("Component %s, %s value %d out of range. "
+ + "Expected minimum is %d and maximum is %d",
+ name,
+ ResourceKeys.COMPONENT_INSTANCES,
+ count,
+ definedMinCount,
+ definedMaxCount);
+ }
+ }
}
// Reads the metainfo.xml in the application package and loads it
private void buildMetainfo(AggregateConf instanceDefinition,
- SliderFileSystem fileSystem) throws IOException, SliderException {
+ SliderFileSystem fileSystem) throws IOException, SliderException {
String appDef = instanceDefinition.getAppConfOperations()
.getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
@@ -206,7 +267,8 @@
"metainfo.xml is required in app package.");
}
commandOrder = new ComponentCommandOrder(metainfo.getApplication()
- .getCommandOrder());
+ .getCommandOrder());
+ defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metainfo);
monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval());
monitor.start();
}
@@ -248,14 +310,14 @@
String workDir = ApplicationConstants.Environment.PWD.$();
launcher.setEnv("AGENT_WORK_ROOT", workDir);
log.info("AGENT_WORK_ROOT set to {}", workDir);
- String logDir = ApplicationConstants.Environment.LOG_DIRS.$();
+ String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR;
launcher.setEnv("AGENT_LOG_ROOT", logDir);
log.info("AGENT_LOG_ROOT set to {}", logDir);
if (System.getenv(HADOOP_USER_NAME) != null) {
launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
}
// for 2-Way SSL
- launcher.setEnv(SLIDER_PASSPHRASE, SliderKeys.PASSPHRASE);
+ launcher.setEnv(SLIDER_PASSPHRASE, instanceDefinition.getPassphrase());
//local resources
@@ -274,11 +336,26 @@
launcher.setEnv(PYTHONPATH, pythonPath);
log.info("PYTHONPATH set to {}", pythonPath);
+ Path agentImagePath = null;
String agentImage = instanceDefinition.getInternalOperations().
get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- if (agentImage != null) {
- LocalResource agentImageRes = fileSystem.createAmResource(new Path(agentImage), LocalResourceType.ARCHIVE);
+ if (SliderUtils.isUnset(agentImage)) {
+ agentImagePath =
+ new Path(new Path(new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR),
+ container.getId().getApplicationAttemptId().getApplicationId().toString()),
+ AgentKeys.PROVIDER_AGENT),
+ SliderKeys.AGENT_TAR);
+ } else {
+ agentImagePath = new Path(agentImage);
+ }
+
+ // TODO: throw exception when agent tarball is not available
+
+ if (fileSystem.getFileSystem().exists(agentImagePath)) {
+ LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE);
launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes);
+ } else {
+ log.error("Required agent image slider-agent.tar.gz is unavailable.");
}
log.info("Using {} for agent.", scriptPath);
@@ -289,10 +366,10 @@
String agentConf = instanceDefinition.getAppConfOperations().
getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
- if (org.apache.commons.lang.StringUtils.isNotEmpty(agentConf)) {
+ if (SliderUtils.isSet(agentConf)) {
LocalResource agentConfRes = fileSystem.createAmResource(fileSystem
- .getFileSystem().resolvePath(new Path(agentConf)),
- LocalResourceType.FILE);
+ .getFileSystem().resolvePath(new Path(agentConf)),
+ LocalResourceType.FILE);
launcher.addLocalResource(AgentKeys.AGENT_CONFIG_FILE, agentConfRes);
}
@@ -305,10 +382,31 @@
launcher.addLocalResource(AgentKeys.AGENT_VERSION_FILE, agentVerRes);
}
+ if (SliderUtils.isHadoopClusterSecure(getConfig())) {
+ localizeServiceKeytabs(launcher, instanceDefinition, fileSystem);
+ }
+
+ MapOperations amComponent = instanceDefinition.
+ getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM);
+ boolean twoWayEnabled = amComponent != null ? Boolean.valueOf(amComponent.
+ getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) : false;
+ if (twoWayEnabled) {
+ localizeContainerSSLResources(launcher, container, fileSystem);
+ }
+
+ //add the configuration resources
+ launcher.addLocalResources(fileSystem.submitDirectory(
+ generatedConfPath,
+ SliderKeys.PROPAGATED_CONF_DIR_NAME));
+
String label = getContainerLabel(container, role);
CommandLineBuilder operation = new CommandLineBuilder();
- operation.add(AgentKeys.PYTHON_EXE);
+ String pythonExec = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().getOption(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH,
+ AgentKeys.PYTHON_EXE);
+
+ operation.add(pythonExec);
operation.add(scriptPath);
operation.add(ARG_LABEL, label);
@@ -318,11 +416,13 @@
operation.add(getZkRegistryPath());
String debugCmd = agentLaunchParameter.getNextLaunchParameter(role);
- if (debugCmd != null && debugCmd.length() != 0) {
+ if (SliderUtils.isSet(debugCmd)) {
operation.add(ARG_DEBUG);
operation.add(debugCmd);
}
+ operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + AgentKeys.AGENT_OUT_FILE + " 2>&1");
+
launcher.addCommand(operation.build());
// initialize the component instance state
@@ -333,61 +433,155 @@
getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
}
- // build the zookeeper registry path
- private String getZkRegistryPath() {
- String zkRegistryRoot = getConfig().get(REGISTRY_PATH,
- DEFAULT_REGISTRY_PATH);
- String appType = APP_TYPE;
- String zkRegistryPath = ZKPaths.makePath(zkRegistryRoot, appType);
- String clusterName = getAmState().getInternalsSnapshot().get(
- OptionKeys.APPLICATION_NAME);
- zkRegistryPath = ZKPaths.makePath(zkRegistryPath, clusterName);
- return zkRegistryPath;
+ private void localizeContainerSSLResources(ContainerLauncher launcher,
+ Container container,
+ SliderFileSystem fileSystem)
+ throws SliderException {
+ try {
+ // localize server cert
+ Path certsDir = new Path(fileSystem.buildClusterDirPath(
+ getClusterName()), "certs");
+ LocalResource certResource = fileSystem.createAmResource(
+ new Path(certsDir, SliderKeys.CRT_FILE_NAME),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(AgentKeys.CERT_FILE_LOCALIZATION_PATH,
+ certResource);
+
+ // generate and localize agent cert
+ CertificateManager certMgr = new CertificateManager();
+ String hostname = container.getNodeId().getHost();
+ String containerId = container.getId().toString();
+ certMgr.generateAgentCertificate(hostname, containerId);
+ LocalResource agentCertResource = fileSystem.createAmResource(
+ uploadSecurityResource(
+ CertificateManager.getAgentCertficateFilePath(containerId),
+ fileSystem), LocalResourceType.FILE);
+ // still using hostname as file name on the agent side, but the files
+ // do end up under the specific container's file space
+ launcher.addLocalResource("certs/" + hostname + ".crt",
+ agentCertResource);
+ LocalResource agentKeyResource = fileSystem.createAmResource(
+ uploadSecurityResource(
+ CertificateManager.getAgentKeyFilePath(containerId), fileSystem),
+ LocalResourceType.FILE);
+ launcher.addLocalResource("certs/" + hostname + ".key",
+ agentKeyResource);
+
+ } catch (Exception e) {
+ throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, e,
+ "Unable to localize certificates. Two-way SSL cannot be enabled");
+ }
}
- @Override
- public void rebuildContainerDetails(List<Container> liveContainers,
- String applicationId, Map<Integer, ProviderRole> providerRoleMap) {
- for (Container container : liveContainers) {
- // get the role name and label
- ProviderRole role = providerRoleMap.get(ContainerPriority
- .extractRole(container));
- if (role != null) {
- String roleName = role.name;
- String label = getContainerLabel(container, roleName);
- log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
- container.getId(), roleName, applicationId);
- getComponentStatuses().put(
- label,
- new ComponentInstanceState(roleName, container.getId(),
- applicationId));
- } else {
- log.warn("Role not found for container {} in cluster {}",
- container.getId(), applicationId);
+ private Path uploadSecurityResource(File resource, SliderFileSystem fileSystem)
+ throws IOException {
+ Path certsDir = new Path(fileSystem.buildClusterDirPath(getClusterName()),
+ "certs");
+ if (!fileSystem.getFileSystem().exists(certsDir)) {
+ fileSystem.getFileSystem().mkdirs(certsDir,
+ new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ }
+ Path destPath = new Path(certsDir, resource.getName());
+ if (!fileSystem.getFileSystem().exists(destPath)) {
+ FSDataOutputStream os = fileSystem.getFileSystem().create(destPath);
+ byte[] contents = FileUtils.readFileToByteArray(resource);
+ os.write(contents, 0, contents.length);
+
+ os.flush();
+ os.close();
+ log.info("Uploaded {} to localization path {}", resource, destPath);
+ }
+
+ while (!fileSystem.getFileSystem().exists(destPath)) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+
+ fileSystem.getFileSystem().setPermission(destPath,
+ new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
+
+ return destPath;
+ }
+
+ private void localizeServiceKeytabs(ContainerLauncher launcher,
+ AggregateConf instanceDefinition,
+ SliderFileSystem fileSystem)
+ throws IOException {
+ String keytabPathOnHost = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+ if (SliderUtils.isUnset(keytabPathOnHost)) {
+ String amKeytabName = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
+ // we need to localize the keytab files in the directory
+ Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
+ getClusterName());
+ FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDirPath);
+ LocalResource keytabRes;
+ boolean serviceKeytabsDeployed = false;
+ for (FileStatus keytab : keytabs) {
+ if (!amKeytabName.equals(keytab.getPath().getName())
+ && keytab.getPath().getName().endsWith(".keytab")) {
+ serviceKeytabsDeployed = true;
+ log.info("Localizing keytab {}", keytab.getPath().getName());
+ keytabRes = fileSystem.createAmResource(keytab.getPath(),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(SliderKeys.KEYTAB_DIR + "/" +
+ keytab.getPath().getName(),
+ keytabRes);
+ }
+ }
+ if (!serviceKeytabsDeployed) {
+ log.warn("No service keytabs for the application have been localized. "
+ + "If the application requires keytabs for secure operation, "
+ + "please ensure that the required keytabs have been uploaded "
+ + "to the folder designated by the property {}: {}",
+ SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR, keytabDirPath);
}
}
}
/**
- * Run this service
- *
- * @param instanceDefinition component description
- * @param confDir local dir with the config
- * @param env environment variables above those generated by
- * @param execInProgress callback for the event notification
- *
- * @throws IOException IO problems
- * @throws SliderException anything internal
+ * build the zookeeper registry path.
+ *
+ * @return the path the service registered at
+ * @throws NullPointerException if the service has not yet registered
*/
- @Override
- public boolean exec(AggregateConf instanceDefinition,
- File confDir,
- Map<String, String> env,
- ProviderCompleted execInProgress) throws
- IOException,
- SliderException {
+ private String getZkRegistryPath() {
+ Preconditions.checkNotNull(yarnRegistry, "Yarn registry not bound");
+ String path = yarnRegistry.getAbsoluteSelfRegistrationPath();
+ Preconditions.checkNotNull(path, "Service record path not defined");
+ return path;
+ }
- return false;
+ @Override
+ public void rebuildContainerDetails(List<Container> liveContainers,
+ String applicationId, Map<Integer, ProviderRole> providerRoleMap) {
+ for (Container container : liveContainers) {
+ // get the role name and label
+ ProviderRole role = providerRoleMap.get(ContainerPriority
+ .extractRole(container));
+ if (role != null) {
+ String roleName = role.name;
+ String label = getContainerLabel(container, roleName);
+ log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
+ container.getId(), roleName, applicationId);
+ getComponentStatuses().put(
+ label,
+ new ComponentInstanceState(roleName, container.getId(),
+ applicationId));
+ } else {
+ log.warn("Role not found for container {} in cluster {}",
+ container.getId(), applicationId);
+ }
+ }
}
@Override
@@ -397,14 +591,16 @@
/**
* Handle registration calls from the agents
+ *
* @param registration
+ *
* @return
*/
@Override
public RegistrationResponse handleRegistration(Register registration) {
log.info("Handling registration: " + registration);
RegistrationResponse response = new RegistrationResponse();
- String label = registration.getHostname();
+ String label = registration.getLabel();
State agentState = registration.getActualState();
if (getComponentStatuses().containsKey(label)) {
response.setResponseStatus(RegistrationStatus.OK);
@@ -412,11 +608,24 @@
componentStatus.heartbeat(System.currentTimeMillis());
updateComponentStatusWithAgentState(componentStatus, agentState);
+ String roleName = getRoleName(label);
+ String containerId = getContainerId(label);
+
+ if (SliderUtils.isSet(registration.getTags())) {
+ tags.recordAssignedTag(roleName, containerId, registration.getTags());
+ } else {
+ response.setTags(tags.getTag(roleName, containerId));
+ }
+
+ String hostFqdn = registration.getPublicHostname();
Map<String, String> ports = registration.getAllocatedPorts();
if (ports != null && !ports.isEmpty()) {
- String roleName = getRoleName(label);
- String containerId = getContainerId(label);
- processAllocatedPorts(registration.getPublicHostname(), roleName, containerId, ports);
+ processAllocatedPorts(hostFqdn, roleName, containerId, ports);
+ }
+
+ Map<String, String> folders = registration.getLogFolders();
+ if (folders != null && !folders.isEmpty()) {
+ publishFolderPaths(folders, containerId, roleName, hostFqdn);
}
} else {
response.setResponseStatus(RegistrationStatus.FAILED);
@@ -429,8 +638,10 @@
/**
* Handle heartbeat response from agents
- * @param heartBeat
- * @return
+ *
+ * @param heartBeat incoming heartbeat from Agent
+ *
+ * @return response to send back
*/
@Override
public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
@@ -444,14 +655,26 @@
String containerId = getContainerId(label);
StateAccessForProviders accessor = getAmState();
- String scriptPath = getScriptPathFromMetainfo(roleName);
+ CommandScript cmdScript = getScriptPathFromMetainfo(roleName);
- if (scriptPath == null) {
+ if (cmdScript == null || cmdScript.getScript() == null) {
log.error("role.script is unavailable for " + roleName + ". Commands will not be sent.");
return response;
}
+ String scriptPath = cmdScript.getScript();
+ long timeout = cmdScript.getTimeout();
+
+ if (timeout == 0L) {
+ timeout = 600L;
+ }
+
if (!getComponentStatuses().containsKey(label)) {
+ // container is completed but still heart-beating, send terminate signal
+ log.info(
+ "Sending terminate signal to completed container (still heartbeating): {}",
+ label);
+ response.setTerminateAgent(true);
return response;
}
@@ -474,7 +697,7 @@
log.info("Component operation. Status: {}", result);
if (command == Command.INSTALL && report.getFolders() != null && report.getFolders().size() > 0) {
- publishLogFolderPaths(report.getFolders(), containerId, heartBeat.getFqdn());
+ publishFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn());
}
}
@@ -492,14 +715,14 @@
if (Command.NOP != command) {
if (command == Command.INSTALL) {
log.info("Installing {} on {}.", roleName, containerId);
- addInstallCommand(roleName, containerId, response, scriptPath);
+ addInstallCommand(roleName, containerId, response, scriptPath, timeout);
componentStatus.commandIssued(command);
} else if (command == Command.START) {
// check against dependencies
boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values());
if (canExecute) {
log.info("Starting {} on {}.", roleName, containerId);
- addStartCommand(roleName, containerId, response, scriptPath, isMarkedAutoRestart(roleName));
+ addStartCommand(roleName, containerId, response, scriptPath, timeout, isMarkedAutoRestart(roleName));
componentStatus.commandIssued(command);
} else {
log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId);
@@ -523,8 +746,8 @@
response.setRestartEnabled(true);
}
} catch (SliderException e) {
- componentStatus.applyCommandResult(CommandResult.FAILED, command);
log.warn("Component instance failed operation.", e);
+ componentStatus.applyCommandResult(CommandResult.FAILED, command);
}
log.debug("Heartbeat response: " + response);
@@ -532,9 +755,9 @@
}
protected void processAllocatedPorts(String fqdn,
- String roleName,
- String containerId,
- Map<String, String> ports) {
+ String roleName,
+ String containerId,
+ Map<String, String> ports) {
RoleInstance instance;
try {
instance = getAmState().getOwnedContainer(containerId);
@@ -546,24 +769,32 @@
String portname = port.getKey();
String portNo = port.getValue();
log.info("Recording allocated port for {} as {}", portname, portNo);
+
+ // add the allocated ports to the global list as well as per container list
+ // per container allocation will over-write each other in the global
this.getAllocatedPorts().put(portname, portNo);
this.getAllocatedPorts(containerId).put(portname, portNo);
- if (instance!=null) {
- try {
- instance.registerPortEndpoint(Integer.valueOf(portNo), portname, "");
- } catch (NumberFormatException e) {
- log.warn("Failed to parse {}: {}", portNo, e);
- }
+ if (instance != null) {
+ try {
+ // if the returned value is not a single port number then there are no
+ // meaningful way for Slider to use it during export
+ // No need to error out as it may not be the responsibility of the component
+ // to allocate port or the component may need an array of ports
+ instance.registerPortEndpoint(Integer.valueOf(portNo), portname);
+ } catch (NumberFormatException e) {
+ log.warn("Failed to parse {}: {}", portNo, e);
}
+ }
}
// component specific publishes
processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
-
+ processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName);
+
// and update registration entries
if (instance != null) {
- queueAccess.put(new RegisterComponentInstance(instance.getId(), 0,
- TimeUnit.MILLISECONDS));
+ queueAccess.put(new RegisterComponentInstance(instance.getId(),
+ roleName, 0, TimeUnit.MILLISECONDS));
}
}
@@ -582,20 +813,28 @@
}
@Override
- public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData instanceData) throws IOException {
- super.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData
- );
+ public void applyInitialRegistryDefinitions(URL amWebURI,
+ URL agentOpsURI,
+ URL agentStatusURI,
+ ServiceRecord serviceRecord)
+ throws IOException {
+ super.applyInitialRegistryDefinitions(amWebURI,
+ agentOpsURI,
+ agentStatusURI,
+ serviceRecord);
try {
- instanceData.internalView.endpoints.put(
- CustomRegistryConstants.AGENT_REST_API,
- new RegisteredEndpoint(
- new URL(secureWebAPI, SLIDER_PATH_AGENTS),
- "Agent REST API"));
+ URL restURL = new URL(agentOpsURI, SLIDER_PATH_AGENTS);
+ URL agentStatusURL = new URL(agentStatusURI, SLIDER_PATH_AGENTS);
+
+ serviceRecord.addInternalEndpoint(
+ new Endpoint(CustomRegistryConstants.AGENT_SECURE_REST_API,
+ ProtocolTypes.PROTOCOL_REST,
+ restURL.toURI()));
+ serviceRecord.addInternalEndpoint(
+ new Endpoint(CustomRegistryConstants.AGENT_ONEWAY_REST_API,
+ ProtocolTypes.PROTOCOL_REST,
+ agentStatusURL.toURI()));
} catch (URISyntaxException e) {
throw new IOException(e);
}
@@ -603,6 +842,8 @@
@Override
public void notifyContainerCompleted(ContainerId containerId) {
+ // containers get allocated and free'ed without being assigned to any
+ // component - so many of the data structures may not be initialized
if (containerId != null) {
String containerIdStr = containerId.toString();
if (getComponentInstanceData().containsKey(containerIdStr)) {
@@ -612,21 +853,55 @@
}
if (this.allocatedPorts.containsKey(containerIdStr)) {
+ Map<String, String> portsByContainerId = getAllocatedPorts(containerIdStr);
this.allocatedPorts.remove(containerIdStr);
+ // free up the allocations from global as well
+ // if multiple containers allocate global ports then last one
+ // wins and similarly first one removes it - its not supported anyway
+ for(String portName : portsByContainerId.keySet()) {
+ getAllocatedPorts().remove(portName);
+ }
+
}
+ String componentName = null;
synchronized (this.componentStatuses) {
for (String label : getComponentStatuses().keySet()) {
if (label.startsWith(containerIdStr)) {
+ componentName = getRoleName(label);
+ log.info("Removing component status for label {}", label);
getComponentStatuses().remove(label);
}
}
}
+
+ tags.releaseTag(componentName, containerIdStr);
+
+ synchronized (this.containerExportsMap) {
+ Set<String> containerExportSets = containerExportsMap.get(containerIdStr);
+ if (containerExportSets != null) {
+ for (String containerExportStr : containerExportSets) {
+ String[] parts = containerExportStr.split(":");
+ Map<String, List<ExportEntry>> exportGroup = getCurrentExports(parts[0]);
+ List<ExportEntry> exports = exportGroup.get(parts[1]);
+ List<ExportEntry> exportToRemove = new ArrayList<ExportEntry>();
+ for (ExportEntry export : exports) {
+ if (containerIdStr.equals(export.getContainerId())) {
+ exportToRemove.add(export);
+ }
+ }
+ exports.removeAll(exportToRemove);
+ }
+ log.info("Removing container exports for {}", containerIdStr);
+ containerExportsMap.remove(containerIdStr);
+ }
+ }
}
}
/**
* Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
+ *
* @param instanceDefinition
*/
private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
@@ -635,7 +910,7 @@
Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
try {
setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
- }catch (NumberFormatException e) {
+ } catch (NumberFormatException e) {
log.warn(
"Bad value {} for {}. Defaulting to ",
hbMonitorInterval,
@@ -646,6 +921,7 @@
/**
* Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
+ *
* @param instanceDefinition
*/
private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
@@ -655,6 +931,16 @@
}
@VisibleForTesting
+ protected Map<String, ExportEntry> getLogFolderExports() {
+ return logFolderExports;
+ }
+
+ @VisibleForTesting
+ protected Map<String, ExportEntry> getWorkFolderExports() {
+ return workFolderExports;
+ }
+
+ @VisibleForTesting
protected Metainfo getMetainfo() {
return this.metainfo;
}
@@ -675,6 +961,43 @@
this.heartbeatMonitorInterval = heartbeatMonitorInterval;
}
+ /**
+ * Read all default configs
+ *
+ * @param fileSystem
+ * @param appDef
+ * @param metainfo
+ *
+ * @return
+ *
+ * @throws IOException
+ */
+ protected Map<String, DefaultConfig> initializeDefaultConfigs(SliderFileSystem fileSystem,
+ String appDef, Metainfo metainfo) throws IOException {
+ Map<String, DefaultConfig> defaultConfigMap = new HashMap<String, DefaultConfig>();
+ if (metainfo.getApplication().getConfigFiles() != null &&
+ metainfo.getApplication().getConfigFiles().size() > 0) {
+ for (ConfigFile configFile : metainfo.getApplication().getConfigFiles()) {
+ DefaultConfig config = null;
+ try {
+ config = AgentUtils.getDefaultConfig(fileSystem, appDef, configFile.getDictionaryName() + ".xml");
+ } catch (IOException e) {
+ log.warn("Default config file not found. Only the config as input during create will be applied for {}",
+ configFile.getDictionaryName());
+ }
+ if (config != null) {
+ defaultConfigMap.put(configFile.getDictionaryName(), config);
+ }
+ }
+ }
+
+ return defaultConfigMap;
+ }
+
+ protected Map<String, DefaultConfig> getDefaultConfigs() {
+ return defaultConfigs;
+ }
+
private int getHeartbeatMonitorInterval() {
return this.heartbeatMonitorInterval;
}
@@ -688,6 +1011,7 @@
/**
* Publish a named property bag that may contain name-value pairs for app configurations such as hbase-site
+ *
* @param name
* @param description
* @param entries
@@ -703,6 +1027,7 @@
/**
* Get a list of all hosts for all role/container per role
+ *
* @return
*/
protected Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping() {
@@ -732,11 +1057,10 @@
}
/**
- * Lost heartbeat from the container - release it and ask for a replacement
- * (async operation)
- * @param label
- * @param containerId
+ * Lost heartbeat from the container - release it and ask for a replacement (async operation)
*
+ * @param label
+ * @param containerId
*/
protected void lostContainer(
String label,
@@ -758,17 +1082,65 @@
/**
* Format the folder locations and publish in the registry service
+ *
* @param folders
* @param containerId
* @param hostFqdn
+ * @param componentName
*/
- private void publishLogFolderPaths(Map<String, String> folders, String containerId, String hostFqdn) {
- for (String key : folders.keySet()) {
- workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
+ protected void publishFolderPaths(
+ Map<String, String> folders, String containerId, String componentName, String hostFqdn) {
+ Date now = new Date();
+ for (Map.Entry<String, String> entry : folders.entrySet()) {
+ ExportEntry exportEntry = new ExportEntry();
+ exportEntry.setValue(String.format(HOST_FOLDER_FORMAT, hostFqdn, entry.getValue()));
+ exportEntry.setContainerId(containerId);
+ exportEntry.setLevel(COMPONENT_TAG);
+ exportEntry.setTag(componentName);
+ exportEntry.setUpdatedTime(now.toString());
+ if (entry.getKey().equals("AGENT_LOG_ROOT")) {
+ synchronized (logFolderExports) {
+ getLogFolderExports().put(containerId, exportEntry);
+ }
+ } else {
+ synchronized (workFolderExports) {
+ getWorkFolderExports().put(containerId, exportEntry);
+ }
+ }
+ log.info("Updating log and pwd folders for container {}", containerId);
}
- publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG,
- (new HashMap<String, String>(this.workFolders)).entrySet());
+ PublishedExports exports = new PublishedExports(CONTAINER_LOGS_TAG);
+ exports.setUpdated(now.getTime());
+ synchronized (logFolderExports) {
+ updateExportsFromList(exports, getLogFolderExports());
+ }
+ getAmState().getPublishedExportsSet().put(CONTAINER_LOGS_TAG, exports);
+
+ exports = new PublishedExports(CONTAINER_PWDS_TAG);
+ exports.setUpdated(now.getTime());
+ synchronized (workFolderExports) {
+ updateExportsFromList(exports, getWorkFolderExports());
+ }
+ getAmState().getPublishedExportsSet().put(CONTAINER_PWDS_TAG, exports);
+ }
+
+ /**
+ * Update the export data from the map
+ * @param exports
+ * @param folderExports
+ */
+ private void updateExportsFromList(PublishedExports exports, Map<String, ExportEntry> folderExports) {
+ Map<String, List<ExportEntry>> perComponentList = new HashMap<String, List<ExportEntry>>();
+ for(Map.Entry<String, ExportEntry> logEntry : folderExports.entrySet())
+ {
+ String componentName = logEntry.getValue().getTag();
+ if(!perComponentList.containsKey(componentName)) {
+ perComponentList.put(componentName, new ArrayList<ExportEntry>());
+ }
+ perComponentList.get(componentName).add(logEntry.getValue());
+ }
+ exports.putValues(perComponentList.entrySet());
}
@@ -779,7 +1151,7 @@
* @param componentStatus
*/
protected void publishConfigAndExportGroups(
- HeartBeat heartBeat, ComponentInstanceState componentStatus, String roleName) {
+ HeartBeat heartBeat, ComponentInstanceState componentStatus, String componentName) {
List<ComponentStatus> statuses = heartBeat.getComponentStatus();
if (statuses != null && !statuses.isEmpty()) {
log.info("Processing {} status reports.", statuses.size());
@@ -789,7 +1161,7 @@
if (status.getConfigs() != null) {
Application application = getMetainfo().getApplication();
- if (canAnyMasterPublishConfig() == false || canPublishConfig(roleName)) {
+ if (canAnyMasterPublishConfig() == false || canPublishConfig(componentName)) {
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
@@ -813,13 +1185,12 @@
}
}
- List<ExportGroup> exportGroups = application.getExportGroups();
- boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty();
+ List<ExportGroup> appExportGroups = application.getExportGroups();
+ boolean hasExportGroups = appExportGroups != null && !appExportGroups.isEmpty();
Set<String> appExports = new HashSet();
- String appExportsStr = getApplicationComponent(roleName).getAppExports();
- boolean hasNoAppExports = appExportsStr == null || appExportsStr.isEmpty();
- if (!hasNoAppExports) {
+ String appExportsStr = getApplicationComponent(componentName).getAppExports();
+ if (SliderUtils.isSet(appExportsStr)) {
for (String appExport : appExportsStr.split(",")) {
if (appExport.trim().length() > 0) {
appExports.add(appExport.trim());
@@ -847,11 +1218,12 @@
}
Set<String> modifiedGroups = new HashSet<String>();
- for (ExportGroup exportGroup : exportGroups) {
+ for (ExportGroup exportGroup : appExportGroups) {
List<Export> exports = exportGroup.getExports();
if (exports != null && !exports.isEmpty()) {
String exportGroupName = exportGroup.getName();
- Map<String, String> map = getCurrentExports(exportGroupName);
+ ConcurrentHashMap<String, List<ExportEntry>> map =
+ (ConcurrentHashMap<String, List<ExportEntry>>)getCurrentExports(exportGroupName);
for (Export export : exports) {
if (canBeExported(exportGroupName, export.getName(), appExports)) {
String value = export.getValue();
@@ -861,7 +1233,12 @@
value = value.replace(token, replaceTokens.get(token));
}
}
- map.put(export.getName(), value);
+ ExportEntry entry = new ExportEntry();
+ entry.setLevel(APPLICATION_TAG);
+ entry.setValue(value);
+ entry.setUpdatedTime(new Date().toString());
+ // over-write, app exports are singletons
+ map.put(export.getName(), new ArrayList(Arrays.asList(entry)));
log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
}
}
@@ -880,26 +1257,40 @@
}
private boolean canBeExported(String exportGroupName, String name, Set<String> appExports) {
- return appExports.contains(String.format("%s-%s", exportGroupName, name));
+ return appExports.contains(String.format("%s-%s", exportGroupName, name));
}
- protected Map<String, String> getCurrentExports(String groupName) {
- if(!this.exportGroups.containsKey(groupName)) {
- synchronized (this.exportGroups) {
- if(!this.exportGroups.containsKey(groupName)) {
- this.exportGroups.put(groupName, new ConcurrentHashMap<String, String>());
- }
- }
+ protected Map<String, List<ExportEntry>> getCurrentExports(String groupName) {
+ if (!this.exportGroups.containsKey(groupName)) {
+ synchronized (this.exportGroups) {
+ if (!this.exportGroups.containsKey(groupName)) {
+ this.exportGroups.put(groupName, new ConcurrentHashMap<String, List<ExportEntry>>());
+ }
+ }
}
return this.exportGroups.get(groupName);
}
private void publishModifiedExportGroups(Set<String> modifiedGroups) {
- synchronized (this.exportGroups) {
- for(String groupName : modifiedGroups) {
- publishApplicationInstanceData(groupName, groupName, this.exportGroups.get(groupName).entrySet());
+ for (String groupName : modifiedGroups) {
+ Map<String, List<ExportEntry>> entries = this.exportGroups.get(groupName);
+
+ // Publish in old format for the time being
+ Map<String, String> simpleEntries = new HashMap<String, String>();
+ for (Map.Entry<String, List<ExportEntry>> entry : entries.entrySet()) {
+ List<ExportEntry> exports = entry.getValue();
+ if(exports != null && exports.size() > 0) {
+ // there is no support for multiple exports per name - so extract only the first one
+ simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue());
+ }
}
+ publishApplicationInstanceData(groupName, groupName, simpleEntries.entrySet());
+
+ PublishedExports exports = new PublishedExports(groupName);
+ exports.setUpdated(new Date().getTime());
+ exports.putValues(entries.entrySet());
+ getAmState().getPublishedExportsSet().put(groupName, exports);
}
}
@@ -907,14 +1298,14 @@
protected void processAndPublishComponentSpecificData(Map<String, String> ports,
String containerId,
String hostFqdn,
- String roleName) {
+ String componentName) {
String portVarFormat = "${site.%s}";
String hostNamePattern = "${THIS_HOST}";
Map<String, String> toPublish = new HashMap<String, String>();
Application application = getMetainfo().getApplication();
for (Component component : application.getComponents()) {
- if (component.getName().equals(roleName)) {
+ if (component.getName().equals(componentName)) {
if (component.getComponentExports().size() > 0) {
for (ComponentExport export : component.getComponentExports()) {
@@ -954,14 +1345,103 @@
}
}
+ /** Publish component instance specific data if the component demands it */
+ protected void processAndPublishComponentSpecificExports(Map<String, String> ports,
+ String containerId,
+ String hostFqdn,
+ String compName) {
+ String portVarFormat = "${site.%s}";
+ String hostNamePattern = "${" + compName + "_HOST}";
+
+ List<ExportGroup> appExportGroups = getMetainfo().getApplication().getExportGroups();
+ Component component = getMetainfo().getApplicationComponent(compName);
+ if (component != null && SliderUtils.isSet(component.getCompExports())
+ && appExportGroups != null && appExportGroups.size() > 0) {
+
+ Set<String> compExports = new HashSet();
+ String compExportsStr = component.getCompExports();
+ for (String compExport : compExportsStr.split(",")) {
+ if (compExport.trim().length() > 0) {
+ compExports.add(compExport.trim());
+ }
+ }
+
+ Date now = new Date();
+ Set<String> modifiedGroups = new HashSet<String>();
+ for (ExportGroup exportGroup : appExportGroups) {
+ List<Export> exports = exportGroup.getExports();
+ if (exports != null && !exports.isEmpty()) {
+ String exportGroupName = exportGroup.getName();
+ ConcurrentHashMap<String, List<ExportEntry>> map =
+ (ConcurrentHashMap<String, List<ExportEntry>>) getCurrentExports(exportGroupName);
+ for (Export export : exports) {
+ if (canBeExported(exportGroupName, export.getName(), compExports)) {
+ log.info("Attempting to publish {} of group {} for component type {}",
+ export.getName(), exportGroupName, compName);
+ String templateToExport = export.getValue();
+ for (String portName : ports.keySet()) {
+ boolean publishData = false;
+ String portValPattern = String.format(portVarFormat, portName);
+ if (templateToExport.contains(portValPattern)) {
+ templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
+ publishData = true;
+ }
+ if (templateToExport.contains(hostNamePattern)) {
+ templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
+ publishData = true;
+ }
+ if (publishData) {
+ ExportEntry entryToAdd = new ExportEntry();
+ entryToAdd.setLevel(COMPONENT_TAG);
+ entryToAdd.setValue(templateToExport);
+ entryToAdd.setUpdatedTime(now.toString());
+ entryToAdd.setContainerId(containerId);
+ entryToAdd.setTag(tags.getTag(compName, containerId));
+
+ List<ExportEntry> existingList =
+ map.putIfAbsent(export.getName(), new CopyOnWriteArrayList(Arrays.asList(entryToAdd)));
+
+ // in-place edit, no lock needed
+ if (existingList != null) {
+ boolean updatedInPlace = false;
+ for (ExportEntry entry : existingList) {
+ if (containerId.equalsIgnoreCase(entry.getContainerId())) {
+ entryToAdd.setValue(templateToExport);
+ entryToAdd.setUpdatedTime(now.toString());
+ updatedInPlace = true;
+ }
+ }
+ if (!updatedInPlace) {
+ existingList.add(entryToAdd);
+ }
+ }
+
+ log.info("Publishing {} for name {} and container {}",
+ templateToExport, export.getName(), containerId);
+ modifiedGroups.add(exportGroupName);
+ synchronized (containerExportsMap) {
+ if (!containerExportsMap.containsKey(containerId)) {
+ containerExportsMap.put(containerId, new HashSet<String>());
+ }
+ Set<String> containerExportMaps = containerExportsMap.get(containerId);
+ containerExportMaps.add(String.format("%s:%s", exportGroupName, export.getName()));
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ publishModifiedExportGroups(modifiedGroups);
+ }
+ }
+
private void publishComponentInstanceData() {
Map<String, String> dataToPublish = new HashMap<String, String>();
- synchronized (this.componentInstanceData) {
- for (String container : getComponentInstanceData().keySet()) {
- for (String prop : getComponentInstanceData().get(container).keySet()) {
- dataToPublish.put(
- container + "." + prop, getComponentInstanceData().get(container).get(prop));
- }
+ for (String container : getComponentInstanceData().keySet()) {
+ for (String prop : getComponentInstanceData().get(container).keySet()) {
+ dataToPublish.put(
+ container + "." + prop, getComponentInstanceData().get(container).get(prop));
}
}
publishApplicationInstanceData(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
@@ -969,21 +1449,13 @@
/**
* Return Component based on name
+ *
* @param roleName
+ *
* @return
*/
protected Component getApplicationComponent(String roleName) {
- Application application = getMetainfo().getApplication();
- if (application == null) {
- log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
- } else {
- for (Component component : application.getComponents()) {
- if (component.getName().equals(roleName)) {
- return component;
- }
- }
- }
- return null;
+ return getMetainfo().getApplicationComponent(roleName);
}
/**
@@ -993,10 +1465,10 @@
*
* @return
*/
- protected String getScriptPathFromMetainfo(String roleName) {
+ protected CommandScript getScriptPathFromMetainfo(String roleName) {
Component component = getApplicationComponent(roleName);
if (component != null) {
- return component.getCommandScript().getScript();
+ return component.getCommandScript();
}
return null;
}
@@ -1050,6 +1522,7 @@
/**
* Can any master publish config explicitly, if not a random master is used
+ *
* @return
*/
protected boolean canAnyMasterPublishConfig() {
@@ -1083,14 +1556,20 @@
/**
* Add install command to the heartbeat response
- * @param roleName
+ *
+ * @param componentName
* @param containerId
* @param response
* @param scriptPath
+ *
* @throws SliderException
*/
@VisibleForTesting
- protected void addInstallCommand(String roleName, String containerId, HeartBeatResponse response, String scriptPath)
+ protected void addInstallCommand(String componentName,
+ String containerId,
+ HeartBeatResponse response,
+ String scriptPath,
+ long timeout)
throws SliderException {
assert getAmState().isApplicationLive();
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
@@ -1101,17 +1580,18 @@
cmd.setClusterName(clusterName);
cmd.setRoleCommand(Command.INSTALL.toString());
cmd.setServiceName(clusterName);
- cmd.setComponentName(roleName);
- cmd.setRole(roleName);
+ cmd.setComponentName(componentName);
+ cmd.setRole(componentName);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
hostLevelParams.put(PACKAGE_LIST, getPackageList());
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- setInstallCommandConfigurations(cmd, containerId);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
+ cmd.setConfigurations(configurations);
- cmd.setCommandParams(setCommandParameters(scriptPath, false));
+ cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
response.addExecutionCommand(cmd);
@@ -1147,26 +1627,24 @@
cmd.setCommandId(cmd.getTaskId() + "-1");
}
- private Map<String, String> setCommandParameters(String scriptPath, boolean recordConfig) {
+ private Map<String, String> setCommandParameters(String scriptPath, long timeout, boolean recordConfig) {
Map<String, String> cmdParams = new TreeMap<String, String>();
cmdParams.put("service_package_folder",
"${AGENT_WORK_ROOT}/work/app/definition/package");
cmdParams.put("script", scriptPath);
cmdParams.put("schema_version", "2.0");
- cmdParams.put("command_timeout", "300");
+ cmdParams.put("command_timeout", Long.toString(timeout));
cmdParams.put("script_type", "PYTHON");
cmdParams.put("record_config", Boolean.toString(recordConfig));
return cmdParams;
}
- private void setInstallCommandConfigurations(ExecutionCommand cmd, String containerId) throws SliderException {
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
- cmd.setConfigurations(configurations);
- }
-
@VisibleForTesting
- protected void addStatusCommand(String roleName, String containerId, HeartBeatResponse response, String scriptPath)
+ protected void addStatusCommand(String componentName,
+ String containerId,
+ HeartBeatResponse response,
+ String scriptPath,
+ long timeout)
throws SliderException {
assert getAmState().isApplicationLive();
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
@@ -1175,7 +1653,7 @@
String clusterName = getClusterName();
cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
- cmd.setComponentName(roleName);
+ cmd.setComponentName(componentName);
cmd.setServiceName(clusterName);
cmd.setClusterName(clusterName);
cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
@@ -1185,9 +1663,9 @@
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- cmd.setCommandParams(setCommandParameters(scriptPath, false));
+ cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
cmd.setConfigurations(configurations);
@@ -1195,7 +1673,7 @@
}
@VisibleForTesting
- protected void addGetConfigCommand(String roleName, String containerId, HeartBeatResponse response)
+ protected void addGetConfigCommand(String componentName, String containerId, HeartBeatResponse response)
throws SliderException {
assert getAmState().isApplicationLive();
@@ -1203,7 +1681,7 @@
String clusterName = getClusterName();
cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
- cmd.setComponentName(roleName);
+ cmd.setComponentName(componentName);
cmd.setServiceName(clusterName);
cmd.setClusterName(clusterName);
cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND);
@@ -1217,8 +1695,8 @@
}
@VisibleForTesting
- protected void addStartCommand(String roleName, String containerId, HeartBeatResponse response,
- String scriptPath, boolean isMarkedAutoRestart)
+ protected void addStartCommand(String componentName, String containerId, HeartBeatResponse response,
+ String scriptPath, long timeout, boolean isMarkedAutoRestart)
throws
SliderException {
assert getAmState().isApplicationLive();
@@ -1233,8 +1711,8 @@
cmd.setClusterName(clusterName);
cmd.setRoleCommand(Command.START.toString());
cmd.setServiceName(clusterName);
- cmd.setComponentName(roleName);
- cmd.setRole(roleName);
+ cmd.setComponentName(componentName);
+ cmd.setRole(componentName);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
hostLevelParams.put(CONTAINER_ID, containerId);
@@ -1244,12 +1722,43 @@
cmd.setRoleParams(roleParams);
cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
- cmd.setCommandParams(setCommandParameters(scriptPath, true));
+ cmd.setCommandParams(setCommandParameters(scriptPath, timeout, true));
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
cmd.setConfigurations(configurations);
response.addExecutionCommand(cmd);
+
+ // With start command, the corresponding command for graceful stop needs to
+ // be sent. This will be used when a particular container is lost as per RM,
+ // but then the agent is still running and heart-beating to the Slider AM.
+ ExecutionCommand cmdStop = new ExecutionCommand(
+ AgentCommandType.EXECUTION_COMMAND);
+ cmdStop.setTaskId(taskId.get());
+ cmdStop.setCommandId(cmdStop.getTaskId() + "-1");
+ cmdStop.setHostname(hostName);
+ cmdStop.setClusterName(clusterName);
+ cmdStop.setRoleCommand(Command.STOP.toString());
+ cmdStop.setServiceName(clusterName);
+ cmdStop.setComponentName(componentName);
+ cmdStop.setRole(componentName);
+ Map<String, String> hostLevelParamsStop = new TreeMap<String, String>();
+ hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions()
+ .getMandatoryOption(JAVA_HOME));
+ hostLevelParamsStop.put(CONTAINER_ID, containerId);
+ cmdStop.setHostLevelParams(hostLevelParamsStop);
+
+ Map<String, String> roleParamsStop = new TreeMap<String, String>();
+ cmdStop.setRoleParams(roleParamsStop);
+ cmdStop.getRoleParams().put("auto_restart",
+ Boolean.toString(isMarkedAutoRestart));
+
+ cmdStop.setCommandParams(setCommandParameters(scriptPath, timeout, true));
+
+ Map<String, Map<String, String>> configurationsStop = buildCommandConfigurations(
+ appConf, containerId, componentName);
+ cmdStop.setConfigurations(configurationsStop);
+ response.addExecutionCommand(cmdStop);
}
protected Map<String, String> getAllocatedPorts() {
@@ -1265,7 +1774,7 @@
synchronized (this.allocatedPorts) {
if (!this.allocatedPorts.containsKey(containerId)) {
this.allocatedPorts.put(containerId,
- new ConcurrentHashMap<String, String>());
+ new ConcurrentHashMap<String, String>());
}
}
}
@@ -1273,48 +1782,93 @@
}
private Map<String, Map<String, String>> buildCommandConfigurations(
- ConfTreeOperations appConf, String containerId)
+ ConfTreeOperations appConf, String containerId, String componentName)
throws SliderException {
Map<String, Map<String, String>> configurations =
new TreeMap<String, Map<String, String>>();
Map<String, String> tokens = getStandardTokenMap(appConf);
- List<String> configs = getApplicationConfigurationTypes(appConf);
+ Set<String> configs = new HashSet<String>();
+ configs.addAll(getApplicationConfigurationTypes());
+ configs.addAll(getSystemConfigurationsRequested(appConf));
- //Add global
for (String configType : configs) {
addNamedConfiguration(configType, appConf.getGlobalOptions().options,
- configurations, tokens, containerId);
+ configurations, tokens, containerId, componentName);
}
+ //do a final replacement of re-used configs
+ dereferenceAllConfigs(configurations);
+
return configurations;
}
+ protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) {
+ Map<String, String> allConfigs = new HashMap<String, String>();
+ String lookupFormat = "${@//site/%s/%s}";
+ for (String configType : configurations.keySet()) {
+ Map<String, String> configBucket = configurations.get(configType);
+ for (String configName : configBucket.keySet()) {
+ allConfigs.put(String.format(lookupFormat, configType, configName), configBucket.get(configName));
+ }
+ }
+
+ for (String configType : configurations.keySet()) {
+ Map<String, String> configBucket = configurations.get(configType);
+ for (Map.Entry<String, String> entry: configBucket.entrySet()) {
+ String configName = entry.getKey();
+ String configValue = entry.getValue();
+ for (String lookUpKey : allConfigs.keySet()) {
+ if (configValue != null && configValue.contains(lookUpKey)) {
+ configValue = configValue.replace(lookUpKey, allConfigs.get(lookUpKey));
+ }
+ }
+ configBucket.put(configName, configValue);
+ }
+ }
+ }
+
private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) throws SliderException {
Map<String, String> tokens = new HashMap<String, String>();
String nnuri = appConf.get("site.fs.defaultFS");
tokens.put("${NN_URI}", nnuri);
tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
- tokens.put("${DEF_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH));
+ tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH));
tokens.put("${DEFAULT_DATA_DIR}", getAmState()
.getInternalsSnapshot()
.getGlobalOptions()
.getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH));
+ tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME));
return tokens;
}
- private List<String> getApplicationConfigurationTypes(ConfTreeOperations appConf) {
- // for now, reading this from appConf. In the future, modify this method to
- // process metainfo.xml
+ @VisibleForTesting
+ protected List<String> getSystemConfigurationsRequested(ConfTreeOperations appConf) {
+ List<String> configList = new ArrayList<String>();
+
+ String configTypes = appConf.get(AgentKeys.SYSTEM_CONFIGS);
+ if (configTypes != null && configTypes.length() > 0) {
+ String[] configs = configTypes.split(",");
+ for (String config : configs) {
+ configList.add(config.trim());
+ }
+ }
+
+ return new ArrayList<String>(new HashSet<String>(configList));
+ }
+
+
+ @VisibleForTesting
+ protected List<String> getApplicationConfigurationTypes() {
List<String> configList = new ArrayList<String>();
configList.add(GLOBAL_CONFIG_TAG);
- String configTypes = appConf.get("config_types");
- if (configTypes != null && configTypes.length() > 0) {
- String[] configs = configTypes.split(",");
- configList.addAll(Arrays.asList(configs));
+ List<ConfigFile> configFiles = getMetainfo().getApplication().getConfigFiles();
+ for (ConfigFile configFile : configFiles) {
+ log.info("Expecting config type {}.", configFile.getDictionaryName());
+ configList.add(configFile.getDictionaryName());
}
// remove duplicates. mostly worried about 'global' being listed
@@ -1323,10 +1877,11 @@
private void addNamedConfiguration(String configName, Map<String, String> sourceConfig,
Map<String, Map<String, String>> configurations,
- Map<String, String> tokens, String containerId) {
+ Map<String, String> tokens, String containerId,
+ String roleName) {
Map<String, String> config = new HashMap<String, String>();
if (configName.equals(GLOBAL_CONFIG_TAG)) {
- addDefaultGlobalConfig(config, containerId);
+ addDefaultGlobalConfig(config, containerId, roleName);
}
// add role hosts to tokens
addRoleRelatedTokens(tokens);
@@ -1337,7 +1892,7 @@
for (String key : config.keySet()) {
String value = config.get(key);
String lookupKey = configName + "." + key;
- if(!value.contains(DO_NOT_PROPAGATE_TAG)) {
+ if (!value.contains(PER_CONTAINER_TAG)) {
// If the config property is shared then pass on the already allocated value
// from any container
if (this.getAllocatedPorts().containsKey(lookupKey)) {
@@ -1350,6 +1905,21 @@
}
}
}
+
+ //apply defaults only if the key is not present and value is not empty
+ if (getDefaultConfigs().containsKey(configName)) {
+ log.info("Adding default configs for type {}.", configName);
+ for (PropertyInfo defaultConfigProp : getDefaultConfigs().get(configName).getPropertyInfos()) {
+ if (!config.containsKey(defaultConfigProp.getName())) {
+ if (!defaultConfigProp.getName().isEmpty() &&
+ defaultConfigProp.getValue() != null &&
+ !defaultConfigProp.getValue().isEmpty()) {
+ config.put(defaultConfigProp.getName(), defaultConfigProp.getValue());
+ }
+ }
+ }
+ }
+
configurations.put(configName, config);
}
@@ -1371,11 +1941,21 @@
return hosts;
}
- private void addDefaultGlobalConfig(Map<String, String> config, String containerId) {
+ private void addDefaultGlobalConfig(Map<String, String> config, String containerId, String roleName) {
config.put("app_log_dir", "${AGENT_LOG_ROOT}");
config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run");
config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
+ config.put("app_input_conf_dir", "${AGENT_WORK_ROOT}/" + SliderKeys.PROPAGATED_CONF_DIR_NAME);
config.put("app_container_id", containerId);
+ config.put("app_container_tag", tags.getTag(roleName, containerId));
+
+ // add optional parameters only if they are not already provided
+ if(!config.containsKey("pid_file")) {
+ config.put("pid_file", "${AGENT_WORK_ROOT}/app/run/component.pid");
+ }
+ if(!config.containsKey("app_root")) {
+ config.put("app_root", "${AGENT_WORK_ROOT}/app/install");
+ }
}
private void buildRoleHostDetails(Map<String, String> details) {
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index f296a95..1d61c15 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -16,14 +16,19 @@
*/
package org.apache.slider.providers.agent;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
+import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.FileNotFoundException;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -33,14 +38,17 @@
public class AgentUtils {
private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
- static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef) throws IOException {
+ public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
+ String appDef) throws IOException {
log.info("Reading metainfo at {}", appDef);
+ FileSystem fs = fileSystem.getFileSystem();
+ Path appPath = new Path(appDef);
InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream(
- fileSystem.getFileSystem(), new Path(appDef), "metainfo.xml");
+ fs, appPath, "metainfo.xml");
if (metainfoStream == null) {
log.error("metainfo.xml is unavailable at {}.", appDef);
- throw new IOException("metainfo.xml is required in app package.");
+ throw new FileNotFoundException("metainfo.xml is required in app package. " +
+ appPath);
}
Metainfo metainfo = new MetainfoParser().parse(metainfoStream);
@@ -48,4 +56,19 @@
return metainfo;
}
+ static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem,
+ String appDef, String configFileName)
+ throws IOException {
+ // this is the path inside the zip file
+ String fileToRead = "configuration/" + configFileName;
+ log.info("Reading default config file {} at {}", fileToRead, appDef);
+ InputStream configStream = SliderUtils.getApplicationResourceInputStream(
+ fileSystem.getFileSystem(), new Path(appDef), fileToRead);
+ if (configStream == null) {
+ log.error("{} is unavailable at {}.", fileToRead, appDef);
+ throw new IOException("Expected config file " + fileToRead + " is not available.");
+ }
+
+ return new DefaultConfigParser().parse(configStream);
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java b/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
index cbeb69d..a851803 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
@@ -22,7 +22,8 @@
public enum Command {
NOP, // do nothing
INSTALL, // Install the component
- START; // Start the component
+ START, // Start the component
+ STOP; // Stop the component
public static Command getCommand(String commandVal) {
if (commandVal.equals(Command.START.toString())) {
@@ -31,6 +32,9 @@
if (commandVal.equals(Command.INSTALL.toString())) {
return Command.INSTALL;
}
+ if (commandVal.equals(Command.STOP.toString())) {
+ return Command.STOP;
+ }
return Command.NOP;
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
index f7f8bf4..a50f3c0 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
@@ -19,7 +19,6 @@
package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,7 +29,7 @@
LoggerFactory.getLogger(ComponentInstanceState.class);
private static int MAX_FAILURE_TOLERATED = 3;
private static String INVALID_TRANSITION_ERROR =
- "Result {0} for command {1} is not expected for component {2} in state {3}.";
+ "Result %s for command %s is not expected for component %s in state %s.";
private final String componentName;
private final ContainerId containerId;
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java
new file mode 100644
index 0000000..68f63fa
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/** A simple tag provider that attempts to associate tags from 1-N to all container of a component */
+public class ComponentTagProvider {
+ private static final Logger log = LoggerFactory.getLogger(ComponentTagProvider.class);
+ private static String FREE = "free";
+ private final ConcurrentHashMap<String, ConcurrentHashMap<String, String>> allTags;
+
+ public ComponentTagProvider() {
+ allTags = new ConcurrentHashMap<String, ConcurrentHashMap<String, String>>();
+ }
+
+ /**
+ * Record an assigned tag to a container
+ *
+ * @param component
+ * @param containerId
+ * @param tag
+ */
+ public void recordAssignedTag(String component, String containerId, String tag) {
+ if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
+ Integer key = null;
+ try {
+ key = Integer.valueOf(tag);
+ } catch (NumberFormatException nfe) {
+ //ignore
+ }
+ if (key != null && key > 0) {
+ ConcurrentHashMap<String, String> compTags = getComponentSpecificTags(component);
+ synchronized (compTags) {
+ for (int index = 1; index <= key.intValue(); index++) {
+ String tempKey = new Integer(index).toString();
+ if (!compTags.containsKey(tempKey)) {
+ compTags.put(tempKey, FREE);
+ }
+ }
+ compTags.put(key.toString(), containerId);
+ }
+ }
+ }
+ }
+
+ /**
+ * Get a tag for container
+ *
+ * @param component
+ * @param containerId
+ *
+ * @return
+ */
+ public String getTag(String component, String containerId) {
+ if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
+ ConcurrentHashMap<String, String> compTags = getComponentSpecificTags(component);
+ synchronized (compTags) {
+ for (String key : compTags.keySet()) {
+ if (compTags.get(key).equals(containerId)) {
+ return key;
+ }
+ }
+ for (String key : compTags.keySet()) {
+ if (compTags.get(key).equals(FREE)) {
+ compTags.put(key, containerId);
+ return key;
+ }
+ }
+ String newKey = new Integer(compTags.size() + 1).toString();
+ compTags.put(newKey, containerId);
+ return newKey;
+ }
+ }
+ return "";
+ }
+
+ /**
+ * Release a tag associated with a container
+ *
+ * @param component
+ * @param containerId
+ */
+ public void releaseTag(String component, String containerId) {
+ if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
+ ConcurrentHashMap<String, String> compTags = allTags.get(component);
+ if (compTags != null) {
+ synchronized (compTags) {
+ for (String key : compTags.keySet()) {
+ if (compTags.get(key).equals(containerId)) {
+ compTags.put(key, FREE);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private ConcurrentHashMap<String, String> getComponentSpecificTags(String component) {
+ if (!allTags.containsKey(component)) {
+ synchronized (allTags) {
+ if (!allTags.containsKey(component)) {
+ allTags.put(component, new ConcurrentHashMap<String, String>());
+ }
+ }
+ }
+ return allTags.get(component);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
index d994e33..bc43d4b 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -31,13 +31,14 @@
List<ExportGroup> exportGroups;
List<OSSpecific> osSpecifics;
List<CommandOrder> commandOrders;
- ConfigurationDependencies configDependencies;
+ List<ConfigFile> configFiles;
public Application() {
exportGroups = new ArrayList<ExportGroup>();
components = new ArrayList<Component>();
osSpecifics = new ArrayList<OSSpecific>();
commandOrders = new ArrayList<CommandOrder>();
+ configFiles = new ArrayList<ConfigFile>();
}
public String getName() {
@@ -72,12 +73,12 @@
this.exportedConfigs = exportedConfigs;
}
- public ConfigurationDependencies getConfigDependencies() {
- return configDependencies;
+ public List<ConfigFile> getConfigFiles() {
+ return configFiles;
}
- public void setConfigDependencies(ConfigurationDependencies configDependencies) {
- this.configDependencies = configDependencies;
+ public void addConfigFile(ConfigFile configFile) {
+ this.configFiles.add(configFile);
}
public void addComponent(Component component) {
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index 03c64d4..418868c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -16,6 +16,10 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+
import java.util.ArrayList;
import java.util.List;
@@ -30,6 +34,7 @@
String maxInstanceCount;
String autoStartOnFailure;
String appExports;
+ String compExports;
CommandScript commandScript;
List<ComponentExport> componentExports;
@@ -78,10 +83,42 @@
this.appExports = appExports;
}
+ public String getCompExports() {
+ return compExports;
+ }
+
+ public void setCompExports(String compExports) {
+ this.compExports = compExports;
+ }
+
public String getMinInstanceCount() {
return minInstanceCount;
}
+ public int getMinInstanceCountInt() throws BadConfigException {
+ if (SliderUtils.isUnset(minInstanceCount)) {
+ return 0;
+ }
+
+ try {
+ return Integer.parseInt(minInstanceCount);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(nfe, "Invalid value for minInstanceCount for %s", name);
+ }
+ }
+
+ public int getMaxInstanceCountInt() throws BadConfigException {
+ if (SliderUtils.isUnset(maxInstanceCount)) {
+ return Integer.MAX_VALUE;
+ }
+
+ try {
+ return Integer.parseInt(maxInstanceCount);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(nfe, "Invalid value for maxInstanceCount for %s", name);
+ }
+ }
+
public void setMinInstanceCount(String minInstanceCount) {
this.minInstanceCount = minInstanceCount;
}
@@ -124,4 +161,10 @@
sb.append('}');
return sb.toString();
}
+
+ class AutoRestartSettings {
+ private boolean requiresAutoRestart;
+ private int maxFailures;
+ private int inThisManyMinutes;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
similarity index 63%
copy from slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java
copy to slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
index 57effac..b9dfb4e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
@@ -16,24 +16,38 @@
*/
package org.apache.slider.providers.agent.application.metadata;
-import java.util.ArrayList;
-import java.util.List;
-
/**
*
*/
-public class ConfigurationDependencies {
- List<String> configTypes;
+public class ConfigFile {
+ String type;
+ String fileName;
+ String dictionaryName;
- public ConfigurationDependencies() {
- configTypes = new ArrayList<String>();
+ public ConfigFile() {
}
- public void setConfigType(String type) {
- configTypes.add(type);
+ public String getType() {
+ return type;
}
- public List<String> getConfigTypes() {
- return configTypes;
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+
+ public String getDictionaryName() {
+ return dictionaryName;
+ }
+
+ public void setDictionaryName(String dictionaryName) {
+ this.dictionaryName = dictionaryName;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
similarity index 73%
rename from slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java
rename to slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
index 57effac..46c8836 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
@@ -20,20 +20,20 @@
import java.util.List;
/**
- *
+ * Application default config
*/
-public class ConfigurationDependencies {
- List<String> configTypes;
+public class DefaultConfig {
+ List<PropertyInfo> propertyInfos;
- public ConfigurationDependencies() {
- configTypes = new ArrayList<String>();
+ public DefaultConfig() {
+ propertyInfos = new ArrayList<PropertyInfo>();
}
- public void setConfigType(String type) {
- configTypes.add(type);
+ public void addPropertyInfo(PropertyInfo propertyInfo) {
+ propertyInfos.add(propertyInfo);
}
- public List<String> getConfigTypes() {
- return configTypes;
+ public List<PropertyInfo> getPropertyInfos() {
+ return propertyInfos;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
new file mode 100644
index 0000000..e136775
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.commons.digester.Digester;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class DefaultConfigParser {
+
+ public DefaultConfig parse(InputStream configFileStream) throws IOException {
+ Digester digester = new Digester();
+ digester.setValidating(false);
+
+ digester.addObjectCreate("configuration", DefaultConfig.class);
+
+ digester.addObjectCreate("*/property", PropertyInfo.class);
+ digester.addBeanPropertySetter("*/property/name");
+ digester.addBeanPropertySetter("*/property/value");
+ digester.addBeanPropertySetter("*/property/description");
+ digester.addSetNext("*/property", "addPropertyInfo");
+
+ try {
+ return (DefaultConfig) digester.parse(configFileStream);
+ } catch (IOException e) {
+
+ } catch (SAXException e) {
+
+ } finally {
+ configFileStream.close();
+ }
+
+ return null;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
index 2455e8e..b34cba1 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
@@ -16,10 +16,16 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Application metainfo uber class
*/
public class Metainfo {
+ protected static final Logger log =
+ LoggerFactory.getLogger(Metainfo.class);
+
String schemaVersion;
Application application;
@@ -41,4 +47,17 @@
public void setApplication(Application application) {
this.application = application;
}
+
+ public Component getApplicationComponent(String roleName) {
+ if (application == null) {
+ log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
+ } else {
+ for (Component component : application.getComponents()) {
+ if (component.getName().equals(roleName)) {
+ return component;
+ }
+ }
+ }
+ return null;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index bc93d6f..1d8403f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -61,6 +61,7 @@
digester.addBeanPropertySetter("*/component/maxInstanceCount");
digester.addBeanPropertySetter("*/component/autoStartOnFailure");
digester.addBeanPropertySetter("*/component/appExports");
+ digester.addBeanPropertySetter("*/component/compExports");
digester.addObjectCreate("*/componentExport", ComponentExport.class);
digester.addBeanPropertySetter("*/componentExport/name");
digester.addBeanPropertySetter("*/componentExport/value");
@@ -81,10 +82,11 @@
digester.addSetNext("*/package", "addOSPackage");
digester.addSetNext("*/osSpecific", "addOSSpecific");
- digester.addObjectCreate("*/configuration-dependencies",
- ConfigurationDependencies.class);
- digester.addBeanPropertySetter("*/config-type", "configType");
- digester.addSetNext("*/configuration-dependencies", "setConfigDependencies");
+ digester.addObjectCreate("*/configFile", ConfigFile.class);
+ digester.addBeanPropertySetter("*/configFile/type");
+ digester.addBeanPropertySetter("*/configFile/fileName");
+ digester.addBeanPropertySetter("*/configFile/dictionaryName");
+ digester.addSetNext("*/configFile", "addConfigFile");
digester.addSetRoot("*/application", "setApplication");
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
similarity index 63%
copy from slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java
copy to slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
index 57effac..62ee0f5 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
@@ -16,24 +16,39 @@
*/
package org.apache.slider.providers.agent.application.metadata;
-import java.util.ArrayList;
-import java.util.List;
-
/**
- *
+ * Application config property info
*/
-public class ConfigurationDependencies {
- List<String> configTypes;
+public class PropertyInfo {
+ String name;
+ String value;
+ String description;
- public ConfigurationDependencies() {
- configTypes = new ArrayList<String>();
+ public PropertyInfo() {
}
- public void setConfigType(String type) {
- configTypes.add(type);
+ public String getName() {
+ return name;
}
- public List<String> getConfigTypes() {
- return configTypes;
+ public void setName(String name) {
+ this.name = name;
}
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index dc84f02..5ce0a78 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -18,25 +18,22 @@
package org.apache.slider.providers.slideram;
-import com.beust.jcommander.JCommander;
-import com.codahale.metrics.MetricRegistry;
-import com.google.gson.GsonBuilder;
-import org.apache.curator.CuratorZookeeperClient;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.server.entity.ServiceNames;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.AbstractLauncher;
@@ -45,7 +42,6 @@
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
-import org.mortbay.jetty.security.SslSelectChannelConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,6 +51,8 @@
import java.util.List;
import java.util.Map;
+import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
+
/**
* handles the setup of the Slider AM.
* This keeps aspects of role, cluster validation and Clusterspec setup
@@ -88,12 +86,15 @@
public static final int KEY_AM = ROLE_AM_PRIORITY_INDEX;
+ public static final ProviderRole APPMASTER =
+ new ProviderRole(COMPONENT_AM, KEY_AM,
+ PlacementPolicy.EXCLUDE_FROM_FLEXING);
+
/**
* Initialize role list
*/
static {
- ROLES.add(new ProviderRole(COMPONENT_AM, KEY_AM,
- PlacementPolicy.EXCLUDE_FROM_FLEXING));
+ ROLES.add(APPMASTER);
}
@Override
@@ -132,6 +133,31 @@
}
/**
+ * Verify that an instance definition is considered valid by the provider
+ * @param instanceDefinition instance definition
+ * @throws SliderException if the configuration is not valid
+ */
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
+ SliderException {
+
+ super.validateInstanceDefinition(instanceDefinition, fs);
+
+ // make sure there is no negative entry in the instance count
+ Map<String, Map<String, String>> instanceMap =
+ instanceDefinition.getResources().components;
+ for (Map.Entry<String, Map<String, String>> entry : instanceMap.entrySet()) {
+ MapOperations mapOperations = new MapOperations(entry);
+ int instances = mapOperations.getOptionInt(COMPONENT_INSTANCES, 0);
+ if (instances < 0) {
+ throw new BadClusterStateException(
+ "Component %s has negative instance count: %d",
+ mapOperations.name,
+ instances);
+ }
+ }
+ }
+
+ /**
* The Slider AM sets up all the dependency JARs above slider.jar itself
* {@inheritDoc}
*/
@@ -158,34 +184,19 @@
libdir,
miniClusterTestRun);
- Class<?>[] classes = {
- JCommander.class,
- GsonBuilder.class,
- SslSelectChannelConnector.class,
+ String libDirProp =
+ System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
+ log.info("Loading all dependencies for AM.");
+ ProviderUtils.addAllDependencyJars(providerResources,
+ fileSystem,
+ tempPath,
+ libdir,
+ libDirProp);
+ addKeytabResourceIfNecessary(fileSystem,
+ launcher,
+ instanceDescription,
+ providerResources);
- CuratorFramework.class,
- CuratorZookeeperClient.class,
- ServiceInstance.class,
- ServiceNames.class,
- MetricRegistry.class
- };
- String[] jars =
- {
- JCOMMANDER_JAR,
- GSON_JAR,
- "jetty-sslengine.jar",
-
- "curator-framework.jar",
- "curator-client.jar",
- "curator-x-discovery.jar",
- "curator-x-discovery-service.jar",
- "metrics-core.jar"
- };
- ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath,
- libdir, jars,
- classes);
-
- launcher.addLocalResources(providerResources);
//also pick up all env variables from a map
launcher.copyEnvVars(
instanceDescription.getInternalOperations().getOrAddComponent(
@@ -193,6 +204,44 @@
}
/**
+ * If the cluster is secure, and an HDFS installed keytab is available for AM
+ * authentication, add this keytab as a local resource for the AM launch.
+ *
+ * @param fileSystem
+ * @param launcher
+ * @param instanceDescription
+ * @param providerResources
+ * @throws IOException
+ */
+ protected void addKeytabResourceIfNecessary(SliderFileSystem fileSystem,
+ AbstractLauncher launcher,
+ AggregateConf instanceDescription,
+ Map<String, LocalResource> providerResources)
+ throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ String keytabPathOnHost = instanceDescription.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+ if (SliderUtils.isUnset(keytabPathOnHost)) {
+ String amKeytabName = instanceDescription.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir = instanceDescription.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
+ Path keytabPath = fileSystem.buildKeytabPath(keytabDir, amKeytabName,
+ instanceDescription.getName());
+ LocalResource keytabRes = fileSystem.createAmResource(keytabPath,
+ LocalResourceType.FILE);
+
+ providerResources.put(SliderKeys.KEYTAB_DIR + "/" +
+ amKeytabName, keytabRes);
+ }
+ }
+ launcher.addLocalResources(providerResources);
+ }
+
+ /**
* Update the AM resource with any local needs
* @param capability capability to update
*/
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index 071fc19..afe6428 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -23,6 +23,8 @@
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -33,13 +35,8 @@
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.info.CommonRegistryConstants;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import org.apache.slider.core.registry.info.RegistryView;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.agent.AgentKeys;
@@ -48,14 +45,13 @@
import java.io.File;
import java.io.IOException;
+import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_MANAGEMENT;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_PUBLISHER;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
/**
* Exists just to move some functionality out of AppMaster into a peer class
@@ -91,14 +87,6 @@
}
@Override
- public boolean exec(AggregateConf instanceDefinition,
- File confDir,
- Map<String, String> env,
- ProviderCompleted execInProgress) throws IOException, SliderException {
- return false;
- }
-
- @Override
public List<ProviderRole> getRoles() {
return new ArrayList<ProviderRole>(0);
}
@@ -110,22 +98,22 @@
}
@Override
- public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData instanceData) throws IOException {
- super.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData
- );
-
+ public void applyInitialRegistryDefinitions(URL amWebURI,
+ URL agentOpsURI,
+ URL agentStatusURI,
+ ServiceRecord serviceRecord)
+ throws IOException {
+ super.applyInitialRegistryDefinitions(amWebURI,
+ agentOpsURI,
+ agentStatusURI,
+ serviceRecord);
// now publish site.xml files
YarnConfiguration defaultYarnConfig = new YarnConfiguration();
amState.getPublishedSliderConfigurations().put(
PublishedArtifacts.COMPLETE_CONFIG,
new PublishedConfiguration(
"Complete slider application settings",
- getConfig(), getConfig())
- );
+ getConfig(), getConfig()));
amState.getPublishedSliderConfigurations().put(
PublishedArtifacts.YARN_SITE_CONFIG,
new PublishedConfiguration(
@@ -148,37 +136,42 @@
try {
- RegistryView externalView = instanceData.externalView;
- RegisteredEndpoint webUI =
- new RegisteredEndpoint(unsecureWebAPI, "Application Master Web UI");
- externalView.endpoints.put(CommonRegistryConstants.WEB_UI, webUI);
+ URL managementAPI = new URL(amWebURI, SLIDER_PATH_MANAGEMENT);
+ URL registryREST = new URL(amWebURI, SLIDER_PATH_REGISTRY );
- externalView.endpoints.put(
- CustomRegistryConstants.MANAGEMENT_REST_API,
- new RegisteredEndpoint(
- new URL(unsecureWebAPI, SLIDER_PATH_MANAGEMENT),
- "Management REST API") );
+ URL publisherURL = new URL(amWebURI, SLIDER_PATH_PUBLISHER);
- externalView.endpoints.put(
- CustomRegistryConstants.REGISTRY_REST_API,
- new RegisteredEndpoint(
- new URL(unsecureWebAPI, RestPaths.SLIDER_PATH_REGISTRY + "/" +
- RestPaths.REGISTRY_SERVICE),
- "Registry Web Service" ) );
+ // Set the configurations URL.
- URL publisherURL = new URL(unsecureWebAPI, SLIDER_PATH_PUBLISHER);
- externalView.endpoints.put(
- CustomRegistryConstants.PUBLISHER_REST_API,
- new RegisteredEndpoint(
- publisherURL,
- "Publisher Service") );
-
- /*
- * Set the configurations URL.
- */
- externalView.configurationsURL = SliderUtils.appendToURL(
+ String configurationsURL = SliderUtils.appendToURL(
publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET);
+ String exportsURL = SliderUtils.appendToURL(
+ publisherURL.toExternalForm(), RestPaths.SLIDER_EXPORTS);
+
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.webEndpoint(
+ CustomRegistryConstants.WEB_UI, amWebURI.toURI()));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.MANAGEMENT_REST_API,
+ managementAPI.toURI()));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.PUBLISHER_REST_API,
+ publisherURL.toURI()));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.REGISTRY_REST_API,
+ registryREST.toURI()));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API,
+ new URI(configurationsURL)));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.PUBLISHER_EXPORTS_API,
+ new URI(exportsURL)));
} catch (URISyntaxException e) {
throw new IOException(e);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java
index 533ee54..32684c6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java
@@ -33,15 +33,14 @@
return LauncherExitCodes.EXIT_SUCCESS;
//remap from a planned shutdown to a failure
case LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN:
- return SliderExitCodes.EXIT_PROCESS_FAILED;
+ return SliderExitCodes.EXIT_SUCCESS;
default:
return exitCode;
}
}
public static boolean isMappedExitAFailure(int mappedExitCode) {
- return mappedExitCode!=LauncherExitCodes.EXIT_SUCCESS
- && mappedExitCode!= LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN;
+ return mappedExitCode != 0;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index e8b6802..9264991 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -22,14 +22,15 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
+import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.server.appmaster.actions.ActionStartContainer;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
@@ -39,9 +40,9 @@
import org.slf4j.LoggerFactory;
import java.util.Map;
-import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
/**
* A service for launching containers
@@ -214,10 +215,26 @@
instance.role = containerRole;
instance.roleId = role.id;
instance.environment = envDescription;
- actionQueue.put(new ActionStartContainer("starting " + containerRole,
- 0, container,
- containerLauncher.completeContainerLaunch(),
- instance));
+ int delay = appComponent.getOptionInt(
+ AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0);
+ int maxDelay =
+ getConfig().getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);
+ if (delay > maxDelay/1000) {
+ log.warn("Container launch delay of {} exceeds the maximum allowed of"
+ + " {} seconds. Delay will not be utilized.",
+ delay, maxDelay/1000);
+ delay = 0;
+ }
+ log.info("Container launch delay for {} set to {} seconds",
+ role.name, delay);
+ actionQueue.schedule(new ActionStartContainer("starting "
+ + containerRole,
+ container,
+ containerLauncher.completeContainerLaunch(),
+ instance,
+ delay,
+ TimeUnit.SECONDS));
} catch (Exception e) {
log.error("Exception thrown while trying to start {}: {}",
containerRole, e);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 93adfb2..7000583 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -20,17 +20,24 @@
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.service.ServiceStateChangeListener;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -50,12 +57,20 @@
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.WebApps;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
@@ -66,11 +81,13 @@
import org.apache.slider.api.proto.SliderClusterAPI;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.SliderAMArgs;
import org.apache.slider.common.params.SliderAMCreateAction;
import org.apache.slider.common.params.SliderActions;
import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.PortScanner;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.common.tools.SliderVersionInfo;
@@ -84,17 +101,15 @@
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
import org.apache.slider.core.main.ExitCodeProvider;
+import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slider.core.persist.ConfTreeSerDeser;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import org.apache.slider.core.registry.info.RegistryNaming;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
+import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
import org.apache.slider.server.appmaster.actions.ActionKillContainer;
@@ -106,6 +121,7 @@
import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
+import org.apache.slider.server.appmaster.actions.ReviewAndFlexApplicationSize;
import org.apache.slider.server.appmaster.actions.UnregisterComponentInstance;
import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
@@ -116,6 +132,7 @@
import org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider;
import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.security.SecurityConfiguration;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.ProviderAppState;
@@ -126,18 +143,17 @@
import org.apache.slider.server.appmaster.web.AgentService;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
import org.apache.slider.server.appmaster.web.SliderAMWebApp;
-import org.apache.slider.server.appmaster.web.SliderAmFilterInitializer;
-import org.apache.slider.server.appmaster.web.SliderAmIpFilter;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
-import org.apache.slider.server.services.registry.SliderRegistryService;
import org.apache.slider.server.services.security.CertificateManager;
+import org.apache.slider.server.services.security.FsDelegationTokenManager;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.slider.server.services.utility.WebAppService;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.apache.slider.server.services.workflow.WorkflowRpcService;
+import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -161,9 +177,6 @@
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.WS_AGENT_CONTEXT_ROOT;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.WS_CONTEXT_ROOT;
-
/**
* This is the AM, which directly implements the callbacks from the AM and NM
*/
@@ -177,6 +190,7 @@
ServiceStateChangeListener,
RoleKeys,
ProviderCompleted {
+
protected static final Logger log =
LoggerFactory.getLogger(SliderAppMaster.class);
@@ -199,13 +213,14 @@
public static final int HEARTBEAT_INTERVAL = 1000;
public static final int NUM_RPC_HANDLERS = 5;
- public static final int SCHEDULED_EXECUTOR_POOL_SIZE = 1;
/**
* Singleton of metrics registry
*/
public static final MetricRegistry metrics = new MetricRegistry();
-
+ public static final String E_TRIGGERED_LAUNCH_FAILURE =
+ "Chaos monkey triggered launch failure";
+
/** YARN RPC to communicate with the Resource Manager or Node Manager */
private YarnRPC yarnRPC;
@@ -214,7 +229,6 @@
private AMRMClientAsync asyncRMClient;
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
-
private RMOperationHandler rmOperationHandler;
private RMOperationHandler providerRMOperationHandler;
@@ -227,7 +241,7 @@
/**
* token blob
*/
- private ByteBuffer allTokens;
+ private Credentials containerCredentials;
private WorkflowRpcService rpcService;
@@ -285,7 +299,10 @@
*/
private final AtomicBoolean amCompletionFlag = new AtomicBoolean(false);
- private volatile boolean success = true;
+ /**
+ * Flag set during the init process
+ */
+ private final AtomicBoolean initCompleted = new AtomicBoolean(false);
/**
* Flag to set if the process exit code was set before shutdown started
@@ -316,11 +333,11 @@
private ProviderService providerService;
/**
- * The registry service
+ * The YARN registry service
*/
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private SliderRegistryService registry;
-
+ private RegistryOperations registryOperations;
+
/**
* Record of the max no. of cores allowed in this cluster
*/
@@ -330,7 +347,12 @@
* limit container memory
*/
private int containerMaxMemory;
- private String amCompletionReason;
+
+ /**
+ * The stop request received...the exit details are extracted
+ * from this
+ */
+ private volatile ActionStopSlider stopAction;
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private RoleLaunchService launchService;
@@ -343,19 +365,26 @@
private SliderAMWebApp webApp;
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private InetSocketAddress rpcServiceAddress;
- private ProviderService sliderAMProvider;
- private String agentAccessUrl;
+ private SliderAMProviderService sliderAMProvider;
private CertificateManager certificateManager;
private WorkflowExecutorService<ExecutorService> executorService;
private final QueueService actionQueues = new QueueService();
-
+ private String agentOpsUrl;
+ private String agentStatusUrl;
+ private YarnRegistryViewForProviders yarnRegistryOperations;
+ private FsDelegationTokenManager fsDelegationTokenManager;
+ private RegisterApplicationMasterResponse amRegistrationData;
+ private PortScanner portScanner;
+
/**
* Service Constructor
*/
public SliderAppMaster() {
super(SERVICE_CLASSNAME_SHORT);
+ new HdfsConfiguration();
+ new YarnConfiguration();
}
/* =================================================================== */
@@ -364,25 +393,31 @@
@Override //AbstractService
public synchronized void serviceInit(Configuration conf) throws Exception {
-
+ // slider client if found
+
+ Configuration customConf = SliderUtils.loadClientConfigurationResource();
// Load in the server configuration - if it is actually on the Classpath
Configuration serverConf =
ConfigHelper.loadFromResource(SERVER_RESOURCE);
- ConfigHelper.mergeConfigurations(conf, serverConf, SERVER_RESOURCE);
+ ConfigHelper.mergeConfigurations(customConf, serverConf, SERVER_RESOURCE, true);
+ serviceArgs.applyDefinitions(customConf);
+ serviceArgs.applyFileSystemBinding(customConf);
+ // conf now contains all customizations
AbstractActionArgs action = serviceArgs.getCoreAction();
SliderAMCreateAction createAction = (SliderAMCreateAction) action;
- //sort out the location of the AM
- serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemBinding(conf);
+ // sort out the location of the AM
String rmAddress = createAction.getRmAddress();
if (rmAddress != null) {
log.debug("Setting rm address from the command line: {}", rmAddress);
- SliderUtils.setRmSchedulerAddress(conf, rmAddress);
+ SliderUtils.setRmSchedulerAddress(customConf, rmAddress);
}
- serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemBinding(conf);
+
+ log.info("AM configuration:\n{}",
+ ConfigHelper.dumpConfigToString(customConf));
+
+ ConfigHelper.mergeConfigurations(conf, customConf, CLIENT_RESOURCE, true);
//init security with our conf
if (SliderUtils.isHadoopClusterSecure(conf)) {
log.info("Secure mode with kerberos realm {}",
@@ -391,23 +426,26 @@
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
log.debug("Authenticating as {}", ugi);
SliderUtils.verifyPrincipalSet(conf,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
- // always enforce protocol to be token-based.
- conf.set(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- SaslRpcServer.AuthMethod.TOKEN.toString());
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ } else {
+ log.info("Cluster is insecure");
}
log.info("Login user is {}", UserGroupInformation.getLoginUser());
//look at settings of Hadoop Auth, to pick up a problem seen once
checkAndWarnForAuthTokenProblems();
+
+ // validate server env
+ boolean dependencyChecks =
+ !conf.getBoolean(KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED,
+ false);
+ SliderUtils.validateSliderServerEnvironment(log, dependencyChecks);
executorService = new WorkflowExecutorService<ExecutorService>("AmExecutor",
- Executors.newCachedThreadPool(
- new ServiceThreadFactory("AmExecutor", true)));
+ Executors.newFixedThreadPool(2,
+ new ServiceThreadFactory("AmExecutor", true)));
addService(executorService);
-
addService(actionQueues);
//init all child services
super.serviceInit(conf);
@@ -416,11 +454,18 @@
@Override
protected void serviceStart() throws Exception {
super.serviceStart();
- executorService.execute(new QueueExecutor(this, actionQueues));
+ }
+
+ /**
+ * Start the queue processing
+ */
+ private void startQueueProcessing() {
+ log.info("Queue Processing started");
executorService.execute(actionQueues);
+ executorService.execute(new QueueExecutor(this, actionQueues));
}
- /* =================================================================== */
+/* =================================================================== */
/* RunService methods called from ServiceLauncher */
/* =================================================================== */
@@ -511,6 +556,7 @@
String sliderClusterDir = serviceArgs.getSliderClusterURI();
URI sliderClusterURI = new URI(sliderClusterDir);
Path clusterDirPath = new Path(sliderClusterURI);
+ log.info("Application defined at {}", sliderClusterURI);
SliderFileSystem fs = getClusterFS();
// build up information about the running application -this
@@ -524,9 +570,20 @@
log.info("Deploying cluster {}:", instanceDefinition);
stateForProviders.setApplicationName(clustername);
-
+
+ Configuration serviceConf = getConfig();
+
+ SecurityConfiguration securityConfiguration = new SecurityConfiguration(
+ serviceConf, instanceDefinition, clustername);
+ // obtain security state
+ boolean securityEnabled = securityConfiguration.isSecurityEnabled();
+ // set the global security flag for the instance definition
+ instanceDefinition.getAppConfOperations().set(
+ KEY_SECURITY_ENABLED, securityEnabled);
+
// triggers resolution and snapshotting in agent
appState.updateInstanceDefinition(instanceDefinition);
+
File confDir = getLocalConfDir();
if (!confDir.exists() || !confDir.isDirectory()) {
log.info("Conf dir {} does not exist.", confDir);
@@ -534,11 +591,8 @@
log.info("Parent dir {}:\n{}", parentFile, SliderUtils.listDir(parentFile));
}
- Configuration serviceConf = getConfig();
- // Try to get the proper filtering of static resources through the yarn proxy working
- serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS,
- SliderAmFilterInitializer.NAME);
- serviceConf.set(SliderAmIpFilter.WS_CONTEXT_ROOT, WS_CONTEXT_ROOT + "|" + WS_AGENT_CONTEXT_ROOT);
+ // IP filtering
+ serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS, AM_FILTER_NAME);
//get our provider
MapOperations globalInternalOptions = getGlobalInternalOptions();
@@ -582,36 +636,6 @@
appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID,
appAttemptID.toString());
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- Credentials credentials =
- currentUser.getCredentials();
- DataOutputBuffer dob = new DataOutputBuffer();
- credentials.writeTokenStorageToStream(dob);
- dob.close();
- // Now remove the AM->RM token so that containers cannot access it.
- Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
- while (iter.hasNext()) {
- Token<?> token = iter.next();
- log.info("Token {}", token.getKind());
- if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
- iter.remove();
- }
- }
- allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
-
- // set up secret manager
- secretManager = new ClientToAMTokenSecretManager(appAttemptID, null);
-
- // if not a secure cluster, extract the username -it will be
- // propagated to workers
- if (!UserGroupInformation.isSecurityEnabled()) {
- hadoop_user_name = System.getenv(HADOOP_USER_NAME);
- service_user_name = hadoop_user_name;
- log.info(HADOOP_USER_NAME + "='{}'", hadoop_user_name);
- } else {
- service_user_name = UserGroupInformation.getCurrentUser().getUserName();
- }
-
Map<String, String> envVars;
List<Container> liveContainers;
/**
@@ -625,8 +649,6 @@
asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval,
this);
addService(asyncRMClient);
- //wrap it for the app state model
- rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient);
//now bring it up
deployChildService(asyncRMClient);
@@ -635,8 +657,18 @@
nmClientAsync = new NMClientAsyncImpl("nmclient", this);
deployChildService(nmClientAsync);
+ // set up secret manager
+ secretManager = new ClientToAMTokenSecretManager(appAttemptID, null);
+
+ if (securityEnabled) {
+ // fix up the ACLs if they are not set
+ String acls = getConfig().get(SliderXmlConfKeys.KEY_PROTOCOL_ACL);
+ if (acls == null) {
+ getConfig().set(SliderXmlConfKeys.KEY_PROTOCOL_ACL, "*");
+ }
+ }
//bring up the Slider RPC service
- startSliderRPCServer();
+ startSliderRPCServer(instanceDefinition);
rpcServiceAddress = rpcService.getConnectAddress();
appMasterHostname = rpcServiceAddress.getHostName();
@@ -647,9 +679,9 @@
appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname);
appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort);
-
- //registry
- registry = startRegistrationService();
+ log.info("Starting Yarn registry");
+ registryOperations = startRegistryOperationsService();
+ log.info(registryOperations.toString());
//build the role map
List<ProviderRole> providerRoles =
@@ -658,22 +690,32 @@
// Start up the WebApp and track the URL for it
certificateManager = new CertificateManager();
- certificateManager.initRootCert(
- instanceDefinition.getAppConfOperations()
- .getComponent(SliderKeys.COMPONENT_AM));
+ MapOperations component = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM);
+ certificateManager.initialize(component);
+ certificateManager.setPassphrase(instanceDefinition.getPassphrase());
+
+ if (component.getOptionBool(
+ AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) {
+ uploadServerCertForLocalization(clustername, fs);
+ }
startAgentWebApp(appInformation, serviceConf);
- webApp = new SliderAMWebApp(registry);
+ int port = getPortToRequest(instanceDefinition);
+
+ webApp = new SliderAMWebApp(registryOperations);
WebApps.$for(SliderAMWebApp.BASE_PATH, WebAppApi.class,
new WebAppApiImpl(this,
stateForProviders,
providerService,
- certificateManager),
+ certificateManager, registryOperations),
RestPaths.WS_CONTEXT)
- .with(serviceConf)
+ .withHttpPolicy(serviceConf, HttpConfig.Policy.HTTP_ONLY)
+ .at(port)
.start(webApp);
- appMasterTrackingUrl = "http://" + appMasterHostname + ":" + webApp.port();
+ String scheme = WebAppUtils.HTTP_PREFIX;
+ appMasterTrackingUrl = scheme + appMasterHostname + ":" + webApp.port();
WebAppService<SliderAMWebApp> webAppService =
new WebAppService<SliderAMWebApp>("slider", webApp);
@@ -689,34 +731,70 @@
// address = SliderUtils.getRmSchedulerAddress(asyncRMClient.getConfig());
log.info("Connecting to RM at {},address tracking URL={}",
appMasterRpcPort, appMasterTrackingUrl);
- RegisterApplicationMasterResponse response = asyncRMClient
+ amRegistrationData = asyncRMClient
.registerApplicationMaster(appMasterHostname,
appMasterRpcPort,
appMasterTrackingUrl);
Resource maxResources =
- response.getMaximumResourceCapability();
+ amRegistrationData.getMaximumResourceCapability();
containerMaxMemory = maxResources.getMemory();
containerMaxCores = maxResources.getVirtualCores();
appState.setContainerLimits(maxResources.getMemory(),
maxResources.getVirtualCores());
+
+ // build the handler for RM request/release operations; this uses
+ // the max value as part of its lookup
+ rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient,
+ maxResources);
+
// set the RM-defined maximum cluster values
appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores));
appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory));
-
- boolean securityEnabled = UserGroupInformation.isSecurityEnabled();
+
+ // process the initial user to obtain the set of user
+ // supplied credentials (tokens were passed in by client). Remove AMRM
+ // token and HDFS delegation token, the latter because we will provide an
+ // up to date token for container launches (getContainerCredentials()).
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ Credentials credentials = currentUser.getCredentials();
+ Iterator<Token<? extends TokenIdentifier>> iter =
+ credentials.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<? extends TokenIdentifier> token = iter.next();
+ log.info("Token {}", token.getKind());
+ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME) ||
+ token.getKind().equals(DelegationTokenIdentifier.HDFS_DELEGATION_KIND)) {
+ iter.remove();
+ }
+ }
+ // at this point this credentials map is probably clear, but leaving this
+ // code to allow for future tokens...
+ containerCredentials = credentials;
+
if (securityEnabled) {
secretManager.setMasterKey(
- response.getClientToAMTokenMasterKey().array());
- applicationACLs = response.getApplicationACLs();
+ amRegistrationData.getClientToAMTokenMasterKey().array());
+ applicationACLs = amRegistrationData.getApplicationACLs();
- //tell the server what the ACLs are
+ //tell the server what the ACLs are
rpcService.getServer().refreshServiceAcl(serviceConf,
new SliderAMPolicyProvider());
+ // perform keytab based login to establish kerberos authenticated
+ // principal. Can do so now since AM registration with RM above required
+ // tokens associated to principal
+ String principal = securityConfiguration.getPrincipal();
+ File localKeytabFile =
+ securityConfiguration.getKeytabFile(instanceDefinition);
+ // Now log in...
+ login(principal, localKeytabFile);
+ // obtain new FS reference that should be kerberos based and different
+ // than the previously cached reference
+ fs = getClusterFS();
}
// extract container list
- liveContainers = response.getContainersFromPreviousAttempts();
+ liveContainers = amRegistrationData.getContainersFromPreviousAttempts();
//now validate the installation
Configuration providerConf =
@@ -781,38 +859,137 @@
appState.noteAMLaunched();
- //Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, registry, actionQueues,
- liveContainers);
- sliderAMProvider.bind(stateForProviders, registry, actionQueues,
- liveContainers);
-
- // now do the registration
- registerServiceInstance(clustername, appid);
+ //Give the provider access to the state, and AM
+ providerService.bind(stateForProviders, actionQueues, liveContainers);
+ sliderAMProvider.bind(stateForProviders, actionQueues, liveContainers);
// chaos monkey
maybeStartMonkey();
- // Start the Slider AM provider
- sliderAMProvider.start();
+ // setup token renewal and expiry handling for long lived apps
+// if (SliderUtils.isHadoopClusterSecure(getConfig())) {
+// fsDelegationTokenManager = new FsDelegationTokenManager(actionQueues);
+// fsDelegationTokenManager.acquireDelegationToken(getConfig());
+// }
- // launch the real provider; this is expected to trigger a callback that
- // starts the node review process
- launchProviderService(instanceDefinition, confDir);
+ // if not a secure cluster, extract the username -it will be
+ // propagated to workers
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ hadoop_user_name = System.getenv(HADOOP_USER_NAME);
+ log.info(HADOOP_USER_NAME + "='{}'", hadoop_user_name);
+ }
+ service_user_name = RegistryUtils.currentUser();
+ log.info("Registry service username ={}", service_user_name);
+
+ // now do the registration
+ registerServiceInstance(clustername, appid);
+
+ // log the YARN and web UIs
+ log.info("RM Webapp address {}", serviceConf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));
+ log.info("slider Webapp address {}", appMasterTrackingUrl);
+
+ // declare the cluster initialized
+ log.info("Application Master Initialization Completed");
+ initCompleted.set(true);
+
try {
+ // start handling any scheduled events
+
+ startQueueProcessing();
+
+ // Start the Slider AM provider
+ sliderAMProvider.start();
+
+ // launch the real provider; this is expected to trigger a callback that
+ // starts the node review process
+ launchProviderService(instanceDefinition, confDir);
+
//now block waiting to be told to exit the process
waitForAMCompletionSignal();
- //shutdown time
- } finally {
- finish();
+ } catch(Exception e) {
+ log.error("Exception : {}", e, e);
+ onAMStop(new ActionStopSlider(e));
+ }
+ //shutdown time
+ return finish();
+ }
+
+ private int getPortToRequest(AggregateConf instanceDefinition)
+ throws SliderException {
+ int portToRequest = 0;
+ String portRange = instanceDefinition.
+ getAppConfOperations().getGlobalOptions().
+ getOption(SliderKeys.KEY_ALLOWED_PORT_RANGE, "0");
+ if (!"0".equals(portRange)) {
+ if (portScanner == null) {
+ portScanner = new PortScanner();
+ portScanner.setPortRange(portRange);
+ }
+ portToRequest = portScanner.getAvailablePort();
}
- return amExitCode;
+ return portToRequest;
+ }
+
+ private void uploadServerCertForLocalization(String clustername,
+ SliderFileSystem fs)
+ throws IOException {
+ Path certsDir = new Path(fs.buildClusterDirPath(clustername),
+ "certs");
+ if (!fs.getFileSystem().exists(certsDir)) {
+ fs.getFileSystem().mkdirs(certsDir,
+ new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ }
+ Path destPath = new Path(certsDir, SliderKeys.CRT_FILE_NAME);
+ if (!fs.getFileSystem().exists(destPath)) {
+ fs.getFileSystem().copyFromLocalFile(
+ new Path(CertificateManager.getServerCertficateFilePath().getAbsolutePath()),
+ destPath);
+ log.info("Uploaded server cert to localization path {}", destPath);
+ }
+
+ fs.getFileSystem().setPermission(destPath,
+ new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
+ }
+
+ protected void login(String principal, File localKeytabFile)
+ throws IOException, SliderException {
+ UserGroupInformation.loginUserFromKeytab(principal,
+ localKeytabFile.getAbsolutePath());
+ validateLoginUser(UserGroupInformation.getLoginUser());
+ }
+
+ /**
+ * Ensure that the user is generated from a keytab and has no HDFS delegation
+ * tokens.
+ *
+ * @param user user to validate
+ * @throws SliderException
+ */
+ protected void validateLoginUser(UserGroupInformation user)
+ throws SliderException {
+ if (!user.isFromKeytab()) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_STATE, "User is "
+ + "not based on a keytab in a secure deployment.");
+ }
+ Credentials credentials =
+ user.getCredentials();
+ Iterator<Token<? extends TokenIdentifier>> iter =
+ credentials.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<? extends TokenIdentifier> token = iter.next();
+ log.info("Token {}", token.getKind());
+ if (token.getKind().equals(
+ DelegationTokenIdentifier.HDFS_DELEGATION_KIND)) {
+ log.info("HDFS delegation token {}. Removing...", token);
+ iter.remove();
+ }
+ }
}
private void startAgentWebApp(MapOperations appInformation,
- Configuration serviceConf) {
+ Configuration serviceConf) throws IOException {
URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader() ).getURLs();
StringBuilder sb = new StringBuilder("AM classpath:");
for (URL url : urls) {
@@ -824,12 +1001,15 @@
new WebAppApiImpl(this,
stateForProviders,
providerService,
- certificateManager),
+ certificateManager, registryOperations),
RestPaths.AGENT_WS_CONTEXT)
.withComponentConfig(getInstanceDefinition().getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM))
.start();
- agentAccessUrl = "https://" + appMasterHostname + ":" + agentWebApp.getSecuredPort();
+ agentOpsUrl =
+ "https://" + appMasterHostname + ":" + agentWebApp.getSecuredPort();
+ agentStatusUrl =
+ "https://" + appMasterHostname + ":" + agentWebApp.getPort();
AgentService agentService =
new AgentService("slider-agent", agentWebApp);
@@ -837,9 +1017,11 @@
agentService.start();
addService(agentService);
- appInformation.put(StatusKeys.INFO_AM_AGENT_URL, agentAccessUrl + "/");
- appInformation.set(StatusKeys.INFO_AM_AGENT_PORT, agentWebApp.getPort());
- appInformation.set(StatusKeys.INFO_AM_SECURED_AGENT_PORT,
+ appInformation.put(StatusKeys.INFO_AM_AGENT_OPS_URL, agentOpsUrl + "/");
+ appInformation.put(StatusKeys.INFO_AM_AGENT_STATUS_URL, agentStatusUrl + "/");
+ appInformation.set(StatusKeys.INFO_AM_AGENT_STATUS_PORT,
+ agentWebApp.getPort());
+ appInformation.set(StatusKeys.INFO_AM_AGENT_OPS_PORT,
agentWebApp.getSecuredPort());
}
@@ -847,85 +1029,136 @@
* This registers the service instance and its external values
* @param instanceName name of this instance
* @param appid application ID
- * @throws Exception
+ * @throws IOException
*/
private void registerServiceInstance(String instanceName,
- ApplicationId appid) throws Exception {
+ ApplicationId appid) throws IOException {
+
+
// the registry is running, so register services
- URL unsecureWebAPI = new URL(appMasterTrackingUrl);
- URL secureWebAPI = new URL(agentAccessUrl);
- String serviceName = SliderKeys.APP_TYPE;
- int id = appid.getId();
- String appServiceType = RegistryNaming.createRegistryServiceType(
- instanceName,
+ URL amWebURI = new URL(appMasterTrackingUrl);
+ URL agentOpsURI = new URL(agentOpsUrl);
+ URL agentStatusURI = new URL(agentStatusUrl);
+
+ //Give the provider restricted access to the state, registry
+ setupInitialRegistryPaths();
+ yarnRegistryOperations = new YarnRegistryViewForProviders(
+ registryOperations,
service_user_name,
- serviceName);
- String registryId =
- RegistryNaming.createRegistryName(instanceName, service_user_name,
- serviceName, id);
+ SliderKeys.APP_TYPE,
+ instanceName,
+ appAttemptID);
+ providerService.bindToYarnRegistry(yarnRegistryOperations);
+ sliderAMProvider.bindToYarnRegistry(yarnRegistryOperations);
- List<String> serviceInstancesRunning = registry.instanceIDs(serviceName);
- log.info("service instances already running: {}", serviceInstancesRunning);
+ // Yarn registry
+ ServiceRecord serviceRecord = new ServiceRecord();
+ serviceRecord.set(YarnRegistryAttributes.YARN_ID, appid.toString());
+ serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.APPLICATION);
+ serviceRecord.description = "Slider Application Master";
+/* SLIDER-531: disable this addition so things compile against versions of
+the registry with/without the new record format
- ServiceInstanceData instanceData = new ServiceInstanceData(registryId,
- appServiceType);
-
-
- // IPC services
- instanceData.externalView.endpoints.put(
- CustomRegistryConstants.AM_IPC_PROTOCOL,
- new RegisteredEndpoint(rpcServiceAddress,
- RegisteredEndpoint.PROTOCOL_HADOOP_PROTOBUF,
- "Slider AM RPC") );
-
-
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.ipcEndpoint(
+ CustomRegistryConstants.AM_IPC_PROTOCOL,
+ rpcServiceAddress));
+
+ */
// internal services
-
- sliderAMProvider.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData
- );
+ sliderAMProvider.applyInitialRegistryDefinitions(amWebURI,
+ agentOpsURI,
+ agentStatusURI,
+ serviceRecord);
// provider service dynamic definitions.
- providerService.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData
- );
+ providerService.applyInitialRegistryDefinitions(amWebURI,
+ agentOpsURI,
+ agentStatusURI,
+ serviceRecord);
-
- // push the registration info to ZK
-
- registry.registerSelf(
- instanceData, unsecureWebAPI
- );
+ // register the service's entry
+ log.info("Service Record \n{}", serviceRecord);
+ yarnRegistryOperations.registerSelf(serviceRecord, true);
+ log.info("Registered service under {}; absolute path {}",
+ yarnRegistryOperations.getSelfRegistrationPath(),
+ yarnRegistryOperations.getAbsoluteSelfRegistrationPath());
+
+ boolean isFirstAttempt = 1 == appAttemptID.getAttemptId();
+ // delete the children in case there are any and this is an AM startup.
+ // just to make sure everything underneath is purged
+ if (isFirstAttempt) {
+ yarnRegistryOperations.deleteChildren(
+ yarnRegistryOperations.getSelfRegistrationPath(),
+ true);
+ }
}
/**
- * Register/re-register a component (that is already in the app state
- * @param id the component
+ * TODO: purge this once RM is doing the work
+ * @throws IOException
*/
- public boolean registerComponent(ContainerId id) {
+ protected void setupInitialRegistryPaths() throws IOException {
+ if (registryOperations instanceof RMRegistryOperationsService) {
+ RMRegistryOperationsService rmRegOperations =
+ (RMRegistryOperationsService) registryOperations;
+ rmRegOperations.initUserRegistryAsync(service_user_name);
+ }
+ }
+
+ /**
+ * Handler for {@link RegisterComponentInstance action}
+ * Register/re-register an ephemeral container that is already in the app state
+ * @param id the component
+ * @param description
+ */
+ public boolean registerComponent(ContainerId id, String description) throws
+ IOException {
RoleInstance instance = appState.getOwnedContainer(id);
if (instance == null) {
return false;
}
- // this is where component registrations will go
+ // this is where component registrations go
log.info("Registering component {}", id);
-
+ String cid = RegistryPathUtils.encodeYarnID(id.toString());
+ ServiceRecord container = new ServiceRecord();
+ container.set(YarnRegistryAttributes.YARN_ID, cid);
+ container.description = description;
+ container.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.CONTAINER);
+ try {
+ yarnRegistryOperations.putComponent(cid, container);
+ } catch (IOException e) {
+ log.warn("Failed to register container {}/{}: {}",
+ id, description, e, e);
+ }
return true;
}
/**
+ * Handler for {@link UnregisterComponentInstance}
+ *
* unregister a component. At the time this message is received,
- * the component may already been deleted from/never added to
- * the app state
+ * the component may not have been registered
* @param id the component
*/
public void unregisterComponent(ContainerId id) {
log.info("Unregistering component {}", id);
+ if (yarnRegistryOperations == null) {
+ log.warn("Processing unregister component event before initialization " +
+ "completed; init flag =" + initCompleted);
+ return;
+ }
+ String cid = RegistryPathUtils.encodeYarnID(id.toString());
+ try {
+ yarnRegistryOperations.deleteComponent(cid);
+ } catch (IOException e) {
+ log.warn("Failed to delete container {} : {}", id, e, e);
+ }
}
-
+
/**
* looks for a specific case where a token file is provided as an environment
* variable, yet the file is not there.
@@ -1013,53 +1246,66 @@
} finally {
AMExecutionStateLock.unlock();
}
- //add a sleep here for about a second. Why? it
- //stops RPC calls breaking so dramatically when the cluster
- //is torn down mid-RPC
- try {
- Thread.sleep(TERMINATION_SIGNAL_PROPAGATION_DELAY);
- } catch (InterruptedException ignored) {
- //ignored
- }
}
/**
- * Declare that the AM is complete
- * @param exitCode exit code for the aM
- * @param reason reason for termination
+ * Signal that the AM is complete .. queues it in a separate thread
+ *
+ * @param stopActionRequest request containing shutdown details
*/
- public synchronized void signalAMComplete(int exitCode, String reason) {
- amCompletionReason = reason;
+ public synchronized void signalAMComplete(ActionStopSlider stopActionRequest) {
+ // this is a queued action: schedule it through the queues
+ schedule(stopActionRequest);
+ }
+
+ /**
+ * Signal that the AM is complete
+ *
+ * @param stopActionRequest request containing shutdown details
+ */
+ public synchronized void onAMStop(ActionStopSlider stopActionRequest) {
+
AMExecutionStateLock.lock();
try {
- amCompletionFlag.set(true);
- amExitCode = exitCode;
- isAMCompleted.signal();
+ if (amCompletionFlag.compareAndSet(false, true)) {
+ // first stop request received
+ this.stopAction = stopActionRequest;
+ isAMCompleted.signal();
+ }
} finally {
AMExecutionStateLock.unlock();
}
}
+
/**
- * shut down the cluster
+ * trigger the YARN cluster termination process
+ * @return the exit code
*/
- private synchronized void finish() {
+ private synchronized int finish() {
+ Preconditions.checkNotNull(stopAction, "null stop action");
FinalApplicationStatus appStatus;
- log.info("Triggering shutdown of the AM: {}", amCompletionReason);
+ log.info("Triggering shutdown of the AM: {}", stopAction);
- String appMessage = amCompletionReason;
+ String appMessage = stopAction.getMessage();
//stop the daemon & grab its exit code
- int exitCode = amExitCode;
- success = exitCode == 0 || exitCode == 3;
+ int exitCode = stopAction.getExitCode();
+ amExitCode = exitCode;
- appStatus = success ? FinalApplicationStatus.SUCCEEDED:
- FinalApplicationStatus.FAILED;
+ appStatus = stopAction.getFinalApplicationStatus();
if (!spawnedProcessExitedBeforeShutdownTriggered) {
//stopped the forked process but don't worry about its exit code
exitCode = stopForkedProcess();
log.debug("Stopped forked process: exit code={}", exitCode);
}
+ // make sure the AM is actually registered. If not, there's no point
+ // trying to unregister it
+ if (amRegistrationData == null) {
+ log.info("Application attempt not yet registered; skipping unregistration");
+ return exitCode;
+ }
+
//stop any launches in progress
launchService.stop();
@@ -1070,17 +1316,6 @@
// signal to the RM
log.info("Application completed. Signalling finish to RM");
- //if there were failed containers and the app isn't already down as failing, it is now
-/*
- int failedContainerCount = appState.getFailedCountainerCount();
- if (failedContainerCount != 0 &&
- appStatus == FinalApplicationStatus.SUCCEEDED) {
- appStatus = FinalApplicationStatus.FAILED;
- appMessage =
- "Completed with exit code = " + exitCode + " - " + getContainerDiagnosticInfo();
- success = false;
- }
-*/
try {
log.info("Unregistering AM status={} message={}", appStatus, appMessage);
asyncRMClient.unregisterApplicationMaster(appStatus, appMessage, null);
@@ -1091,16 +1326,21 @@
*/
} catch (IOException e) {
log.info("Failed to unregister application: " + e, e);
+ } catch (InvalidApplicationMasterRequestException e) {
+ log.info("Application not found in YARN application list;" +
+ " it may have been terminated/YARN shutdown in progress: " + e, e);
} catch (YarnException e) {
log.info("Failed to unregister application: " + e, e);
}
+ return exitCode;
}
- /**
- * Get diagnostics info about containers
- */
+ /**
+ * Get diagnostics info about containers
+ */
private String getContainerDiagnosticInfo() {
- return appState.getContainerDiagnosticInfo();
+
+ return appState.getContainerDiagnosticInfo();
}
public Object getProxy(Class protocol, InetSocketAddress addr) {
@@ -1110,22 +1350,45 @@
/**
* Start the slider RPC server
*/
- private void startSliderRPCServer() throws IOException {
- SliderClusterProtocolPBImpl protobufRelay = new SliderClusterProtocolPBImpl(this);
- BlockingService blockingService = SliderClusterAPI.SliderClusterProtocolPB
- .newReflectiveBlockingService(
- protobufRelay);
+ private void startSliderRPCServer(AggregateConf instanceDefinition)
+ throws IOException, SliderException {
+ verifyIPCAccess();
- rpcService = new WorkflowRpcService("SliderRPC", RpcBinder.createProtobufServer(
- new InetSocketAddress("0.0.0.0", 0),
- getConfig(),
- secretManager,
- NUM_RPC_HANDLERS,
- blockingService,
- null));
+
+ SliderClusterProtocolPBImpl protobufRelay =
+ new SliderClusterProtocolPBImpl(this);
+ BlockingService blockingService = SliderClusterAPI.SliderClusterProtocolPB
+ .newReflectiveBlockingService(
+ protobufRelay);
+
+ int port = getPortToRequest(instanceDefinition);
+ rpcService =
+ new WorkflowRpcService("SliderRPC", RpcBinder.createProtobufServer(
+ new InetSocketAddress("0.0.0.0", port),
+ getConfig(),
+ secretManager,
+ NUM_RPC_HANDLERS,
+ blockingService,
+ null));
deployChildService(rpcService);
}
+ /**
+ * verify that if the cluster is authed, the ACLs are set.
+ * @throws BadConfigException if Authorization is set without any ACL
+ */
+ private void verifyIPCAccess() throws BadConfigException {
+ boolean authorization = getConfig().getBoolean(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
+ false);
+ String acls = getConfig().get(SliderXmlConfKeys.KEY_PROTOCOL_ACL);
+ if (authorization && SliderUtils.isUnset(acls)) {
+ throw new BadConfigException("Application has IPC authorization enabled in " +
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION +
+ " but no ACLs in " + SliderXmlConfKeys.KEY_PROTOCOL_ACL);
+ }
+ }
+
/* =================================================================== */
/* AMRMClientAsync callbacks */
@@ -1189,15 +1452,14 @@
log.error("Role instance {} failed ", ri);
}
- getProviderService().notifyContainerCompleted(containerId);
- queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
+ // known nodes trigger notifications
+ if(!result.unknownNode) {
+ getProviderService().notifyContainerCompleted(containerId);
+ queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
+ }
}
- try {
- reviewRequestAndReleaseNodes();
- } catch (SliderInternalStateException e) {
- log.warn("Exception while flexing nodes", e);
- }
+ reviewRequestAndReleaseNodes("onContainersCompleted");
}
/**
@@ -1205,22 +1467,27 @@
* It should be the only way that anything -even the AM itself on startup-
* asks for nodes.
* @param resources the resource tree
- * @return true if the any requests were made
- * @throws IOException
+ * @throws SliderException slider problems, including invalid configs
+ * @throws IOException IO problems
*/
- private boolean flexCluster(ConfTree resources)
- throws IOException, SliderInternalStateException, BadConfigException {
+ private void flexCluster(ConfTree resources)
+ throws IOException, SliderException {
+
+ AggregateConf newConf =
+ new AggregateConf(appState.getInstanceDefinitionSnapshot());
+ newConf.setResources(resources);
+ // verify the new definition is valid
+ sliderAMProvider.validateInstanceDefinition(newConf);
+ providerService.validateInstanceDefinition(newConf);
appState.updateResourceDefinitions(resources);
// reset the scheduled windows...the values
// may have changed
appState.resetFailureCounts();
-
-
// ask for more containers if needed
- return reviewRequestAndReleaseNodes();
+ reviewRequestAndReleaseNodes("flexCluster");
}
/**
@@ -1251,13 +1518,47 @@
/**
* Look at where the current node state is -and whether it should be changed
+ * @param reason
*/
- private synchronized boolean reviewRequestAndReleaseNodes()
+ private synchronized void reviewRequestAndReleaseNodes(String reason) {
+ log.debug("reviewRequestAndReleaseNodes({})", reason);
+ queue(new ReviewAndFlexApplicationSize(reason, 0, TimeUnit.SECONDS));
+ }
+
+ /**
+ * Handle the event requesting a review ... look at the queue and decide
+ * whether to act or not
+ * @param action action triggering the event. It may be put
+ * back into the queue
+ * @throws SliderInternalStateException
+ */
+ public void handleReviewAndFlexApplicationSize(ReviewAndFlexApplicationSize action)
throws SliderInternalStateException {
- log.debug("in reviewRequestAndReleaseNodes()");
+
+ if ( actionQueues.hasQueuedActionWithAttribute(
+ AsyncAction.ATTR_REVIEWS_APP_SIZE | AsyncAction.ATTR_HALTS_APP)) {
+ // this operation isn't needed at all -existing duplicate or shutdown due
+ return;
+ }
+ // if there is an action which changes cluster size, wait
+ if (actionQueues.hasQueuedActionWithAttribute(
+ AsyncAction.ATTR_CHANGES_APP_SIZE)) {
+ // place the action at the back of the queue
+ actionQueues.put(action);
+ }
+
+ executeNodeReview(action.name);
+ }
+
+ /**
+ * Look at where the current node state is -and whether it should be changed
+ */
+ public synchronized void executeNodeReview(String reason)
+ throws SliderInternalStateException {
+
+ log.debug("in executeNodeReview({})", reason);
if (amCompletionFlag.get()) {
log.info("Ignoring node review operation: shutdown in progress");
- return false;
}
try {
List<AbstractRMOperation> allOperations = appState.reviewRequestAndReleaseNodes();
@@ -1265,13 +1566,10 @@
providerRMOperationHandler.execute(allOperations);
//now apply the operations
executeRMOperations(allOperations);
- return !allOperations.isEmpty();
} catch (TriggerClusterTeardownException e) {
-
//App state has decided that it is time to exit
- log.error("Cluster teardown triggered %s", e);
- signalAMComplete(e.getExitCode(), e.toString());
- return false;
+ log.error("Cluster teardown triggered {}", e, e);
+ queue(new ActionStopSlider(e));
}
}
@@ -1289,7 +1587,10 @@
@Override //AMRMClientAsync
public void onShutdownRequest() {
LOG_YARN.info("Shutdown Request received");
- signalAMComplete(EXIT_CLIENT_INITIATED_SHUTDOWN, "Shutdown requested from RM");
+ signalAMComplete(new ActionStopSlider("stop",
+ EXIT_SUCCESS,
+ FinalApplicationStatus.SUCCEEDED,
+ "Shutdown requested from RM"));
}
/**
@@ -1298,7 +1599,10 @@
*/
@Override //AMRMClientAsync
public void onNodesUpdated(List<NodeReport> updatedNodes) {
- LOG_YARN.info("Nodes updated");
+ LOG_YARN.info("onNodesUpdated({})", updatedNodes.size());
+ log.info("Updated nodes {}", updatedNodes);
+ // Check if any nodes are lost or revived and update state accordingly
+ appState.onNodesUpdated(updatedNodes);
}
/**
@@ -1315,8 +1619,10 @@
public void onError(Throwable e) {
//callback says it's time to finish
LOG_YARN.error("AMRMClientAsync.onError() received " + e, e);
- signalAMComplete(EXIT_EXCEPTION_THROWN,
- "AMRMClientAsync.onError() received " + e);
+ signalAMComplete(new ActionStopSlider("stop",
+ EXIT_EXCEPTION_THROWN,
+ FinalApplicationStatus.FAILED,
+ "AMRMClientAsync.onError() received " + e));
}
/* =================================================================== */
@@ -1345,14 +1651,35 @@
/* SliderClusterProtocol */
/* =================================================================== */
+ /**
+ * General actions to perform on a slider RPC call coming in
+ * @param operation operation to log
+ * @throws IOException problems
+ */
+ protected void onRpcCall(String operation) throws IOException {
+ // it's not clear why this is here —it has been present since the
+ // code -> git change. Leaving it in
+ SliderUtils.getCurrentUser();
+ log.debug("Received call to {}", operation);
+ }
+
@Override //SliderClusterProtocol
public Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request) throws
IOException,
YarnException {
- SliderUtils.getCurrentUser();
+ onRpcCall("stopCluster()");
String message = request.getMessage();
- log.info("SliderAppMasterApi.stopCluster: {}", message);
- schedule(new ActionStopSlider(message, 1000, TimeUnit.MILLISECONDS));
+ if (message == null) {
+ message = "application frozen by client";
+ }
+ ActionStopSlider stopSlider =
+ new ActionStopSlider(message,
+ 1000, TimeUnit.MILLISECONDS,
+ LauncherExitCodes.EXIT_SUCCESS,
+ FinalApplicationStatus.SUCCEEDED,
+ message);
+ log.info("SliderAppMasterApi.stopCluster: {}", stopSlider);
+ schedule(stopSlider);
return Messages.StopClusterResponseProto.getDefaultInstance();
}
@@ -1360,13 +1687,13 @@
public Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) throws
IOException,
YarnException {
- SliderUtils.getCurrentUser();
-
+ onRpcCall("flexCluster()");
String payload = request.getClusterSpec();
ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
ConfTree updatedResources = confTreeSerDeser.fromJson(payload);
- boolean flexed = flexCluster(updatedResources);
- return Messages.FlexClusterResponseProto.newBuilder().setResponse(flexed).build();
+ flexCluster(updatedResources);
+ return Messages.FlexClusterResponseProto.newBuilder().setResponse(
+ true).build();
}
@Override //SliderClusterProtocol
@@ -1374,7 +1701,7 @@
Messages.GetJSONClusterStatusRequestProto request) throws
IOException,
YarnException {
- SliderUtils.getCurrentUser();
+ onRpcCall("getJSONClusterStatus()");
String result;
//quick update
//query and json-ify
@@ -1386,14 +1713,13 @@
.build();
}
-
@Override
public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition(
Messages.GetInstanceDefinitionRequestProto request) throws
IOException,
YarnException {
- log.info("Received call to getInstanceDefinition()");
+ onRpcCall("getInstanceDefinition()");
String internal;
String resources;
String app;
@@ -1406,7 +1732,7 @@
assert internal != null;
assert resources != null;
assert app != null;
- log.info("Generating getInstanceDefinition Response");
+ log.debug("Generating getInstanceDefinition Response");
Messages.GetInstanceDefinitionResponseProto.Builder builder =
Messages.GetInstanceDefinitionResponseProto.newBuilder();
builder.setInternal(internal);
@@ -1415,12 +1741,11 @@
return builder.build();
}
-
@Override //SliderClusterProtocol
public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request) throws
IOException,
YarnException {
- SliderUtils.getCurrentUser();
+ onRpcCall("listNodeUUIDsByRole()");
String role = request.getRole();
Messages.ListNodeUUIDsByRoleResponseProto.Builder builder =
Messages.ListNodeUUIDsByRoleResponseProto.newBuilder();
@@ -1435,9 +1760,9 @@
public Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request) throws
IOException,
YarnException {
- SliderUtils.getCurrentUser();
+ onRpcCall("getNode()");
RoleInstance instance = appState.getLiveInstanceByContainerID(
- request.getUuid());
+ request.getUuid());
return Messages.GetNodeResponseProto.newBuilder()
.setClusterNode(instance.toProtobuf())
.build();
@@ -1447,10 +1772,10 @@
public Messages.GetClusterNodesResponseProto getClusterNodes(Messages.GetClusterNodesRequestProto request) throws
IOException,
YarnException {
- SliderUtils.getCurrentUser();
+ onRpcCall("getClusterNodes()");
List<RoleInstance>
clusterNodes = appState.getLiveInstancesByContainerIDs(
- request.getUuidList());
+ request.getUuidList());
Messages.GetClusterNodesResponseProto.Builder builder =
Messages.GetClusterNodesResponseProto.newBuilder();
@@ -1465,6 +1790,7 @@
public Messages.EchoResponseProto echo(Messages.EchoRequestProto request) throws
IOException,
YarnException {
+ onRpcCall("echo()");
Messages.EchoResponseProto.Builder builder =
Messages.EchoResponseProto.newBuilder();
String text = request.getText();
@@ -1479,6 +1805,7 @@
public Messages.KillContainerResponseProto killContainer(Messages.KillContainerRequestProto request) throws
IOException,
YarnException {
+ onRpcCall("killContainer()");
String containerID = request.getId();
log.info("Kill Container {}", containerID);
//throws NoSuchNodeException if it is missing
@@ -1518,7 +1845,7 @@
return Messages.AMSuicideResponseProto.getDefaultInstance();
}
- /* =================================================================== */
+/* =================================================================== */
/* END */
/* =================================================================== */
@@ -1555,7 +1882,6 @@
}
}
-
/* =================================================================== */
/* EventCallback from the child or ourselves directly */
/* =================================================================== */
@@ -1568,10 +1894,10 @@
try {
flexCluster(getInstanceDefinition().getResources());
} catch (Exception e) {
- //this may happen in a separate thread, so the ability to act is limited
- log.error("Failed to flex cluster nodes", e);
- //declare a failure
- finish();
+ // cluster flex failure: log
+ log.error("Failed to flex cluster nodes: {}", e, e);
+ // then what? exit
+ queue(new ActionStopSlider(e));
}
}
@@ -1591,7 +1917,7 @@
executeRMOperations(appState.releaseContainer(containerId));
// ask for more containers if needed
log.info("Container released; triggering review");
- reviewRequestAndReleaseNodes();
+ reviewRequestAndReleaseNodes("Loss of container");
} else {
log.info("Container not in active set - ignoring");
}
@@ -1610,12 +1936,19 @@
if (service == providerService && service.isInState(STATE.STOPPED)) {
//its the current master process in play
int exitCode = providerService.getExitCode();
- int mappedProcessExitCode =
- AMUtils.mapProcessExitCodeToYarnExitCode(exitCode);
+ int mappedProcessExitCode = exitCode;
+
boolean shouldTriggerFailure = !amCompletionFlag.get()
- && (AMUtils.isMappedExitAFailure(mappedProcessExitCode));
-
+ && (mappedProcessExitCode != 0);
+
if (shouldTriggerFailure) {
+ String reason =
+ "Spawned process failed with raw " + exitCode + " mapped to " +
+ mappedProcessExitCode;
+ ActionStopSlider stop = new ActionStopSlider("stop",
+ mappedProcessExitCode,
+ FinalApplicationStatus.FAILED,
+ reason);
//this wasn't expected: the process finished early
spawnedProcessExitedBeforeShutdownTriggered = true;
log.info(
@@ -1624,9 +1957,7 @@
mappedProcessExitCode);
//tell the AM the cluster is complete
- signalAMComplete(mappedProcessExitCode,
- "Spawned master exited with raw " + exitCode + " mapped to " +
- mappedProcessExitCode);
+ signalAMComplete(stop);
} else {
//we don't care
log.info(
@@ -1656,18 +1987,44 @@
*/
public void startContainer(Container container,
ContainerLaunchContext ctx,
- RoleInstance instance) {
+ RoleInstance instance) throws IOException {
// Set up tokens for the container too. Today, for normal shell commands,
// the container in distribute-shell doesn't need any tokens. We are
// populating them mainly for NodeManagers to be able to download any
// files in the distributed file-system. The tokens are otherwise also
// useful in cases, for e.g., when one is running a "hadoop dfs" command
// inside the distributed shell.
- ctx.setTokens(allTokens.duplicate());
+
+ // add current HDFS delegation token with an up to date token
+ ByteBuffer tokens = getContainerCredentials();
+
+ if (tokens != null) {
+ ctx.setTokens(tokens);
+ } else {
+ log.warn("No delegation tokens obtained and set for launch context");
+ }
appState.containerStartSubmitted(container, instance);
nmClientAsync.startContainerAsync(container, ctx);
}
+ private ByteBuffer getContainerCredentials() throws IOException {
+ // a delegation token can be retrieved from filesystem since
+ // the login is via a keytab (see above)
+ Credentials credentials = new Credentials(containerCredentials);
+ ByteBuffer tokens = null;
+ Token<? extends TokenIdentifier>[] hdfsTokens =
+ getClusterFS().getFileSystem().addDelegationTokens(
+ UserGroupInformation.getLoginUser().getShortUserName(), credentials);
+ if (hdfsTokens.length > 0) {
+ DataOutputBuffer dob = new DataOutputBuffer();
+ credentials.writeTokenStorageToStream(dob);
+ dob.close();
+ tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ }
+
+ return tokens;
+ }
+
@Override // NMClientAsync.CallbackHandler
public void onContainerStopped(ContainerId containerId) {
// do nothing but log: container events from the AM
@@ -1687,7 +2044,8 @@
nmClientAsync.getContainerStatusAsync(containerId,
cinfo.container.getNodeId());
// push out a registration
- queue(new RegisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
+ queue(new RegisterComponentInstance(containerId, cinfo.role,
+ 0, TimeUnit.MILLISECONDS));
} else {
//this is a hypothetical path not seen. We react by warning
@@ -1772,11 +2130,20 @@
*/
public void onExceptionInThread(Thread thread, Exception exception) {
log.error("Exception in {}: {}", thread.getName(), exception, exception);
- int exitCode = EXIT_EXCEPTION_THROWN;
- if (exception instanceof ExitCodeProvider) {
- exitCode = ((ExitCodeProvider) exception).getExitCode();
+
+ // if there is a teardown in progress, ignore it
+ if (amCompletionFlag.get()) {
+ log.info("Ignoring exception: shutdown in progress");
+ } else {
+ int exitCode = EXIT_EXCEPTION_THROWN;
+ if (exception instanceof ExitCodeProvider) {
+ exitCode = ((ExitCodeProvider) exception).getExitCode();
+ }
+ signalAMComplete(new ActionStopSlider("stop",
+ exitCode,
+ FinalApplicationStatus.FAILED,
+ exception.toString()));
}
- signalAMComplete(exitCode, exception.toString());
}
/**
@@ -1791,6 +2158,7 @@
InternalKeys.DEFAULT_CHAOS_MONKEY_ENABLED);
if (!enabled) {
log.info("Chaos monkey disabled");
+ return false;
}
long monkeyInterval = internals.getTimeRange(
@@ -1799,30 +2167,61 @@
InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
0);
- log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)",
- monkeyInterval, monkeyInterval/(60*60));
+ if (monkeyInterval == 0) {
+ log.debug(
+ "Chaos monkey not configured with a time interval...not enabling");
+ return false;
+ }
+
+ long monkeyDelay = internals.getTimeRange(
+ InternalKeys.CHAOS_MONKEY_DELAY,
+ 0,
+ 0,
+ 0,
+ (int)monkeyInterval);
+
+ log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours -delay {}",
+ monkeyInterval, monkeyInterval/(60*60), monkeyDelay);
monkey = new ChaosMonkeyService(metrics, actionQueues);
+ initAndAddService(monkey);
+
+ // configure the targets
+
+ // launch failure: special case with explicit failure triggered now
+ int amLaunchFailProbability = internals.getOptionInt(
+ InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE,
+ 0);
+ if (amLaunchFailProbability> 0 && monkey.chaosCheck(amLaunchFailProbability)) {
+ log.info("Chaos Monkey has triggered AM Launch failure");
+ // trigger a failure
+ ActionStopSlider stop = new ActionStopSlider("stop",
+ 0, TimeUnit.SECONDS,
+ LauncherExitCodes.EXIT_FALSE,
+ FinalApplicationStatus.FAILED,
+ E_TRIGGERED_LAUNCH_FAILURE);
+ queue(stop);
+ }
+
int amKillProbability = internals.getOptionInt(
InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE);
- if (amKillProbability > 0) {
- monkey.addTarget("AM killer",
- new ChaosKillAM(actionQueues, -1), amKillProbability
- );
- }
+ monkey.addTarget("AM killer",
+ new ChaosKillAM(actionQueues, -1), amKillProbability);
int containerKillProbability = internals.getOptionInt(
InternalKeys.CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE);
- if (containerKillProbability > 0) {
- monkey.addTarget("Container killer",
- new ChaosKillContainer(appState, actionQueues, rmOperationHandler),
- containerKillProbability
- );
- }
- initAndAddService(monkey);
+ monkey.addTarget("Container killer",
+ new ChaosKillContainer(appState, actionQueues, rmOperationHandler),
+ containerKillProbability);
+
// and schedule it
- schedule(monkey.getChaosAction(monkeyInterval, TimeUnit.SECONDS));
- return true;
+ if (monkey.schedule(monkeyDelay, monkeyInterval, TimeUnit.SECONDS)) {
+ log.info("Chaos Monkey is running");
+ return true;
+ } else {
+ log.info("Chaos monkey not started");
+ return false;
+ }
}
/**
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
index c21e249..e2ad559 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -25,7 +25,7 @@
import java.util.concurrent.TimeUnit;
/**
- * Exit a JVM halt.
+ * Exit an emergency JVM halt.
* @see ExitUtil#halt(int, String)
*/
public class ActionHalt extends AsyncAction {
@@ -37,7 +37,9 @@
int status,
String text,
long delay, TimeUnit timeUnit) {
- super("Halt", delay, ActionAttributes.HALTS_CLUSTER);
+
+ // do not declare that this action halts the cluster ... keep it a surprise
+ super("Halt", delay, timeUnit);
this.status = status;
this.text = text;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index c1e7e6e..95bf067 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -30,16 +30,34 @@
import java.util.List;
import java.util.concurrent.TimeUnit;
+/**
+ * Kill a specific container
+ */
public class ActionKillContainer extends AsyncAction {
+ /**
+ * container to kill
+ */
private final ContainerId containerId;
+
+ /**
+ * handler for the operation
+ */
private final RMOperationHandler operationHandler;
+
+ /**
+ * Kill a container
+ * @param containerId container to kill
+ * @param delay
+ * @param timeUnit
+ * @param operationHandler
+ */
public ActionKillContainer(
ContainerId containerId,
long delay,
TimeUnit timeUnit,
RMOperationHandler operationHandler) {
- super("kill container", delay, timeUnit);
+ super("kill container", delay, timeUnit, ATTR_CHANGES_APP_SIZE);
this.operationHandler = operationHandler;
Preconditions.checkArgument(containerId != null);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
index d95dc74..358c844 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
@@ -25,6 +25,7 @@
import org.apache.slider.server.appmaster.state.RoleInstance;
import java.util.Locale;
+import java.util.concurrent.TimeUnit;
/**
* Start a container
@@ -37,15 +38,16 @@
private final RoleInstance instance;
public ActionStartContainer(String name,
- long delay,
Container container,
ContainerLaunchContext ctx,
- RoleInstance instance) {
+ RoleInstance instance,
+ long delay, TimeUnit timeUnit) {
super(
String.format(Locale.ENGLISH,
"%s %s: /",
name , container.getId().toString()),
- delay);
+ delay,
+ timeUnit);
this.container = container;
this.ctx = ctx;
this.instance = instance;
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
index 66a3961..08e8086 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
@@ -20,6 +20,8 @@
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.state.AppState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.concurrent.TimeUnit;
@@ -27,7 +29,9 @@
* Action to tell a queue executor to stop -after handing this on/executing it
*/
public class ActionStopQueue extends AsyncAction {
-
+ private static final Logger log =
+ LoggerFactory.getLogger(ActionStopQueue.class);
+
public ActionStopQueue(long delay) {
super("stop queue", delay);
}
@@ -47,6 +51,6 @@
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
AppState appState) throws Exception {
- // no-op
+ log.warn("STOP");
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
index f084383..d2f23a2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
@@ -18,33 +18,135 @@
package org.apache.slider.server.appmaster.actions;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.state.AppState;
import java.util.concurrent.TimeUnit;
+/**
+ * Trigger an AM exit. This is used to build the exit status message for YARN
+ */
public class ActionStopSlider extends AsyncAction {
- public ActionStopSlider(String message,
- long delay) {
- super(message, delay, ActionAttributes.HALTS_CLUSTER);
- }
+ private int exitCode;
+ private FinalApplicationStatus finalApplicationStatus;
+ private String message;
+
+ /**
+ * Simple constructor
+ * @param name action name
+ */
+ public ActionStopSlider(String name) {
+ super(name);
+ }
+
+
+ /**
+ * Stop slider
+ * @param name action name
+ * @param delay execution delay
+ * @param timeUnit delay time unit
+ * @param exitCode process exit code
+ * @param finalApplicationStatus yarn status
+ * @param message message for AM
+ */
public ActionStopSlider(String name,
long delay,
- TimeUnit timeUnit) {
- super(name, delay, timeUnit, ActionAttributes.HALTS_CLUSTER);
+ TimeUnit timeUnit,
+ int exitCode,
+ FinalApplicationStatus finalApplicationStatus,
+ String message) {
+ super(name, delay, timeUnit, ATTR_HALTS_APP);
+ this.exitCode = exitCode;
+ this.finalApplicationStatus = finalApplicationStatus;
+ this.message = message;
}
+ /**
+ * Stop slider
+ * @param name action name
+ * @param exitCode process exit code
+ * @param finalApplicationStatus yarn status
+ * @param message message for AM
+ */
+ public ActionStopSlider(String name,
+ int exitCode,
+ FinalApplicationStatus finalApplicationStatus, String message) {
+ super(name);
+ this.exitCode = exitCode;
+ this.finalApplicationStatus = finalApplicationStatus;
+ this.message = message;
+ }
+
+ /**
+ * Simple constructor
+ * @param name action name
+ */
+ public ActionStopSlider(TriggerClusterTeardownException ex) {
+ this("stop",
+ ex.getExitCode(),
+ ex.getFinalApplicationStatus(),
+ ex.getMessage());
+ }
+
+ /**
+ * Build from an exception.
+ * <p>
+ * If the exception implements
+ * {@link ExitCodeProvider} then the exit code is extracted from that
+ * @param ex exception.
+ */
+ public ActionStopSlider(Exception ex) {
+ super("stop");
+ if (ex instanceof ExitCodeProvider) {
+ setExitCode(((ExitCodeProvider)ex).getExitCode());
+ } else {
+ setExitCode(LauncherExitCodes.EXIT_EXCEPTION_THROWN);
+ }
+ setFinalApplicationStatus(FinalApplicationStatus.FAILED);
+ setMessage(ex.getMessage());
+ }
+
@Override
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
AppState appState) throws Exception {
- String message = name;
SliderAppMaster.getLog().info("SliderAppMasterApi.stopCluster: {}",
message);
- appMaster.signalAMComplete(
- LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN,
- message);
+ appMaster.onAMStop(this);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s: exit code = %d, %s: %s;",
+ name, exitCode, finalApplicationStatus, message) ;
+ }
+
+ public int getExitCode() {
+ return exitCode;
+ }
+
+ public void setExitCode(int exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ return finalApplicationStatus;
+ }
+
+ public void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus) {
+ this.finalApplicationStatus = finalApplicationStatus;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
index 996390d..f9a1fd5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
@@ -23,8 +23,6 @@
import org.apache.slider.server.appmaster.state.AppState;
import java.io.IOException;
-import java.util.Collections;
-import java.util.EnumSet;
import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -35,7 +33,7 @@
public final String name;
private long nanos;
- private final EnumSet<ActionAttributes> attrs;
+ public final int attrs;
private final long sequenceNumber = sequencer.incrementAndGet();
@@ -51,34 +49,18 @@
protected AsyncAction(String name,
long delay,
TimeUnit timeUnit) {
- this.name = name;
- this.setNanos(convertAndOffset(delay, timeUnit));
- attrs = EnumSet.noneOf(ActionAttributes.class);
+ this(name, delay, timeUnit, 0);
}
protected AsyncAction(String name,
long delay,
TimeUnit timeUnit,
- EnumSet<ActionAttributes> attrs) {
+ int attrs) {
this.name = name;
this.setNanos(convertAndOffset(delay, timeUnit));
this.attrs = attrs;
}
- protected AsyncAction(String name,
- long delay,
- TimeUnit timeUnit,
- ActionAttributes... attributes) {
- this(name, delay, timeUnit);
- Collections.addAll(attrs, attributes);
- }
-
- protected AsyncAction(String name,
- long delayMillis,
- ActionAttributes... attributes) {
- this(name, delayMillis, TimeUnit.MILLISECONDS);
- }
-
protected long convertAndOffset(long delay, TimeUnit timeUnit) {
return now() + TimeUnit.NANOSECONDS.convert(delay, timeUnit);
}
@@ -111,24 +93,25 @@
final StringBuilder sb =
new StringBuilder(super.toString());
sb.append(" name='").append(name).append('\'');
- sb.append(", nanos=").append(getNanos());
+ sb.append(", delay=").append(getDelay(TimeUnit.SECONDS));
sb.append(", attrs=").append(attrs);
sb.append(", sequenceNumber=").append(sequenceNumber);
sb.append('}');
return sb.toString();
}
- protected EnumSet<ActionAttributes> getAttrs() {
+ protected int getAttrs() {
return attrs;
}
/**
- * Ask if an action has a specific attribute
+ * Ask if an action has an of the specified bits set.
+ * This is not an equality test.
* @param attr attribute
- * @return true iff the action has the specific attribute
+ * @return true iff the action has any of the bits in the attr arg set
*/
- public boolean hasAttr(ActionAttributes attr) {
- return attrs.contains(attr);
+ public boolean hasAttr(int attr) {
+ return (attrs & attr) != 0;
}
/**
@@ -148,12 +131,8 @@
public void setNanos(long nanos) {
this.nanos = nanos;
}
-
- public enum ActionAttributes {
- SHRINKS_CLUSTER,
- EXPANDS_CLUSTER,
- HALTS_CLUSTER,
- }
-
-
+
+ public static final int ATTR_CHANGES_APP_SIZE = 1;
+ public static final int ATTR_HALTS_APP = 2;
+ public static final int ATTR_REVIEWS_APP_SIZE = 4;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
index 2aa67bb..41fe494 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
@@ -22,6 +22,8 @@
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.state.AppState;
+import java.util.concurrent.TimeUnit;
+
/**
* Report container loss to the AM
* {@link SliderAppMaster#providerLostContainer(ContainerId)}
@@ -31,13 +33,14 @@
private final ContainerId containerId;
public ProviderReportedContainerLoss(ContainerId containerId) {
- super("lost container " + containerId);
- this.containerId = containerId;
+ this("lost container", 0, TimeUnit.MILLISECONDS, containerId);
}
- public ProviderReportedContainerLoss(
- ContainerId containerId, long delayMillis) {
- super("lost container " + containerId, delayMillis);
+ public ProviderReportedContainerLoss(String name,
+ long delay,
+ TimeUnit timeUnit,
+ ContainerId containerId) {
+ super(name, delay, timeUnit);
this.containerId = containerId;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
index 4577025..957a35f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
@@ -27,10 +27,6 @@
super("ProviderStartupCompleted");
}
- public ProviderStartupCompleted(long delayMillis) {
- super("ProviderStartupCompleted", delayMillis);
- }
-
@Override
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
index cffaf5e..0396891 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
@@ -64,4 +64,9 @@
* @return true if the action was found and removed.
*/
boolean removeRenewingAction(String name);
+
+ /**
+ * Look in the immediate queue for any actions of a specific attribute
+ */
+ boolean hasQueuedActionWithAttribute(int attr);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
index 87956db..1bc933f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
@@ -25,6 +25,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.concurrent.atomic.AtomicBoolean;
+
/**
* Executor for async actions - hands them off to the AM as
* appropriate
@@ -37,6 +39,7 @@
private final QueueService actionQueues;
private final AppState appState;
+
public QueueExecutor(SliderAppMaster appMaster,
QueueService actionQueues) {
Preconditions.checkNotNull(appMaster);
@@ -68,13 +71,20 @@
log.debug("Executing {}", take);
take.execute(appMaster, actionQueues, appState);
+ log.debug("Completed {}", take);
+
} while (!(take instanceof ActionStopQueue));
log.info("Queue Executor run() stopped");
+ } catch (InterruptedException e) {
+ // interrupted: exit
} catch (Exception e) {
log.error("Exception processing {}: {}", take, e, e);
if (appMaster != null) {
appMaster.onExceptionInThread(Thread.currentThread(), e);
}
}
+ // tag completed
+ actionQueues.complete();
}
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 6ad579d..146dea4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -19,7 +19,6 @@
package org.apache.slider.server.appmaster.actions;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.slf4j.Logger;
@@ -33,6 +32,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* The Queue service provides immediate and scheduled queues, as well
@@ -52,6 +52,7 @@
private static final Logger log =
LoggerFactory.getLogger(QueueService.class);
public static final String NAME = "Action Queue";
+ private final AtomicBoolean completed = new AtomicBoolean(false);
/**
* Immediate actions.
@@ -149,6 +150,16 @@
}
}
}
+
+ @Override
+ public boolean hasQueuedActionWithAttribute(int attr) {
+ for (AsyncAction action : actionQueue) {
+ if (action.hasAttr(attr)) {
+ return true;
+ }
+ }
+ return false;
+ }
/**
* Run until the queue has been told to stop
@@ -167,7 +178,25 @@
} while (!(take instanceof ActionStopQueue));
log.info("QueueService processor terminated");
} catch (InterruptedException e) {
- //game over
+ // interrupted during actions
}
+ // the thread exits, but does not tag the service as complete. That's expected
+ // to be done by the stop queue
+ }
+
+
+ /**
+ * Check to see if the queue executor has completed
+ * @return the status
+ */
+ public boolean isCompleted() {
+ return completed.get();
+ }
+
+ /**
+ * Package scoped method to mark the queue service as finished
+ */
+ void complete() {
+ completed.set(true);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
index a8a6fe2..3145ecb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
@@ -25,15 +25,23 @@
import java.util.concurrent.TimeUnit;
+/**
+ * Notify the app master that it should register a component instance
+ * in the registry
+ * {@link SliderAppMaster#registerComponent(ContainerId)}
+ */
public class RegisterComponentInstance extends AsyncAction {
-
public final ContainerId containerId;
+ public final String description;
- public RegisterComponentInstance(ContainerId containerId, long delay,
+ public RegisterComponentInstance(ContainerId containerId,
+ String description,
+ long delay,
TimeUnit timeUnit) {
super("RegisterComponentInstance :" + containerId,
delay, timeUnit);
+ this.description = description;
Preconditions.checkArgument(containerId != null);
this.containerId = containerId;
}
@@ -43,6 +51,6 @@
QueueAccess queueService,
AppState appState) throws Exception {
- appMaster.registerComponent(containerId);
+ appMaster.registerComponent(containerId, description);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
index c62582f..1164df9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
@@ -26,6 +26,8 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* This action executes then reschedules an inner action; a limit
@@ -36,9 +38,12 @@
private static final Logger log =
LoggerFactory.getLogger(RenewingAction.class);
private final A action;
- private final long interval;
- private final TimeUnit timeUnit;
+ private long interval;
+ private TimeUnit timeUnit;
public final AtomicInteger executionCount = new AtomicInteger();
+ private final ReentrantReadWriteLock intervalLock = new ReentrantReadWriteLock();
+ private final Lock intervalReadLock = intervalLock.readLock();
+ private final Lock intervalWriteLock = intervalLock.writeLock();
public final int limit;
@@ -59,6 +64,7 @@
// slightly superfluous as the super init above checks these values...retained
// in case that code is ever changed
Preconditions.checkArgument(action != null, "null actions");
+ Preconditions.checkArgument(interval > 0, "invalid interval: " + interval);
this.action = action;
this.interval = interval;
this.timeUnit = timeUnit;
@@ -85,7 +91,7 @@
reschedule = limit > exCount;
}
if (reschedule) {
- this.setNanos(convertAndOffset(interval, timeUnit));
+ this.setNanos(convertAndOffset(getInterval(), getTimeUnit()));
log.debug("{}: rescheduling, new offset {} mS ", this,
getDelay(TimeUnit.MILLISECONDS));
queueService.schedule(this);
@@ -101,11 +107,31 @@
}
public long getInterval() {
- return interval;
+ intervalReadLock.lock();
+ try {
+ return interval;
+ } finally {
+ intervalReadLock.unlock();
+ }
+ }
+
+ public void updateInterval(long delay, TimeUnit timeUnit) {
+ intervalWriteLock.lock();
+ try {
+ interval = delay;
+ this.timeUnit = timeUnit;
+ } finally {
+ intervalWriteLock.unlock();
+ }
}
public TimeUnit getTimeUnit() {
- return timeUnit;
+ intervalReadLock.lock();
+ try {
+ return timeUnit;
+ } finally {
+ intervalReadLock.unlock();
+ }
}
public int getExecutionCount() {
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java
new file mode 100644
index 0000000..273f599
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tell the AM to execute the full flex review operation
+ */
+public class ReviewAndFlexApplicationSize extends AsyncAction{
+
+ public ReviewAndFlexApplicationSize(String name,
+ long delay,
+ TimeUnit timeUnit) {
+ super(name, delay, timeUnit, ATTR_REVIEWS_APP_SIZE);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster,
+ QueueAccess queueService,
+ AppState appState) throws Exception {
+ appMaster.handleReviewAndFlexApplicationSize(this);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
index 78d9c1c..575fe8f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
@@ -24,12 +24,17 @@
import java.util.concurrent.TimeUnit;
+/**
+ * Tell AM to unregister this component instance
+ * {@link SliderAppMaster#unregisterComponent(ContainerId)}
+ */
public class UnregisterComponentInstance extends AsyncAction {
public final ContainerId containerId;
- public UnregisterComponentInstance(ContainerId containerId, long delay,
+ public UnregisterComponentInstance(ContainerId containerId,
+ long delay,
TimeUnit timeUnit) {
super("UnregisterComponentInstance :" + containerId.toString(),
delay, timeUnit);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
index 5905d2f..87a0aaa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -22,6 +22,7 @@
import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
+import org.apache.slider.api.InternalKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,17 +43,17 @@
/**
* Constructor -includes validation of all arguments
- * @param name
- * @param target
- * @param probability
+ * @param name entry name
+ * @param target target
+ * @param probability probability of occurring
*/
public ChaosEntry(String name, ChaosTarget target, long probability,
MetricRegistry metrics) {
Preconditions.checkArgument(!StringUtils.isEmpty(name), "missing name");
Preconditions.checkArgument(target != null, "null target");
Preconditions.checkArgument(probability > 0, "negative probability");
- Preconditions.checkArgument(probability <= ChaosMonkeyService.PERCENT_100,
- "probability over 100%");
+ Preconditions.checkArgument(probability <= InternalKeys.PROBABILITY_PERCENT_100,
+ "probability over 100%: "+ probability);
this.name = name;
this.target = target;
this.probability = probability;
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
index daf2590..a5cdfc6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.monkey;
import com.google.common.base.Preconditions;
+import org.apache.slider.common.SliderKeys;
import org.apache.slider.server.appmaster.actions.ActionKillContainer;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.operations.RMOperationHandler;
@@ -28,6 +29,7 @@
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.ListIterator;
import java.util.Random;
import java.util.concurrent.TimeUnit;
@@ -60,8 +62,17 @@
public void chaosAction() {
List<RoleInstance> liveContainers =
appState.cloneLiveContainerInfoList();
+ // purge any and all components which declare that they are an AM
+ ListIterator<RoleInstance> containers =
+ liveContainers.listIterator();
+ while (containers.hasNext()) {
+ RoleInstance instance = containers.next();
+ if (SliderKeys.COMPONENT_AM.equals(instance.role)) {
+ containers.remove();
+ }
+ }
int size = liveContainers.size();
- if (size == 0) {
+ if (size <= 0) {
log.info("No containers to kill");
return;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index 592889c..27219e4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -20,6 +20,7 @@
import com.codahale.metrics.MetricRegistry;
import org.apache.hadoop.service.AbstractService;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.slf4j.Logger;
@@ -36,18 +37,12 @@
public class ChaosMonkeyService extends AbstractService {
protected static final Logger log =
LoggerFactory.getLogger(ChaosMonkeyService.class);
- public static final int PERCENT_1 = 100;
- public static final double PERCENT_1D = 100.0;
-
- /**
- * the percentage value as multiplied up
- */
- public static final int PERCENT_100 = 100 * PERCENT_1;
+
private final MetricRegistry metrics;
private final QueueAccess queues;
private final Random random = new Random();
- private static final List<ChaosEntry> chaosEntries =
+ private final List<ChaosEntry> chaosEntries =
new ArrayList<ChaosEntry>();
public ChaosMonkeyService(MetricRegistry metrics, QueueAccess queues) {
@@ -56,28 +51,84 @@
this.queues = queues;
}
-
+ /**
+ * Add a target ... it is only added if <code>probability > 0</code>
+ * @param name name
+ * @param target chaos target
+ * @param probability probability
+ */
public synchronized void addTarget(String name,
ChaosTarget target, long probability) {
- log.info("Adding {} with probability {}", name, probability / PERCENT_1);
- chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
+ if (probability > 0) {
+ log.info("Adding {} with probability {}", name, probability / InternalKeys.PROBABILITY_PERCENT_1);
+ chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
+ } else {
+ log.debug("Action {} not enabled", name);
+ }
}
/**
+ * Get the number of targets in the list
+ * @return the count of added targets
+ */
+ public int getTargetCount() {
+ return chaosEntries.size();
+ }
+
+ /**
* Iterate through all the entries and invoke chaos on those wanted
*/
public void play() {
for (ChaosEntry chaosEntry : chaosEntries) {
- long p = random.nextInt(PERCENT_100);
+ long p = randomPercentage();
chaosEntry.maybeInvokeChaos(p);
}
}
- public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) {
+ public int randomPercentage() {
+ return random.nextInt(InternalKeys.PROBABILITY_PERCENT_100);
+ }
+
+ /**
+ * Check for callers to see if chaos should be triggered; shares the
+ * same random number source as the rest of the monkey entries
+ * @param probability probability
+ * @return true if the action should happen
+ */
+ public boolean chaosCheck(long probability) {
+ return randomPercentage() < probability;
+ }
+
+ /**
+ * Schedule the monkey
+ *
+ * @param delay initial delay
+ * @param timeUnit time unit
+ * @return true if it was scheduled (i.e. 1+ action) and interval > 0
+ */
+ public boolean schedule(long delay, long interval, TimeUnit timeUnit) {
+ if (interval > 0 && !chaosEntries.isEmpty()) {
+ queues.schedule(getChaosAction(delay, interval, timeUnit));
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Get the chaos action
+ *
+ * @param delay
+ * @param timeUnit time unit
+ * @return the action to schedule
+ */
+ public RenewingAction<MonkeyPlayAction> getChaosAction(long delay,
+ long interval,
+ TimeUnit timeUnit) {
RenewingAction<MonkeyPlayAction> action = new RenewingAction<MonkeyPlayAction>(
new MonkeyPlayAction(this, 0, TimeUnit.MILLISECONDS),
- time,
- time,
+ delay,
+ interval,
timeUnit,
0
);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
index f7a95a7..1cbb960 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
@@ -19,21 +19,70 @@
package org.apache.slider.server.appmaster.operations;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
+import java.util.List;
+
/**
- * Hands off RM operations to the Resource Manager
+ * Hands off RM operations to the Resource Manager.
*/
public class AsyncRMOperationHandler extends RMOperationHandler {
protected static final Logger log =
LoggerFactory.getLogger(AsyncRMOperationHandler.class);
private final AMRMClientAsync client;
+ private final Resource maxResources;
- public AsyncRMOperationHandler(AMRMClientAsync client) {
+ public AsyncRMOperationHandler(AMRMClientAsync client, Resource maxResources) {
this.client = client;
+ this.maxResources = maxResources;
+ }
+
+ @Override
+ public int cancelContainerRequests(Priority priority1,
+ Priority priority2,
+ int count) {
+ // need to revoke a previously issued container request
+ // so enum the sets and pick some
+ int remaining = cancelSinglePriorityRequests(priority1, count);
+ remaining = cancelSinglePriorityRequests(priority2, remaining);
+
+ return remaining;
+ }
+
+ /**
+ * Cancel just one of the priority levels
+ * @param priority priority to cancel
+ * @param count count to cancel
+ * @return number of requests cancelled
+ */
+ protected int cancelSinglePriorityRequests(Priority priority,
+ int count) {
+ List<Collection<AMRMClient.ContainerRequest>> requestSets =
+ client.getMatchingRequests(priority, "", maxResources);
+ if (count <= 0) {
+ return 0;
+ }
+ int remaining = count;
+ for (Collection<AMRMClient.ContainerRequest> requestSet : requestSets) {
+ if (remaining == 0) {
+ break;
+ }
+ for (AMRMClient.ContainerRequest request : requestSet) {
+ if (remaining == 0) {
+ break;
+ }
+ // a single release
+ client.removeContainerRequest(request);
+ remaining --;
+ }
+ }
+ return remaining;
}
@Override
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java
new file mode 100644
index 0000000..be5dbab
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelRequestOperation.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.operations;
+
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
+
+/**
+ * Cancel a container request
+ */
+public class CancelRequestOperation extends AbstractRMOperation {
+
+ private final Priority priority1;
+ private final Priority priority2;
+ private final int count;
+
+ public CancelRequestOperation(Priority priority1, Priority priority2, int count) {
+ this.priority1 = priority1;
+ this.priority2 = priority2;
+ this.count = count;
+ }
+
+ @Override
+ public void execute(RMOperationHandler handler) {
+ handler.cancelContainerRequests(priority1, priority2, count);
+ }
+
+ @Override
+ public String toString() {
+ return "release " + count
+ + " requests for " + ContainerPriority.toString(priority1)
+ + " and " + ContainerPriority.toString(priority2);
+ }
+
+ /**
+ * Get the number to release
+ * @return the number of containers to release
+ */
+ public int getCount() {
+ return count;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
index 711bb98..203f898 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.operations;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
public class ContainerRequestOperation extends AbstractRMOperation {
@@ -39,6 +40,6 @@
@Override
public String toString() {
- return "request container ";
+ return "request container for " + ContainerPriority.toString(request.getPriority());
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java
index a24d9e5..66df566 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java
@@ -19,12 +19,13 @@
package org.apache.slider.server.appmaster.operations;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.providers.ProviderService;
public class ProviderNotifyingOperationHandler extends RMOperationHandler {
- final ProviderService providerService;
+ private final ProviderService providerService;
public ProviderNotifyingOperationHandler(ProviderService providerService) {
this.providerService = providerService;
@@ -38,6 +39,12 @@
@Override
public void addContainerRequest(AMRMClient.ContainerRequest req) {
providerService.addContainerRequest(req);
+ }
+ @Override
+ public int cancelContainerRequests(Priority priority1,
+ Priority priority2,
+ int count) {
+ return providerService.cancelContainerRequests(priority1, priority2, count);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java
index 2b6e9e2..3ab9d89 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java
@@ -18,11 +18,12 @@
package org.apache.slider.server.appmaster.operations;
+import org.apache.hadoop.yarn.api.records.Priority;
+
import java.util.List;
public abstract class RMOperationHandler implements RMOperationHandlerActions {
-
/**
* Execute an entire list of operations
* @param operations ops
@@ -32,4 +33,5 @@
operation.execute(this);
}
}
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
index 6659cc9..e6d6c9d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
@@ -19,10 +19,19 @@
package org.apache.slider.server.appmaster.operations;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.client.api.AMRMClient;
public interface RMOperationHandlerActions {
void releaseAssignedContainer(ContainerId containerId);
void addContainerRequest(AMRMClient.ContainerRequest req);
+
+ /**
+ * Remove a container request
+ * @param priority1 priority to remove at
+ * @param priority2 second priority to target
+ * @param count number to remove
+ */
+ int cancelContainerRequests(Priority priority1, Priority priority2, int count);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
index 0cef1d8..4cb6ee1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
@@ -130,7 +130,7 @@
rpcTimeout,
null);
SliderClusterProtocolPB endpoint = protoProxy.getProxy();
- return new SliderClusterProtocolProxy(endpoint);
+ return new SliderClusterProtocolProxy(endpoint, addr);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
index d16b9c8..c1e0886 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
@@ -23,23 +23,35 @@
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.proto.Messages;
import java.io.IOException;
+import java.net.InetSocketAddress;
public class SliderClusterProtocolProxy implements SliderClusterProtocol {
- final SliderClusterProtocolPB endpoint;
private static final RpcController NULL_CONTROLLER = null;
-
- public SliderClusterProtocolProxy(SliderClusterProtocolPB endpoint) {
+ private final SliderClusterProtocolPB endpoint;
+ private final InetSocketAddress address;
+
+ public SliderClusterProtocolProxy(SliderClusterProtocolPB endpoint,
+ InetSocketAddress address) {
this.endpoint = endpoint;
+ this.address = address;
}
private IOException convert(ServiceException se) {
- return ProtobufHelper.getRemoteException(se);
+ IOException ioe = ProtobufHelper.getRemoteException(se);
+ if (ioe instanceof RemoteException) {
+ RemoteException remoteException = (RemoteException) ioe;
+ return new RemoteException(
+ remoteException.getClassName(),
+ address.toString() + ": " + remoteException.getMessage());
+ }
+ return ioe;
}
@Override
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
new file mode 100644
index 0000000..63a7543
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.security;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ *
+ */
+public class SecurityConfiguration {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(SecurityConfiguration.class);
+ private final Configuration configuration;
+ private final AggregateConf instanceDefinition;
+ private String clusterName;
+
+ public SecurityConfiguration(Configuration configuration,
+ AggregateConf instanceDefinition,
+ String clusterName) throws SliderException {
+ Preconditions.checkNotNull(configuration);
+ Preconditions.checkNotNull(instanceDefinition);
+ Preconditions.checkNotNull(clusterName);
+ this.configuration = configuration;
+ this.instanceDefinition = instanceDefinition;
+ this.clusterName = clusterName;
+ validate();
+ }
+
+ private void validate() throws SliderException {
+ if (isSecurityEnabled()) {
+ String principal = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
+ if(SliderUtils.isUnset(principal)) {
+ // if no login identity is available, fail
+ UserGroupInformation loginUser = null;
+ try {
+ loginUser = getLoginUser();
+ } catch (IOException e) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_STATE, e,
+ "No principal configured for the application and "
+ + "exception raised during retrieval of login user. "
+ + "Unable to proceed with application "
+ + "initialization. Please ensure a value "
+ + "for %s exists in the application "
+ + "configuration or the login issue is addressed",
+ SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
+ }
+ if (loginUser == null) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "No principal configured for the application "
+ + "and no login user found. "
+ + "Unable to proceed with application "
+ + "initialization. Please ensure a value "
+ + "for %s exists in the application "
+ + "configuration or the login issue is addressed",
+ SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
+ }
+ }
+ // ensure that either local or distributed keytab mechanism is enabled,
+ // but not both
+ String keytabFullPath = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM)
+ .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+ String keytabName = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM)
+ .get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+ if (SliderUtils.isUnset(keytabFullPath) && SliderUtils.isUnset(keytabName)) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Either a keytab path on the cluster host (%s) or a"
+ + " keytab to be retrieved from HDFS (%s) are"
+ + " required. Please configure one of the keytab"
+ + " retrieval mechanisms.",
+ SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
+ SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+ }
+ if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Both a keytab on the cluster host (%s) and a"
+ + " keytab to be retrieved from HDFS (%s) are"
+ + " specified. Please configure only one keytab"
+ + " retrieval mechanism.",
+ SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
+ SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+
+ }
+ }
+ }
+
+ protected UserGroupInformation getLoginUser() throws IOException {
+ return UserGroupInformation.getLoginUser();
+ }
+
+ public boolean isSecurityEnabled () {
+ return SliderUtils.isHadoopClusterSecure(configuration);
+ }
+
+ public String getPrincipal () throws IOException {
+ String principal = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
+ if (SliderUtils.isUnset(principal)) {
+ principal = UserGroupInformation.getLoginUser().getShortUserName();
+ log.info("No principal set in the slider configuration. Will use AM login"
+ + " identity {} to attempt keytab-based login", principal);
+ }
+
+ return principal;
+ }
+
+ public File getKeytabFile(AggregateConf instanceDefinition)
+ throws SliderException, IOException {
+ String keytabFullPath = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM)
+ .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+ File localKeytabFile;
+ if (SliderUtils.isUnset(keytabFullPath)) {
+ // get the keytab
+ String keytabName = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).
+ get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+ log.info("No host keytab file path specified. Will attempt to retrieve"
+ + " keytab file {} as a local resource for the container",
+ keytabName);
+ // download keytab to local, protected directory
+ localKeytabFile = new File(SliderKeys.KEYTAB_DIR, keytabName);
+ } else {
+ log.info("Leveraging host keytab file {} for login",
+ keytabFullPath);
+ localKeytabFile = new File(keytabFullPath);
+ }
+ return localKeytabFile;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 07976ef..9956db2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -26,7 +26,10 @@
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.client.api.AMRMClient;
@@ -36,6 +39,7 @@
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterDescriptionOperations;
import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
@@ -54,8 +58,10 @@
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.CancelRequestOperation;
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
import org.slf4j.Logger;
@@ -74,14 +80,8 @@
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.slider.api.ResourceKeys.DEF_YARN_CORES;
-import static org.apache.slider.api.ResourceKeys.DEF_YARN_MEMORY;
-import static org.apache.slider.api.ResourceKeys.YARN_CORES;
-import static org.apache.slider.api.ResourceKeys.YARN_MEMORY;
-import static org.apache.slider.api.RoleKeys.ROLE_FAILED_INSTANCES;
-import static org.apache.slider.api.RoleKeys.ROLE_FAILED_STARTING_INSTANCES;
-import static org.apache.slider.api.RoleKeys.ROLE_RELEASING_INSTANCES;
-import static org.apache.slider.api.RoleKeys.ROLE_REQUESTED_INSTANCES;
+import static org.apache.slider.api.ResourceKeys.*;
+import static org.apache.slider.api.RoleKeys.*;
/**
@@ -478,19 +478,16 @@
Set<String> confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf);
-// Add the -site configuration properties
+ // Add the -site configuration properties
for (String key : confKeys) {
String val = publishedProviderConf.get(key);
clientProperties.put(key, val);
}
-
// set the cluster specification (once its dependency the client properties
// is out the way
-
updateInstanceDefinition(instanceDefinition);
-
//build the initial role list
for (ProviderRole providerRole : providerRoles) {
buildRole(providerRole);
@@ -503,7 +500,7 @@
for (String name : roleNames) {
if (!roles.containsKey(name)) {
// this is a new value
- log.info("Adding new role {}", name);
+ log.info("Adding role {}", name);
MapOperations resComponent = resources.getComponent(name);
ProviderRole dynamicRole =
createDynamicProviderRole(name, resComponent);
@@ -520,8 +517,8 @@
instanceDefinition.getResourceOperations().getGlobalOptions();
startTimeThreshold = globalResOpts.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_SHORTLIFE,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE);
+ InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
+ InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
failureThreshold = globalResOpts.getOptionInt(
ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
@@ -578,22 +575,23 @@
* @return a new provider role
* @throws BadConfigException bad configuration
*/
+ @VisibleForTesting
public ProviderRole createDynamicProviderRole(String name,
MapOperations component) throws
BadConfigException {
String priOpt = component.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY);
- int pri = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PRIORITY,
- priOpt, 0, 1, -1
- );
- log.info("Role {} assigned priority {}", name, pri);
+ int priority = SliderUtils.parseAndValidate("value of " + name + " " +
+ ResourceKeys.COMPONENT_PRIORITY,
+ priOpt, 0, 1, -1);
String placementOpt = component.getOption(
- ResourceKeys.COMPONENT_PLACEMENT_POLICY, "0");
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ Integer.toString(PlacementPolicy.DEFAULT));
int placement = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PLACEMENT_POLICY,
- placementOpt, 0, 0, -1
- );
- return new ProviderRole(name, pri, placement);
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ placementOpt, 0, 0, -1);
+ ProviderRole newRole = new ProviderRole(name, priority, placement);
+ log.info("New {} ", newRole);
+ return newRole;
}
@@ -608,6 +606,14 @@
IOException {
instanceDefinition.resolve();
+ // force in the AM desired state values
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+ if (resources.getComponent(SliderKeys.COMPONENT_AM) != null) {
+ resources.setComponentOpt(
+ SliderKeys.COMPONENT_AM, ResourceKeys.COMPONENT_INSTANCES, "1");
+ }
+
//note the time
snapshotTime = now();
//snapshot all three sectons
@@ -638,52 +644,58 @@
/**
* The resource configuration is updated -review and update state.
* @param resources updated resources specification
+ * @return a list of any dynamically added provider roles
+ * (purely for testing purposes)
*/
- public synchronized void updateResourceDefinitions(ConfTree resources) throws
- BadConfigException,
- IOException {
+ public synchronized List<ProviderRole> updateResourceDefinitions(ConfTree resources)
+ throws BadConfigException, IOException {
log.debug("Updating resources to {}", resources);
instanceDefinition.setResources(resources);
onInstanceDefinitionUpdated();
-
-
+
//propagate the role table
-
Map<String, Map<String, String>> updated = resources.components;
getClusterStatus().roles = SliderUtils.deepClone(updated);
getClusterStatus().updateTime = now();
- buildRoleRequirementsFromResources();
+ return buildRoleRequirementsFromResources();
}
/**
* build the role requirements from the cluster specification
+ * @return a list of any dynamically added provider roles
*/
- private void buildRoleRequirementsFromResources() throws
- BadConfigException {
+ private List<ProviderRole> buildRoleRequirementsFromResources() throws BadConfigException {
+
+ List<ProviderRole> newRoles = new ArrayList<ProviderRole>(0);
+
//now update every role's desired count.
//if there are no instance values, that role count goes to zero
ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
+ instanceDefinition.getResourceOperations();
// Add all the existing roles
for (RoleStatus roleStatus : getRoleStatusMap().values()) {
+ if (roleStatus.getExcludeFromFlexing()) {
+ // skip inflexible roles, e.g AM itself
+ continue;
+ }
int currentDesired = roleStatus.getDesired();
String role = roleStatus.getName();
MapOperations comp =
- resources.getComponent(role);
- int desiredInstanceCount =
- resources.getComponentOptInt(role, ResourceKeys.COMPONENT_INSTANCES, 0);
+ resources.getComponent(role);
+ int desiredInstanceCount = getDesiredInstanceCount(resources, role);
if (desiredInstanceCount == 0) {
- log.warn("Role {} has 0 instances specified", role);
+ log.info("Role {} has 0 instances specified", role);
}
if (currentDesired != desiredInstanceCount) {
log.info("Role {} flexed from {} to {}", role, currentDesired,
- desiredInstanceCount);
+ desiredInstanceCount);
roleStatus.setDesired(desiredInstanceCount);
}
}
+
//now the dynamic ones. Iterate through the the cluster spec and
//add any role status entries not in the role status
Set<String> roleNames = resources.getComponentNames();
@@ -691,12 +703,39 @@
if (!roles.containsKey(name)) {
// this is a new value
log.info("Adding new role {}", name);
+ MapOperations component = resources.getComponent(name);
ProviderRole dynamicRole = createDynamicProviderRole(name,
- resources.getComponent(name));
- buildRole(dynamicRole);
+ component);
+ RoleStatus roleStatus = buildRole(dynamicRole);
+ roleStatus.setDesired(getDesiredInstanceCount(resources, name));
+ log.info("New role {}", roleStatus);
roleHistory.addNewProviderRole(dynamicRole);
+ newRoles.add(dynamicRole);
}
}
+ return newRoles;
+ }
+
+ /**
+ * Get the desired instance count of a role, rejecting negative values
+ * @param resources resource map
+ * @param role role name
+ * @return the instance count
+ * @throws BadConfigException if the count is negative
+ */
+ private int getDesiredInstanceCount(ConfTreeOperations resources,
+ String role) throws BadConfigException {
+ int desiredInstanceCount =
+ resources.getComponentOptInt(role, ResourceKeys.COMPONENT_INSTANCES, 0);
+
+ if (desiredInstanceCount < 0) {
+ log.error("Role {} has negative desired instances : {}", role,
+ desiredInstanceCount);
+ throw new BadConfigException(
+ "Negative instance count (%) requested for component %s",
+ desiredInstanceCount, role);
+ }
+ return desiredInstanceCount;
}
/**
@@ -705,8 +744,10 @@
* should be used while setting up the system state -before servicing
* requests.
* @param providerRole role to add
+ * @return the role status built up
+ * @throws BadConfigException if a role of that priority already exists
*/
- public void buildRole(ProviderRole providerRole) throws BadConfigException {
+ public RoleStatus buildRole(ProviderRole providerRole) throws BadConfigException {
//build role status map
int priority = providerRole.id;
if (roleStatusMap.containsKey(priority)) {
@@ -714,18 +755,20 @@
providerRole,
roleStatusMap.get(priority));
}
- roleStatusMap.put(priority,
- new RoleStatus(providerRole));
+ RoleStatus roleStatus = new RoleStatus(providerRole);
+ roleStatusMap.put(priority, roleStatus);
roles.put(providerRole.name, providerRole);
rolePriorityMap.put(priority, providerRole);
+ return roleStatus;
}
/**
* build up the special master node, which lives
* in the live node set but has a lifecycle bonded to the AM
* @param containerId the AM master
- * @param host
- * @param nodeHttpAddress
+ * @param host hostname
+ * @param amPort port
+ * @param nodeHttpAddress http address: may be null
*/
public void buildAppMasterNode(ContainerId containerId,
String host,
@@ -741,6 +784,13 @@
appMasterNode = am;
//it is also added to the set of live nodes
getLiveNodes().put(containerId, am);
+
+ // patch up the role status
+ RoleStatus roleStatus = roleStatusMap.get(
+ (SliderKeys.ROLE_AM_PRIORITY_INDEX));
+ roleStatus.setDesired(1);
+ roleStatus.incActual();
+ roleStatus.incStarted();
}
/**
@@ -781,6 +831,23 @@
return lookupRoleStatus(ContainerPriority.extractRole(c));
}
+ /**
+ * Get a clone of the role status list. Concurrent events may mean this
+ * list (or indeed, some of the role status entries) may be inconsistent
+ * @return a snapshot of the role status entries
+ */
+ public List<RoleStatus> cloneRoleStatusList() {
+ Collection<RoleStatus> statuses = roleStatusMap.values();
+ List<RoleStatus> statusList = new ArrayList<RoleStatus>(statuses.size());
+ try {
+ for (RoleStatus status : statuses) {
+ statusList.add((RoleStatus)(status.clone()));
+ }
+ } catch (CloneNotSupportedException e) {
+ log.warn("Unexpected cloning failure: {}", e, e);
+ }
+ return statusList;
+ }
public RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException {
@@ -934,17 +1001,17 @@
/**
- * enum nodes by role ID, from either the active or live node list
+ * enum nodes by role ID, from either the owned or live node list
* @param roleId role the container must be in
- * @param active flag to indicate "use active list" rather than the smaller
+ * @param owned flag to indicate "use owned list" rather than the smaller
* "live" list
* @return a list of nodes, may be empty
*/
public synchronized List<RoleInstance> enumNodesWithRoleId(int roleId,
- boolean active) {
+ boolean owned) {
List<RoleInstance> nodes = new ArrayList<RoleInstance>();
Collection<RoleInstance> allRoleInstances;
- allRoleInstances = active? ownedContainers.values() : liveNodes.values();
+ allRoleInstances = owned ? ownedContainers.values() : liveNodes.values();
for (RoleInstance node : allRoleInstances) {
if (node.roleId == roleId) {
nodes.add(node);
@@ -1048,9 +1115,10 @@
RoleStatus role,
Resource capability) {
buildResourceRequirements(role, capability);
+ String labelExpression = getLabelExpression(role);
//get the role history to select a suitable node, if available
AMRMClient.ContainerRequest containerRequest =
- createContainerRequest(role, capability);
+ createContainerRequest(role, capability, labelExpression);
return containerRequest;
}
@@ -1060,15 +1128,16 @@
* This is where role history information will be used for placement decisions -
* @param role role
* @param resource requirements
+ * @param labelExpression label expression to satisfy
* @return the container request to submit
*/
public AMRMClient.ContainerRequest createContainerRequest(RoleStatus role,
- Resource resource) {
+ Resource resource,
+ String labelExpression) {
AMRMClient.ContainerRequest request;
- int key = role.getKey();
- request = roleHistory.requestNode(role, resource);
+ request = roleHistory.requestNode(role, resource, labelExpression);
role.incRequested();
return request;
@@ -1103,6 +1172,7 @@
}
return intVal;
}
+
/**
* Build up the resource requirements for this role from the
@@ -1129,6 +1199,17 @@
}
/**
+ * Extract the label expression for this role.
+ * @param role role
+ */
+ public String getLabelExpression(RoleStatus role) {
+ // Set up resource requirements from role values
+ String name = role.getName();
+ ConfTreeOperations resources = getResourcesSnapshot();
+ return resources.getComponentOpt(name, YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION);
+ }
+
+ /**
* add a launched container to the node map for status responses
* @param container id
* @param node node details
@@ -1224,10 +1305,14 @@
}
}
+ public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) {
+ roleHistory.onNodesUpdated(updatedNodes);
+ }
+
/**
* Is a role short lived by the threshold set for this application
* @param instance instance
- * @return true if the instance is considered short live
+ * @return true if the instance is considered short lived
*/
@VisibleForTesting
public boolean isShortLived(RoleInstance instance) {
@@ -1236,7 +1321,7 @@
boolean shortlived;
if (started > 0) {
long duration = time - started;
- shortlived = duration < startTimeThreshold;
+ shortlived = duration < (startTimeThreshold * 1000);
} else {
// never even saw a start event
shortlived = true;
@@ -1254,13 +1339,14 @@
}
/**
- * This is a very small class to send a triple result back from
+ * This is a very small class to send a multiple result back from
* the completion operation
*/
public static class NodeCompletionResult {
public boolean surplusNode = false;
public RoleInstance roleInstance;
public boolean containerFailed;
+ public boolean unknownNode = false;
public String toString() {
@@ -1269,6 +1355,7 @@
sb.append("surplusNode=").append(surplusNode);
sb.append(", roleInstance=").append(roleInstance);
sb.append(", containerFailed=").append(containerFailed);
+ sb.append(", unknownNode=").append(unknownNode);
sb.append('}');
return sb.toString();
}
@@ -1352,6 +1439,7 @@
log.error("Notified of completed container {} that is not in the list" +
" of active or failed containers", containerId);
completionOfUnknownContainerEvent.incrementAndGet();
+ result.unknownNode = true;
}
}
@@ -1532,16 +1620,16 @@
throws TriggerClusterTeardownException {
int failures = role.getFailed();
int threshold = getFailureThresholdForRole(role);
- log.debug("Failure count of role: {}: {}, threshold={}",
+ log.debug("Failure count of component: {}: {}, threshold={}",
role.getName(), failures, threshold);
if (failures > threshold) {
throw new TriggerClusterTeardownException(
SliderExitCodes.EXIT_DEPLOYMENT_FAILED,
- ErrorStrings.E_UNSTABLE_CLUSTER +
- " - failed with role %s failing %d times (%d in startup);" +
+ FinalApplicationStatus.FAILED, ErrorStrings.E_UNSTABLE_CLUSTER +
+ " - failed with component %s failing %d times (%d in startup);" +
" threshold is %d - last failure: %s",
- role.getName(),
+ role.getName(),
role.getFailed(),
role.getStartFailed(),
threshold,
@@ -1552,14 +1640,14 @@
/**
* Get the failure threshold for a specific role, falling back to
* the global one if not
- * @param roleStatus
+ * @param roleStatus role
* @return the threshold for failures
*/
private int getFailureThresholdForRole(RoleStatus roleStatus) {
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
return resources.getComponentOptInt(roleStatus.getName(),
- ResourceKeys.CONTAINER_FAILURE_SHORTLIFE,
+ ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
failureThreshold);
}
@@ -1593,13 +1681,21 @@
String name = role.getName();
synchronized (role) {
delta = role.getDelta();
- details = role.toString();
expected = role.getDesired();
}
- log.info(details);
+ log.info("Reviewing {} : expected {}", role, expected);
checkFailureThreshold(role);
+ if (expected < 0 ) {
+ // negative value: fail
+ throw new TriggerClusterTeardownException(
+ SliderExitCodes.EXIT_DEPLOYMENT_FAILED,
+ FinalApplicationStatus.FAILED,
+ "Negative component count of %d desired for component %s",
+ expected, role);
+ }
+
if (delta > 0) {
log.info("{}: Asking for {} more nodes(s) for a total of {} ", name,
delta, expected);
@@ -1608,13 +1704,11 @@
Resource capability = recordFactory.newResource();
AMRMClient.ContainerRequest containerAsk =
buildContainerResourceAndRequest(role, capability);
- log.info("Container ask is {}", containerAsk);
- if (containerAsk.getCapability().getMemory() >
- this.containerMaxMemory) {
- log.warn(
- "Memory requested: " + containerAsk.getCapability().getMemory() +
- " > " +
- this.containerMaxMemory);
+ log.info("Container ask is {} and label = {}", containerAsk,
+ containerAsk.getNodeLabelExpression());
+ int askMemory = containerAsk.getCapability().getMemory();
+ if (askMemory > this.containerMaxMemory) {
+ log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory);
}
operations.add(new ContainerRequestOperation(containerAsk));
}
@@ -1627,47 +1721,76 @@
//then pick some containers to kill
int excess = -delta;
- // get the nodes to release
- int roleId = role.getKey();
-
- // enum all active nodes that aren't being released
- List<RoleInstance> containersToRelease = enumNodesWithRoleId(roleId, true);
-
- // cut all release-in-progress nodes
- ListIterator<RoleInstance> li = containersToRelease.listIterator();
- while (li.hasNext()) {
- RoleInstance next = li.next();
- if (next.released) {
- li.remove();
+ // how many requests are outstanding
+ int outstandingRequests = role.getRequested();
+ if (outstandingRequests > 0) {
+ // outstanding requests.
+ int toCancel = Math.min(outstandingRequests, excess);
+ Priority p1 =
+ ContainerPriority.createPriority(role.getPriority(), true);
+ Priority p2 =
+ ContainerPriority.createPriority(role.getPriority(), false);
+ operations.add(new CancelRequestOperation(p1, p2, toCancel));
+ role.cancel(toCancel);
+ excess -= toCancel;
+ assert excess >= 0 : "Attempted to cancel too many requests";
+ log.info("Submitted {} cancellations, leaving {} to release",
+ toCancel, excess);
+ if (excess == 0) {
+ log.info("After cancelling requests, application is at desired size");
}
}
- // warn if the desired state can't be reaced
- if (containersToRelease.size() < excess) {
- log.warn("Not enough nodes to release...short of {} nodes",
- containersToRelease.size() - excess);
- }
-
- // ask the release selector to sort the targets
- containersToRelease = containerReleaseSelector.sortCandidates(
- roleId,
- containersToRelease,
- excess);
-
- //crop to the excess
- List<RoleInstance> finalCandidates = (excess < containersToRelease.size())
- ? containersToRelease.subList(0, excess)
- : containersToRelease;
-
+ // after the cancellation there may be no excess
+ if (excess > 0) {
+ // get the nodes to release
+ int roleId = role.getKey();
- // then build up a release operation, logging each container as released
- for (RoleInstance possible : finalCandidates) {
- log.debug("Targeting for release: {}", possible);
- containerReleaseSubmitted(possible.container);
- operations.add(new ContainerReleaseOperation(possible.getId()));
+ // enum all active nodes that aren't being released
+ List<RoleInstance> containersToRelease = enumNodesWithRoleId(roleId, true);
+ if (containersToRelease.isEmpty()) {
+ log.info("No containers for component {}", roleId);
+ }
+
+ // cut all release-in-progress nodes
+ ListIterator<RoleInstance> li = containersToRelease.listIterator();
+ while (li.hasNext()) {
+ RoleInstance next = li.next();
+ if (next.released) {
+ li.remove();
+ }
+ }
+
+ // warn if the desired state can't be reaced
+ int numberAvailableForRelease = containersToRelease.size();
+ if (numberAvailableForRelease < excess) {
+ log.warn("Not enough containers to release, have {} and need {} more",
+ numberAvailableForRelease,
+ excess - numberAvailableForRelease);
+ }
+
+ // ask the release selector to sort the targets
+ containersToRelease = containerReleaseSelector.sortCandidates(
+ roleId,
+ containersToRelease,
+ excess);
+
+ //crop to the excess
+
+ List<RoleInstance> finalCandidates = (excess < numberAvailableForRelease)
+ ? containersToRelease.subList(0, excess)
+ : containersToRelease;
+
+
+ // then build up a release operation, logging each container as released
+ for (RoleInstance possible : finalCandidates) {
+ log.debug("Targeting for release: {}", possible);
+ containerReleaseSubmitted(possible.container);
+ operations.add(new ContainerReleaseOperation(possible.getId()));
+ }
}
-
+
}
return operations;
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
index 369a932..3cc2106 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
@@ -18,10 +18,13 @@
package org.apache.slider.server.appmaster.state;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.util.Records;
+import java.util.Locale;
+
/**
* Class containing the logic to build/split container priorities into the
* different fields used by Slider
@@ -61,14 +64,12 @@
}
/**
- * Map from a container to a role key by way of its priority
- * @param container container
- * @return role key
+ * Does the priority have location
+ * @param priority priority index
+ * @return true if the priority has the location marker
*/
- public static int extractRole(Container container) {
- Priority priority = container.getPriority();
- assert priority != null;
- return extractRole(priority.getPriority());
+ public static boolean hasLocation(int priority) {
+ return (priority ^ NOLOCATION ) != 0;
}
/**
@@ -76,8 +77,34 @@
* @param container container
* @return role key
*/
+ public static int extractRole(Container container) {
+ Priority priority = container.getPriority();
+ return extractRole(priority);
+ }
+
+ /**
+ * Priority record to role mapper
+ * @param priorityRecord priority record
+ * @return the role #
+ */
public static int extractRole(Priority priorityRecord) {
+ Preconditions.checkNotNull(priorityRecord);
return extractRole(priorityRecord.getPriority());
}
+ /**
+ * Convert a priority record to a string, extracting role and locality
+ * @param priorityRecord priority record. May be null
+ * @return a string value
+ */
+ public static String toString(Priority priorityRecord) {
+ if (priorityRecord==null) {
+ return "(null)";
+ } else {
+ return String.format(Locale.ENGLISH,
+ "role %d (locality=%b)",
+ extractRole(priorityRecord),
+ hasLocation(priorityRecord.getPriority()));
+ }
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
index c8ab2a7..ebddaf9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
@@ -23,7 +23,7 @@
* No fields are synchronized; sync on the instance to work with it
*
The two fields `releasing` and `requested` are used to track the ongoing
- state of YARN requests; they do not need to be persisted across freeze/thaw
+ state of YARN requests; they do not need to be persisted across stop/start
cycles. They may be relevant across AM restart, but without other data
structures in the AM, not enough to track what the AM was up to before
it was restarted. The strategy will be to ignore unexpected allocation
@@ -37,10 +37,10 @@
*/
public class NodeEntry {
- public final int index;
+ public final int rolePriority;
- public NodeEntry(int index) {
- this.index = index;
+ public NodeEntry(int rolePriority) {
+ this.rolePriority = rolePriority;
}
/**
@@ -132,8 +132,7 @@
public synchronized boolean onStartFailed() {
decStarting();
++startFailed;
- ++failed;
- return isAvailable();
+ return containerCompleted(false);
}
/**
@@ -211,7 +210,8 @@
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("NodeEntry{");
- sb.append("requested=").append(requested);
+ sb.append("priority=").append(rolePriority);
+ sb.append(", requested=").append(requested);
sb.append(", starting=").append(starting);
sb.append(", live=").append(live);
sb.append(", failed=").append(failed);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index 1ba2282..bc79b71 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -52,7 +52,7 @@
*/
public synchronized NodeEntry get(int role) {
for (NodeEntry nodeEntry : nodeEntries) {
- if (nodeEntry.index == role) {
+ if (nodeEntry.rolePriority == role) {
return nodeEntry;
}
}
@@ -146,7 +146,7 @@
new StringBuilder(toString());
int i = 0;
for (NodeEntry entry : nodeEntries) {
- sb.append(String.format("\n [%02d] ", i++));
+ sb.append(String.format("\n [%02d] ", entry.rolePriority));
sb.append(entry.toString());
}
return sb.toString();
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 0d8b56c..d6022e0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -97,17 +97,18 @@
* @param resource resource
* @param role role
* @param time: time to record
+ * @param labelExpression label to satisfy
* @return the request to raise
*/
public AMRMClient.ContainerRequest buildContainerRequest(Resource resource,
- RoleStatus role, long time) {
+ RoleStatus role, long time, String labelExpression) {
String[] hosts;
boolean relaxLocality;
requestedTime = time;
if (node != null) {
hosts = new String[1];
hosts[0] = node.hostname;
- relaxLocality = false;
+ relaxLocality = !role.isStrictPlacement();
// tell the node it is in play
node.getOrCreate(roleId);
log.info("Submitting request for container on {}", hosts[0]);
@@ -122,8 +123,9 @@
hosts,
null,
pri,
- relaxLocality);
-
+ relaxLocality,
+ labelExpression);
+
return request;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index a0871ae..d5a041b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -26,6 +26,7 @@
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.services.utility.PatternValidator;
@@ -40,6 +41,7 @@
private final Map<String, PublishedConfigSet> publishedConfigSets =
new ConcurrentHashMap<String, PublishedConfigSet>(5);
+ private final PublishedExportsSet publishedExportsSets = new PublishedExportsSet();
private static final PatternValidator validator = new PatternValidator(
RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP);
private String applicationName;
@@ -66,6 +68,11 @@
}
@Override
+ public PublishedExportsSet getPublishedExportsSet() {
+ return publishedExportsSets;
+ }
+
+ @Override
public PublishedConfigSet getPublishedConfigSet(String name) {
return publishedConfigSets.get(name);
}
@@ -204,4 +211,8 @@
appState.refreshClusterStatus();
}
+ @Override
+ public List<RoleStatus> cloneRoleStatusList() {
+ return appState.cloneRoleStatusList();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index edcf7ea..ce2ab0a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -18,10 +18,21 @@
package org.apache.slider.server.appmaster.state;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.common.tools.SliderUtils;
@@ -32,14 +43,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import com.google.common.annotations.VisibleForTesting;
/**
* The Role History.
@@ -59,8 +63,6 @@
protected static final Logger log =
LoggerFactory.getLogger(RoleHistory.class);
private final List<ProviderRole> providerRoles;
- private final Map<String, ProviderRole> providerRoleMap =
- new HashMap<String, ProviderRole>();
private long startTime;
/**
* Time when saved
@@ -88,13 +90,19 @@
*/
private Map<Integer, LinkedList<NodeInstance>> availableNodes;
+ /**
+ * Track the failed nodes. Currently used to make wiser decision of container
+ * ask with/without locality. Has other potential uses as well.
+ */
+ private Set<String> failedNodes = new HashSet<String>();
+
+ // dummy to be used in maps for faster lookup where we don't care about values
+ private final Object DUMMY_VALUE = new Object();
+
public RoleHistory(List<ProviderRole> providerRoles) throws
BadConfigException {
this.providerRoles = providerRoles;
roleSize = providerRoles.size();
- for (ProviderRole providerRole : providerRoles) {
- providerRoleMap.put(providerRole.name, providerRole);
- }
reset();
}
@@ -108,18 +116,24 @@
resetAvailableNodeLists();
outstandingRequests = new OutstandingRequestTracker();
+
Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
-
-
for (ProviderRole providerRole : providerRoles) {
- addProviderRole(roleStats, providerRole);
+ checkProviderRole(roleStats, providerRole);
}
}
-
- private void addProviderRole(Map<Integer, RoleStatus> roleStats,
- ProviderRole providerRole)
- throws ArrayIndexOutOfBoundsException, BadConfigException {
+ /**
+ * safety check: make sure the provider role is unique amongst
+ * the role stats...which is extended with the new role
+ * @param roleStats role stats
+ * @param providerRole role
+ * @throws ArrayIndexOutOfBoundsException
+ * @throws BadConfigException
+ */
+ protected void checkProviderRole(Map<Integer, RoleStatus> roleStats,
+ ProviderRole providerRole)
+ throws BadConfigException {
int index = providerRole.id;
if (index < 0) {
throw new BadConfigException("Provider " + providerRole
@@ -140,14 +154,17 @@
*/
public void addNewProviderRole(ProviderRole providerRole)
throws BadConfigException {
+ log.debug("Validating/adding new provider role to role history: {} ",
+ providerRole);
Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
-
for (ProviderRole role : providerRoles) {
roleStats.put(role.id, new RoleStatus(role));
}
- addProviderRole(roleStats, providerRole);
+ checkProviderRole(roleStats, providerRole);
+ log.debug("Check successful; adding role");
+ this.providerRoles.add(providerRole);
}
/**
@@ -233,7 +250,7 @@
/**
* Get the node instance for the specific node -creating it if needed
- * @param nodeAddr node address
+ * @param hostname node address
* @return the instance
*/
public synchronized NodeInstance getOrCreateNodeInstance(String hostname) {
@@ -350,11 +367,11 @@
* Handler for bootstrap event
*/
public void onBootstrap() {
- log.info("Role history bootstrapped");
+ log.debug("Role history bootstrapped");
}
/**
- * Handle the thaw process <i>after the history has been rebuilt</i>,
+ * Handle the start process <i>after the history has been rebuilt</i>,
* and after any gc/purge
*/
@VisibleForTesting
@@ -371,7 +388,7 @@
}
if (loaded != null) {
thawSuccessful = true;
- log.info("loaded history from {}", loaded);
+ log.debug("loaded history from {}", loaded);
// delete any old entries
try {
int count = historyWriter.purgeOlderHistoryEntries(filesystem, loaded);
@@ -381,7 +398,7 @@
e);
}
- //thaw is then completed
+ //start is then completed
buildAvailableNodeLists();
} else {
//fallback to bootstrap procedure
@@ -392,7 +409,7 @@
/**
- * (After the thaw), rebuild the availability datastructures
+ * (After the start), rebuild the availability data structures
*/
@VisibleForTesting
public synchronized void buildAvailableNodeLists() {
@@ -418,9 +435,10 @@
/**
* Get the nodes for an ID -may be null
* @param id role ID
- * @return list
+ * @return potenially null list
*/
- private LinkedList<NodeInstance> getNodesForRoleId(int id) {
+ @VisibleForTesting
+ public List<NodeInstance> getNodesForRoleId(int id) {
return availableNodes.get(id);
}
@@ -450,10 +468,6 @@
}
}
- public synchronized void onAMRestart() {
- //TODO once AM restart is implemented and we know what to expect
- }
-
/**
* Find a node for use
* @param role role
@@ -469,7 +483,7 @@
List<NodeInstance> targets = getNodesForRoleId(roleKey);
int cnt = targets == null ? 0 : targets.size();
- log.info("There're {} nodes to consider for {}", cnt, role.getName());
+ log.debug("There are {} node(s) to consider for {}", cnt, role.getName());
while (targets != null && !targets.isEmpty() && nodeInstance == null) {
NodeInstance head = targets.remove(0);
if (head.getActiveRoleInstances(roleKey) == 0) {
@@ -495,12 +509,28 @@
*
* @param node node to target or null for "any"
* @param role role to request
+ * @param labelExpression label to satisfy
* @return the container priority
*/
public synchronized AMRMClient.ContainerRequest requestInstanceOnNode(
- NodeInstance node, RoleStatus role, Resource resource) {
+ NodeInstance node, RoleStatus role, Resource resource, String labelExpression) {
OutstandingRequest outstanding = outstandingRequests.addRequest(node, role.getKey());
- return outstanding.buildContainerRequest(resource, role, now());
+ return outstanding.buildContainerRequest(resource, role, now(), labelExpression);
+ }
+
+ /**
+ * Find a node for a role and request an instance on that (or a location-less
+ * instance) with a label expression
+ * @param role role status
+ * @param resource resource capabilities
+ * @param labelExpression label to satisfy
+ * @return a request ready to go
+ */
+ public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role,
+ Resource resource,
+ String labelExpression) {
+ NodeInstance node = findNodeForNewInstance(role);
+ return requestInstanceOnNode(node, role, resource, labelExpression);
}
/**
@@ -513,7 +543,7 @@
public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role,
Resource resource) {
NodeInstance node = findNodeForNewInstance(role);
- return requestInstanceOnNode(node, role, resource);
+ return requestInstanceOnNode(node, role, resource, null);
}
/**
@@ -586,6 +616,8 @@
public synchronized boolean onContainerAllocated(Container container, int desiredCount, int actualCount) {
int role = ContainerPriority.extractRole(container);
String hostname = RoleHistoryUtils.hostnameOf(container);
+ LinkedList<NodeInstance> nodeInstances =
+ getOrCreateNodesForRoleId(role);
boolean requestFound =
outstandingRequests.onContainerAllocated(role, hostname);
if (desiredCount <= actualCount) {
@@ -595,7 +627,7 @@
if (!hosts.isEmpty()) {
//add the list
log.debug("Adding {} hosts for role {}", hosts.size(), role);
- getOrCreateNodesForRoleId(role).addAll(hosts);
+ nodeInstances.addAll(hosts);
sortAvailableNodeList(role);
}
}
@@ -644,6 +676,28 @@
}
/**
+ * Update failedNodes and nodemap based on the node state
+ *
+ * @param updatedNodes list of updated nodes
+ */
+ public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) {
+ for (NodeReport updatedNode : updatedNodes) {
+ String hostname = updatedNode.getNodeId() == null ? null : updatedNode
+ .getNodeId().getHost();
+ if (hostname == null) {
+ continue;
+ }
+ if (updatedNode.getNodeState() != null
+ && updatedNode.getNodeState().isUnusable()) {
+ failedNodes.add(hostname);
+ nodemap.remove(hostname);
+ } else {
+ failedNodes.remove(hostname);
+ }
+ }
+ }
+
+ /**
* A container release request was issued
* @param container container submitted
*/
@@ -688,6 +742,8 @@
boolean wasReleased,
boolean shortLived) {
NodeEntry nodeEntry = getOrCreateNodeEntry(container);
+ log.debug("Finished container for node {}, released={}, shortlived={}",
+ nodeEntry.rolePriority, wasReleased, shortLived);
boolean available;
if (shortLived) {
nodeEntry.onStartFailed();
@@ -731,13 +787,16 @@
List<NodeInstance> instances =
getOrCreateNodesForRoleId(role.id);
log.info(" available: " + instances.size()
- + " " + SliderUtils.joinWithInnerSeparator(", ", instances));
+ + " " + SliderUtils.joinWithInnerSeparator(" ", instances));
}
log.info("Nodes in Cluster: {}", getClusterSize());
for (NodeInstance node : nodemap.values()) {
log.info(node.toFullString());
}
+
+ log.info("Failed nodes: {}",
+ SliderUtils.joinWithInnerSeparator(" ", failedNodes));
}
/**
@@ -759,5 +818,15 @@
return outstandingRequests.listOutstandingRequests();
}
+ /**
+ * Get a clone of the failedNodes
+ *
+ * @return the list
+ */
+ public List<String> cloneFailedNodes() {
+ List<String> lst = new ArrayList<String>();
+ lst.addAll(failedNodes);
+ return lst;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index e373843..c8ddc6f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -22,12 +22,13 @@
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ProtocolTypes;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -50,8 +51,13 @@
* already been targeted for termination
*/
public boolean released;
+
+ /**
+ * Name of the role
+ */
public String role;
public int roleId;
+
/**
* state from {@link ClusterDescription}
*/
@@ -90,8 +96,8 @@
/**
* A list of registered endpoints.
*/
- private List<RegisteredEndpoint> endpoints =
- new ArrayList<RegisteredEndpoint>(2);
+ private List<Endpoint> endpoints =
+ new ArrayList<Endpoint>(2);
public RoleInstance(Container container) {
Preconditions.checkNotNull(container, "Null container");
@@ -194,7 +200,7 @@
public Object clone() throws CloneNotSupportedException {
RoleInstance cloned = (RoleInstance) super.clone();
// clone the endpoint list, but not the values
- cloned.endpoints = new ArrayList<RegisteredEndpoint>(this.endpoints);
+ cloned.endpoints = new ArrayList<Endpoint>(this.endpoints);
return cloned;
}
@@ -202,15 +208,15 @@
* Get the list of endpoints.
* @return the endpoint list.
*/
- public List<RegisteredEndpoint> getEndpoints() {
+ public List<Endpoint> getEndpoints() {
return endpoints;
}
/**
* Add an endpoint registration
- * @param endpoint
+ * @param endpoint endpoint (non-null)
*/
- public void addEndpoint(RegisteredEndpoint endpoint) {
+ public void addEndpoint(Endpoint endpoint) {
Preconditions.checkArgument(endpoint != null);
endpoints.add(endpoint);
}
@@ -218,13 +224,13 @@
/**
* Register a port endpoint as an inet-addr formatted endpoint, using the
* hostname as the first part of the address
- * @param port
- * @param protocol
- * @param text
+ * @param port port port
+ * @param api API API name
*/
- public void registerPortEndpoint(int port, String protocol, String text) {
- InetSocketAddress addr = new InetSocketAddress(host, port);
- RegisteredEndpoint epr = new RegisteredEndpoint(addr, protocol, text);
+ public void registerPortEndpoint(int port, String api) {
+ Endpoint epr =
+ RegistryTypeUtils.inetAddrEndpoint(api,
+ ProtocolTypes.PROTOCOL_TCP, host, port);
addEndpoint(epr);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index df4ab8e..3c860d6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -22,6 +22,8 @@
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
+import java.io.Serializable;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@@ -32,10 +34,8 @@
*/
public final class RoleStatus implements Cloneable {
-
private final String name;
-
/**
* Role key in the container details stored in the AM,
* currently mapped to priority
@@ -45,8 +45,25 @@
private final ProviderRole providerRole;
private int desired, actual, requested, releasing;
- private volatile int failed, started, startFailed, completed, totalRequested;
+ private int failed, started, startFailed, completed, totalRequested;
+ /**
+ * value to use when specifiying "no limit" for instances: {@value}
+ */
+ public static final int UNLIMITED_INSTANCES = 1;
+
+ /**
+ * minimum number of instances of a role permitted in a valid
+ * configuration. Default: 0.
+ */
+ private int minimum = 0;
+
+ /**
+ * maximum number of instances of a role permitted in a valid
+ * configuration. Default: unlimited.
+ */
+ private int maximum = UNLIMITED_INSTANCES;
+
private String failureMessage = "";
public RoleStatus(ProviderRole providerRole) {
@@ -54,7 +71,7 @@
this.name = providerRole.name;
this.key = providerRole.id;
}
-
+
public String getName() {
return name;
}
@@ -83,27 +100,29 @@
public boolean getNoDataLocality() {
return 0 != (getPlacementPolicy() & PlacementPolicy.NO_DATA_LOCALITY);
}
+
+ public boolean isStrictPlacement() {
+ return 0 != (getPlacementPolicy() & PlacementPolicy.STRICT);
+ }
- public int getDesired() {
+ public synchronized int getDesired() {
return desired;
}
- public void setDesired(int desired) {
+ public synchronized void setDesired(int desired) {
this.desired = desired;
}
- public int getActual() {
+ public synchronized int getActual() {
return actual;
}
- public int incActual() {
+ public synchronized int incActual() {
return ++actual;
}
- public int decActual() {
- if (0 > --actual) {
- actual = 0;
- }
+ public synchronized int decActual() {
+ actual = Math.max(0, actual - 1);
return actual;
}
@@ -116,29 +135,29 @@
return ++requested;
}
- public synchronized int decRequested() {
- if (0 > --requested) {
- requested = 0;
- }
+ public synchronized int cancel(int count) {
+ requested = Math.max(0, requested - count);
return requested;
}
+
+ public synchronized int decRequested() {
+ return cancel(1);
+ }
- public int getReleasing() {
+ public synchronized int getReleasing() {
return releasing;
}
- public int incReleasing() {
+ public synchronized int incReleasing() {
return ++releasing;
}
- public int decReleasing() {
- if (0 > --releasing) {
- releasing = 0;
- }
+ public synchronized int decReleasing() {
+ releasing = Math.max(0, releasing - 1);
return releasing;
}
- public int getFailed() {
+ public synchronized int getFailed() {
return failed;
}
@@ -146,7 +165,7 @@
* Reset the failure counts
* @return the total number of failures up to this point
*/
- public int resetFailed() {
+ public synchronized int resetFailed() {
int total = failed + startFailed;
failed = 0;
startFailed = 0;
@@ -161,7 +180,7 @@
* @return the number of failures
* @param text text about the failure
*/
- public int noteFailed(boolean startupFailure, String text) {
+ public synchronized int noteFailed(boolean startupFailure, String text) {
int current = ++failed;
if (text != null) {
failureMessage = text;
@@ -173,41 +192,42 @@
return current;
}
- public int getStartFailed() {
+ public synchronized int getStartFailed() {
return startFailed;
}
- public void incStartFailed() {
+ public synchronized void incStartFailed() {
startFailed++;
}
- public String getFailureMessage() {
+ public synchronized String getFailureMessage() {
return failureMessage;
}
- public int getCompleted() {
+ public synchronized int getCompleted() {
return completed;
}
- public void setCompleted(int completed) {
+ public synchronized void setCompleted(int completed) {
this.completed = completed;
}
- public int incCompleted() {
+ public synchronized int incCompleted() {
return completed ++;
}
- public int getStarted() {
+ public synchronized int getStarted() {
return started;
}
- public void incStarted() {
+ public synchronized void incStarted() {
started++;
}
- public int getTotalRequested() {
+ public synchronized int getTotalRequested() {
return totalRequested;
}
+
/**
* Get the number of roles we are short of.
* nodes released are ignored.
@@ -215,9 +235,8 @@
* 0 means "do nothing".
*/
public synchronized int getDelta() {
- int inuse = actual + requested;
+ int inuse = getActualAndRequested();
//don't know how to view these. Are they in-use or not?
- //inuse += releasing;
int delta = desired - inuse;
if (delta < 0) {
//if we are releasing, remove the number that are already released.
@@ -228,11 +247,21 @@
return delta;
}
+ /**
+ * Get count of actual and requested containers
+ * @return the size of the application when outstanding requests are included
+ */
+ public synchronized int getActualAndRequested() {
+ return actual + requested;
+ }
+
@Override
- public String toString() {
+ public synchronized String toString() {
return "RoleStatus{" +
"name='" + name + '\'' +
", key=" + key +
+ ", minimum=" + minimum +
+ ", maximum=" + maximum +
", desired=" + desired +
", actual=" + actual +
", requested=" + requested +
@@ -242,18 +271,17 @@
", startFailed=" + startFailed +
", completed=" + completed +
", failureMessage='" + failureMessage + '\'' +
-
'}';
}
@Override
- public Object clone() throws CloneNotSupportedException {
+ public synchronized Object clone() throws CloneNotSupportedException {
return super.clone();
}
/**
* Get the provider role
- * @return
+ * @return the provider role
*/
public ProviderRole getProviderRole() {
return providerRole;
@@ -263,7 +291,7 @@
* Build the statistics map from the current data
* @return a map for use in statistics reports
*/
- public Map<String, Integer> buildStatistics() {
+ public synchronized Map<String, Integer> buildStatistics() {
Map<String, Integer> stats = new HashMap<String, Integer>();
stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, getRequested());
stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, getCompleted());
@@ -275,4 +303,27 @@
stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, getStartFailed());
return stats;
}
+
+ /**
+ * Compare two role status entries by name
+ */
+ public static class CompareByName implements Comparator<RoleStatus>,
+ Serializable {
+ @Override
+ public int compare(RoleStatus o1, RoleStatus o2) {
+ return o1.getName().compareTo(o2.getName());
+ }
+ }
+
+ /**
+ * Compare two role status entries by key
+ */
+ public static class CompareByKey implements Comparator<RoleStatus>,
+ Serializable {
+ @Override
+ public int compare(RoleStatus o1, RoleStatus o2) {
+ return (o1.getKey() < o2.getKey() ? -1 : (o1.getKey() == o2.getKey() ? 0 : 1));
+ }
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index 1714f75..75076ed 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -26,6 +26,7 @@
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import java.util.Collection;
import java.util.List;
@@ -51,6 +52,12 @@
PublishedConfigSet getPublishedSliderConfigurations();
/**
+ * Get the published exports set
+ * @return
+ */
+ PublishedExportsSet getPublishedExportsSet();
+
+ /**
* Get a named published config set
* @param name name to look up
* @return the instance or null
@@ -194,9 +201,8 @@
/**
* Update the cluster description with anything interesting
- * @param providerStatus status from the provider for the cluster info section
*/
void refreshClusterStatus();
-
+ List<RoleStatus> cloneRoleStatusList();
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
index 08338e8..f840035 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
@@ -16,38 +16,22 @@
*/
package org.apache.slider.server.appmaster.web;
-import org.apache.hadoop.service.AbstractService;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
+import org.apache.slider.server.services.workflow.ClosingService;
+import org.apache.slider.server.services.workflow.WorkflowCompositeService;
/**
- *
+ * agent service gives the agent webapp lifecycle integration
*/
-public class AgentService extends AbstractService {
- private volatile AgentWebApp webApp;
+public class AgentService extends ClosingService<AgentWebApp> {
+
public AgentService(String name) {
super(name);
}
public AgentService(String name, AgentWebApp app) {
- super(name);
- webApp = app;
+ super(name, app);
}
- @Override
- protected void serviceStart() throws Exception {
-
- }
-
- /**
- * Stop operation stops the webapp; sets the reference to null
- * @throws Exception
- */
- @Override
- protected void serviceStop() throws Exception {
- if (webApp != null) {
- webApp.stop();
- webApp = null;
- }
- }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
index 9192efe..7f30440 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -21,18 +21,13 @@
import com.sun.jersey.api.core.ResourceConfig;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.spi.container.servlet.ServletContainer;
-import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.yarn.webapp.Dispatcher;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig;
import org.apache.slider.server.appmaster.web.rest.AMWebServices;
import org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider;
-import org.apache.slider.server.services.curator.CuratorHelper;
-import org.apache.slider.server.services.registry.RegistryDiscoveryContext;
-import org.apache.slider.server.services.registry.RegistryRestResources;
-import org.apache.slider.server.services.registry.SliderRegistryService;
import java.util.HashMap;
import java.util.Map;
@@ -47,9 +42,9 @@
public static final String CONTAINER_STATS = "/stats";
public static final String CLUSTER_SPEC = "/spec";
- public final SliderRegistryService registry;
+ private final RegistryOperations registry;
- public SliderAMWebApp(SliderRegistryService registry) {
+ public SliderAMWebApp(RegistryOperations registry) {
Preconditions.checkArgument(registry != null, "registry null");
this.registry = registry;
}
@@ -66,18 +61,6 @@
// bind the REST interface
bind(AMWebServices.class);
//bind(AMAgentWebServices.class);
-
- CuratorHelper curatorHelper = registry.getCuratorHelper();
- ServiceDiscovery<ServiceInstanceData> discovery = registry.getDiscovery();
- RegistryDiscoveryContext discoveryContext = curatorHelper
- .createDiscoveryContext(
- discovery);
-
- bind(RegistryDiscoveryContext.class).toInstance(discoveryContext);
- RegistryRestResources registryRestResources =
- new RegistryRestResources(discoveryContext, registry);
- bind(RegistryRestResources.class).toInstance(registryRestResources);
-
route("/", SliderAMController.class);
route(CONTAINER_STATS, SliderAMController.class, "containerStats");
route(CLUSTER_SPEC, SliderAMController.class, "specification");
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
deleted file mode 100644
index 5fffa4a..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.slider.server.appmaster.web;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.FilterContainer;
-import org.apache.hadoop.http.FilterInitializer;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class SliderAmFilterInitializer extends FilterInitializer {
- private static final String FILTER_NAME = "AM_PROXY_FILTER";
- private static final String FILTER_CLASS = SliderAmIpFilter.class.getCanonicalName();
- private static final String HTTPS_PREFIX = "https://";
- private static final String HTTP_PREFIX = "http://";
- private Configuration configuration;
-
- public static final String NAME =
- "org.apache.slider.server.appmaster.web.SliderAmFilterInitializer";
-
- @Override
- public void initFilter(FilterContainer container, Configuration conf) {
- configuration = conf;
- Map<String, String> params = new HashMap<String, String>();
- String proxy = WebAppUtils.getProxyHostAndPort(conf);
- String[] parts = proxy.split(":");
- params.put(SliderAmIpFilter.PROXY_HOST, parts[0]);
- // todo: eventually call WebAppUtils.getHttpSchemePrefix
- params.put(SliderAmIpFilter.PROXY_URI_BASE, getHttpSchemePrefix()
- + proxy + getApplicationWebProxyBase());
- params.put(SliderAmIpFilter.WS_CONTEXT_ROOT,
- conf.get(SliderAmIpFilter.WS_CONTEXT_ROOT));
- container.addFilter(FILTER_NAME, FILTER_CLASS, params);
- }
-
- @VisibleForTesting
- protected String getApplicationWebProxyBase() {
- return System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV);
- }
-
- private String getHttpSchemePrefix() {
- return HttpConfig.Policy.HTTPS_ONLY ==
- HttpConfig.Policy.fromString(configuration
- .get(
- YarnConfiguration.YARN_HTTP_POLICY_KEY,
- YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))
- ? HTTPS_PREFIX : HTTP_PREFIX;
- }
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
deleted file mode 100644
index 4c66876..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.slider.server.appmaster.web;
-
-import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
-import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpPrincipal;
-import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.Cookie;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public class SliderAmIpFilter implements Filter {
- protected static final Logger log =
- LoggerFactory.getLogger(SliderAmIpFilter.class);
-
- public static final String PROXY_HOST = "PROXY_HOST";
- public static final String PROXY_URI_BASE = "PROXY_URI_BASE";
- //update the proxy IP list about every 5 min
- private static final long updateInterval = 5 * 60 * 1000;
- public static final String WS_CONTEXT_ROOT = "slider.rest.context.root";
-
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private String proxyHost;
- private Set<String> proxyAddresses = null;
- private long lastUpdate;
- private String proxyUriBase;
- private List<String> wsContextRoots;
-
- @Override
- public void init(FilterConfig conf) throws ServletException {
- proxyHost = conf.getInitParameter(PROXY_HOST);
- proxyUriBase = conf.getInitParameter(PROXY_URI_BASE);
- wsContextRoots = Arrays.asList(conf.getInitParameter(WS_CONTEXT_ROOT).split("\\|"));
- }
-
- protected Set<String> getProxyAddresses() throws ServletException {
- long now = System.currentTimeMillis();
- synchronized(this) {
- if(proxyAddresses == null || (lastUpdate + updateInterval) >= now) {
- try {
- proxyAddresses = new HashSet<String>();
- for(InetAddress add : InetAddress.getAllByName(proxyHost)) {
- if (log.isDebugEnabled()) {
- log.debug("proxy address is: " + add.getHostAddress());
- }
- proxyAddresses.add(add.getHostAddress());
- }
- lastUpdate = now;
- } catch (UnknownHostException e) {
- throw new ServletException("Could not locate "+proxyHost, e);
- }
- }
- return proxyAddresses;
- }
- }
-
- @Override
- public void destroy() {
- //Empty
- }
-
- @Override
- public void doFilter(ServletRequest req, ServletResponse resp,
- FilterChain chain) throws IOException, ServletException {
- if(!(req instanceof HttpServletRequest)) {
- throw new ServletException("This filter only works for HTTP/HTTPS");
- }
-
- HttpServletRequest httpReq = (HttpServletRequest)req;
- HttpServletResponse httpResp = (HttpServletResponse)resp;
- if (log.isDebugEnabled()) {
- log.debug("Remote address for request is: " + httpReq.getRemoteAddr());
- }
- String requestURI = httpReq.getRequestURI();
- if(!isWsRequest(requestURI) &&
- !getProxyAddresses().contains(httpReq.getRemoteAddr())) {
- String redirectUrl = httpResp.encodeRedirectURL(proxyUriBase +
- requestURI);
- httpResp.sendRedirect(redirectUrl);
- return;
- }
-
- String user = null;
-
- if (httpReq.getCookies() != null) {
- for(Cookie c: httpReq.getCookies()) {
- if(WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())){
- user = c.getValue();
- break;
- }
- }
- }
- try {
- if (user == null) {
- log.debug("Could not find " + WebAppProxyServlet.PROXY_USER_COOKIE_NAME
- + " cookie, so user will not be set");
- chain.doFilter(req, resp);
- } else {
- final AmIpPrincipal principal = new AmIpPrincipal(user);
- ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq,
- principal);
- chain.doFilter(requestWrapper, resp);
- }
-// JKD7 } catch (IOException | ServletException e) {
- } catch (IOException e) {
- log.warn("When fetching {}: {}", requestURI, e);
- throw e;
- } catch (ServletException e) {
- log.warn("When fetching {}: {}", requestURI, e);
- throw e;
- }
- }
-
- private boolean isWsRequest(String requestURI) {
- boolean isWsReq = false;
- for (String wsContext : wsContextRoots) {
- isWsReq = requestURI.startsWith(wsContext);
- if (isWsReq) break;
- }
-
- return isWsReq;
- }
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
index aa20baa..c8c47c9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.server.appmaster.web;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.state.AppState;
@@ -56,10 +57,13 @@
* Generate a mapping from role name to its {@link RoleStatus}. Be aware that this
* is a computed value and not just a getter
*/
- public Map<String,RoleStatus> getRoleStatusByName();
+ public Map<String, RoleStatus> getRoleStatusByName();
/**
* Returns an interface that can support the agent-based REST operations.
*/
public AgentRestOperations getAgentRestOperations();
+
+
+ RegistryOperations getRegistryOperations();
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index 4d595a9..3b47ed1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -16,9 +16,8 @@
*/
package org.apache.slider.server.appmaster.web;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -27,10 +26,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.TreeMap;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -41,17 +38,18 @@
public class WebAppApiImpl implements WebAppApi {
private static final Logger log = LoggerFactory.getLogger(WebAppApiImpl.class);
- protected static final ProviderRole AM_ROLE_NAME = new ProviderRole("Slider Application Master", SliderKeys.ROLE_AM_PRIORITY_INDEX);
-
protected final SliderClusterProtocol clusterProto;
protected final StateAccessForProviders appState;
protected final ProviderService provider;
protected final CertificateManager certificateManager;
-
+ private final RegistryOperations registryOperations;
+
public WebAppApiImpl(SliderClusterProtocol clusterProto,
- StateAccessForProviders appState,
- ProviderService provider,
- CertificateManager certificateManager) {
+ StateAccessForProviders appState,
+ ProviderService provider,
+ CertificateManager certificateManager,
+ RegistryOperations registryOperations) {
+ this.registryOperations = registryOperations;
checkNotNull(clusterProto);
checkNotNull(appState);
checkNotNull(provider);
@@ -95,54 +93,23 @@
* @see org.apache.slider.server.appmaster.web.WebAppApi#getRoleStatusByName()
*/
@Override
- public TreeMap<String,RoleStatus> getRoleStatusByName() {
- Map<Integer,ProviderRole> rolesById = rolesById(provider.getRoles());
- Map<Integer,RoleStatus> status = appState.getRoleStatusMap();
-
- return getRoleStatusesByName(rolesById, status);
- }
-
- /**
- * Get the ProviderRoles by their index
- * @param roles
- * @return
- */
- private Map<Integer,ProviderRole> rolesById(List<ProviderRole> roles) {
- Map<Integer,ProviderRole> rolesById = new HashMap<Integer,ProviderRole>();
- rolesById.put(SliderKeys.ROLE_AM_PRIORITY_INDEX, AM_ROLE_NAME);
-
- for (ProviderRole role : roles) {
- rolesById.put(role.id, role);
+ public Map<String,RoleStatus> getRoleStatusByName() {
+ List<RoleStatus> roleStatuses = appState.cloneRoleStatusList();
+ TreeMap<String, RoleStatus> map =
+ new TreeMap<String, RoleStatus>();
+ for (RoleStatus status : roleStatuses) {
+ map.put(status.getName(), status);
}
-
- return rolesById;
- }
-
- /**
- * Join the ProviderRole by their ID with the RoleStatus by their ID, to get the RoleStatus by role name.
- * @param rolesById
- * @param statusById
- * @return A Map of RoleStatus by the role name
- */
- private TreeMap<String, RoleStatus> getRoleStatusesByName(Map<Integer, ProviderRole> rolesById,
- Map<Integer, RoleStatus> statusById) {
- TreeMap<String, RoleStatus> statusByName = new TreeMap<String, RoleStatus>();
- for (Entry<Integer, ProviderRole> role : rolesById.entrySet()) {
- final RoleStatus status = statusById.get(role.getKey());
-
- if (null == status) {
- log.error("Found ID ({}) which has no known ProviderRole",
- role.getKey());
- } else {
- statusByName.put(role.getValue().name, status);
- }
- }
-
- return statusByName;
+ return map;
}
@Override
public AgentRestOperations getAgentRestOperations() {
return provider.getAgentRestOperations();
}
+
+ @Override
+ public RegistryOperations getRegistryOperations() {
+ return registryOperations;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
index 4f068f3..30db98e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
@@ -22,6 +22,7 @@
import org.apache.slider.server.appmaster.web.rest.agent.AgentResource;
import org.apache.slider.server.appmaster.web.rest.management.ManagementResource;
import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource;
+import org.apache.slider.server.appmaster.web.rest.registry.RegistryResource;
import javax.ws.rs.Path;
@@ -29,6 +30,7 @@
@Singleton
@Path(RestPaths.SLIDER_CONTEXT_ROOT)
public class AMWebServices {
+
/** AM/WebApp info object */
private WebAppApi slider;
@@ -46,4 +48,11 @@
public PublisherResource getPublisherResource() {
return new PublisherResource(slider);
}
+
+ @Path(RestPaths.SLIDER_SUBPATH_REGISTRY)
+ public RegistryResource getRegistryResource() {
+ return new RegistryResource(slider);
+ }
+
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 0571ca1..94f1e4c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -42,10 +42,12 @@
+ SLIDER_SUBPATH_PUBLISHER;
public static final String SLIDER_SUBPATH_REGISTRY = "/registry";
- public static final String SLIDER_PATH_REGISTRY = WS_CONTEXT_ROOT
+ public static final String SLIDER_PATH_REGISTRY = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_REGISTRY;
+ @Deprecated
public static final String REGISTRY_SERVICE = "v1/service";
+ @Deprecated
public static final String REGISTRY_ANYSERVICE = "v1/anyservice";
/**
@@ -59,6 +61,7 @@
= "[a-z0-9][a-z0-9_.\\+-]*";
public static final String SLIDER_CONFIGSET = "slider";
+ public static final String SLIDER_EXPORTS = "exports";
public static final String SLIDER_CLASSPATH = "classpath";
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
index 54d2b1f..f8d7b88 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
@@ -23,6 +23,7 @@
import com.sun.jersey.spi.container.servlet.WebConfig;
import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider;
import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.services.security.SecurityUtils;
@@ -36,13 +37,15 @@
import org.slf4j.LoggerFactory;
import javax.ws.rs.ext.Provider;
+import java.io.Closeable;
import java.io.File;
+import java.io.IOException;
import java.util.Set;
/**
*
*/
-public class AgentWebApp {
+public class AgentWebApp implements Closeable {
protected static final Logger LOG = LoggerFactory.getLogger(AgentWebApp.class);
private int port;
private int securedPort;
@@ -66,7 +69,7 @@
return this;
}
- public AgentWebApp start() {
+ public AgentWebApp start() throws IOException {
if (configsMap == null) {
throw new IllegalStateException("No SSL Configuration Available");
}
@@ -80,7 +83,8 @@
SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false);
SslSelectChannelConnector ssl2WayConnector =
createSSLConnector(Boolean.valueOf(
- configsMap.getOption("ssl.server.client.auth","false")));
+ configsMap.getOption(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED,
+ "false")));
agentServer.setConnectors(new Connector[]{ssl1WayConnector,
ssl2WayConnector});
@@ -100,8 +104,12 @@
try {
agentServer.start();
+ } catch (IOException e) {
+ LOG.error("Unable to start agent server", e);
+ throw e;
} catch (Exception e) {
LOG.error("Unable to start agent server", e);
+ throw new IOException("Unable to start agent server: " + e, e);
}
AgentWebApp webApp = new AgentWebApp();
@@ -191,12 +199,14 @@
return securedPort;
}
- public void stop() {
+ public void close() throws IOException{
//need to stop server and reset injector
try {
agentServer.stop();
+ } catch (IOException e) {
+ throw e;
} catch (Exception e) {
- LOG.warn("Unable to stop agent server", e);
+ throw new IOException(e.toString(), e);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
index c184e63..5fb3b5e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
@@ -36,6 +36,7 @@
private String clusterName;
private long taskId;
private String commandId;
+ //TODO Remove hostname from being set in the command
private String hostname;
private String role;
private Map<String, String> hostLevelParams = new HashMap<String, String>();
@@ -179,4 +180,20 @@
public void setComponentName(String componentName) {
this.componentName = componentName;
}
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("ExecutionCommand [commandType=").append(commandType)
+ .append(", clusterName=").append(clusterName).append(", taskId=")
+ .append(taskId).append(", commandId=").append(commandId)
+ .append(", hostname=").append(hostname).append(", role=").append(role)
+ .append(", hostLevelParams=").append(hostLevelParams)
+ .append(", roleParams=").append(roleParams).append(", roleCommand=")
+ .append(roleCommand).append(", configurations=").append(configurations)
+ .append(", commandParams=").append(commandParams)
+ .append(", serviceName=").append(serviceName)
+ .append(", componentName=").append(componentName).append("]");
+ return builder.toString();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
index 0545499..c118840 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
@@ -42,6 +42,7 @@
boolean restartAgent = false;
boolean restartEnabled = true;
boolean hasMappedComponents = false;
+ boolean terminateAgent = false;
@JsonProperty("responseId")
public long getResponseId() {
@@ -113,6 +114,16 @@
this.hasMappedComponents = hasMappedComponents;
}
+ @JsonProperty("terminateAgent")
+ public boolean isTerminateAgent() {
+ return terminateAgent;
+ }
+
+ @JsonProperty("terminateAgent")
+ public void setTerminateAgent(boolean terminateAgent) {
+ this.terminateAgent = terminateAgent;
+ }
+
public void addExecutionCommand(ExecutionCommand execCmd) {
executionCommands.add(execCmd);
}
@@ -129,6 +140,7 @@
", statusCommands=" + statusCommands +
", registrationCommand=" + registrationCommand +
", restartAgent=" + restartAgent +
+ ", terminateAgent=" + terminateAgent +
'}';
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
index a44c3a4..842e5a9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
@@ -29,15 +29,17 @@
public class Register {
private int responseId = -1;
private long timestamp;
- private String hostname;
+ private String label;
private int currentPingPort;
private HostInfo hardwareProfile;
private String publicHostname;
+ private String tags;
private AgentEnv agentEnv;
private String agentVersion;
private State actualState;
private State expectedState;
private Map<String, String> allocatedPorts;
+ private Map<String, String> logFolders;
@JsonProperty("responseId")
public int getResponseId() {
@@ -57,12 +59,20 @@
this.timestamp = timestamp;
}
- public String getHostname() {
- return hostname;
+ public String getLabel() {
+ return label;
}
- public void setHostname(String hostname) {
- this.hostname = hostname;
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
+ public String getTags() {
+ return tags;
+ }
+
+ public void setTags(String tags) {
+ this.tags = tags;
}
public HostInfo getHardwareProfile() {
@@ -133,11 +143,24 @@
this.allocatedPorts = ports;
}
+ /** @return the log folders, or <code>null</code> if none are present */
+ @JsonProperty("logFolders")
+ public Map<String, String> getLogFolders() {
+ return logFolders;
+ }
+
+ /** @param logFolders assigned log folders */
+ @JsonProperty("logFolders")
+ public void setLogFolders(Map<String, String> logFolders) {
+ this.logFolders = logFolders;
+ }
+
@Override
public String toString() {
String ret = "responseId=" + responseId + "\n" +
"timestamp=" + timestamp + "\n" +
- "hostname=" + hostname + "\n" +
+ "label=" + label + "\n" +
+ "hostname=" + publicHostname + "\n" +
"expectedState=" + expectedState + "\n" +
"actualState=" + actualState + "\n";
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
index 734119d..fd852e2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
@@ -37,10 +37,14 @@
@JsonProperty("exitstatus")
private int exitstatus;
- /** log - message, which will be printed to agents log */
+ /** log - message, which will be printed to agents log */
@JsonProperty("log")
private String log;
+ /** tags - tags associated with the container */
+ @JsonProperty("tags")
+ private String tags;
+
//Response id to start with, usually zero.
@JsonProperty("responseId")
private long responseId;
@@ -75,6 +79,14 @@
this.responseId = responseId;
}
+ public String getTags() {
+ return tags;
+ }
+
+ public void setTags(String tags) {
+ this.tags = tags;
+ }
+
public void setExitstatus(int exitstatus) {
this.exitstatus = exitstatus;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
index 0610cac..52fb8dd 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
@@ -125,4 +125,17 @@
public void setRoleCommand(String roleCommand) {
this.roleCommand = roleCommand;
}
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("StatusCommand [agentCommandType=").append(agentCommandType)
+ .append(", clusterName=").append(clusterName).append(", serviceName=")
+ .append(serviceName).append(", componentName=").append(componentName)
+ .append(", configurations=").append(configurations)
+ .append(", commandParams=").append(commandParams)
+ .append(", hostLevelParams=").append(hostLevelParams)
+ .append(", roleCommand=").append(roleCommand).append("]");
+ return builder.toString();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
index 5d8b657..d2ad09e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
@@ -23,6 +23,8 @@
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import org.apache.slider.core.registry.docstore.UriMap;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
@@ -42,7 +44,6 @@
import java.net.URLClassLoader;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
@@ -56,7 +57,10 @@
protected static final Logger log =
LoggerFactory.getLogger(PublisherResource.class);
private final WebAppApi slider;
- public static final String SET_NAME =
+ public static final String EXPORTS_NAME = "exports";
+ public static final String EXPORTS_RESOURCES_PATH = "/" + EXPORTS_NAME;
+ public static final String EXPORT_RESOURCE_PATH = EXPORTS_RESOURCES_PATH + "/{exportname}" ;
+ public static final String SET_NAME =
"{setname: " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
private static final String CONFIG =
SET_NAME + "/{config: " + PUBLISHED_CONFIGURATION_REGEXP + "}";
@@ -101,7 +105,9 @@
UriMap uriMap = new UriMap();
for (String name : appState.listConfigSets()) {
uriMap.put(name, baseURL + name);
+ log.info("Tick tack {} and {}", name, baseURL);
}
+ uriMap.put(EXPORTS_NAME, baseURL + EXPORTS_NAME);
return uriMap;
}
@@ -114,6 +120,26 @@
}
@GET
+ @Path(EXPORTS_RESOURCES_PATH)
+ @Produces({MediaType.APPLICATION_JSON})
+ public PublishedExportsSet gePublishedExports() {
+
+ PublishedExportsSet set = appState.getPublishedExportsSet();
+ return set.shallowCopy();
+ }
+
+ @GET
+ @Path(EXPORT_RESOURCE_PATH)
+ @Produces({MediaType.APPLICATION_JSON})
+ public PublishedExports getAMExports2(@PathParam("exportname") String exportname,
+ @Context UriInfo uriInfo,
+ @Context HttpServletResponse res) {
+ init(res, uriInfo);
+ PublishedExportsSet set = appState.getPublishedExportsSet();
+ return set.get(exportname);
+ }
+
+ @GET
@Path("/"+ SET_NAME)
@Produces({MediaType.APPLICATION_JSON})
public PublishedConfigSet getPublishedConfiguration(
@@ -129,7 +155,7 @@
}
private void logRequest(UriInfo uriInfo) {
- log.debug(uriInfo.getRequestUri().toString());
+ log.info(uriInfo.getRequestUri().toString());
}
@GET
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
similarity index 70%
rename from slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
rename to slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
index bdf70a2..efb09a8 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
@@ -16,26 +16,30 @@
* limitations under the License.
*/
-package org.apache.slider.core.registry.info;
+package org.apache.slider.server.appmaster.web.rest.registry;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.List;
+/**
+ * Representation of a path entry
+ */
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class RegistryView {
+public class PathEntryResource {
/**
- * Endpoints
+ * Child nodes: as the short path to each element
*/
- public Map<String, RegisteredEndpoint> endpoints =
- new HashMap<String, RegisteredEndpoint>(2);
+ public List<String> nodes;
- public String configurationsURL;
-
- public String documentsURL;
+ /**
+ * Service record: if null —there is no resolvable service
+ * record at this node.
+ */
+ public ServiceRecord service;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
new file mode 100644
index 0000000..70c0826
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.registry;
+
+import com.google.inject.Singleton;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.exceptions.AuthenticationFailedException;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.exceptions.NoPathPermissionsException;
+import org.apache.hadoop.registry.client.exceptions.NoRecordException;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.io.IOException;
+
+/**
+ * This is the read-only view of the YARN registry.
+ *
+ * Model:
+ * <ol>
+ * <li>a tree of nodes</li>
+ * <li>Default view is of children + record</li>
+ * </ol>
+ *
+ */
+@Singleton
+public class RegistryResource {
+ protected static final Logger log =
+ LoggerFactory.getLogger(RegistryResource.class);
+ public static final String SERVICE_PATH =
+ "/{path:.*}";
+
+ private final RegistryOperations registry;
+
+ /**
+ * Construct an instance bonded to a registry
+ * @param slider slider API
+ */
+ public RegistryResource(WebAppApi slider) {
+ this.registry = slider.getRegistryOperations();
+ }
+
+ /**
+ * Internal init code, per request
+ * @param request incoming request
+ * @param uriInfo URI details
+ */
+ private void init(HttpServletRequest request, UriInfo uriInfo) {
+ log.debug(uriInfo.getRequestUri().toString());
+ }
+
+ @GET
+ @Produces({MediaType.APPLICATION_JSON})
+ public PathEntryResource getRoot(@Context HttpServletRequest request,
+ @Context UriInfo uriInfo) {
+ return lookup("/", request, uriInfo);
+ }
+
+// {path:.*}
+
+ @Path(SERVICE_PATH)
+ @GET
+ @Produces({MediaType.APPLICATION_JSON})
+ public PathEntryResource lookup(
+ @PathParam("path") String path,
+ @Context HttpServletRequest request,
+ @Context UriInfo uriInfo) {
+ init(request, uriInfo);
+ return resolvePath(path);
+ }
+
+ /**
+ * Do the actual processing of requests to responses; can be directly
+ * invoked for testing.
+ * @param path path to query
+ * @return the entry
+ * @throws WebApplicationException on any failure.
+ */
+ public PathEntryResource resolvePath(String path) throws
+ WebApplicationException {
+ try {
+ PathEntryResource pathEntry =
+ fromRegistry(path);
+ if (log.isDebugEnabled()) {
+ log.debug("Resolved:\n{}", pathEntry);
+ }
+ return pathEntry;
+ } catch (WebApplicationException e) {
+ // rethrow direct
+ throw e;
+ } catch (PathNotFoundException e) {
+ throw new NotFoundException("Not found: " + path);
+ } catch (AuthenticationFailedException e) {
+ throw new ForbiddenException(path);
+ } catch (NoPathPermissionsException e) {
+ throw new ForbiddenException(path);
+ } catch (Exception e) {
+ log.error("Error during generation of response: {}", e, e);
+ throw new WebApplicationException(e);
+ }
+ }
+
+
+ /**
+ * Build from the registry, filling up the children and service records.
+ * If there is no service record at the end of the path, that entry is
+ * null
+ * @param path path to query
+ * @return the built up record
+ * @throws IOException problems
+ *
+ */
+ private PathEntryResource fromRegistry(String path) throws IOException {
+ PathEntryResource entry = new PathEntryResource();
+ try {
+ entry.service = registry.resolve(path);
+ } catch (NoRecordException e) {
+ // ignoring
+ log.debug("No record at {}", path);
+ } catch (InvalidRecordException e) {
+ // swallowing this exception, the sign of "no entry present"
+ // "nothing parseable"
+ log.warn("Failed to resolve {}: {}", path, e, e);
+ }
+ entry.nodes = registry.list(path);
+ return entry;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
index 54bdb09..59a03f9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -16,20 +16,25 @@
*/
package org.apache.slider.server.appmaster.web.view;
+import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
@@ -37,15 +42,14 @@
*
*/
public class IndexBlock extends HtmlBlock {
- private static final String HBASE = "HBase";
private static final Logger log = LoggerFactory.getLogger(IndexBlock.class);
- private StateAccessForProviders appState;
+ private StateAccessForProviders appView;
private ProviderService providerService;
@Inject
public IndexBlock(WebAppApi slider) {
- this.appState = slider.getAppState();
+ this.appView = slider.getAppState();
this.providerService = slider.getProviderService();
}
@@ -57,45 +61,73 @@
}
// An extra method to make testing easier since you can't make an instance of Block
+ @VisibleForTesting
protected void doIndex(Hamlet html, String providerName) {
- DIV<Hamlet> div = html.div("general_info").h1("index_header", providerName + " cluster: '" + appState.getClusterStatus().name + "'");
+ ClusterDescription clusterStatus = appView.getClusterStatus();
+ DIV<Hamlet> div = html.div("general_info")
+ .h1("index_header",
+ "Application: '" + clusterStatus.name + "'");
UL<DIV<Hamlet>> ul = div.ul();
- ul.li("Total number of containers for cluster: " + appState.getNumOwnedContainers());
- ul.li("Cluster created: " + getInfoAvoidingNulls(StatusKeys.INFO_CREATE_TIME_HUMAN));
- ul.li("Cluster last flexed: " + getInfoAvoidingNulls(StatusKeys.INFO_FLEX_TIME_HUMAN));
- ul.li("Cluster running since: " + getInfoAvoidingNulls(StatusKeys.INFO_LIVE_TIME_HUMAN));
- ul.li("Cluster HDFS storage path: " + appState.getClusterStatus().dataPath);
- ul.li("Cluster configuration path: " + appState.getClusterStatus().originConfigurationPath);
+ ul.li("Total number of containers for application: " + appView.getNumOwnedContainers());
+ ul.li("Application created: " +
+ getInfoAvoidingNulls(StatusKeys.INFO_CREATE_TIME_HUMAN));
+ ul.li("Application last flexed: " + getInfoAvoidingNulls(StatusKeys.INFO_FLEX_TIME_HUMAN));
+ ul.li("Application running since: " + getInfoAvoidingNulls(StatusKeys.INFO_LIVE_TIME_HUMAN));
+ ul.li("Application HDFS storage path: " + clusterStatus.dataPath);
+ ul.li("Application configuration path: " + clusterStatus.originConfigurationPath);
ul._()._();
html.div("provider_info").h3(providerName + " specific information");
ul = div.ul();
- addProviderServiceOptions(providerService, ul);
+ addProviderServiceOptions(providerService, ul, clusterStatus);
ul._()._();
+
+ html.div("container_instances").h3("Component Instances");
+
+ Hamlet.TABLE<DIV<Hamlet>> table = div.table();
+ table.tr()
+ .th("Component")
+ .th("Desired")
+ .th("Actual")
+ .th("Outstanding Requests")
+ .th("Failed")
+ .th("Failed to start")
+ ._();
+
+ List<RoleStatus> roleStatuses = appView.cloneRoleStatusList();
+ Collections.sort(roleStatuses, new RoleStatus.CompareByName());
+ for (RoleStatus status : roleStatuses) {
+ table.tr()
+ .td(status.getName())
+ .th(String.format("%d", status.getDesired()))
+ .th(String.format("%d", status.getActual()))
+ .th(String.format("%d", status.getRequested()))
+ .th(String.format("%d", status.getFailed()))
+ .th(String.format("%d", status.getStartFailed()))
+ ._();
+ }
+
+ table._()._();
}
private String getProviderName() {
- String providerServiceName = providerService.getName().toLowerCase();
-
- // Get HBase properly capitalized
- if (providerServiceName.contains("hbase")) {
- return HBASE;
- }
-
- return StringUtils.capitalize(providerServiceName);
+ String providerServiceName = providerService.getName().toLowerCase(Locale.ENGLISH);
+ return providerServiceName;
}
private String getInfoAvoidingNulls(String key) {
- String createTime = appState.getClusterStatus().getInfo(key);
+ String createTime = appView.getClusterStatus().getInfo(key);
return null == createTime ? "N/A" : createTime;
}
- protected void addProviderServiceOptions(ProviderService providerService, UL<DIV<Hamlet>> ul) {
- Map<String, String> details = providerService.buildMonitorDetails(appState.getClusterStatus());
+ protected void addProviderServiceOptions(ProviderService providerService,
+ UL<DIV<Hamlet>> ul, ClusterDescription clusterStatus) {
+ Map<String, String> details = providerService.buildMonitorDetails(
+ clusterStatus);
if (null == details) {
return;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
deleted file mode 100644
index a6316d5..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.curator;
-
-import com.google.common.base.Preconditions;
-import org.apache.curator.RetryPolicy;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.curator.x.discovery.ServiceDiscovery;
-import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
-import org.apache.curator.x.discovery.strategies.RandomStrategy;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
-import org.apache.slider.server.services.registry.RegistryServiceConstants;
-import org.apache.slider.server.services.registry.RegistryDiscoveryContext;
-import org.apache.slider.server.services.registry.SliderRegistryService;
-
-/**
- * This class creates a curator -but does not start or close it. That
- * is the responsbility of the owner
- */
-public class CuratorHelper extends Configured {
-
- private final CuratorFramework curator;
- private final String connectionString;
-
- public CuratorHelper(Configuration conf, String connectionString) {
- super(conf);
- this.connectionString = connectionString;
- curator = createCurator(this.connectionString);
- }
-
-
- public CuratorFramework getCurator() {
- return curator;
- }
-
- public String getConnectionString() {
- return connectionString;
- }
-
- /**
- * Create a retry policy for this configuration
- * @param conf
- * @return
- */
- public RetryPolicy createRetryPolicy() {
- RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
- return retryPolicy;
- }
-
- private CuratorFramework createCurator(String connectionString) {
- CuratorFramework curator =
- CuratorFrameworkFactory.newClient(connectionString,
- createRetryPolicy());
- return curator;
- }
-
- /**
- * Create an (united) curator client service
- * @param connectionString ZK binding
- * @return the service
- */
- public CuratorService createCuratorClientService() {
- CuratorService curatorService =
- new CuratorService("Curator ", curator, connectionString);
- return curatorService;
- }
-
- /**
- * Create a discovery builder bonded to this curator
- * @return
- */
- public ServiceDiscoveryBuilder<ServiceInstanceData> createDiscoveryBuilder() {
- ServiceDiscoveryBuilder<ServiceInstanceData> discoveryBuilder =
- ServiceDiscoveryBuilder.builder(ServiceInstanceData.class);
- discoveryBuilder.client(curator);
- return discoveryBuilder;
- }
-
- /**
- * Create an instance
- * @param discoveryBuilder builder to create the discovery from
- */
-
- public SliderRegistryService createRegistryBinderService(
- String basePath,
- ServiceDiscoveryBuilder<ServiceInstanceData> discoveryBuilder) {
- discoveryBuilder.basePath(basePath);
- return new SliderRegistryService(curator,
- basePath,
- discoveryBuilder.build());
- }
-
-
- /**
- * Create an instance -including the initial binder
- * @param basePath base path
- * @return the binder service
- */
- public SliderRegistryService createRegistryBinderService(
- String basePath) {
- ServiceDiscoveryBuilder<ServiceInstanceData> discoveryBuilder =
- createDiscoveryBuilder();
- //registry will start curator as well as the binder, in the correct order
- SliderRegistryService registryBinderService =
- createRegistryBinderService(basePath, discoveryBuilder);
- return registryBinderService;
- }
-
- public RegistryDiscoveryContext createDiscoveryContext(
- ServiceDiscovery<ServiceInstanceData> discovery) {
- Preconditions.checkNotNull(discovery);
- return new RegistryDiscoveryContext(discovery,
- new RandomStrategy<ServiceInstanceData>(),
- RegistryServiceConstants.INSTANCE_REFRESH_MS,
- ServiceInstanceData.class);
-
- }
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
deleted file mode 100644
index 645bc8f..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.curator;
-
-import com.google.common.base.Preconditions;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.utils.ZKPaths;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.service.AbstractService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-
-public class CuratorService extends AbstractService {
- private static final Logger log =
- LoggerFactory.getLogger(CuratorService.class);
- protected final String basePath;
-
- private final CuratorFramework curator;
- private CuratorHelper curatorHelper;
-
-
- public CuratorService(String name,
- CuratorFramework curator,
- String basePath) {
- super(name);
- this.curator = Preconditions.checkNotNull(curator, "null curator");
- this.basePath = basePath;
- }
-
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- curatorHelper = new CuratorHelper(conf, basePath);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- curator.start();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- closeCuratorComponent(curator);
- }
-
- public CuratorFramework getCurator() {
- return curator;
- }
-
- protected void closeCuratorComponent(Closeable closeable) {
- try {
- IOUtils.closeStream(closeable);
- } catch (Throwable ignored) {
- //triggered on an attempt to close before started
- log.debug("Error when closing {}", ignored);
- }
- }
-
- public String pathForServicetype(String servicetype) {
- return ZKPaths.makePath(getBasePath(), servicetype);
- }
-
- protected String pathForInstance(String servicetype, String id) {
- Preconditions.checkNotNull(servicetype);
- Preconditions.checkNotNull(id);
- return ZKPaths.makePath(pathForServicetype(servicetype), id);
- }
-
- public String getBasePath() {
- return basePath;
- }
-
- public CuratorHelper getCuratorHelper() {
- return curatorHelper;
- }
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
deleted file mode 100644
index 61efde2..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.curator;
-
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.ServiceType;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonTypeInfo;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class CuratorServiceInstance<T> {
-
- public String name;
- public String id;
- public String address;
- public Integer port;
- public Integer sslPort;
- public T payload;
- public long registrationTimeUTC;
- public ServiceType serviceType;
- public CuratorUriSpec uriSpec;
-
- public CuratorServiceInstance() {
- }
-
- public CuratorServiceInstance(ServiceInstance<T> si) {
- this.name = si.getName();
- this.id = si.getId();
- this.address = si.getAddress();
- this.port = si.getPort();
- this.sslPort = si.getSslPort();
- this.payload = si.getPayload();
- this.registrationTimeUTC = si.getRegistrationTimeUTC();
- this.serviceType = si.getServiceType();
- this.uriSpec = new CuratorUriSpec();
- this.uriSpec.addAll(si.getUriSpec().getParts());
- }
-
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("CuratorServiceInstance{");
- sb.append("name='").append(name).append('\'');
- sb.append(", id='").append(id).append('\'');
- sb.append(", address='").append(address).append('\'');
- sb.append(", port=").append(port);
- sb.append(", sslPort=").append(sslPort);
- sb.append(", payload=").append(payload);
- sb.append(", registrationTimeUTC=").append(registrationTimeUTC);
- sb.append(", serviceType=").append(serviceType);
- sb.append(", uriSpec=").append(uriSpec);
- sb.append('}');
- return sb.toString();
- }
-
- @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS)
- public T getPayload() {
- return payload;
- }
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
deleted file mode 100644
index 8923e63..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.curator;
-
-import com.google.common.collect.Lists;
-
-import java.util.Collection;
-import java.util.List;
-
-/**
- *
- */
-public class CuratorServiceInstances<T> {
- private final List<CuratorServiceInstance<T>> services;
-
- public CuratorServiceInstances() {
- services = Lists.newArrayList();
- }
-
- public CuratorServiceInstances(Collection<? extends CuratorServiceInstance<T>> c) {
- services = Lists.newArrayList(c);
- }
-
- public List<CuratorServiceInstance<T>> getServices() {
- return Lists.newArrayList(services);
- }
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
deleted file mode 100644
index adda359..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.curator;
-
-import com.google.common.collect.Lists;
-import org.apache.curator.x.discovery.UriSpec;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-
-import java.util.List;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class CuratorUriSpec extends UriSpec{
-
- private final List<Part> parts = Lists.newArrayList();
-
- public CuratorUriSpec() {
- super();
- }
-
- @Override
- public List<Part> getParts() {
- return Lists.newArrayList(parts);
- }
-
- @Override
- public void add(Part part) {
- super.add(part);
- parts.add(part);
- }
-
- public void addAll(List<Part> parts) {
- parts.addAll(parts);
- }
-
- @Override
- public void remove(Part part) {
- super.remove(part);
- parts.remove(part);
- }
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
deleted file mode 100644
index b3e2ff2..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.curator;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.x.discovery.ServiceDiscovery;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.ServiceInstanceBuilder;
-import org.apache.curator.x.discovery.ServiceType;
-import org.apache.curator.x.discovery.UriSpec;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.persist.JsonSerDeser;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * YARN service for Curator service discovery; the discovery instance's
- * start/close methods are tied to the lifecycle of this service
- * @param <Payload> the payload of the operation
- */
-public class RegistryBinderService<Payload> extends CuratorService {
- private static final Logger log =
- LoggerFactory.getLogger(RegistryBinderService.class);
-
- private final ServiceDiscovery<Payload> discovery;
-
- private final Map<String, ServiceInstance<Payload>> entries =
- new HashMap<String, ServiceInstance<Payload>>();
-
- private JsonSerDeser<CuratorServiceInstance<Payload>> deser =
- new JsonSerDeser<CuratorServiceInstance<Payload>>(CuratorServiceInstance.class);
-
- /**
- * Create an instance
- * @param curator Does not need to be started
- * @param basePath base directory
- * @param discovery discovery instance -not yet started
- */
- public RegistryBinderService(CuratorFramework curator,
- String basePath,
- ServiceDiscovery<Payload> discovery) {
- super("RegistryBinderService", curator, basePath);
-
- this.discovery =
- Preconditions.checkNotNull(discovery, "null discovery arg");
- }
-
-
- public ServiceDiscovery<Payload> getDiscovery() {
- return discovery;
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- discovery.start();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- closeCuratorComponent(discovery);
- super.serviceStop();
- }
-
- /**
- * register an instance -only valid once the service is started
- * @param id ID -must be unique
- * @param name name
- * @param url URL
- * @param payload payload (may be null)
- * @return the instance
- * @throws Exception on registration problems
- */
- public ServiceInstance<Payload> register(String name,
- String id,
- URL url,
- Payload payload) throws Exception {
- Preconditions.checkNotNull(id, "null `id` arg");
- Preconditions.checkNotNull(name, "null `name` arg");
- Preconditions.checkState(isInState(STATE.STARTED), "Not started: " + this);
-
- ServiceInstanceBuilder<Payload> instanceBuilder = builder()
- .name(name)
- .id(id)
- .payload(payload)
- .serviceType(ServiceType.DYNAMIC);
- if (url != null) {
- UriSpec uriSpec = new UriSpec(url.toString());
-
- int port = url.getPort();
- if (port == 0) {
- throw new IOException("Port undefined in " + url);
- }
- instanceBuilder
- .uriSpec(uriSpec)
- .port(port);
- }
- ServiceInstance<Payload> instance = instanceBuilder.build();
- log.info("registering {}", instance.toString());
- discovery.registerService(instance);
- log.info("registration completed {}", instance.toString());
- synchronized (this) {
- entries.put(id, instance);
- }
- return instance;
- }
-
- /**
- * Create a builder. This is already pre-prepared with address, registration
- * time and a (random) UUID
- * @return a builder
- * @throws IOException IO problems, including enumerating network ports
- */
- public ServiceInstanceBuilder<Payload> builder() throws
- IOException {
- try {
- return ServiceInstance.builder();
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
-
- /**
- * List all instance IDs of a service type
- * @param servicetype service type
- * @return list of matches
- * @throws Exception
- */
- public List<String> instanceIDs(String servicetype) throws Exception {
- Preconditions.checkNotNull(servicetype);
- List<String> instanceIds;
- try {
- instanceIds =
- getCurator().getChildren().forPath(pathForServicetype(servicetype));
- } catch (KeeperException.NoNodeException e) {
- instanceIds = Lists.newArrayList();
- }
- return instanceIds;
- }
-
- /**
- * List all service types registered
- * @return a list of service types
- * @throws Exception
- */
- public List<String> serviceTypes() throws Exception {
- List<String> types;
- try {
- types =
- getCurator().getChildren().forPath(getBasePath());
- } catch (KeeperException.NoNodeException e) {
- types = Lists.newArrayList();
- }
- return types;
- }
-
-
- /**
- * Return a service instance POJO
- *
- * @param servicetype name of the service
- * @param id ID of the instance
- * @return the instance or <code>null</code> if not found
- * @throws Exception errors
- */
- public CuratorServiceInstance<Payload> queryForInstance(String servicetype, String id)
- throws Exception {
- CuratorServiceInstance<Payload> instance = null;
- String path = pathForInstance(servicetype, id);
- try {
- byte[] bytes = getCurator().getData().forPath(path);
- if (bytes!=null && bytes.length>0) {
- instance = deser.fromBytes(bytes);
- }
- } catch (KeeperException.NoNodeException ignore) {
- // ignore
- }
- return instance;
- }
-
- /**
- * List all the instances
- * @param servicetype name of the service
- * @return a list of instances and their payloads
- * @throws IOException any problem
- */
- public List<CuratorServiceInstance<Payload>> listInstances(String servicetype) throws
- IOException {
- try {
- List<String> instanceIDs = instanceIDs(servicetype);
- List<CuratorServiceInstance<Payload>> instances =
- new ArrayList<CuratorServiceInstance<Payload>>(instanceIDs.size());
- for (String instanceID : instanceIDs) {
- CuratorServiceInstance<Payload> instance =
- queryForInstance(servicetype, instanceID);
- if (instance != null) {
- instances.add(instance);
- }
- }
- return instances;
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- /**
- * Find an instance with a given ID
- * @param instances instances
- * @param name ID to look for
- * @return the discovered instance or null
- */
- public CuratorServiceInstance<Payload> findByID(List<CuratorServiceInstance<Payload>> instances, String name) {
- Preconditions.checkNotNull(name);
- for (CuratorServiceInstance<Payload> instance : instances) {
- if (instance.id.equals(name)) {
- return instance;
- }
- }
- return null;
- }
-
- /**
- * Find a single instance -return that value or raise an exception
- * @param serviceType service type
- * @param name the name (required(
- * @return the instance that matches the criteria
- * @throws FileNotFoundException if there were no matches
- * @throws IOException any network problem
- */
- public CuratorServiceInstance<Payload> findInstance(String serviceType,
- String name) throws IOException {
- Preconditions.checkArgument(StringUtils.isNotEmpty(name), "name");
- return findInstances(serviceType, name).get(0);
- }
- /**
- * List registry entries. If a name was given, then the single match is returned
- * -otherwise all entries matching the service type
- * @param serviceType service type
- * @param name an optional name
- * @return the (non-empty) list of instances that match the criteria
- * @throws FileNotFoundException if there were no matches
- * @throws IOException any network problem
- */
- public List<CuratorServiceInstance<Payload>> findInstances(String serviceType,
- String name)
- throws FileNotFoundException, IOException {
- List<CuratorServiceInstance<Payload>> instances =
- listInstances(serviceType);
- if (instances.isEmpty()) {
- throw new FileNotFoundException(
- "No registry entries for service type " + serviceType);
- }
- if (StringUtils.isNotEmpty(name)) {
- CuratorServiceInstance<Payload> foundInstance = findByID(instances, name);
- if (foundInstance == null) {
- throw new FileNotFoundException(
- "No registry entries for service name " + name
- + " and service type " + serviceType);
- }
- instances.clear();
- instances.add(foundInstance);
- }
- return instances;
- }
-
- /**
- * Enum all service types in the registry
- * @return a possibly empty collection of service types
- * @throws IOException networking
- */
- public Collection<String> getServiceTypes() throws IOException {
- try {
- return getDiscovery().queryForNames();
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java
deleted file mode 100644
index a7c35e8..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.registry;
-
-import org.apache.curator.x.discovery.ProviderStrategy;
-import org.apache.curator.x.discovery.ServiceDiscovery;
-import org.apache.curator.x.discovery.server.contexts.GenericDiscoveryContext;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
-
-public class RegistryDiscoveryContext extends GenericDiscoveryContext<ServiceInstanceData> {
-
- public RegistryDiscoveryContext(ServiceDiscovery<ServiceInstanceData> serviceDiscovery,
- ProviderStrategy<ServiceInstanceData> providerStrategy,
- int instanceRefreshMs,
- Class<ServiceInstanceData> payloadType) {
- super(serviceDiscovery, providerStrategy, instanceRefreshMs, payloadType);
- }
-
-
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
deleted file mode 100644
index e4e8523..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.registry;
-
-import com.google.inject.Singleton;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.server.rest.DiscoveryContext;
-import org.apache.curator.x.discovery.server.rest.DiscoveryResource;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
-import org.apache.slider.server.services.curator.CuratorServiceInstance;
-import org.apache.slider.server.services.curator.CuratorServiceInstances;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import java.net.URI;
-import java.net.URL;
-import java.util.List;
-import java.util.Random;
-
-@Singleton
-@Path(RestPaths.SLIDER_PATH_REGISTRY)
-public class RegistryRestResources extends DiscoveryResource<ServiceInstanceData> {
- public static final String SERVICE_NAME = RestPaths.REGISTRY_SERVICE +"/{name}";
- public static final String SERVICE_NAME_ID = SERVICE_NAME + "/{id}";
- protected static final Logger log =
- LoggerFactory.getLogger(RegistryRestResources.class);
- private final SliderRegistryService registry;
- private DiscoveryContext<ServiceInstanceData> context;
- private Random randomizer = new Random();
-
- public RegistryRestResources(@Context DiscoveryContext<ServiceInstanceData> context,
- SliderRegistryService registry) {
- super(context);
- this.context = context;
- this.registry = registry;
- }
-
-// @GET
- public Response getWadl (@Context HttpServletRequest request) {
- try {
- URI location = new URL(request.getScheme(),
- request.getServerName(),
- request.getServerPort(),
- "/application.wadl").toURI();
- return Response.temporaryRedirect(location).build();
- } catch (Exception e) {
- log.error("Error during redirect to WADL", e);
- throw new WebApplicationException(Response.serverError().build());
- }
- }
-
- @javax.ws.rs.GET
- @javax.ws.rs.Produces({MediaType.APPLICATION_JSON})
- public Response getAtRoot() {
- try {
- List<String>
- instances = registry.serviceTypes();
- return Response.ok(instances).build();
- } catch (Exception e) {
- log.error("Error during generation of response", e);
- return Response.serverError().build();
- }
- }
-
-
- @Override
- @javax.ws.rs.GET
- @javax.ws.rs.Path(SERVICE_NAME)
- @javax.ws.rs.Produces({MediaType.APPLICATION_JSON})
- public Response getAll(@PathParam("name") String name) {
- try {
- List<CuratorServiceInstance<ServiceInstanceData>>
- instances = registry.listInstances(name);
- return Response.ok(
- new CuratorServiceInstances<ServiceInstanceData>(instances)).build();
- } catch (Exception e) {
- log.error("Error during generation of response", e);
- return Response.serverError().build();
- }
- }
-
- @Override
- @GET
- @Path(SERVICE_NAME_ID)
- @Produces(MediaType.APPLICATION_JSON)
- public Response get(@PathParam("name") String name,
- @PathParam("id") String id) {
- try {
- CuratorServiceInstance<ServiceInstanceData> instance = registry.queryForInstance(name, id);
- if ( instance == null )
- {
- return Response.status(Response.Status.NOT_FOUND).build();
- }
- Response.ResponseBuilder builder = Response.ok(instance);
- return builder.build();
- } catch (Exception e) {
- log.error("Trying to get instance {} from service {}: {})",
- id,
- name,
- e);
- return Response.serverError().build();
- }
- }
-
- @Override
- @GET
- @Path("v1/anyservice/{name}")
- @Produces(MediaType.APPLICATION_JSON)
- public Response getAny(@PathParam("name") String name) {
- try {
- List<CuratorServiceInstance<ServiceInstanceData>>
- instances = registry.listInstances(name);
- if (instances == null || instances.isEmpty()) {
- return Response.status(Response.Status.NOT_FOUND).build();
- }
-
- CuratorServiceInstance<ServiceInstanceData> randomInstance =
- instances.get(randomizer.nextInt(instances.size()));
- if ( randomInstance == null )
- {
- return Response.status(Response.Status.NOT_FOUND).build();
- }
- return Response.ok(randomInstance).build();
- } catch (Exception e) {
- log.error(String.format("Trying to get any instance from service (%s)", name), e);
- return Response.serverError().build();
- }
- }
-
- @Override
- @PUT
- @Path(SERVICE_NAME_ID)
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public Response putService(ServiceInstance<ServiceInstanceData> instance,
- @PathParam("name") String name,
- @PathParam("id") String id) {
- throw new UnsupportedOperationException("putService not supported");
- }
-
- @Override
- @DELETE
- @Path(SERVICE_NAME_ID)
- public Response removeService(@PathParam("name") String name,
- @PathParam("id") String id) {
- throw new UnsupportedOperationException("removeService not supported");
- }
-}
-
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryViewForProviders.java
deleted file mode 100644
index 22ba066..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryViewForProviders.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.registry;
-
-import org.apache.slider.core.registry.info.ServiceInstanceData;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.List;
-
-/**
- * This offers restricted access to the registry for providers
- */
-public interface RegistryViewForProviders {
- List<ServiceInstanceData> listInstancesByType(String serviceType) throws
- IOException;
-
- /**
- * Get the registration of slider itself
- * @return the registration of slider
- */
- ServiceInstanceData getSelfRegistration();
-
- /**
- * Register the service, raising IOExceptions when anything fails
- * @param instanceData instance data
- * @param url URL to register
- * @throws IOException on registration problems
- */
- void registerServiceInstance(
- ServiceInstanceData instanceData, URL url) throws IOException;
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
deleted file mode 100644
index ca4d180..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.registry;
-
-import com.google.common.base.Preconditions;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.x.discovery.ServiceDiscovery;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
-import org.apache.slider.server.services.curator.CuratorServiceInstance;
-import org.apache.slider.server.services.curator.RegistryBinderService;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * This is the registry service, which tries to hide exactly how the
- * registry is implemented
- */
-
-public class SliderRegistryService
- extends RegistryBinderService<ServiceInstanceData>
- implements RegistryViewForProviders {
-
- private ServiceInstanceData selfRegistration;
-
- public SliderRegistryService(CuratorFramework curator,
- String basePath,
- ServiceDiscovery<ServiceInstanceData> discovery) {
- super(curator, basePath, discovery);
- }
-
-
- @Override
- public List<ServiceInstanceData> listInstancesByType(String serviceType) throws
- IOException {
- List<CuratorServiceInstance<ServiceInstanceData>> services =
- listInstances(serviceType);
- List<ServiceInstanceData> payloads = new ArrayList<ServiceInstanceData>(services.size());
- for (CuratorServiceInstance<ServiceInstanceData> instance : services) {
- payloads.add(instance.payload);
- }
- return payloads;
- }
-
- @Override
- public ServiceInstanceData getSelfRegistration() {
- return selfRegistration;
- }
-
- private void setSelfRegistration(ServiceInstanceData selfRegistration) {
- this.selfRegistration = selfRegistration;
- }
-
- /**
- * register an instance -only valid once the service is started.
- * This sets the selfRegistration field
- * @param instanceData instance data
- * @param url URL to register
- * @throws IOException on registration problems
- */
- public void registerSelf(ServiceInstanceData instanceData, URL url) throws IOException {
- registerServiceInstance(instanceData, url);
- setSelfRegistration(instanceData);
- }
-
- @Override
- public void registerServiceInstance(
- ServiceInstanceData instanceData, URL url) throws IOException {
- Preconditions.checkNotNull(instanceData);
- Preconditions.checkNotNull(instanceData.id);
- Preconditions.checkNotNull(instanceData.serviceType);
-
- try {
- register(instanceData.serviceType, instanceData.id, url, instanceData);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
index 3771208..257f8f9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -19,16 +19,16 @@
import com.google.inject.Singleton;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.text.MessageFormat;
@@ -55,10 +55,15 @@
"-passin pass:{3} -cert {0}/{5}";
private static final String SIGN_AGENT_CRT = "openssl ca -config " +
"{0}/ca.config -in {0}/{1} -out {0}/{2} -batch -passin pass:{3} " +
- "-keyfile {0}/{4} -cert {0}/{5}"; /**
- * Verify that root certificate exists, generate it otherwise.
- */
- public void initRootCert(MapOperations compOperations) {
+ "-keyfile {0}/{4} -cert {0}/{5}";
+ private static final String GEN_AGENT_KEY="openssl req -new -newkey " +
+ "rsa:1024 -nodes -keyout {0}/{2}.key -subj /OU={1}/CN={2} -out {0}/{2}.csr";
+ private String passphrase;
+
+ /**
+ * Verify that root certificate exists, generate it otherwise.
+ */
+ public void initialize(MapOperations compOperations) {
SecurityUtils.initializeSecurityParameters(compOperations);
LOG.info("Initialization of root certificate");
@@ -87,28 +92,73 @@
return certFile.exists();
}
+ public void setPassphrase(String passphrase) {
+ this.passphrase = passphrase;
+ }
+
+ class StreamConsumer extends Thread
+ {
+ InputStream is;
+ boolean logOutput;
+
+ StreamConsumer(InputStream is, boolean logOutput)
+ {
+ this.is = is;
+ this.logOutput = logOutput;
+ }
+
+ StreamConsumer(InputStream is)
+ {
+ this(is, false);
+ }
+
+ public void run()
+ {
+ try
+ {
+ InputStreamReader isr = new InputStreamReader(is,
+ Charset.forName("UTF8"));
+ BufferedReader br = new BufferedReader(isr);
+ String line;
+ while ( (line = br.readLine()) != null)
+ if (logOutput) {
+ LOG.info(line);
+ }
+ } catch (IOException e)
+ {
+ LOG.error("Error during processing of process stream", e);
+ }
+ }
+ }
+
+
/**
* Runs os command
*
* @return command execution exit code
*/
- private int runCommand(String command) {
+ private int runCommand(String command) throws SliderException {
+ int exitCode = -1;
String line = null;
Process process = null;
BufferedReader br= null;
try {
process = Runtime.getRuntime().exec(command);
- br = new BufferedReader(new InputStreamReader(
- process.getInputStream(), Charset.forName("UTF8")));
+ StreamConsumer outputConsumer =
+ new StreamConsumer(process.getInputStream(), true);
+ StreamConsumer errorConsumer =
+ new StreamConsumer(process.getErrorStream());
- while ((line = br.readLine()) != null) {
- LOG.info(line);
- }
+ outputConsumer.start();
+ errorConsumer.start();
try {
process.waitFor();
SecurityUtils.logOpenSslExitCode(command, process.exitValue());
- return process.exitValue(); //command is executed
+ exitCode = process.exitValue();
+ if (exitCode != 0) {
+ throw new SliderException(exitCode, "Error running command %s", command);
+ }
} catch (InterruptedException e) {
e.printStackTrace();
}
@@ -124,11 +174,28 @@
}
}
- return -1;//some exception occurred
+ return exitCode;//some exception occurred
}
- private void generateServerCertificate() {
+ public synchronized void generateAgentCertificate(String agentHostname, String containerId) {
+ LOG.info("Generation of agent certificate for {}", agentHostname);
+
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ Object[] scriptArgs = {srvrKstrDir, agentHostname, containerId};
+
+ try {
+ String command = MessageFormat.format(GEN_AGENT_KEY, scriptArgs);
+ runCommand(command);
+
+ signAgentCertificate(containerId);
+
+ } catch (SliderException e) {
+ LOG.error("Error generating the agent certificate", e);
+ }
+ }
+
+ private void generateServerCertificate(){
LOG.info("Generation of server certificate");
String srvrKstrDir = SecurityUtils.getSecurityDir();
@@ -141,17 +208,21 @@
Object[] scriptArgs = {srvrCrtPass, srvrKstrDir, srvrKeyName,
srvrCrtName, kstrName, srvrCsrName};
- String command = MessageFormat.format(GEN_SRVR_KEY,scriptArgs);
- runCommand(command);
+ try {
+ String command = MessageFormat.format(GEN_SRVR_KEY,scriptArgs);
+ runCommand(command);
- command = MessageFormat.format(GEN_SRVR_REQ,scriptArgs);
- runCommand(command);
+ command = MessageFormat.format(GEN_SRVR_REQ,scriptArgs);
+ runCommand(command);
- command = MessageFormat.format(SIGN_SRVR_CRT,scriptArgs);
- runCommand(command);
+ command = MessageFormat.format(SIGN_SRVR_CRT,scriptArgs);
+ runCommand(command);
- command = MessageFormat.format(EXPRT_KSTR,scriptArgs);
- runCommand(command);
+ command = MessageFormat.format(EXPRT_KSTR,scriptArgs);
+ runCommand(command);
+ } catch (SliderException e) {
+ LOG.error("Error generating the server certificate", e);
+ }
}
@@ -160,8 +231,7 @@
* @return string with server certificate content
*/
public String getServerCert() {
- File certFile = new File(SecurityUtils.getSecurityDir() +
- File.separator + SliderKeys.CRT_FILE_NAME);
+ File certFile = getServerCertficateFilePath();
String srvrCrtContent = null;
try {
srvrCrtContent = FileUtils.readFileToString(certFile);
@@ -171,6 +241,21 @@
return srvrCrtContent;
}
+ public static File getServerCertficateFilePath() {
+ return new File(SecurityUtils.getSecurityDir() +
+ File.separator + SliderKeys.CRT_FILE_NAME);
+ }
+
+ public static File getAgentCertficateFilePath(String containerId) {
+ return new File(SecurityUtils.getSecurityDir() +
+ File.separator + containerId + ".crt");
+ }
+
+ public static File getAgentKeyFilePath(String containerId) {
+ return new File(SecurityUtils.getSecurityDir() +
+ File.separator + containerId + ".key");
+ }
+
/**
* Signs agent certificate
* Adds agent certificate to server keystore
@@ -183,9 +268,7 @@
LOG.info("Signing of agent certificate");
LOG.info("Verifying passphrase");
- String passphraseSrvr = SliderKeys.PASSPHRASE;
-
- if (!passphraseSrvr.equals(passphraseAgent.trim())) {
+ if (!this.passphrase.equals(passphraseAgent.trim())) {
LOG.warn("Incorrect passphrase from the agent");
response.setResult(SignCertResponse.ERROR_STATUS);
response.setMessage("Incorrect passphrase from the agent");
@@ -205,11 +288,14 @@
//Revoke previous agent certificate if exists
File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
+ String command = null;
if (agentCrtFile.exists()) {
LOG.info("Revoking of " + agentHostname + " certificate.");
- String command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
- int commandExitCode = runCommand(command);
- if (commandExitCode != 0) {
+ command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
+ try {
+ runCommand(command);
+ } catch (SliderException e) {
+ int commandExitCode = e.getExitCode();
response.setResult(SignCertResponse.ERROR_STATUS);
response.setMessage(
SecurityUtils.getOpenSslCommandResult(command, commandExitCode));
@@ -226,16 +312,16 @@
e1.printStackTrace();
}
- String command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
+ command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
LOG.debug(SecurityUtils.hideOpenSslPassword(command));
-
- int commandExitCode = runCommand(command); // ssl command execution
- if (commandExitCode != 0) {
+ try {
+ runCommand(command);
+ } catch (SliderException e) {
+ int commandExitCode = e.getExitCode();
response.setResult(SignCertResponse.ERROR_STATUS);
response.setMessage(
SecurityUtils.getOpenSslCommandResult(command, commandExitCode));
- //LOG.warn(ShellCommandUtil.getOpenSslCommandResult(command, commandExitCode));
return response;
}
@@ -254,4 +340,35 @@
//LOG.info(ShellCommandUtil.getOpenSslCommandResult(command, commandExitCode));
return response;
}
+
+ private String signAgentCertificate (String containerId)
+ throws SliderException {
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ String srvrCrtPass = SecurityUtils.getKeystorePass();
+ String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+ String srvrKeyName = SliderKeys.KEY_FILE_NAME;
+ String agentCrtReqName = containerId + ".csr";
+ String agentCrtName = containerId + ".crt";
+
+ Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName,
+ srvrCrtPass, srvrKeyName, srvrCrtName};
+
+ //Revoke previous agent certificate if exists
+ File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
+
+ String command;
+ if (agentCrtFile.exists()) {
+ LOG.info("Revoking of " + containerId + " certificate.");
+ command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
+ runCommand(command);
+ }
+
+ command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
+
+ LOG.debug(SecurityUtils.hideOpenSslPassword(command));
+ runCommand(command);
+
+ return agentCrtName;
+
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
new file mode 100644
index 0000000..c892b10
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.util.Time;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.actions.AsyncAction;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.actions.RenewingAction;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.security.PrivilegedExceptionAction;
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public class FsDelegationTokenManager {
+ private final QueueAccess queue;
+ private RenewingAction<RenewAction> renewingAction;
+ private UserGroupInformation remoteUser;
+ private UserGroupInformation currentUser;
+ private static final Logger
+ log = LoggerFactory.getLogger(FsDelegationTokenManager.class);
+ private long renewInterval;
+ private RenewAction renewAction;
+ private String tokenName;
+
+ public FsDelegationTokenManager(QueueAccess queue) throws IOException {
+ this.queue = queue;
+ this.currentUser = UserGroupInformation.getCurrentUser();
+ }
+
+ private void createRemoteUser(Configuration configuration) throws IOException {
+ Configuration loginConfig = new Configuration(configuration);
+ loginConfig.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ // using HDFS principal...
+ this.remoteUser = UserGroupInformation
+ .loginUserFromKeytabAndReturnUGI(
+ SecurityUtil.getServerPrincipal(
+ loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
+ InetAddress.getLocalHost().getCanonicalHostName()),
+ loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
+ log.info("Created remote user {}. UGI reports current user is {}",
+ this.remoteUser, UserGroupInformation.getCurrentUser());
+ }
+
+ public void acquireDelegationToken(Configuration configuration)
+ throws IOException, InterruptedException {
+ if (remoteUser == null) {
+ createRemoteUser(configuration);
+ }
+ if (SliderUtils.isHadoopClusterSecure(configuration) &&
+ renewingAction == null) {
+ renewInterval = configuration.getLong(
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+ // constructor of action will retrieve initial token. One may already be
+ // associated with user, but its lifecycle/management is not clear so let's
+ // create and manage a token explicitly
+ renewAction = new RenewAction("HDFS renew",
+ configuration);
+ // set retrieved token as the user associated delegation token and
+ // start a renewing action to renew
+ Token<?> token = renewAction.getToken();
+ currentUser.addToken(token.getService(), token);
+ log.info("HDFS delegation token {} acquired and set as credential for current user", token);
+ renewingAction = new RenewingAction<RenewAction>(renewAction,
+ (int) renewInterval,
+ (int) renewInterval,
+ TimeUnit.MILLISECONDS,
+ getRenewingLimit());
+ log.info("queuing HDFS delegation token renewal interval of {} milliseconds",
+ renewInterval);
+ queue(renewingAction);
+ }
+ }
+
+ public void cancelDelegationToken(Configuration configuration)
+ throws IOException, InterruptedException {
+ queue.removeRenewingAction(getRenewingActionName());
+ if (renewAction != null) {
+ renewAction.getToken().cancel(configuration);
+ }
+ log.info("Renewing action {} removed and HDFS delegation token renewal "
+ + "cancelled", getRenewingActionName());
+ }
+
+ protected int getRenewingLimit() {
+ return 0;
+ }
+
+ protected void queue(RenewingAction<RenewAction> action) {
+ queue.renewing(getRenewingActionName(),
+ action);
+ }
+
+ protected String getRenewingActionName() {
+ if (tokenName == null) {
+ tokenName = "HDFS renewing token " + UUID.randomUUID();
+ }
+ return tokenName;
+ }
+
+ class RenewAction extends AsyncAction {
+ Configuration configuration;
+ Token<?> token;
+ private long tokenExpiryTime;
+ private final FileSystem fs;
+
+ RenewAction(String name,
+ Configuration configuration)
+ throws IOException, InterruptedException {
+ super(name);
+ this.configuration = configuration;
+ fs = getFileSystem();
+ // get initial token by creating a kerberos authenticated user and
+ // invoking token methods as that user
+ synchronized (fs) {
+ this.token = remoteUser.doAs(new PrivilegedExceptionAction<Token<?>>() {
+ @Override
+ public Token<?> run() throws Exception {
+ log.info("Obtaining HDFS delgation token with user {}",
+ remoteUser.getShortUserName());
+ Token token = fs.getDelegationToken(
+ remoteUser.getShortUserName());
+ tokenExpiryTime = getTokenExpiryTime(token);
+ log.info("Initial delegation token obtained with expiry time of {}", getPrintableExirationTime(tokenExpiryTime));
+ return token;
+ }
+ });
+ }
+ log.info("Initial request returned delegation token {}", token);
+ }
+
+ private long getTokenExpiryTime(Token token) throws IOException {
+ AbstractDelegationTokenIdentifier id =
+ (AbstractDelegationTokenIdentifier)token.decodeIdentifier();
+ return id.getMaxDate();
+ }
+
+ protected FileSystem getFileSystem()
+ throws IOException, InterruptedException {
+ // return non-cache FS reference
+ return remoteUser.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws Exception {
+ Configuration config = new Configuration(configuration);
+ config.setBoolean("fs.hdfs.impl.disable.cache", true);
+ return getRemoteFileSystemForRenewal(config);
+ }
+ });
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster, QueueAccess queueService,
+ AppState appState)
+ throws Exception {
+ if (fs != null) {
+ synchronized(fs) {
+ try {
+ long expires = remoteUser.doAs(new PrivilegedExceptionAction<Long>() {
+ @Override
+ public Long run() throws Exception {
+ long expires = token.renew(fs.getConf());
+ log.info("HDFS delegation token renewed. Renewal cycle ends at {}",
+ getPrintableExirationTime(expires));
+ return expires;
+ }
+ });
+ long calculatedInterval = tokenExpiryTime - Time.now();
+ if ( calculatedInterval < renewInterval ) {
+ // time to get a new token since the token will expire before
+ // next renewal interval. Could modify this to be closer to expiry
+ // time if deemed necessary....
+ log.info("Interval of {} less than renew interval. Getting new token",
+ calculatedInterval);
+ getNewToken();
+ } else {
+ updateRenewalTime(renewInterval);
+ }
+ } catch (IOException ie) {
+ // token has expired. get a new one...
+ log.info("Exception raised by renew", ie);
+ getNewToken();
+ }
+ }
+ }
+ }
+
+ private String getPrintableExirationTime(long expires) {
+ Date d = new Date(expires);
+ return DateFormat.getDateTimeInstance().format(d);
+ }
+
+ private void getNewToken()
+ throws InterruptedException, IOException {
+ try {
+ Text service = token.getService();
+ Token<?>[] tokens = remoteUser.doAs(new PrivilegedExceptionAction<Token<?>[]>() {
+ @Override
+ public Token<?>[] run() throws Exception {
+ return fs.addDelegationTokens(remoteUser.getShortUserName(), null);
+ }
+ });
+ if (tokens.length == 0) {
+ throw new IOException("addDelegationTokens returned no tokens");
+ }
+ token = findMatchingToken(service, tokens);
+ currentUser.addToken(token.getService(), token);
+
+ tokenExpiryTime = getTokenExpiryTime(token);
+
+ log.info("Expired HDFS delegation token replaced and added as credential"
+ + " to current user. Token expires at {}",
+ getPrintableExirationTime(tokenExpiryTime));
+ updateRenewalTime(renewInterval);
+ } catch (IOException ie2) {
+ throw new IOException("Can't get new delegation token ", ie2);
+ }
+ }
+
+ private void updateRenewalTime(long interval) {
+ long delay = interval - interval/10;
+ renewingAction.updateInterval(delay, TimeUnit.MILLISECONDS);
+ log.info("Token renewal set for {} ms from now", delay);
+ }
+
+ private Token<?> findMatchingToken(Text service, Token<?>[] tokens) {
+ Token<?> token = null;
+ int i = 0;
+ while (token == null && i < tokens.length) {
+ if (tokens[i].getService().equals(service)) {
+ token = tokens[i];
+ }
+ i++;
+ }
+
+ return token;
+ }
+
+ Token<?> getToken() {
+ synchronized (fs) {
+ return token;
+ }
+ }
+ }
+
+ protected FileSystem getRemoteFileSystemForRenewal(Configuration config)
+ throws IOException {
+ return FileSystem.get(config);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
index 56ee199..527d4e6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.conf.MapOperations;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,7 +35,7 @@
//import java.nio.file.Paths;
//import java.nio.file.attribute.PosixFilePermission;
//import java.nio.file.attribute.PosixFilePermissions;
-import java.util.Set;
+
/**
*
@@ -139,8 +140,13 @@
}
public static void initializeSecurityParameters(MapOperations configMap) {
+ initializeSecurityParameters(configMap, false);
+ }
+
+ public static void initializeSecurityParameters(MapOperations configMap,
+ boolean persistPassword) {
String keyStoreLocation = configMap.getOption(
- SliderKeys.KEYSTORE_LOCATION, getDefaultKeystoreLocation());
+ SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, getDefaultKeystoreLocation());
File secDirFile = new File(keyStoreLocation).getParentFile();
if (!secDirFile.exists()) {
// create entire required directory structure
@@ -166,26 +172,28 @@
}
// need to create the password
}
- keystorePass = getKeystorePassword(secDirFile);
+ keystorePass = getKeystorePassword(secDirFile, persistPassword);
securityDir = secDirFile.getAbsolutePath();
}
- private static String getKeystorePassword(File secDirFile) {
+ private static String getKeystorePassword(File secDirFile,
+ boolean persistPassword) {
File passFile = new File(secDirFile, SliderKeys.CRT_PASS_FILE_NAME);
String password = null;
-
if (!passFile.exists()) {
- LOG.info("Generation of file with password");
- try {
- password = RandomStringUtils.randomAlphanumeric(
- Integer.valueOf(SliderKeys.PASS_LEN));
- FileUtils.writeStringToFile(passFile, password);
- passFile.setWritable(true);
- passFile.setReadable(true);
- } catch (IOException e) {
- e.printStackTrace();
- throw new RuntimeException(
- "Error creating certificate password file");
+ LOG.info("Generating keystore password");
+ password = RandomStringUtils.randomAlphanumeric(
+ Integer.valueOf(SliderKeys.PASS_LEN));
+ if (persistPassword) {
+ try {
+ FileUtils.writeStringToFile(passFile, password);
+ passFile.setWritable(true);
+ passFile.setReadable(true);
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException(
+ "Error creating certificate password file");
+ }
}
} else {
LOG.info("Reading password from existing file");
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index 6c0edb8..10af3d3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -18,21 +18,20 @@
package org.apache.slider.server.services.utility;
-
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.zk.ZookeeperUtils;
-import org.apache.slider.server.services.curator.CuratorHelper;
-import org.apache.slider.server.services.registry.SliderRegistryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_REGISTRY_PATH;
-import static org.apache.slider.common.SliderXmlConfKeys.REGISTRY_PATH;
-
/**
* Base service for the standard slider client/server services
*/
@@ -44,20 +43,8 @@
public AbstractSliderLaunchedService(String name) {
super(name);
// make sure all the yarn configs get loaded
- new YarnConfiguration();
- }
-
- /**
- * Start the registration service
- * @return the instance
- * @throws BadConfigException
- */
- protected SliderRegistryService startRegistrationService()
- throws BadConfigException {
-
- String registryQuorum = lookupZKQuorum();
- String zkPath = getConfig().get(REGISTRY_PATH, DEFAULT_REGISTRY_PATH);
- return startRegistrationService(registryQuorum, zkPath);
+ YarnConfiguration conf = new YarnConfiguration();
+ ConfigHelper.registerDeprecatedConfigItems();
}
/**
@@ -66,12 +53,14 @@
* @throws BadConfigException if it is not there or invalid
*/
public String lookupZKQuorum() throws BadConfigException {
- String registryQuorum = getConfig().get(
- SliderXmlConfKeys.REGISTRY_ZK_QUORUM);
+
+ String registryQuorum = getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+
+ // though if neither is set: trouble
if (SliderUtils.isUnset(registryQuorum)) {
throw new BadConfigException(
"No Zookeeper quorum provided in the"
- + " configuration property " + SliderXmlConfKeys.REGISTRY_ZK_QUORUM
+ + " configuration property " + RegistryConstants.KEY_REGISTRY_ZK_QUORUM
);
}
ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
@@ -79,29 +68,41 @@
}
/**
- * Start the registration service
- * @param zkConnection
- * @param zkPath
- * @return
+ * Create, adopt ,and start the YARN registration service
+ * @return the registry operations service, already deployed as a child
+ * of the AbstractSliderLaunchedService instance.
*/
- public SliderRegistryService startRegistrationService(
- String zkConnection, String zkPath) {
- CuratorHelper curatorHelper =
- new CuratorHelper(getConfig(), zkConnection);
+ public RegistryOperations startRegistryOperationsService()
+ throws BadConfigException {
- //registry will start curator as well as the binder, in the correct order
- SliderRegistryService registryBinderService =
- curatorHelper.createRegistryBinderService(zkPath);
- deployChildService(registryBinderService);
- return registryBinderService;
+ // push back the slider registry entry if needed
+ String quorum = lookupZKQuorum();
+ RegistryOperations registryWriterService =
+ createRegistryOperationsInstance();
+ deployChildService(registryWriterService);
+ return registryWriterService;
}
- protected void requireArgumentSet(String argname, String argfield)
+ /**
+ * Create the registry operations instance. This is to allow
+ * subclasses to instantiate a subclass service
+ * @return an instance to match to the lifecycle of this service
+ */
+ protected RegistryOperations createRegistryOperationsInstance() {
+ return RegistryOperationsFactory.createInstance("YarnRegistry", getConfig());
+ }
+
+ /**
+ * Utility method to require an argument to be set (non null, non-empty)
+ * @param argname argument name
+ * @param value value
+ * @throws BadCommandArgumentsException if the condition is not met
+ */
+ protected static void requireArgumentSet(String argname, String value)
throws BadCommandArgumentsException {
- if (isUnset(argfield)) {
- throw new BadCommandArgumentsException("Required argument "
- + argname
- + " missing");
+ if (isUnset(value)) {
+ throw new BadCommandArgumentsException(
+ "Required argument " + argname + " missing");
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
new file mode 100644
index 0000000..40ceab8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceStateChangeListener;
+
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Wait for a service to stop.
+ *
+ * WARNING: the notification may come in as soon as the service enters
+ * the stopped state: it may take some time for the actual stop operation
+ * to complete.
+ */
+public class EndOfServiceWaiter implements ServiceStateChangeListener {
+
+ private final AtomicBoolean finished = new AtomicBoolean(false);
+ private final String name;
+ private Service service;
+
+ /**
+ * Wait for a service; use the service name as this instance's name
+ * @param service service
+ */
+ public EndOfServiceWaiter(Service service) {
+ this(service.getName(), service);
+ }
+
+
+ /**
+ * Wait for a service
+ * @param name name for messages
+ * @param service service
+ */
+ public EndOfServiceWaiter(String name, Service service) {
+ this.name = name;
+ this.service = service;
+ service.registerServiceListener(this);
+ }
+
+ public synchronized void waitForServiceToStop(long timeout) throws
+ InterruptedException, TimeoutException {
+ service.waitForServiceToStop(timeout);
+ if (!finished.get()) {
+ wait(timeout);
+ if (!finished.get()) {
+ throw new TimeoutException(name
+ + " did not finish after " + timeout +
+ " milliseconds");
+ }
+ }
+ }
+
+ /**
+ * Wait for service state change callbacks; notify self if the service has
+ * now stopped
+ * @param service service
+ */
+ @Override
+ public synchronized void stateChanged(Service service) {
+ if (service.isInState(Service.STATE.STOPPED)) {
+ finished.set(true);
+ notify();
+ }
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
index 0d47c3b..e939c20 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
@@ -28,6 +28,10 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * This is a workflow compositoe service which can be launched from the CLI
+ * ... catches the arguments and implements a stub runService operation.
+ */
public class LaunchedWorkflowCompositeService extends WorkflowCompositeService
implements RunService {
private static final Logger log = LoggerFactory.getLogger(
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
index 7a475cc..8b711aa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
@@ -33,6 +33,9 @@
private C closeable;
+ public ClosingService(String name) {
+ super(name);
+ }
/**
* Construct an instance of the service
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
index 46c724c..9f6b327 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
@@ -81,7 +81,11 @@
private LongLivedProcess process;
private int executionTimeout = -1;
private int timeoutCode = 1;
-
+ /**
+ log to log to; defaults to this service log
+ */
+ private Logger processLog = LOG;
+
/**
* Exit code set when the spawned process exits
*/
@@ -102,7 +106,8 @@
* @param env environment variables above those generated by
* @throws IOException IO problems
*/
- public ForkedProcessService(String name, Map<String, String> env,
+ public ForkedProcessService(String name,
+ Map<String, String> env,
List<String> commandList) throws IOException {
super(name);
build(env, commandList);
@@ -130,6 +135,15 @@
}
/**
+ * Set the process log. This may be null for "do not log"
+ * @param processLog process log
+ */
+ public void setProcessLog(Logger processLog) {
+ this.processLog = processLog;
+ process.setProcessLog(processLog);
+ }
+
+ /**
* Set the timeout by which time a process must have finished -or -1 for forever
* @param timeout timeout in milliseconds
*/
@@ -148,7 +162,8 @@
List<String> commandList)
throws IOException {
assert process == null;
- process = new LongLivedProcess(getName(), LOG, commandList);
+
+ process = new LongLivedProcess(getName(), processLog, commandList);
process.setLifecycleCallback(this);
//set the env variable mapping
process.putEnvMap(env);
@@ -249,7 +264,7 @@
}
- public int getExitCode() {
+ public Integer getExitCode() {
return process.getExitCode();
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
index c8ff758..cebb035 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -24,6 +24,7 @@
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
@@ -50,6 +51,7 @@
* Key Features:
* <ol>
* <li>Output is streamed to the output logger provided</li>.
+ * <li>the input stream is closed as soon as the process starts.</li>
* <li>The most recent lines of output are saved to a linked list</li>.
* <li>A synchronous callback, {@link LongLivedProcessLifecycleEvent}, is raised on the start
* and finish of a process.</li>
@@ -90,7 +92,7 @@
* Log supplied in the constructor for the spawned process -accessible
* to inner classes
*/
- private final Logger processLog;
+ private Logger processLog;
/**
* Class log -accessible to inner classes
@@ -102,10 +104,15 @@
*/
private final AtomicBoolean finished = new AtomicBoolean(false);
+ /**
+ * Create an instance
+ * @param name process name
+ * @param processLog log for output (or null)
+ * @param commands command list
+ */
public LongLivedProcess(String name,
Logger processLog,
List<String> commands) {
- Preconditions.checkArgument(processLog != null, "processLog");
Preconditions.checkArgument(commands != null, "commands");
this.name = name;
@@ -168,6 +175,14 @@
}
/**
+ * Set the process log. Ignored once the process starts
+ * @param processLog new log ... may be null
+ */
+ public void setProcessLog(Logger processLog) {
+ this.processLog = processLog;
+ }
+
+ /**
* Get the process reference
* @return the process -null if the process is not started
*/
@@ -270,7 +285,8 @@
/**
* Exec the process
* @return the process
- * @throws IOException
+ * @throws IOException on aany failure to start the process
+ * @throws FileNotFoundException if the process could not be found
*/
private Process spawnChildProcess() throws IOException {
if (process != null) {
@@ -279,7 +295,20 @@
if (LOG.isDebugEnabled()) {
LOG.debug("Spawning process:\n " + describeBuilder());
}
- process = processBuilder.start();
+ try {
+ process = processBuilder.start();
+ } catch (IOException e) {
+ // on windows, upconvert DOS error 2 from ::CreateProcess()
+ // to its real meaning: FileNotFound
+ if (e.toString().contains("CreateProcess error=2")) {
+ FileNotFoundException fnfe =
+ new FileNotFoundException(e.toString());
+ fnfe.initCause(e);
+ throw fnfe;
+ } else {
+ throw e;
+ }
+ }
return process;
}
@@ -295,6 +324,8 @@
lifecycleCallback.onProcessStarted(this);
}
try {
+ //close stdin for the process
+ IOUtils.closeStream(process.getOutputStream());
exitCode = process.waitFor();
} catch (InterruptedException e) {
LOG.debug("Process wait interrupted -exiting thread", e);
@@ -388,6 +419,7 @@
}
return getRecentOutput();
}
+
/**
* add the recent line to the list of recent lines; deleting
* an earlier on if the limit is reached.
@@ -398,10 +430,11 @@
* something that is only called once per line of IO?
* @param line line to record
* @param isErrorStream is the line from the error stream
- * @param logger logger to log to
+ * @param logger logger to log to - null for no logging
*/
private synchronized void recordRecentLine(String line,
- boolean isErrorStream, Logger logger) {
+ boolean isErrorStream,
+ Logger logger) {
if (line == null) {
return;
}
@@ -410,10 +443,12 @@
if (recentLines.size() > recentLineLimit) {
recentLines.remove(0);
}
- if (isErrorStream) {
- logger.warn(line);
- } else {
- logger.info(line);
+ if (logger != null) {
+ if (isErrorStream) {
+ logger.warn(line);
+ } else {
+ logger.info(line);
+ }
}
}
@@ -428,6 +463,12 @@
private final Logger streamLog;
private final int sleepTime;
+ /**
+ * Create an instance
+ * @param streamLog log -or null to disable logging (recent entries
+ * will still be retained)
+ * @param sleepTime time to sleep when stopping
+ */
private ProcessStreamReader(Logger streamLog, int sleepTime) {
this.streamLog = streamLog;
this.sleepTime = sleepTime;
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowSequenceService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowSequenceService.java
index ca07f99..e584e63 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowSequenceService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowSequenceService.java
@@ -79,6 +79,8 @@
null if one did not finish yet
*/
private volatile Service previousService;
+
+ private boolean stopIfNoChildServicesAtStartup = true;
/**
* Construct an instance
@@ -133,13 +135,17 @@
return previousService;
}
+ protected void setStopIfNoChildServicesAtStartup(boolean stopIfNoChildServicesAtStartup) {
+ this.stopIfNoChildServicesAtStartup = stopIfNoChildServicesAtStartup;
+ }
+
/**
* When started
* @throws Exception
*/
@Override
protected void serviceStart() throws Exception {
- if (!startNextService()) {
+ if (!startNextService() && stopIfNoChildServicesAtStartup) {
//nothing to start -so stop
stop();
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
new file mode 100644
index 0000000..254bf27
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.yarnregistry;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.slider.common.tools.SliderUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
+
+/**
+ * Registry view for providers. This tracks where the service
+ * is registered, offers access to the record and other things.
+ */
+public class YarnRegistryViewForProviders {
+
+ private final RegistryOperations registryOperations;
+
+ private final String user;
+
+ private final String sliderServiceClass;
+ private final String instanceName;
+ private final ApplicationAttemptId applicationAttemptId;
+ /**
+ * Record used where the service registered itself.
+ * Null until the service is registered
+ */
+ private ServiceRecord selfRegistration;
+
+ /**
+ * Path where record was registered
+ * Null until the service is registered
+ */
+ private String selfRegistrationPath;
+
+ public YarnRegistryViewForProviders(RegistryOperations registryOperations,
+ String user,
+ String sliderServiceClass,
+ String instanceName,
+ ApplicationAttemptId applicationAttemptId) {
+ Preconditions.checkArgument(registryOperations != null,
+ "null registry operations");
+ Preconditions.checkArgument(user != null, "null user");
+ Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
+ "unset service class");
+ Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+ "instanceName");
+ Preconditions.checkArgument(applicationAttemptId != null,
+ "null applicationAttemptId");
+ this.registryOperations = registryOperations;
+ this.user = user;
+ this.sliderServiceClass = sliderServiceClass;
+ this.instanceName = instanceName;
+ this.applicationAttemptId = applicationAttemptId;
+ }
+
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return applicationAttemptId;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public String getSliderServiceClass() {
+ return sliderServiceClass;
+ }
+
+ public String getInstanceName() {
+ return instanceName;
+ }
+
+ public RegistryOperations getRegistryOperations() {
+ return registryOperations;
+ }
+
+ public ServiceRecord getSelfRegistration() {
+ return selfRegistration;
+ }
+
+ private void setSelfRegistration(ServiceRecord selfRegistration) {
+ this.selfRegistration = selfRegistration;
+ }
+
+ /**
+ * Get the path to where the service has registered itself.
+ * Null until the service is registered
+ * @return the service registration path.
+ */
+ public String getSelfRegistrationPath() {
+ return selfRegistrationPath;
+ }
+
+ /**
+ * Get the absolute path to where the service has registered itself.
+ * This includes the base registry path
+ * Null until the service is registered
+ * @return the service registration path.
+ */
+ public String getAbsoluteSelfRegistrationPath() {
+ if (selfRegistrationPath == null) {
+ return null;
+ }
+ String root = registryOperations.getConfig().getTrimmed(
+ RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
+ return RegistryPathUtils.join(root, selfRegistrationPath);
+ }
+
+ /**
+ * Add a component under the slider name/entry
+ * @param componentName component name
+ * @param record record to put
+ * @throws IOException
+ */
+ public void putComponent(String componentName,
+ ServiceRecord record) throws
+ IOException {
+ putComponent(sliderServiceClass, instanceName,
+ componentName,
+ record);
+ }
+
+ /**
+ * Add a component
+ * @param serviceClass service class to use under ~user
+ * @param componentName component name
+ * @param record record to put
+ * @throws IOException
+ */
+ public void putComponent(String serviceClass,
+ String serviceName,
+ String componentName,
+ ServiceRecord record) throws IOException {
+ String path = RegistryUtils.componentPath(
+ user, serviceClass, serviceName, componentName);
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ }
+
+ /**
+ * Add a service under a path, optionally purging any history
+ * @param username user
+ * @param serviceClass service class to use under ~user
+ * @param serviceName name of the service
+ * @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first.
+ * @return the path the service was created at
+ * @throws IOException
+ */
+ public String putService(String username,
+ String serviceClass,
+ String serviceName,
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ String path = RegistryUtils.servicePath(
+ username, serviceClass, serviceName);
+ if (deleteTreeFirst) {
+ registryOperations.delete(path, true);
+ }
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ return path;
+ }
+
+ /**
+ * Add a service under a path for the current user
+ * @param serviceClass service class to use under ~user
+ * @param serviceName name of the service
+ * @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first
+ * @return the path the service was created at
+ * @throws IOException
+ */
+ public String putService(
+ String serviceClass,
+ String serviceName,
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ return putService(user, serviceClass, serviceName, record, deleteTreeFirst);
+ }
+
+
+ /**
+ * Add a service under a path for the current user
+ * @param serviceClass service class to use under ~user
+ * @param serviceName name of the service
+ * @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first
+ * @return the path the service was created at
+ * @throws IOException
+ */
+ public String registerSelf(
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ selfRegistrationPath =
+ putService(user, sliderServiceClass, instanceName, record, deleteTreeFirst);
+ setSelfRegistration(record);
+ return selfRegistrationPath;
+ }
+
+ /**
+ * Update the self record by pushing out the latest version of the service
+ * registration record.
+ * @throws IOException any failure.
+ */
+ public void updateSelf() throws IOException {
+ putService(user, sliderServiceClass, instanceName, selfRegistration, false);
+ }
+
+ /**
+ * Delete a component
+ * @param componentName component name
+ * @throws IOException
+ */
+ public void deleteComponent(String componentName) throws IOException {
+ String path = RegistryUtils.componentPath(
+ user, sliderServiceClass, instanceName,
+ componentName);
+ registryOperations.delete(path, false);
+ }
+
+ /**
+ * Delete the children of a path -but not the path itself.
+ * It is not an error if the path does not exist
+ * @param path path to delete
+ * @param recursive flag to request recursive deletes
+ * @throws IOException IO problems
+ */
+ public void deleteChildren(String path, boolean recursive) throws IOException {
+ List<String> childNames = null;
+ try {
+ childNames = registryOperations.list(path);
+ } catch (PathNotFoundException e) {
+ return;
+ }
+ for (String childName : childNames) {
+ String child = join(path, childName);
+ registryOperations.delete(child, recursive);
+ }
+ }
+
+}
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json b/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json
index 1bc8381..197a046 100644
--- a/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json
+++ b/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json
@@ -56,7 +56,6 @@
},
"global": {
"hbase_root": "/share/hbase/hbase-0.96.1-hadoop2",
- "security_enabled": "false",
"hbase_pid_dir": "/var/run/hbase",
"proxyuser_group": "users",
"syncLimit": "5",
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json b/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
index cab952e..da06c13 100644
--- a/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
+++ b/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
@@ -56,7 +56,6 @@
},
"global": {
"hbase_root": "{{HBASE_HOME}}",
- "security_enabled": "false",
"hbase_pid_dir": "{{PID_DIR}}",
"proxyuser_group": "users",
"syncLimit": "5",
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
index 89095b1..81239a2 100644
--- a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
+++ b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
@@ -12,8 +12,7 @@
"components": {
"slider-appmaster" : {
- "jvm.heapsize": "256M",
- "ssl.server.keystore.location": "/tmp/work/security/keystore.p12"
+ "jvm.heapsize": "256M"
}
}
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json
index 5ed462b..478ab7e 100644
--- a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json
+++ b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json
@@ -12,7 +12,7 @@
"slider-appmaster": {
"yarn.component.instances": "1",
"yarn.vcores": "1",
- "yarn.memory": "256"
+ "yarn.memory": "1024"
}
}
}
\ No newline at end of file
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig.json b/slider-core/src/test/app_packages/test_command_log/appConfig.json
index e7f9700..a8d3d4b 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig.json
@@ -3,19 +3,13 @@
"metadata": {
},
"global": {
- "agent.conf": "agent.ini",
- "application.def": "apache-slider-command-logger.zip",
- "config_types": "cl-site",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/command-logger.tar",
- "site.global.app_user": "yarn",
+ "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.application_id": "CommandLogger",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log",
- "site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
+ "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/operations.log",
+ "site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p",
+ "site.cl-site.pattern.for.test.to.verify": "verify this pattern"
},
"components": {
"COMMAND_LOGGER": {
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json b/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
index 57c935c..ac11c08 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
@@ -5,18 +5,11 @@
"global": {
"heartbeat.monitor.interval": "20000",
"agent.instance.debug.data": "ANY:DO_NOT_REGISTER:NONE",
- "agent.conf": "agent.ini",
- "application.def": "apache-slider-command-logger.zip",
- "config_types": "cl-site",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/command-logger.tar",
- "site.global.app_user": "yarn",
+ "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.application_id": "CommandLogger",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log",
+ "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/operations.log",
"site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
},
"components": {
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json b/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
index e028140..9d70879 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
@@ -5,18 +5,11 @@
"global": {
"heartbeat.monitor.interval": "20000",
"agent.instance.debug.data": "ANY:DO_NOT_HEARTBEAT:DO_NOT_HEARTBEAT:NONE",
- "agent.conf": "agent.ini",
- "application.def": "apache-slider-command-logger.zip",
- "config_types": "cl-site",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/command-logger.tar",
- "site.global.app_user": "yarn",
+ "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
"site.global.application_id": "CommandLogger",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log",
+ "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/operations.log",
"site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
},
"components": {
diff --git a/slider-core/src/test/app_packages/test_command_log/resources.json b/slider-core/src/test/app_packages/test_command_log/resources.json
index 8345661..18c505f 100644
--- a/slider-core/src/test/app_packages/test_command_log/resources.json
+++ b/slider-core/src/test/app_packages/test_command_log/resources.json
@@ -6,10 +6,12 @@
},
"components": {
"COMMAND_LOGGER": {
- "yarn.role.priority": "1",
+ "yarn.memory": "128",
+ "yarn.role.priority": "1",
"yarn.component.instances": "1"
},
"slider-appmaster": {
+ "yarn.memory": "256"
}
}
}
diff --git a/slider-core/src/test/app_packages/test_command_log/resources_no_role.json b/slider-core/src/test/app_packages/test_command_log/resources_no_role.json
index 7913fe2..b25c8c2 100644
--- a/slider-core/src/test/app_packages/test_command_log/resources_no_role.json
+++ b/slider-core/src/test/app_packages/test_command_log/resources_no_role.json
@@ -6,10 +6,12 @@
},
"components": {
"COMMAND_LOGGER": {
+ "yarn.memory": "128",
"yarn.role.priority": "1",
"yarn.component.instances": "0"
},
"slider-appmaster": {
+ "yarn.memory": "256"
}
}
}
diff --git a/slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json b/slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json
new file mode 100644
index 0000000..1357f9f
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json
@@ -0,0 +1,19 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ },
+ "components": {
+ "COMMAND_LOGGER": {
+ "yarn.memory": "128",
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.label.expression":"blue"
+ },
+ "slider-appmaster": {
+ "yarn.label.expression":"red",
+ "yarn.memory": "256"
+ }
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index 74f7a3f..56beeaf 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -27,6 +27,7 @@
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXMLConfKeysForTesting
import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.agent.AgentKeys
import org.apache.slider.test.YarnZKMiniClusterTestBase
@@ -43,10 +44,17 @@
extends YarnZKMiniClusterTestBase {
protected static File agentConf
protected static File agentDef
- protected static File imagePath
protected static Map<String, String> agentDefOptions
private static TemporaryFolder tempFolder = new TemporaryFolder();
+ /**
+ * Server side test: validate system env before launch
+ */
+ @BeforeClass
+ public static void checkSystem() {
+// SliderUtils.validateSliderServerEnvironment(LOG)
+ }
+
@BeforeClass
public static void createSubConfFiles() {
@@ -80,8 +88,16 @@
@AfterClass
public static void cleanSubConfFiles() {
- if (tempFolder.getRoot().exists()) {
- FileUtils.deleteDirectory(tempFolder.getRoot());
+ def tempRoot
+ try {
+ tempRoot = tempFolder.root
+ if (tempRoot.exists()) {
+ FileUtils.deleteDirectory(tempRoot);
+ }
+ } catch (IOException e) {
+ log.info("Failed to delete $tempRoot :$e", e)
+ } catch (IllegalStateException e) {
+ log.warn("Temp folder deletion failed: $e")
}
}
@@ -126,7 +142,7 @@
}
/**
- * Create an AM without a master
+ * Create a standalone AM
* @param clustername AM name
* @param size # of nodes
* @param deleteExistingData should any existing cluster data be deleted
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
index 9bfeb8c..202734c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
@@ -21,9 +21,11 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionExistsArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
@@ -107,10 +109,14 @@
assertSucceeded(launcher)
// assert that the cluster exists
-
assert 0 == sliderClient.actionExists(clustername, true)
+
+ // assert that the cluster is in the running state
+ ActionExistsArgs running = new ActionExistsArgs()
+ running.state = YarnApplicationState.RUNNING.toString()
+ assert 0 == sliderClient.actionExists(clustername, running)
- // freeze the cluster
+ // stop the cluster
clusterActionFreeze(sliderClient, clustername)
//verify that exists(live) is now false
@@ -118,6 +124,15 @@
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
+ assert LauncherExitCodes.EXIT_FALSE == sliderClient.actionExists(clustername, running)
+
+ // verify that on a cluster thaw the existence probes still work, that is
+ // they do not discover the previous instance and return false when there
+ // is actually a live cluster
+ ServiceLauncher launcher2 = thawCluster(clustername, [], true);
+ addToTeardown(launcher2)
+ assert 0 == sliderClient.actionExists(clustername, running)
+ assert 0 == sliderClient.actionExists(clustername, true)
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
new file mode 100644
index 0000000..064c7a9
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.actions
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * existence tests
+ */
+@CompileStatic
+@Slf4j
+
+class TestActionInstallPackage extends AgentMiniClusterTestBase {
+
+ @Before
+ public void setup() {
+ super.setup()
+ createMiniCluster("", configuration, 1, false)
+ }
+
+ @Test
+ public void testInstallPackageFailsWithNoPackageName() throws Throwable {
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE
+ ],
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (BadCommandArgumentsException e) {
+ assert e.message.contains("A valid application type name is required (e.g. HBASE)")
+ }
+ }
+
+ @Test
+ public void testInstallPackageFailsWithNoPackagePath() throws Throwable {
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase"
+ ],
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (BadCommandArgumentsException e) {
+ assert e.message.contains("A valid application package location required")
+ }
+ }
+
+ @Test
+ public void testInstallPackageFailsWithInvalidPackagePath() throws Throwable {
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase",
+ Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
+ ],
+ )
+ launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase",
+ Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
+ ],
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (BadCommandArgumentsException e) {
+ assert e.message.contains("Use --replacepkg to overwrite")
+ }
+ }
+
+ @Test
+ public void testInstallPackageFailsWithNeedingReplaceFlag() throws Throwable {
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase",
+ Arguments.ARG_PACKAGE, "unlikely_to_be_a_file_path",
+ ],
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (BadCommandArgumentsException e) {
+ assert e.message.contains("Unable to access supplied pkg file at")
+ }
+ }
+
+ @Test
+ public void testInstallPackageWithReplace() throws Throwable {
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase",
+ Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
+ ],
+ )
+ launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase",
+ Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
+ Arguments.ARG_REPLACE_PKG
+ ],
+ )
+ } catch (BadCommandArgumentsException e) {
+ log.info(e.message)
+ }
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index b7196ac..6baf5b1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -20,11 +20,15 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionListArgs
+import org.apache.slider.common.params.ActionThawArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Before
@@ -34,13 +38,12 @@
* Test List operations
*/
@Slf4j
-
class TestActionList extends AgentMiniClusterTestBase {
@Before
public void setup() {
super.setup()
- createMiniCluster("", configuration, 1, false)
+ createMiniCluster("", configuration, 1, true)
}
/**
@@ -52,9 +55,9 @@
@Test
public void testSuite() throws Throwable {
testListThisUserNoClusters()
- testListAllUsersNoClusters()
testListLiveCluster()
testListMissingCluster()
+ testActionListStates()
}
public void testListThisUserNoClusters() throws Throwable {
@@ -70,28 +73,15 @@
)
assert launcher.serviceExitCode == 0
}
-
- public void testListAllUsersNoClusters() throws Throwable {
- log.info("RM address = ${RMAddr}")
- ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
- //config includes RM binding info
- new YarnConfiguration(miniCluster.config),
- //varargs list of command line params
- [
- SliderActions.ACTION_LIST,
- Arguments.ARG_MANAGER, RMAddr,
- ]
- )
- assert launcher.serviceExitCode == 0
- }
public void testListLiveCluster() throws Throwable {
//launch the cluster
- String clustername = createClusterName()
+ String clustername = "testlistlivecluster"
ServiceLauncher<SliderClient> launcher = createStandaloneAM(
clustername,
true,
false)
+
addToTeardown(launcher)
//do the low level operations to get a better view of what is going on
SliderClient sliderClient = launcher.service
@@ -110,7 +100,7 @@
//now look for the explicit sevice
- def serviceRegistryClient = sliderClient.YARNRegistryClient
+ def serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport instance = serviceRegistryClient.findInstance(clustername)
assert instance != null
log.info(instance.toString())
@@ -124,11 +114,11 @@
SliderActions.ACTION_LIST, clustername
]
)
-
+ clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
}
public void testListMissingCluster() throws Throwable {
- describe("exec the status command against an unknown cluster")
+ describe("exec the list command against an unknown cluster")
ServiceLauncher<SliderClient> launcher
try {
@@ -138,7 +128,7 @@
//varargs list of command line params
[
SliderActions.ACTION_LIST,
- createClusterName()
+ "no-instance"
]
)
fail("expected an exception, got a status code " + launcher.serviceExitCode)
@@ -148,4 +138,80 @@
}
+ public void testActionListStates() {
+ String clustername = "testactionliststates"
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ true)
+ addToTeardown(launcher)
+ SliderClient sliderClient = launcher.service
+ waitForClusterLive(sliderClient)
+
+ describe "listing"
+ //Listing only live instances
+ assert sliderClient.actionList(clustername, new ActionListArgs(live: true)) == 0;
+ assert sliderClient.actionList(clustername,
+ new ActionListArgs(live: true, verbose:true)) == 0;
+ clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
+ waitForAppToFinish(sliderClient)
+
+
+ try {
+ // unknown yarn state
+ int e= sliderClient.actionList(clustername,
+ new ActionListArgs(state: "undefined"));
+ fail("expected failure, got return code of $e")
+ } catch (BadCommandArgumentsException expected) {
+
+ }
+
+ try {
+ // state and --live options
+ int e= sliderClient.actionList(clustername,
+ new ActionListArgs(state: "running", live: true));
+ fail("expected failure, got return code of $e")
+ } catch (BadCommandArgumentsException expected) {
+
+ }
+ //Listing only live instances but prints nothing since instance is frozen/stopped
+
+ describe("after freeze")
+ // listing finished will work
+ assert 0 == sliderClient.actionList("",
+ new ActionListArgs(state: YarnApplicationState.FINISHED.toString()));
+ assert 0 == sliderClient.actionList(clustername,
+ new ActionListArgs(state: YarnApplicationState.FINISHED.toString(),
+ verbose: true));
+
+ assert -1 == sliderClient.actionList("", new ActionListArgs(live: true));
+ assert -1 == sliderClient.actionList(clustername,
+ new ActionListArgs(live: true));
+
+ assert -1 == sliderClient.actionList(clustername,
+ new ActionListArgs(state: YarnApplicationState.RUNNING.toString()));
+
+ assert -1 == sliderClient.actionList("",
+ new ActionListArgs(state: YarnApplicationState.RUNNING.toString()));
+
+ // thaw
+ sliderClient.actionThaw(clustername, new ActionThawArgs());
+ waitForClusterLive(sliderClient)
+
+ describe("Post-thaw listing")
+ assert 0 == sliderClient.actionList(clustername,
+ new ActionListArgs(state: YarnApplicationState.RUNNING.toString()));
+
+ //Listing only live instances
+ assert 0 == sliderClient.actionList(clustername,
+ new ActionListArgs(live: true));
+
+ //Listing all the instance both history (previously freezed instance) and live
+ assert 0 == sliderClient.actionList("", new ActionListArgs(live: true));
+
+ maybeStopCluster(sliderClient, "", "forced", true)
+ assert 0 == sliderClient.actionList(clustername,
+ new ActionListArgs(state: "killed"));
+ }
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionStatus.groovy
index bae8cea..fea07af 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionStatus.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionStatus.groovy
@@ -20,6 +20,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.api.ClusterDescription
@@ -43,11 +44,6 @@
@Slf4j
class TestActionStatus extends AgentMiniClusterTestBase {
- @Before
- public void setup() {
- super.setup()
- createMiniCluster("", configuration, 1, false)
- }
/**
* This is a test suite to run the tests against a single cluster instance
@@ -57,6 +53,8 @@
@Test
public void testSuite() throws Throwable {
+ super.setup()
+ createMiniCluster("testactionstatus", configuration, 1, true)
testStatusLiveCluster()
testStatusMissingCluster()
}
@@ -115,7 +113,7 @@
assert statusLauncher.serviceExitCode == 0
//status to a file
- File tfile = new File("target/" + clustername + "/status.json")
+ File tfile = new File("target/$clustername-status.json")
statusArgs.output = tfile.absolutePath
sliderClient.actionStatus(clustername, statusArgs)
def text = tfile.text
@@ -136,12 +134,13 @@
]
)
assert statusLauncher.serviceExitCode == 0
- tfile = new File(path)
ClusterDescription cd2 = new ClusterDescription();
cd2.fromJson(text)
clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
- waitForAppToFinish(sliderClient)
+ def finishedAppReport = waitForAppToFinish(sliderClient)
+ assert finishedAppReport.finalApplicationStatus ==
+ FinalApplicationStatus.SUCCEEDED
//now expect the status to fail
try {
@@ -151,6 +150,7 @@
assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE,
ErrorStrings.E_APPLICATION_NOT_RUNNING)
}
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy
index 1d50b71..7a2820f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy
@@ -20,10 +20,9 @@
import groovy.util.logging.Slf4j
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.test.YarnMiniClusterTestBase
+import org.apache.slider.test.SliderTestBase
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Before
import org.junit.Test
/**
@@ -31,7 +30,7 @@
*/
@Slf4j
-class TestActionVersion extends YarnMiniClusterTestBase {
+class TestActionVersion extends SliderTestBase {
@Test
public void testVersion() throws Throwable {
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
index f5eff25..0473e02 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
@@ -45,7 +45,7 @@
YarnConfiguration conf = configuration
String clustername = createMiniCluster("", conf, 1, 1, 1, true, false)
- describe "create a masterless AM, freeze it, try to freeze again"
+ describe "create a masterless AM, stop it, try to stop again"
ServiceLauncher<SliderClient> launcher = createStandaloneAM(
clustername
@@ -59,7 +59,7 @@
assertSucceeded(execSliderCommand(conf,
[SliderActions.ACTION_LIST, clustername]))
- log.info("First Freeze command");
+ log.info("First stop command");
ServiceLauncher freezeCommand = execSliderCommand(conf,
[
SliderActions.ACTION_FREEZE, clustername,
@@ -67,7 +67,7 @@
]);
assertSucceeded(freezeCommand)
- log.info("Second Freeze command");
+ log.info("Second stop command");
ServiceLauncher<SliderClient> freeze2 = execSliderCommand(conf,
[
@@ -94,7 +94,7 @@
assert e.exitCode == LauncherExitCodes.EXIT_FALSE;
}
- log.info("First Thaw");
+ log.info("First Start");
def commands = [
@@ -107,11 +107,12 @@
ServiceLauncher thawCommand = execSliderCommand(conf, commands);
assertSucceeded(thawCommand)
assertSucceeded(execSliderCommand(conf,
- [SliderActions.ACTION_LIST, clustername]))
+ [SliderActions.ACTION_LIST, clustername,
+ Arguments.ARG_LIVE]))
assertSucceeded(execSliderCommand(conf,
[SliderActions.ACTION_EXISTS, clustername]))
- log.info("Freeze 3");
+ log.info("stop 3");
ServiceLauncher<SliderClient> freeze3 = execSliderCommand(conf,
[
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy
new file mode 100644
index 0000000..70d5ba5
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.freezethaw
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionLookupArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
+import org.apache.slider.core.exceptions.NotFoundException
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.main.ServiceLaunchException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.ApplicationReportSerDeser
+import org.junit.Test
+
+/**
+ * stop and start an AM. For fun authorization is turned on, so this
+ * verifies that the AM comes up successfully
+ */
+@CompileStatic
+@Slf4j
+
+class TestFreezeThawFlexStandaloneAM extends AgentMiniClusterTestBase {
+
+ File getConfDirFile() {
+ return new File("target/testFreezeThawFlexStandaloneAM/conf")
+ }
+
+ @Override
+ String getConfDir() {
+ return confDirFile.toURI().toString()
+ }
+
+ @Test
+ public void testFreezeThawFlexStandaloneAM() throws Throwable {
+ YarnConfiguration conf = configuration
+ conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
+ false);
+ String clustername = createMiniCluster("", conf, 1, 1, 1, true, false)
+
+ describe "create a standalone AM, stop it, start it"
+ //copy the confdir somewhere
+ Path resConfPath = new Path(resourceConfDirURI)
+ Path tempConfPath = new Path(confDir)
+ SliderUtils.copyDirectory(conf, resConfPath, tempConfPath, null)
+
+
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ assert 0 == clusterActionFreeze(sliderClient, clustername)
+ def report = sliderClient.applicationReport
+ assert report.finalApplicationStatus == FinalApplicationStatus.SUCCEEDED
+
+ // here we do something devious: delete our copy of the configuration
+ // this makes sure the remote config gets picked up
+ HadoopFS localFS = HadoopFS.get(tempConfPath.toUri(), conf)
+ localFS.delete(tempConfPath,true)
+
+ //now start the cluster
+ File appreport = new File("target/$clustername/appreport.json")
+ ServiceLauncher launcher2 = thawCluster(clustername,
+ [Arguments.ARG_OUTPUT, appreport.absolutePath],
+ true);
+
+ SliderClient newCluster = launcher2.service
+ addToTeardown(newCluster);
+ ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser();
+ def sar = serDeser.fromFile(appreport)
+ log.info(sar.toString())
+ assert sar.applicationId != null
+
+ describe("lookup")
+
+ // now via lookup
+ appreport.delete()
+ def lookup1 = new ActionLookupArgs()
+ lookup1.id = sar.applicationId
+
+ assert 0 == newCluster.actionLookup(lookup1)
+ lookup1.outputFile = appreport
+ assert 0 == newCluster.actionLookup(lookup1)
+ sar = serDeser.fromFile(appreport)
+ assert sar.state == YarnApplicationState.RUNNING.toString()
+
+
+ newCluster.getClusterDescription(clustername);
+
+ describe("no change flex")
+ // while running, flex it with no changes
+ newCluster.flex(clustername, [:]);
+
+ // force freeze now
+
+ assert 0 == clusterActionFreeze(newCluster, clustername, "forced", true)
+ report = newCluster.applicationReport
+ assert report.finalApplicationStatus == FinalApplicationStatus.KILLED
+
+ assert 0 == newCluster.actionLookup(lookup1)
+ sar = serDeser.fromFile(appreport)
+ assert sar.finalStatus == FinalApplicationStatus.KILLED.toString()
+
+ //stop again
+ assert 0 == clusterActionFreeze(newCluster, clustername)
+
+ // and add some invalid lookup operations for
+
+ def lookup2 = new ActionLookupArgs()
+ lookup2.id = "invalid"
+ try {
+ newCluster.actionLookup(lookup2)
+ fail("expected $lookup2 to fail")
+ } catch (BadCommandArgumentsException expected) {
+ }
+ try {
+ lookup2.id = "application_1414593568640_0002"
+ newCluster.actionLookup(lookup2)
+ fail("expected $lookup2 to fail")
+ } catch (NotFoundException expected) {
+ }
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
deleted file mode 100644
index 04be7c0..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.freezethaw
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.FileSystem as HadoopFS
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.tools.SliderUtils
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-/**
- * freeze and thaw an AM
- */
-@CompileStatic
-@Slf4j
-
-class TestFreezeThawMasterlessAM extends AgentMiniClusterTestBase {
-
- File getConfDirFile() {
- return new File("target/TestFreezeThawMasterlessAM/conf")
- }
-
- @Override
- String getConfDir() {
- return confDirFile.toURI().toString()
- }
-
- @Test
- public void testFreezeThawMasterlessAM() throws Throwable {
- YarnConfiguration conf = configuration
- String clustername = createMiniCluster("", conf, 1, 1, 1, true, false)
-
- describe "create a masterless AM, freeze it, thaw it"
- //copy the confdir somewhere
- Path resConfPath = new Path(resourceConfDirURI)
- Path tempConfPath = new Path(confDir)
- SliderUtils.copyDirectory(conf, resConfPath, tempConfPath, null)
-
-
- ServiceLauncher<SliderClient> launcher = createStandaloneAM(
- clustername,
- true,
- true)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
-
- assert 0 == clusterActionFreeze(sliderClient, clustername)
-
-
- // here we do something devious: delete our copy of the configuration
- // this makes sure the remote config gets picked up
- HadoopFS localFS = HadoopFS.get(tempConfPath.toUri(), conf)
- localFS.delete(tempConfPath,true)
-
- //now start the cluster
- ServiceLauncher launcher2 = thawCluster(clustername, [], true);
- SliderClient newCluster = launcher2.service
- addToTeardown(newCluster);
-
-// ApplicationReport report = waitForClusterLive(newCluster)
- newCluster.getClusterDescription(clustername);
- //freeze
- assert 0 == clusterActionFreeze(sliderClient, clustername)
-
- //freeze again
- assert 0 == clusterActionFreeze(sliderClient, clustername)
-
- }
-
-}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeUnknownCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeUnknownCluster.groovy
index b81bc77..a7bab01 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeUnknownCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeUnknownCluster.groovy
@@ -42,7 +42,7 @@
YarnConfiguration conf = configuration
String clustername = createMiniCluster("", conf, 1, true)
- describe "try to freeze a cluster that isn't defined"
+ describe "try to stop a cluster that isn't defined"
try {
ServiceLauncher<SliderClient> command = execSliderCommand(conf,
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
index 0665a9a..b6ac657 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
@@ -30,6 +30,7 @@
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.registry.YarnAppListClient
import org.junit.Test
import static org.apache.slider.common.params.Arguments.ARG_COMP_OPT
@@ -65,14 +66,12 @@
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
-
-
-
+
// verify the YARN registry doesn't know of it
- def serviceRegistryClient = sliderClient.YARNRegistryClient
+ YarnAppListClient serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport report = serviceRegistryClient.findInstance(clustername)
assert report == null;
-
+
// verify that global resource options propagate from the CLI
def aggregateConf = sliderClient.loadPersistedClusterDescription(clustername)
def windowDays = aggregateConf.resourceOperations.globalOptions.getMandatoryOptionInt(
@@ -94,13 +93,27 @@
assertExceptionDetails(e, SliderExitCodes.EXIT_INSTANCE_EXISTS, "")
}
-
-
- //thaw time
+ //start time
ServiceLauncher<SliderClient> l2 = thawCluster(clustername, [], true)
SliderClient thawed = l2.service
addToTeardown(thawed);
waitForClusterLive(thawed)
+
+ // in the same code (for speed), create an illegal cluster
+ try {
+ ServiceLauncher<SliderClient> cluster3 = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ "illegalcluster",
+ ["role1": -1],
+ [],
+ false,
+ false,
+ agentDefOptions)
+ fail("expected an exception, got $cluster3.service")
+ } catch (SliderException e) {
+ assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "role1")
+ }
+
}
@Test
@@ -127,7 +140,7 @@
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
- def serviceRegistryClient = sliderClient.YARNRegistryClient
+ def serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport report = serviceRegistryClient.findInstance(clustername)
assert report == null;
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
index 17e4ff2..164f609 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -20,6 +20,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
@@ -34,7 +35,7 @@
import org.junit.Test
/**
- * destroy a masterless AM
+ * destroy a standalone AM
*/
@CompileStatic
@Slf4j
@@ -42,8 +43,10 @@
class TestStandaloneAMDestroy extends AgentMiniClusterTestBase {
@Test
- public void testDestroyStandaloneAM() throws Throwable {
- String clustername = createMiniCluster("", configuration, 1, false)
+ public void testStandaloneAMDestroy() throws Throwable {
+ skipOnWindows()
+
+ String clustername = createMiniCluster("", configuration, 1, true)
describe "create a Standalone AM, stop it, try to create" +
"a second cluster with the same name, destroy it, try a third time"
@@ -57,6 +60,22 @@
])
assert launcher1.serviceExitCode == 0
+ // try to list it and expect failures
+ try {
+ launchClientAgainstMiniMR(
+ configuration,
+ [
+ SliderActions.ACTION_LIST,
+ "no-cluster-of-this-name",
+ Arguments.ARG_LIVE
+ ])
+ fail("expected a failure")
+ } catch (UnknownApplicationInstanceException e) {
+ assertExceptionDetails(e,
+ SliderExitCodes.EXIT_UNKNOWN_INSTANCE,
+ ErrorStrings.E_UNKNOWN_INSTANCE)
+ }
+
ServiceLauncher<SliderClient> launcher = createStandaloneAM(
clustername,
true,
@@ -74,16 +93,17 @@
instanceDir)
sliderFileSystem.locateInstanceDefinition(clustername)
- clusterActionFreeze(sliderClient, clustername,"stopping first cluster")
- waitForAppToFinish(sliderClient)
+ clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
+ def finishedAppReport = waitForAppToFinish(sliderClient)
+ assert finishedAppReport.finalApplicationStatus == FinalApplicationStatus.SUCCEEDED
describe "Warnings below are expected"
//now try to create instance #2, and expect an in-use failure
try {
- createStandaloneAM(clustername, false, false)
- fail("expected a failure, got an AM")
+ SliderClient am = createStandaloneAM(clustername, false, false).service
+ fail("expected a failure, got an AM: $am")
} catch (SliderException e) {
assertExceptionDetails(e,
SliderExitCodes.EXIT_INSTANCE_EXISTS,
@@ -92,18 +112,29 @@
describe "END EXPECTED WARNINGS"
-
describe "destroying $clustername"
//now: destroy it
int exitCode = sliderClient.actionDestroy(clustername);
assert 0 == exitCode
+ sleep(1000)
+ // twice, not expecting an error the second time
+ exitCode = sliderClient.actionDestroy(clustername);
+ assert 0 == exitCode
describe "post destroy checks"
+ if (fs.exists(instanceDir)) {
+ log.warn("Destroy operation did not delete $instanceDir")
+ rigorousDelete(sliderFileSystem, instanceDir, 60000)
+ }
+
sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
- describe "thaw expected to fail"
- //expect thaw to now fail
+ // look up app report and verify exit code is good
+
+
+ describe "start expected to fail"
+ //expect start to now fail
def ex = launchExpectingException(SliderClient,
configuration,
"",
@@ -115,7 +146,7 @@
])
assert ex instanceof UnknownApplicationInstanceException
- describe "thaw completed, checking dir is still absent"
+ describe "start completed, checking dir is still absent"
sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
@@ -148,6 +179,8 @@
//and try to destroy a completely different cluster just for the fun of it
assert 0 == sliderClient.actionDestroy("no-cluster-of-this-name")
+
+ maybeStopCluster(cluster2, "", "Teardown at end of test case", false);
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
index 75f9a2c..d6b3929 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
@@ -27,10 +27,11 @@
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Assume
import org.junit.Test
/**
- * kill a masterless AM and verify it shuts down. This test
+ * kill a standalone AM and verify it shuts down. This test
* also sets the retry count to 1 to stop recreation attempts
*/
@CompileStatic
@@ -38,9 +39,9 @@
class TestStandaloneAMKill extends AgentMiniClusterTestBase {
-
@Test
public void testKillStandaloneAM() throws Throwable {
+ Assume.assumeTrue(kill_supported)
String clustername = createMiniCluster("", configuration, 1, true)
describe "kill a Standalone AM and verify that it shuts down"
@@ -54,11 +55,12 @@
SliderClient sliderClient = launcher.service
addToTeardown(sliderClient);
ApplicationReport report = waitForClusterLive(sliderClient)
+ assert report.yarnApplicationState == YarnApplicationState.RUNNING
describe("listing Java processes")
lsJavaProcesses();
describe("killing AM")
- killAM(SIGTERM);
+ assert 0 == killAM(SIGTERM);
waitWhileClusterLive(sliderClient);
//give yarn some time to notice
sleep(10000)
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
index 162bab0..5136092 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
@@ -20,32 +20,22 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.SleepJob
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.InternalKeys
-import org.apache.slider.api.ResourceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.ActionAMSuicideArgs
import org.apache.slider.common.params.Arguments
-import org.apache.slider.core.exceptions.ErrorStrings
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
-/**
- * kill a masterless AM and verify it shuts down. This test
- * also sets the retry count to 1 to stop recreation attempts
- */
@CompileStatic
@Slf4j
-
class TestStandaloneAMMonkeyRestart extends AgentMiniClusterTestBase {
-
@Test
public void testStandaloneAMMonkeyRestart() throws Throwable {
describe "Run a Standalone AM with the Chaos monkey set to kill it"
@@ -59,7 +49,7 @@
[
Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_ENABLED, "true",
Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "8",
- Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "75000",
+ Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "7500",
],
true,
false)
@@ -69,13 +59,11 @@
ApplicationReport report
report = waitForClusterLive(sliderClient, 30000)
describe "Waiting for the cluster to fail"
- sleep(40000)
- // end of process
- report = sliderClient.applicationReport
- log.info(report.diagnostics)
- assert report.currentApplicationAttemptId.attemptId == threshold
- assert YarnApplicationState.FAILED == report.yarnApplicationState
- assert FinalApplicationStatus.FAILED == report.finalApplicationStatus
+ def finishedReport = waitForAppToFinish(sliderClient, 90000)
+ log.info(finishedReport.diagnostics)
+ assert finishedReport.currentApplicationAttemptId.attemptId == threshold
+ assert YarnApplicationState.FAILED == finishedReport.yarnApplicationState
+ assert FinalApplicationStatus.FAILED == finishedReport.finalApplicationStatus
}
/**
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
index 8d9318a..bdcf615 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
@@ -21,18 +21,19 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.ActionAMSuicideArgs
+import org.apache.slider.common.params.ActionDiagnosticArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
/**
- * kill a masterless AM and verify it shuts down. This test
- * also sets the retry count to 1 to stop recreation attempts
+ * kill an AM and verify it is restarted
*/
@CompileStatic
@Slf4j
@@ -41,16 +42,18 @@
@Test
- public void testRestartStandaloneAM() throws Throwable {
+ public void testStandaloneAMRestart() throws Throwable {
describe "kill a Standalone AM and verify that it restarts"
// patch the configuration for AM restart
YarnConfiguration conf = getRestartableConfiguration(5)
+ int restartLimit = 3;
String clustername = createMiniCluster("", conf, 1, true)
ServiceLauncher<SliderClient> launcher =
createStandaloneAMWithArgs(clustername,
[
- Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "4"
+ Arguments.ARG_DEFINE,
+ SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=" + restartLimit
],
true,
false)
@@ -62,22 +65,66 @@
waitUntilClusterLive(sliderClient, 30000)
- ActionAMSuicideArgs args = new ActionAMSuicideArgs()
- args.message = "test AM iteration"
- args.waittime = 100
- args.exitcode = 1
- sliderClient.actionAmSuicide(clustername, args)
- waitWhileClusterLive(sliderClient);
- //give yarn some time to notice
- sleep(20000)
- waitUntilClusterLive(sliderClient, 40000)
+ def diagnosticArgs = new ActionDiagnosticArgs()
+ diagnosticArgs.client = true
+ diagnosticArgs.yarn = true
+ sliderClient.actionDiagnostic(diagnosticArgs)
+
+ int iteration = 1;
+ killAMAndWaitForRestart(sliderClient, iteration, clustername)
+ killAMAndWaitForRestart(sliderClient, iteration++, clustername)
// app should be running here
assert 0 == sliderClient.actionExists(clustername, true)
-
-
- clusterActionFreeze(sliderClient, clustername)
+
+ // kill again & expect it to be considered a failure
+ killAmAndWaitForDeath(sliderClient, iteration++, clustername)
+ sleep(40000)
+
+ report = sliderClient.applicationReport
+ assert report.finalApplicationStatus == FinalApplicationStatus.FAILED
+
+ logReport(report)
+ describe("Kill worked, freezing again")
+ assert 0 == clusterActionFreeze(sliderClient, clustername, "force", true)
+ assert 0 == clusterActionFreeze(sliderClient, clustername, "force", true)
+ assert 0 == clusterActionFreeze(sliderClient, clustername, "force", true)
+ assert 0 == clusterActionFreeze(sliderClient, clustername, "force", true)
+ assert 0 == clusterActionFreeze(sliderClient, clustername, "force", true)
+ }
+
+ /**
+ * Kill an AM. take an iteration count for the message sent to the
+ * AM (hence its logs)
+ * @param iteration
+ * @param sliderClient
+ * @param clustername
+ * @return
+ */
+ public ActionAMSuicideArgs killAMAndWaitForRestart(
+ SliderClient sliderClient, int iteration, String clustername) {
+ ActionAMSuicideArgs args = killAmAndWaitForDeath(
+ sliderClient,
+ iteration,
+ clustername)
+ //give yarn some time to notice
+ sleep(40000)
+ waitUntilClusterLive(sliderClient, 40000)
+ return args
+ }
+
+ public ActionAMSuicideArgs killAmAndWaitForDeath(
+ SliderClient sliderClient,
+ int iteration,
+ String clustername) {
+ ActionAMSuicideArgs args = new ActionAMSuicideArgs()
+ args.waittime = 100
+ args.exitcode = 1
+ args.message = "kill AM iteration #$iteration"
+ sliderClient.actionAmSuicide(clustername, args)
+ waitWhileClusterLive(sliderClient);
+ return args
}
/**
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
index 73e9b07..c31e9d3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -20,42 +20,59 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.exceptions.YarnException
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.ClusterNode
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.ActionRegistryArgs
+import org.apache.slider.core.build.InstanceBuilder
+import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.launch.LaunchedApplication
+import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.apache.slider.server.services.registry.SliderRegistryService
+import org.apache.slider.core.persist.LockAcquireFailedException
+import org.junit.After
import org.junit.Test
@CompileStatic
@Slf4j
class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
+ @After
+ void fixclientname() {
+ sliderClientClassName = DEFAULT_SLIDER_CLIENT
+ }
+
@Test
public void testStandaloneAgentAM() throws Throwable {
- describe "create a masterless AM then get the service and look it up via the AM"
-
+ describe "create a standalone AM then perform actions on it"
+ sliderClientClassName = ExtendedSliderClient.name
//launch fake master
String clustername = createMiniCluster("", configuration, 1, true)
+
+
ServiceLauncher<SliderClient> launcher =
createStandaloneAM(clustername, true, false)
SliderClient client = launcher.service
addToTeardown(client);
ApplicationReport report = waitForClusterLive(client)
+ URI uri = new URI(report.originalTrackingUrl)
+ assert uri.port in [60000, 60001, 60002, 60003]
+ assert report.rpcPort in [60000, 60001, 60002, 60003]
+
logReport(report)
List<ApplicationReport> apps = client.applications;
//get some of its status
- dumpClusterStatus(client, "masterless application status")
+ dumpClusterStatus(client, "standalone application status")
List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
SliderKeys.COMPONENT_AM)
assert clusterNodes.size() == 1
@@ -77,10 +94,8 @@
assert nodes[0].role == SliderKeys.COMPONENT_AM
-
-
String username = client.username
- def serviceRegistryClient = client.YARNRegistryClient
+ def serviceRegistryClient = client.yarnAppListClient
describe("list of all applications")
logApplications(apps)
describe("apps of user $username")
@@ -92,12 +107,7 @@
logReport(instance)
assert instance != null
- //switch to the ZK-based registry
-
- describe "service registry names"
- SliderRegistryService registry = client.registry
- def names = registry.getServiceTypes();
- dumpRegistryServiceTypes(names)
+ //switch to the slider ZK-based registry
describe "service registry instance IDs"
def instanceIds = client.listRegisteredSliderInstances()
@@ -105,14 +115,9 @@
log.info("number of instanceIds: ${instanceIds.size()}")
instanceIds.each { String it -> log.info(it) }
- describe "service registry slider instances"
- List<CuratorServiceInstance<ServiceInstanceData>> instances = client.listRegistryInstances(
- )
- instances.each { CuratorServiceInstance<ServiceInstanceData> svc ->
- log.info svc.toString()
- }
- describe "end list service registry slider instances"
-
+ describe "Yarn registry"
+ def yarnRegistry = client.registryOperations
+
describe "teardown of cluster instance #1"
//now kill that cluster
assert 0 == clusterActionFreeze(client, clustername)
@@ -122,8 +127,10 @@
assert oldInstance != null
assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
+ sleep(5000)
//create another AM
- launcher = createStandaloneAM(clustername, true, true)
+ def newcluster = clustername + "2"
+ launcher = createStandaloneAM(newcluster, true, true)
client = launcher.service
ApplicationId i2AppID = client.applicationId
@@ -134,22 +141,27 @@
//but when we look up an instance, we get the new App ID
ApplicationReport instance2 = serviceRegistryClient.findInstance(
- clustername)
+ newcluster)
assert i2AppID == instance2.applicationId
describe("attempting to create instance #3")
//now try to create instance #3, and expect an in-use failure
try {
- createStandaloneAM(clustername, false, true)
- fail("expected a failure, got a masterless AM")
+ createStandaloneAM(newcluster, false, true)
+ fail("expected a failure, got a standalone AM")
} catch (SliderException e) {
assertFailureClusterInUse(e);
}
- describe("Stopping instance #2")
+ // do a quick registry listing here expecting a usage failure.
+ ActionRegistryArgs registryArgs = new ActionRegistryArgs()
+ registryArgs.name=clustername;
+ def exitCode = client.actionRegistry(registryArgs)
+ assert LauncherExitCodes.EXIT_USAGE == exitCode
+ describe("Stopping instance #2")
//now stop that cluster
- assert 0 == clusterActionFreeze(client, clustername)
+ assert 0 == clusterActionFreeze(client, newcluster)
logApplications(client.listSliderInstances(username))
@@ -162,11 +174,36 @@
ApplicationReport instance3 = serviceRegistryClient.findInstance(
- clustername)
+ newcluster)
assert instance3.yarnApplicationState >= YarnApplicationState.FINISHED
-
+ // destroy it
+ client.actionDestroy(newcluster)
+
}
+ static class ExtendedSliderClient extends SliderClient {
+ @Override
+ protected void persistInstanceDefinition(boolean overwrite,
+ Path appconfdir,
+ InstanceBuilder builder)
+ throws IOException, SliderException, LockAcquireFailedException {
+ AggregateConf conf = builder.instanceDescription
+ conf.appConfOperations.
+ globalOptions[SliderKeys.KEY_ALLOWED_PORT_RANGE]= "60000-60003"
+ super.persistInstanceDefinition(overwrite, appconfdir, builder)
+ }
+
+ @Override
+ LaunchedApplication launchApplication(String clustername,
+ Path clusterDirectory,
+ AggregateConf instanceDefinition,
+ boolean debugAM)
+ throws YarnException, IOException {
+ instanceDefinition.appConfOperations.
+ globalOptions[SliderKeys.KEY_ALLOWED_PORT_RANGE] ="60000-60003"
+ return super.launchApplication(clustername, clusterDirectory, instanceDefinition, debugAM)
+ }
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
deleted file mode 100644
index b8a590e..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
+++ /dev/null
@@ -1,362 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.api.ClusterNode
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.params.ActionRegistryArgs
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.persist.JsonSerDeser
-import org.apache.slider.core.registry.docstore.PublishedConfigSet
-import org.apache.slider.core.registry.docstore.PublishedConfiguration
-import org.apache.slider.core.registry.docstore.UriMap
-import org.apache.slider.core.registry.info.CustomRegistryConstants
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.core.registry.retrieve.RegistryRetriever
-import org.apache.slider.server.appmaster.PublishedArtifacts
-import org.apache.slider.server.appmaster.web.rest.RestPaths
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.apache.slider.server.services.registry.SliderRegistryService
-import org.junit.Test
-
-/**
- * create masterless AMs and work with them. This is faster than
- * bringing up full clusters
- */
-@CompileStatic
-@Slf4j
-
-class TestStandaloneRegistryAM extends AgentMiniClusterTestBase {
-
-
- public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG
-
- @Test
- public void testRegistryAM() throws Throwable {
-
-
- describe "create a masterless AM then perform registry operations on it"
-
- //launch fake master
- String clustername = createMiniCluster(configuration, 1, true)
- ServiceLauncher<SliderClient> launcher
- launcher = createStandaloneAM(clustername, true, false)
- SliderClient client = launcher.service
- addToTeardown(client);
-
- ApplicationReport report = waitForClusterLive(client)
- logReport(report)
- List<ApplicationReport> apps = client.applications;
-
- List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
- SliderKeys.COMPONENT_AM)
- assert ((List<ClusterNode>)clusterNodes).size() == 1
-
- ClusterNode masterNode = clusterNodes[0]
- log.info("Master node = ${masterNode}");
-
- List<ClusterNode> nodes
- String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM)
- assert uuids.length == 1;
- nodes = client.listClusterNodes(uuids);
- assert ((List<ClusterNode>)nodes).size() == 1;
- describe "AM Node UUID=${uuids[0]}"
-
- nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM)
- assert ((List<ClusterNode>)nodes).size() == 1;
- nodes = listNodesInRole(client, "")
- assert ((List<ClusterNode>)nodes).size() == 1;
- ClusterNode master = nodes[0]
- assert master.role == SliderKeys.COMPONENT_AM
-
-
-
-
- String username = client.username
- def yarnRegistryClient = client.YARNRegistryClient
- describe("list of all applications")
- logApplications(apps)
- describe("apps of user $username")
- List<ApplicationReport> userInstances = yarnRegistryClient.listInstances()
- logApplications(userInstances)
- assert userInstances.size() == 1
- describe("named app $clustername")
- ApplicationReport instance = yarnRegistryClient.findInstance(clustername)
- logReport(instance)
- assert instance != null
-
- //switch to the ZK-based registry
-
- describe "service registry names"
- SliderRegistryService registryService = client.registry
- def serviceTypes = registryService.serviceTypes;
- dumpRegistryServiceTypes(serviceTypes)
-
- List<String> instanceIds = client.listRegisteredSliderInstances()
-
-
- dumpRegistryInstanceIDs(instanceIds)
- assert instanceIds.size() == 1
-
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
- client.listRegistryInstances()
- dumpRegistryInstances(instances)
-
- assert instances.size() == 1
-
- def amInstance = instances[0]
- def serviceInstanceData = amInstance.payload
-
- def externalEndpoints = serviceInstanceData.externalView.endpoints
-
- // hit the registry web page
-
- def registryEndpoint = externalEndpoints.get(
- CustomRegistryConstants.REGISTRY_REST_API)
- assert registryEndpoint != null
- def registryURL = registryEndpoint.asURL()
- describe("Registry WADL @ $registryURL")
-
- def publisherEndpoint = externalEndpoints.get(CustomRegistryConstants.PUBLISHER_REST_API)
- assert publisherEndpoint != null
- def publisherURL = publisherEndpoint.asURL()
- def publisher = publisherURL.toString()
- describe("Publisher")
-
- JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap)
- def setlisting = GET(publisherURL)
-
- log.info(setlisting)
-
- UriMap uris = uriMapDeser.fromJson(setlisting)
- assert uris.uris[RestPaths.SLIDER_CONFIGSET]
- def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET)
- JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>(
- PublishedConfigSet)
- def configSet = serDeser.fromJson(publishedJSON)
- assert configSet.size() >= 1
- assert configSet.contains(ARTIFACT_NAME)
- PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME)
-
- assert publishedYarnSite.empty
-
- //get the full URL
- def yarnSitePublisher = appendToURL(publisher,
- RestPaths.SLIDER_CONFIGSET,
- ARTIFACT_NAME)
-
- String confJSON = GET(yarnSitePublisher)
-// log.info(confJSON)
- JsonSerDeser< PublishedConfiguration> confSerDeser =
- new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
-
- publishedYarnSite = confSerDeser.fromJson(confJSON)
-
- assert !publishedYarnSite.empty
-
-
- //get the XML
- def yarnSiteXML = yarnSitePublisher + ".xml"
-
-
- String confXML = GET(yarnSiteXML)
- log.info("Conf XML at $yarnSiteXML = \n $confXML")
-
- String properties = GET(yarnSitePublisher + ".properties")
- Properties parsedProps = new Properties()
- parsedProps.load(new StringReader(properties))
- assert parsedProps.size() > 0
- def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS)
- def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME)
- assert rmAddrFromDownloadedProperties
- assert rmHostnameFromDownloadedProperties
-
- String json = GET(yarnSitePublisher + ".json")
-
-
-
- describe("Registry List")
- log.info(GET(registryURL))
-
-
- describe "Registry Retrieval Class"
- // retrieval
-
- RegistryRetriever retriever = new RegistryRetriever(serviceInstanceData)
- log.info retriever.toString()
-
- assert retriever.hasConfigurations(true)
- PublishedConfigSet externalConfSet = retriever.getConfigurations(true)
- dumpConfigurationSet(externalConfSet)
- assert externalConfSet[ARTIFACT_NAME]
-
-
- describe "verify SLIDER-52 processing"
- def yarnSite = retriever.retrieveConfiguration(
- externalConfSet,
- ARTIFACT_NAME,
- true)
- assert !yarnSite.empty
- def siteXML = yarnSite.asConfiguration()
- def rmHostnameViaClientSideXML = parsedProps.get(
- YarnConfiguration.RM_HOSTNAME)
- assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties
- def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS)
-
- log.info("RM from downloaded props = $rmAddrFromDownloadedProperties")
- assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties
-
- describe "fetch missing artifact"
- try {
- retriever.retrieveConfiguration(externalConfSet, "no-such-artifact", true)
- fail("expected a failure")
- } catch (FileNotFoundException expected) {
- // expected
- }
- describe "Internal configurations"
- assert !retriever.hasConfigurations(false)
- try {
- retriever.getConfigurations(false)
- fail("expected a failure")
- } catch (FileNotFoundException expected) {
- // expected
- }
-
-
- // retrieval via API
- ActionRegistryArgs registryArgs = new ActionRegistryArgs()
- registryArgs.verbose = true
-
- // list all
- registryArgs.list = true;
- describe registryArgs.toString()
- client.actionRegistry(registryArgs)
-
- // list a named instance and expect a failure
- registryArgs.list = true;
- registryArgs.name = "unknown"
- try {
- client.actionRegistryList(registryArgs)
- } catch (FileNotFoundException expected) {
- // expected
- }
-
- // list all instances of an alternate type and expect failure
- registryArgs.list = true;
- registryArgs.name = null
- registryArgs.serviceType = "org.apache.hadoop"
- try {
- client.actionRegistryList(registryArgs)
- } catch (FileNotFoundException expected) {
- // expected
- }
-
- registryArgs.serviceType = ""
- try {
- client.actionRegistryList(registryArgs)
- } catch (FileNotFoundException expected) {
- // expected
- }
- //set the name
- registryArgs.name = serviceInstanceData.id;
- registryArgs.serviceType = SliderKeys.APP_TYPE
-
-
- //now expect list to work
- describe registryArgs.toString()
-
- def listedInstance = client.actionRegistryList(registryArgs)
- assert listedInstance[0].id == serviceInstanceData.id
-
-
- // listconf
- registryArgs.list = false;
- registryArgs.listConf = true
- describe registryArgs.toString()
-
- client.actionRegistry(registryArgs)
-
- // listconf --internal
- registryArgs.list = false;
- registryArgs.listConf = true
- registryArgs.internal = true
- describe registryArgs.toString()
- assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
-
- registryArgs.list = false;
- registryArgs.listConf = false
- registryArgs.internal = false
-
- def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
- registryArgs.getConf = yarn_site_config
-
- //properties format
- registryArgs.format = "properties"
- describe registryArgs.toString()
-
- client.actionRegistry(registryArgs)
-
-
- File outputDir = new File("target/test_standalone_registry_am/output")
- outputDir.mkdirs()
-
- // create a new registry args with the defaults back in
- registryArgs = new ActionRegistryArgs(serviceInstanceData.id)
- registryArgs.getConf = yarn_site_config
- registryArgs.dest = outputDir
- describe registryArgs.toString()
- client.actionRegistry(registryArgs)
- assert new File(outputDir, yarn_site_config + ".xml").exists()
-
- registryArgs.format = "properties"
- client.actionRegistry(registryArgs)
- assert new File(outputDir, yarn_site_config + ".properties").exists()
-
- describe registryArgs.toString()
-
- def unknownFilename = "undefined-file"
- registryArgs.getConf = unknownFilename
- assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
-
- describe "freeze cluster"
- //now kill that cluster
- assert 0 == clusterActionFreeze(client, clustername)
- //list it & See if it is still there
- ApplicationReport oldInstance = yarnRegistryClient.findInstance(
- clustername)
- assert oldInstance != null
- assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
-
-
- sleep(20000)
-
- // now verify that the service is not in the registry
- instances = client.listRegistryInstances()
- assert instances.size() == 0
-
- }
-}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
new file mode 100644
index 0000000..705430f
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -0,0 +1,555 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.fs.PathNotFoundException
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.registry.client.api.RegistryConstants
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils
+import org.apache.hadoop.registry.client.impl.RegistryOperationsClient
+import org.apache.hadoop.registry.client.types.RegistryPathStatus
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.ActionResolveArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.exceptions.NotFoundException
+import org.apache.slider.core.main.LauncherExitCodes
+
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.*
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.ActionRegistryArgs
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.core.registry.docstore.PublishedConfiguration
+import org.apache.slider.core.registry.docstore.UriMap
+import org.apache.slider.core.registry.retrieve.RegistryRetriever
+import org.apache.slider.server.appmaster.PublishedArtifacts
+import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.apache.slider.test.Outcome;
+import org.junit.Test
+
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+
+/**
+ * work with a YARN registry
+ */
+//@CompileStatic
+@Slf4j
+class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
+
+ public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG
+ public static final String HBASE = "hbase/localhost@HADOOP.APACHE.ORG"
+
+ @Test
+ public void testStandaloneYarnRegistryAM() throws Throwable {
+
+
+ describe "create a standalone AM then perform YARN registry operations on it"
+
+
+ String clustername = createMiniCluster(configuration, 1, true)
+
+ // get local binding
+ def registryOperations = microZKCluster.registryOperations
+ try {
+ registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
+ } catch (PathNotFoundException e) {
+ log.warn(" RM is not apparently running registry services: {}", e, e)
+ }
+
+ ServiceLauncher<SliderClient> launcher
+ launcher = createStandaloneAM(clustername, true, false)
+ SliderClient client = launcher.service
+ addToTeardown(client);
+
+ ApplicationReport report = waitForClusterLive(client)
+ logReport(report)
+ List<ApplicationReport> apps = client.applications;
+
+ List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
+ SliderKeys.COMPONENT_AM)
+ assert ((List<ClusterNode>)clusterNodes).size() == 1
+
+ ClusterNode masterNode = clusterNodes[0]
+ log.info("Master node = ${masterNode}");
+
+ List<ClusterNode> nodes
+ String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM)
+ assert uuids.length == 1;
+ nodes = client.listClusterNodes(uuids);
+ assert nodes.size() == 1;
+ describe "AM Node UUID=${uuids[0]}"
+
+ nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM)
+ assert nodes.size() == 1;
+ nodes = listNodesInRole(client, "")
+ assert nodes.size() == 1;
+ ClusterNode master = nodes[0]
+ assert master.role == SliderKeys.COMPONENT_AM
+
+ String username = client.username
+ def yarnRegistryClient = client.yarnAppListClient
+ describe("list of all applications")
+ logApplications(apps)
+ describe("apps of user $username")
+ List<ApplicationReport> userInstances = yarnRegistryClient.listInstances()
+ logApplications(userInstances)
+ assert userInstances.size() == 1
+ describe("named app $clustername")
+ ApplicationReport instance = yarnRegistryClient.findInstance(clustername)
+ logReport(instance)
+ assert instance != null
+
+ // sleep to allow registration to complete
+ sleep(5000)
+
+
+ describe "service registry names"
+ def registryService = client.registryOperations
+
+ RegistryOperationsClient registryOperationsClient =
+ registryService as RegistryOperationsClient
+ try {
+ def yarnRegistryDump = registryOperationsClient.dumpPath(false)
+ log.info("yarn service registry: \n${yarnRegistryDump}\n")
+ } catch (IOException ignored) {
+
+ }
+
+
+ def self = currentUser()
+ def children = statChildren(registryService, homePathForUser(self));
+ Collection<RegistryPathStatus> serviceTypes = children.values()
+ dumpCollection(serviceTypes)
+
+ def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE)
+
+ Map<String, ServiceRecord> recordMap = extractServiceRecords(
+ registryService,
+ recordsPath);
+ def serviceRecords = recordMap.values();
+ dumpCollection(serviceRecords)
+ assert serviceRecords.size() == 1
+
+ def serviceRecord = serviceRecords[0]
+ log.info(serviceRecord.toString())
+
+ assert serviceRecord[YarnRegistryAttributes.YARN_ID] != null
+ assert serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] != ""
+ def externalEndpoints = serviceRecord.external;
+ assert externalEndpoints.size() > 0
+/*
+
+ def am_ipc_protocol = AM_IPC_PROTOCOL
+ serviceRecord.getExternalEndpoint(am_ipc_protocol)
+ assert null != am_ipc_protocol;
+*/
+
+ assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API)
+ assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
+ // internals
+ assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API)
+ assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API)
+
+ // use the resolve operation
+
+ describe "resolve CLI action"
+ File destDir= new File("target/resolve")
+ ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal()
+ FileUtil.fullyDelete(destDir)
+ File resolveListDir= new File(destDir, "list")
+ ActionResolveArgs resolveList = new ActionResolveArgs(
+ path:recordsPath,
+ list:true)
+
+ // to stdout
+ assert 0 == client.actionResolve(resolveList)
+ // to a file
+ resolveList.out = resolveListDir;
+
+ assertFailsWithException(
+ LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR,
+ Arguments.ARG_OUTPUT) {
+ client.actionResolve(resolveList)
+ }
+ // list to a destination dir
+ resolveList.out = null
+ resolveList.destdir = resolveListDir
+ assert 0 == client.actionResolve(resolveList)
+ File resolvedFile = new File(resolveListDir, clustername + ".json")
+ assertFileExists("resolved file", resolvedFile)
+ serviceRecordMarshal.fromFile(resolvedFile)
+
+ // list the parent path, expect success and no entries
+ File listParentDir = new File(destDir, "listParent")
+ String parentPath = RegistryPathUtils.parentOf(recordsPath)
+ assert 0 == client.actionResolve(new ActionResolveArgs(
+ path: parentPath,
+ list: true,
+ destdir: listParentDir))
+ assertFileExists("output dir", listParentDir)
+ assert null != listParentDir.list()
+ assert 0 == listParentDir.list().length
+
+ // look for a record a path not in the registry expect failure
+ def unknownPath = recordsPath + "/unknown"
+ // the record is not there, even if the path is
+ assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND, unknownPath) {
+ client.actionResolve(new ActionResolveArgs(
+ path: unknownPath,
+ list: true))
+ }
+
+ // there's always a root path for listing
+ client.actionResolve(new ActionResolveArgs(
+ path: "/",
+ list: true))
+
+ // but not usually a root entry
+ assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND, "") {
+ client.actionResolve(new ActionResolveArgs(
+ path: "/"))
+ }
+
+
+ // look for a record at the same path as the listing; expect failure
+ // the record is not there, even if the path is
+ assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND, recordsPath) {
+ client.actionResolve(new ActionResolveArgs(
+ path: recordsPath,
+ list: false))
+ }
+
+ // look at a single record
+ def instanceRecordPath = recordsPath + "/" + clustername
+ ActionResolveArgs resolveRecordCommand = new ActionResolveArgs(
+ path: instanceRecordPath)
+ // to stdout
+ client.actionResolve(resolveRecordCommand)
+
+ // then to a file
+ FileUtil.fullyDelete(destDir)
+ File singleFile = new File(destDir, "singlefile.json")
+ singleFile.delete()
+ resolveRecordCommand.out = singleFile
+ assert 0 == client.actionResolve(resolveRecordCommand)
+ assertFileExists("\"slider $resolveRecordCommand\"", singleFile)
+ def recordFromFile = serviceRecordMarshal.fromFile(singleFile)
+ assert recordFromFile[YarnRegistryAttributes.YARN_ID] ==
+ serviceRecord[YarnRegistryAttributes.YARN_ID]
+ assert recordFromFile[YarnRegistryAttributes.YARN_PERSISTENCE] ==
+ serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE]
+
+ //resolve to the home directory
+ assert 0 == client.actionResolve(new ActionResolveArgs(
+ path: "~", list:true))
+
+ //resolve to the instance
+ assert 0 == client.actionResolve(new ActionResolveArgs(
+ path: "~/services/org-apache-slider/$clustername",
+ list: true))
+
+ // hit the registry web page
+ def registryEndpoint = serviceRecord.getExternalEndpoint(REGISTRY_REST_API)
+ assert registryEndpoint != null
+ def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
+
+ // list the path at the record, expect success and no entries
+ File listUnderRecordDir = new File(destDir, "listUnderRecord")
+ ActionResolveArgs listUnderRecordCommand = new ActionResolveArgs(
+ path: instanceRecordPath,
+ list:true,
+ destdir: listUnderRecordDir)
+ assert 0 == client.actionResolve(listUnderRecordCommand)
+ assert 0 == listUnderRecordDir.list().length
+
+
+ // Look at the Registry WADL
+ describe("Registry WADL @ $registryURL")
+ def publisherEndpoint = serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
+
+ def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0]
+ def publisher = publisherURL.toString()
+ describe("Publisher")
+
+ JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap)
+ def setlisting = GET(publisherURL)
+
+ log.info(setlisting)
+
+ UriMap uris = uriMapDeser.fromJson(setlisting)
+ assert uris.uris[RestPaths.SLIDER_CONFIGSET]
+ def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET)
+ JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>(
+ PublishedConfigSet)
+ def configSet = serDeser.fromJson(publishedJSON)
+ assert configSet.size() >= 1
+ assert configSet.contains(ARTIFACT_NAME)
+ PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME)
+
+ assert publishedYarnSite.empty
+
+ //get the full URL
+ def yarnSitePublisher = appendToURL(publisher,
+ RestPaths.SLIDER_CONFIGSET,
+ ARTIFACT_NAME)
+
+ String confJSON = GET(yarnSitePublisher)
+// log.info(confJSON)
+ JsonSerDeser< PublishedConfiguration> confSerDeser =
+ new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
+
+ publishedYarnSite = confSerDeser.fromJson(confJSON)
+
+ assert !publishedYarnSite.empty
+
+
+ //get the XML
+ def yarnSiteXML = yarnSitePublisher + ".xml"
+
+
+ String confXML = GET(yarnSiteXML)
+ log.info("Conf XML at $yarnSiteXML = \n $confXML")
+
+ String properties = GET(yarnSitePublisher + ".properties")
+ Properties parsedProps = new Properties()
+ parsedProps.load(new StringReader(properties))
+ assert parsedProps.size() > 0
+ def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS)
+ def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME)
+ assert rmAddrFromDownloadedProperties
+ assert rmHostnameFromDownloadedProperties
+
+ String json = GET(yarnSitePublisher + ".json")
+
+
+ describe("Registry List")
+ log.info(GET(registryURL))
+
+
+ describe "Registry Retrieval Class"
+ // retrieval
+
+ RegistryRetriever retriever = new RegistryRetriever(serviceRecord)
+ log.info retriever.toString()
+
+ assert retriever.hasConfigurations(true)
+ PublishedConfigSet externalConfSet = retriever.getConfigurations(true)
+ dumpConfigurationSet(externalConfSet)
+ assert externalConfSet[ARTIFACT_NAME]
+
+
+ describe "verify SLIDER-52 processing"
+ def yarnSite = retriever.retrieveConfiguration(
+ externalConfSet,
+ ARTIFACT_NAME,
+ true)
+ assert !yarnSite.empty
+ def siteXML = yarnSite.asConfiguration()
+ def rmHostnameViaClientSideXML = parsedProps.get(
+ YarnConfiguration.RM_HOSTNAME)
+ assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties
+ def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS)
+
+ log.info("RM from downloaded props = $rmAddrFromDownloadedProperties")
+ assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties
+
+ describe "fetch missing artifact"
+ assertFailsWithExceptionClass(
+ FileNotFoundException, "") {
+ retriever.retrieveConfiguration(externalConfSet, "no-such-artifact", true)
+ }
+
+ describe "Internal configurations"
+ assert !retriever.hasConfigurations(false)
+ assertFailsWithExceptionClass(
+ FileNotFoundException, "") {
+ retriever.getConfigurations(false)
+ }
+
+ // retrieval via API
+ ActionRegistryArgs registryArgs = new ActionRegistryArgs()
+ registryArgs.verbose = true
+
+ // list all
+ registryArgs.list = true;
+ describe registryArgs.toString()
+ client.actionRegistry(registryArgs)
+
+ // list a named instance and expect a failure
+ registryArgs.list = true;
+ registryArgs.name = "unknown"
+ assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND, "") {
+ client.actionRegistryList(registryArgs)
+ }
+
+ // list all instances of an alternate type and expect failure
+ registryArgs.list = true;
+ registryArgs.name = null
+ registryArgs.serviceType = "org-apache-hadoop"
+ assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND,"") {
+ client.actionRegistryList(registryArgs)
+ }
+
+ registryArgs.serviceType = ""
+
+ //set the name
+ registryArgs.name = clustername;
+ registryArgs.serviceType = SliderKeys.APP_TYPE
+
+
+ //now expect list to work
+ describe registryArgs.toString()
+
+ def listedInstance = client.actionRegistryList(registryArgs)
+
+ def resolvedRecord = listedInstance[0]
+ assert resolvedRecord[YarnRegistryAttributes.YARN_ID] ==
+ serviceRecord[YarnRegistryAttributes.YARN_ID]
+ assert resolvedRecord[YarnRegistryAttributes.YARN_PERSISTENCE] ==
+ serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE]
+
+ // listconf
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ describe registryArgs.toString()
+
+ client.actionRegistry(registryArgs)
+
+ // listconf --internal
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ registryArgs.internal = true
+ describe registryArgs.toString()
+ assert LauncherExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
+
+ registryArgs.list = false;
+ registryArgs.listConf = false
+ registryArgs.internal = false
+
+ def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
+ registryArgs.getConf = yarn_site_config
+ registryArgs.user = currentUser()
+
+ //properties format
+ registryArgs.format = "properties"
+ describe registryArgs.toString()
+
+ client.actionRegistry(registryArgs)
+
+
+ File outputDir = new File("target/test_standalone_registry_am/output")
+ outputDir.mkdirs()
+
+ // create a new registry args with the defaults back in
+ registryArgs = new ActionRegistryArgs(clustername)
+ registryArgs.getConf = yarn_site_config
+ registryArgs.out = outputDir
+ registryArgs.user = ""
+ describe registryArgs.toString()
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".xml").exists()
+
+ registryArgs.format = "properties"
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".properties").exists()
+
+ describe registryArgs.toString()
+
+ def unknownFilename = "undefined-file"
+ registryArgs.getConf = unknownFilename
+ assert LauncherExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
+
+ //look for a different user
+ assertFailsWithException(SliderExitCodes.EXIT_NOT_FOUND, "") {
+ def args = new ActionRegistryArgs(
+ name: clustername,
+ user: "unknown",
+ serviceType: SliderKeys.APP_TYPE,
+ listConf: true,
+ verbose: true
+ )
+ client.actionRegistry(args)
+ log.warn("Success on $args")
+ }
+
+ describe "stop cluster"
+ //now kill that cluster
+ assert 0 == clusterActionFreeze(client, clustername)
+ //list it & See if it is still there
+ ApplicationReport oldInstance = yarnRegistryClient.findInstance(
+ clustername)
+ assert oldInstance != null
+ assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
+
+
+ // verify hbase to path generation filters things
+ def hbase = homePathForUser(HBASE)
+ def hbaseServices = serviceclassPath(hbase, SliderKeys.APP_TYPE)
+
+ // only check this if the YARN registry renaming logic is in
+ if (!hbase.contains("@")) {
+ assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND, "") {
+ client.actionResolve(
+ new ActionResolveArgs(
+ path: hbaseServices,
+ list: true))
+ }
+ assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND, "") {
+ client.actionResolve(
+ new ActionResolveArgs(
+ path: hbaseServices))
+ }
+
+ }
+/* SLIDER-531 disabled until YARN-2571 patch in everywhere
+
+ // now expect the AM to have had its service record deleted
+ ActionResolveArgs finalResolve = new ActionResolveArgs(
+ path: instanceRecordPath)
+
+ repeatUntilSuccess(this.&probeForEntryMissing, 10000, 1000,
+ [client:client,
+ resolve:finalResolve],
+ true,
+ "registry entry never deleted") {}
+ */
+ }
+
+ Outcome probeForEntryMissing(Map args) {
+ SliderClient client = (SliderClient)args["client"]
+ ActionResolveArgs resolveArgs = (ActionResolveArgs) args["resolve"]
+ try {
+ client.actionResolve(resolveArgs)
+ return Outcome.Retry
+ } catch (NotFoundException ignored) {
+ return Outcome.Success
+ }
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
index 088fdb3..8aacf6a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
@@ -19,10 +19,15 @@
package org.apache.slider.client
import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.slider.common.params.ActionRegistryArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.exceptions.UsageException
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.main.ServiceLauncherBaseTest
import org.junit.Test
@@ -30,12 +35,13 @@
* Test the argument parsing/validation logic
*/
@CompileStatic
+@Slf4j
class TestClientBadArgs extends ServiceLauncherBaseTest {
@Test
public void testNoAction() throws Throwable {
launchExpectingException(SliderClient,
new Configuration(),
- ErrorStrings.ERROR_NO_ACTION,
+ "Usage: slider COMMAND",
[])
}
@@ -47,6 +53,14 @@
"not-a-known-action",
["not-a-known-action"])
}
+
+ @Test
+ public void testActionWithoutOptions() throws Throwable {
+ launchExpectingException(SliderClient,
+ new Configuration(),
+ "Usage: slider build <application>",
+ [SliderActions.ACTION_BUILD])
+ }
@Test
public void testActionWithoutEnoughArgs() throws Throwable {
@@ -73,5 +87,56 @@
[SliderActions.ACTION_HELP,
Arguments.ARG_IMAGE])
}
+
+ @Test
+ public void testRegistryUsage() throws Throwable {
+ def exception = launchExpectingException(SliderClient,
+ new Configuration(),
+ "org.apache.slider.core.exceptions.UsageException: Argument --name missing",
+ [SliderActions.ACTION_REGISTRY])
+ assert exception instanceof UsageException
+ log.info(exception.toString())
+ }
+ @Test
+ public void testRegistryExportBadUsage1() throws Throwable {
+ def exception = launchExpectingException(SliderClient,
+ new Configuration(),
+ "Expected a value after parameter --getexp",
+ [SliderActions.ACTION_REGISTRY,
+ Arguments.ARG_NAME,
+ "cl1",
+ Arguments.ARG_GETEXP])
+ assert exception instanceof BadCommandArgumentsException
+ log.info(exception.toString())
+ }
+
+ @Test
+ public void testRegistryExportBadUsage2() throws Throwable {
+ def exception = launchExpectingException(SliderClient,
+ new Configuration(),
+ "Expected a value after parameter --getexp",
+ [SliderActions.ACTION_REGISTRY,
+ Arguments.ARG_NAME,
+ "cl1",
+ Arguments.ARG_LISTEXP,
+ Arguments.ARG_GETEXP])
+ assert exception instanceof BadCommandArgumentsException
+ log.info(exception.toString())
+ }
+
+ @Test
+ public void testRegistryExportBadUsage3() throws Throwable {
+ def exception = launchExpectingException(SliderClient,
+ new Configuration(),
+ "Usage: registry",
+ [SliderActions.ACTION_REGISTRY,
+ Arguments.ARG_NAME,
+ "cl1",
+ Arguments.ARG_LISTEXP,
+ Arguments.ARG_GETEXP,
+ "export1"])
+ assert exception instanceof UsageException
+ log.info(exception.toString())
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy
index 1e03030..23c1ffa 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy
@@ -18,6 +18,8 @@
package org.apache.slider.client
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.ClientArgs
@@ -47,21 +49,20 @@
@Test
public void testNoArgs() throws Throwable {
- try {
- ServiceLauncher launcher = launch(SliderClient,
+ launchExpectingException(SliderClient,
SliderUtils.createConfiguration(),
+ "Usage: slider COMMAND",
[])
- assert SliderExitCodes.EXIT_COMMAND_ARGUMENT_ERROR == launcher.serviceExitCode
- } catch (BadCommandArgumentsException ignored) {
- // expected
- }
}
@Test
- public void testListUnknownHost() throws Throwable {
+ public void testListUnknownRM() throws Throwable {
try {
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 1000)
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1000)
ServiceLauncher launcher = launch(SliderClient,
- SliderUtils.createConfiguration(),
+ conf,
[
ClientArgs.ACTION_LIST,
"cluster",
@@ -74,4 +75,5 @@
}
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
index d94cd54..8b013e5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
@@ -74,7 +74,8 @@
@Test
public void testSliderBasePath() throws Throwable {
- ClientArgs clientArgs = createClientArgs([ACTION_LIST, "--basepath", "/projects/slider/clusters"])
+ ClientArgs clientArgs = createClientArgs([ACTION_LIST,
+ ARG_BASE_PATH, "/projects/slider/clusters"])
assert clientArgs.basePath == new Path("/projects/slider/clusters")
}
@@ -154,7 +155,7 @@
}
/**
- * Test a thaw command
+ * Test a start command
* @throws Throwable
*/
@Test
@@ -234,37 +235,6 @@
}
@Test
- public void testGetConfFailsNoArg() throws Throwable {
- assertParseFails([
- ACTION_GETCONF,
- ])
- }
-
- @Test
- public void testGetConfWorks1Arg() throws Throwable {
- ClientArgs ca = createClientArgs([
- ACTION_GETCONF,
- CLUSTERNAME,
- ])
- assert ca.clusterName == CLUSTERNAME
- assert ca.coreAction instanceof ActionGetConfArgs
- }
-
- @Test
- public void testGetConfWorksOut() throws Throwable {
- ClientArgs ca = createClientArgs([
- ACTION_GETCONF,
- CLUSTERNAME,
- ARG_FORMAT,"xml",
- ARG_OUTPUT,"file.xml"
- ])
- assert ca.clusterName == CLUSTERNAME
- assert ca.coreAction instanceof ActionGetConfArgs
- assert ca.actionGetConfArgs.format == "xml"
- assert ca.actionGetConfArgs.output == "file.xml"
- }
-
- @Test
public void testGetStatusWorks1Arg() throws Throwable {
ClientArgs ca = createClientArgs([
ACTION_STATUS,
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
new file mode 100644
index 0000000..04f019f
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.params.ActionDiagnosticArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.test.SliderTestBase
+import org.apache.slider.test.YarnMiniClusterTestBase
+import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestDiagnostics extends YarnZKMiniClusterTestBase {
+
+ @Test
+ public void testClientDiags() throws Throwable {
+ //launch fake master
+ String clustername = createMiniCluster("", configuration, 1, true)
+ ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [SliderActions.ACTION_DIAGNOSTICS,
+ Arguments.ARG_CLIENT]
+ )
+ def client = launcher.service
+ def diagnostics = new ActionDiagnosticArgs()
+ diagnostics.client = true
+ diagnostics.verbose = true
+ describe("Verbose diagnostics")
+
+
+ client.actionDiagnostic(diagnostics)
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestInstallKeytab.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestInstallKeytab.groovy
new file mode 100644
index 0000000..2bbad6a
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestInstallKeytab.groovy
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import org.apache.commons.io.FileUtils
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.RawLocalFileSystem
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.ClientArgs
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * Test a keytab installation
+ */
+class TestInstallKeytab extends ServiceLauncherBaseTest {
+ final shouldFail = new GroovyTestCase().&shouldFail
+
+ private static SliderFileSystem testFileSystem
+
+ @Before
+ public void setupFilesystem() {
+ org.apache.hadoop.fs.FileSystem fileSystem = new RawLocalFileSystem()
+ YarnConfiguration configuration = SliderUtils.createConfiguration()
+ fileSystem.setConf(configuration)
+ testFileSystem = new SliderFileSystem(fileSystem, configuration)
+ }
+
+ @Test
+ public void testInstallKeytab() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(tempLocation, "test", true);
+ String contents = UUID.randomUUID().toString()
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+ ServiceLauncher launcher = launch(TestSliderClient,
+ conf,
+ [
+ ClientArgs.ACTION_INSTALL_KEYTAB,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.absolutePath,
+ Arguments.ARG_FOLDER,
+ "testFolder"])
+ Path installedPath = new Path(testFileSystem.buildKeytabInstallationDirPath("testFolder"), localKeytab.getName())
+ File installedKeytab = new File(installedPath.toUri().path)
+ assert installedKeytab.exists()
+ assert FileUtils.readFileToString(installedKeytab).equals(
+ FileUtils.readFileToString(localKeytab))
+ }
+
+ @Test
+ public void testInstallKeytabWithNoFolder() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(tempLocation, "test", true);
+ String contents = UUID.randomUUID().toString()
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+ shouldFail(BadCommandArgumentsException) {
+ ServiceLauncher launcher = launch(TestSliderClient,
+ conf,
+ [
+ ClientArgs.ACTION_INSTALL_KEYTAB,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.absolutePath])
+ }
+ }
+
+ @Test
+ public void testInstallKeytabWithNoKeytab() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(tempLocation, "test", true);
+ String contents = UUID.randomUUID().toString()
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+ shouldFail(BadCommandArgumentsException) {
+ ServiceLauncher launcher = launch(TestSliderClient,
+ conf,
+ [
+ ClientArgs.ACTION_INSTALL_KEYTAB,
+ ClientArgs.ARG_FOLDER,
+ "testFolder"])
+ }
+ }
+
+ @Test
+ public void testInstallKeytabAllowingOverwrite() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(tempLocation, "test", true);
+ String contents = UUID.randomUUID().toString()
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+ ServiceLauncher launcher = launch(TestSliderClient,
+ conf,
+ [
+ ClientArgs.ACTION_INSTALL_KEYTAB,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.absolutePath,
+ Arguments.ARG_FOLDER,
+ "testFolder"])
+ Path installedPath = new Path(testFileSystem.buildKeytabInstallationDirPath("testFolder"), localKeytab.getName())
+ File installedKeytab = new File(installedPath.toUri().path)
+ assert installedKeytab.exists()
+ assert FileUtils.readFileToString(installedKeytab).equals(FileUtils.readFileToString(localKeytab))
+ launcher = launch(TestSliderClient,
+ conf,
+ [
+ ClientArgs.ACTION_INSTALL_KEYTAB,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.absolutePath,
+ Arguments.ARG_FOLDER,
+ "testFolder",
+ Arguments.ARG_OVERWRITE]
+ )
+ assert installedKeytab.exists()
+ assert FileUtils.readFileToString(installedKeytab).equals(
+ FileUtils.readFileToString(localKeytab))
+ }
+
+ @Test
+ public void testInstallKeytabNotAllowingOverwrite() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(tempLocation, "test", true);
+ String contents = UUID.randomUUID().toString()
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+ ServiceLauncher launcher = launch(TestSliderClient,
+ conf,
+ [
+ ClientArgs.ACTION_INSTALL_KEYTAB,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.absolutePath,
+ Arguments.ARG_FOLDER,
+ "testFolder"])
+ Path installedPath = new Path(testFileSystem.buildKeytabInstallationDirPath("testFolder"), localKeytab.getName())
+ File installedKeytab = new File(installedPath.toUri().path)
+ assert installedKeytab.exists()
+ assert FileUtils.readFileToString(installedKeytab).equals(FileUtils.readFileToString(localKeytab))
+ shouldFail(BadCommandArgumentsException) {
+ launcher = launch(TestSliderClient,
+ conf,
+ [
+ ClientArgs.ACTION_INSTALL_KEYTAB,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.absolutePath,
+ Arguments.ARG_FOLDER,
+ "testFolder"])
+ }
+ }
+
+ @Test
+ public void testInstallKeytabWithMissingKeytab() throws Throwable {
+ // create a mock keytab file
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+ shouldFail(BadCommandArgumentsException) {
+ ServiceLauncher launcher = launch(TestSliderClient,
+ conf,
+ [
+ ClientArgs.ACTION_INSTALL_KEYTAB,
+ ClientArgs.ARG_KEYTAB,
+ "HeyIDontExist.keytab",
+ Arguments.ARG_FOLDER,
+ "testFolder"])
+ }
+ }
+
+ private File getTempLocation () {
+ return new File(System.getProperty("user.dir") + "/target");
+ }
+
+ static class TestSliderClient extends SliderClient {
+ public TestSliderClient() {
+ super()
+ }
+
+ @Override
+ protected void initHadoopBinding() throws IOException, SliderException {
+ sliderFileSystem = testFileSystem
+ }
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy
new file mode 100644
index 0000000..c4c755d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.ClientArgs
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.apache.slider.core.persist.ConfTreeSerDeser
+import org.apache.slider.core.persist.JsonSerDeser
+import org.junit.Assert
+import org.junit.Test
+
+/**
+ * Test bad argument handling
+ */
+//@CompileStatic
+class TestReplaceTokens extends Assert {
+
+ static final String PACKAGE = "/org/apache/slider/core/conf/examples/"
+ static final String app_configuration = "app_configuration_tokenized.json"
+ static final String app_configuration_processed =
+ "app_configuration_processed.json"
+
+ /**
+ * help should print out help string and then succeed
+ * @throws Throwable
+ */
+ @Test
+ public void testHelp() throws Throwable {
+ JsonSerDeser<ConfTree> confTreeJsonSerDeser =
+ new JsonSerDeser<ConfTree>(ConfTree)
+ def confTree = confTreeJsonSerDeser.fromResource(PACKAGE + app_configuration)
+ SliderClient.replaceTokens(confTree, "testUser", "testCluster")
+ assert confTree.global.get("site.fs.defaultFS") == "hdfs://testCluster:8020"
+ assert confTree.global.get("site.fs.default.name") == "hdfs://testCluster:8020"
+ assert confTree.global.get("site.hbase.user_name") == "testUser"
+ assert confTree.global.get("site.hbase.another.user") == "testUser"
+
+
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
index 7150b3c..b217329 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
@@ -18,12 +18,15 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.junit.Test
@Slf4j
+@CompileStatic
+
class TestClientResourceRegistration {
/**
@@ -50,15 +53,31 @@
conf1.set("key1", "conf1")
conf1.set("key2", "conf1")
Configuration conf2 = new Configuration(false)
- conf1.set("key1", "conf2")
- conf1.set("key3", "conf2")
- ConfigHelper.mergeConfigurations(conf1, conf2, "test")
+ conf2.set("key1", "conf2")
+ conf2.set("key3", "conf2")
+ ConfigHelper.mergeConfigurations(conf1, conf2, "test", true)
log.info(ConfigHelper.dumpConfigToString(conf1))
assert conf1.get("key1").equals("conf2")
assert conf1.get("key2").equals("conf1")
assert conf1.get("key3").equals("conf2")
}
+
+ @Test
+ public void testMergeConfigsNoOverwrite() throws Throwable {
+ Configuration conf1 = new Configuration(false)
+ conf1.set("key1", "conf1")
+ conf1.set("key2", "conf1")
+ Configuration conf2 = new Configuration(false)
+ conf2.set("key1", "conf2")
+ conf2.set("key3", "conf2")
+ ConfigHelper.mergeConfigurations(conf1, conf2, "test", false)
+ log.info(ConfigHelper.dumpConfigToString(conf1))
+
+ assert conf1.get("key1").equals("conf1")
+ assert conf1.get("key2").equals("conf1")
+ assert conf1.get("key3").equals("conf2")
+ }
/**
* This tests the situation where a yarn-config creation forces
@@ -73,7 +92,7 @@
String hostname = "nosuchhost:0"
conf.set(YarnConfiguration.RM_ADDRESS, hostname)
YarnConfiguration yc = new YarnConfiguration()
- ConfigHelper.mergeConfigurations(yc, conf, "slider-client")
+ ConfigHelper.mergeConfigurations(yc, conf, "slider-client", true)
InetSocketAddress addr = SliderUtils.getRmAddress(yc)
assert SliderUtils.isAddressDefined(addr)
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelper.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelper.groovy
new file mode 100644
index 0000000..bdab926
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelper.groovy
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.registry.client.api.RegistryConstants
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.test.YarnMiniClusterTestBase
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+
+class TestConfigHelper extends YarnMiniClusterTestBase {
+
+
+ @Test
+ public void testConfigLoaderIteration() throws Throwable {
+
+ String xml =
+ """<?xml version="1.0" encoding="UTF-8" standalone="no"?><configuration>
+<property><name>key</name><value>value</value><source>programatically</source></property>
+</configuration>
+ """
+ InputStream ins = new ByteArrayInputStream(xml.bytes);
+ Configuration conf = new Configuration(false);
+ conf.addResource(ins);
+ Configuration conf2 = new Configuration(false);
+ for (Map.Entry<String, String> entry : conf) {
+ conf2.set(entry.key, entry.value, "src")
+ }
+
+ }
+
+ @Test
+ public void testConfigDeprecation() throws Throwable {
+ ConfigHelper.registerDeprecatedConfigItems()
+ Configuration conf = new Configuration(false);
+ conf.set(SliderXmlConfKeys.REGISTRY_PATH, "path")
+ assert "path" == conf.get(SliderXmlConfKeys.REGISTRY_PATH)
+ assert "path" == conf.get(RegistryConstants.KEY_REGISTRY_ZK_ROOT)
+
+ conf.set(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, "localhost")
+ assert "localhost" == conf.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM)
+ assert "localhost" == conf.get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
index 0d21d6c..6558437 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
@@ -27,9 +28,10 @@
import org.junit.Test
@Slf4j
+@CompileStatic
+
class TestConfigHelperHDFS extends YarnMiniClusterTestBase {
- //diabled for now;
@Test
public void testConfigHelperHDFS() throws Throwable {
YarnConfiguration config = getConfiguration()
@@ -48,23 +50,4 @@
assert loaded.get("key") == "value"
}
- @Test
- public void testConfigLoaderIteration() throws Throwable {
-
- String xml =
- """<?xml version="1.0" encoding="UTF-8" standalone="no"?><configuration>
-<property><name>key</name><value>value</value><source>programatically</source></property>
-</configuration>
- """
- InputStream ins = new ByteArrayInputStream(xml.bytes);
- Configuration conf = new Configuration(false);
- conf.addResource(ins);
- Configuration conf2 = new Configuration(false);
- for (Map.Entry<String, String> entry : conf) {
- String key = entry.getKey();
- String val = entry.getValue();
- conf2.set(key, val, "src")
- }
-
- }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
new file mode 100644
index 0000000..3b2f992
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestExecutionEnvironment extends SliderTestBase {
+
+ @Test
+ public void testClientEnv() throws Throwable {
+ SliderUtils.validateSliderClientEnvironment(log)
+ }
+
+ @Test
+ public void testWinutils() throws Throwable {
+ SliderUtils.maybeVerifyWinUtilsValid();
+ }
+
+ @Test
+ public void testServerEnv() throws Throwable {
+ SliderUtils.validateSliderServerEnvironment(log, true)
+ }
+
+ @Test
+ public void testServerEnvNoDependencies() throws Throwable {
+ SliderUtils.validateSliderServerEnvironment(log, false)
+ }
+
+ @Test
+ public void testopenSSLEnv() throws Throwable {
+ SliderUtils.validateOpenSSLEnv(log)
+ }
+
+ @Test
+ public void testValidatePythonEnv() throws Throwable {
+ SliderUtils.validatePythonEnv(log)
+ }
+
+ @Test
+ public void testNativeLibs() throws Throwable {
+ assertNativeLibrariesPresent()
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
index 24367a3..9042af5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
@@ -18,12 +18,15 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.slider.test.SliderTestBase
import org.junit.Test
+@CompileStatic
+
class TestMiscSliderUtils extends SliderTestBase {
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
index 49bd58e..f009e25 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
@@ -18,11 +18,11 @@
package org.apache.slider.common.tools
-import groovy.transform.CompileStatic
+import org.apache.slider.core.exceptions.SliderException
import org.junit.Test
-@CompileStatic
class TestPortScan {
+ final shouldFail = new GroovyTestCase().&shouldFail
@Test
public void testScanPorts() throws Throwable {
@@ -38,4 +38,91 @@
server.close()
}
}
+
+ @Test
+ public void testRequestedPortsLogic() throws Throwable {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("5,6,8-10, 11,14 ,20 - 22")
+ List<Integer> ports = portScanner.remainingPortsToCheck
+ def expectedPorts = [5,6,8,9,10,11,14,20,21,22]
+ assert ports == expectedPorts
+ }
+
+ @Test
+ public void testRequestedPortsOutOfOrder() throws Throwable {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("8-10,5,6, 11,20 - 22, 14 ")
+ List<Integer> ports = portScanner.remainingPortsToCheck
+ def expectedPorts = [5,6,8,9,10,11,14,20,21,22]
+ assert ports == expectedPorts
+ }
+
+ @Test
+ public void testFindAvailablePortInRange() throws Throwable {
+ ServerSocket server = new ServerSocket(0)
+ try {
+ int serverPort = server.getLocalPort()
+
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("" + (serverPort-1) + "-" + (serverPort + 3))
+ int port = portScanner.availablePort
+ assert port != serverPort
+ assert port >= serverPort -1 && port <= serverPort + 3
+ } finally {
+ server.close()
+ }
+ }
+
+ @Test
+ public void testFindAvailablePortInList() throws Throwable {
+ ServerSocket server = new ServerSocket(0)
+ try {
+ int serverPort = server.getLocalPort()
+
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("" + (serverPort-1) + ", " + (serverPort + 1))
+ int port = portScanner.availablePort
+ assert port != serverPort
+ assert port == serverPort -1 || port == serverPort + 1
+ } finally {
+ server.close()
+ }
+ }
+
+ @Test
+ public void testNoAvailablePorts() throws Throwable {
+ ServerSocket server1 = new ServerSocket(0)
+ ServerSocket server2 = new ServerSocket(0)
+ try {
+ int serverPort1 = server1.getLocalPort()
+ int serverPort2 = server2.getLocalPort()
+
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("" + serverPort1+ ", " + serverPort2)
+ shouldFail(SliderException) {
+ portScanner.availablePort
+ }
+ } finally {
+ server1.close()
+ server2.close()
+ }
+ }
+
+ @Test
+ public void testPortRemovedFromRange() throws Throwable {
+ ServerSocket server = new ServerSocket(0)
+ try {
+ int serverPort = server.getLocalPort()
+
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("" + (serverPort-1) + "-" + (serverPort + 3))
+ int port = portScanner.availablePort
+ assert port != serverPort
+ assert port >= serverPort -1 && port <= serverPort + 3
+ def isPortInList = port in portScanner.remainingPortsToCheck
+ assert !isPortInList
+ } finally {
+ server.close()
+ }
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
new file mode 100644
index 0000000..198bd82
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.ChecksumFileSystem
+import org.apache.hadoop.fs.FSDataInputStream
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.service.ServiceStateException
+import org.apache.hadoop.util.Shell
+import org.apache.slider.providers.agent.AgentUtils
+import org.apache.slider.test.YarnMiniClusterTestBase
+import org.junit.Test
+
+import java.util.regex.Pattern
+
+@CompileStatic
+@Slf4j
+class TestWindowsSupport extends YarnMiniClusterTestBase {
+
+ private static final Pattern hasDriveLetterSpecifier =
+ Pattern.compile("^/?[a-zA-Z]:");
+ public static
+ final String windowsFile = "C:\\Users\\Administrator\\AppData\\Local\\Temp\\junit3180177850133852404\\testpkg\\appdef_1.zip"
+
+
+ private static boolean hasWindowsDrive(String path) {
+ return hasDriveLetterSpecifier.matcher(path).find();
+ }
+
+ private static int startPositionWithoutWindowsDrive(String path) {
+ if (hasWindowsDrive(path)) {
+ return path.charAt(0) == '/' ? 3 : 2;
+ } else {
+ return 0;
+ }
+ }
+
+ @Test
+ public void testHasWindowsDrive() throws Throwable {
+ assert hasWindowsDrive(windowsFile)
+ }
+
+ @Test
+ public void testStartPosition() throws Throwable {
+ assert 2 == startPositionWithoutWindowsDrive(windowsFile)
+ }
+
+ @Test
+ public void testPathHandling() throws Throwable {
+ assumeWindows()
+
+ Path path = new Path(windowsFile);
+ def uri = path.toUri()
+// assert "file" == uri.scheme
+ assert uri.authority == null;
+
+ Configuration conf = new Configuration()
+
+ def localfs = HadoopFS.get(uri, conf)
+ assert localfs instanceof ChecksumFileSystem
+ try {
+ def stat = localfs.getFileStatus(path)
+ fail("expected an exception, got $stat")
+ } catch (FileNotFoundException fnfe) {
+ // expected
+ }
+
+ try {
+ FSDataInputStream appStream = localfs.open(path);
+ } catch (FileNotFoundException fnfe) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testSliderFS() throws Throwable {
+ assumeWindows()
+ SliderFileSystem sfs = new SliderFileSystem(new Configuration())
+ try {
+ def metainfo = AgentUtils.getApplicationMetainfo(sfs, windowsFile)
+ } catch (FileNotFoundException fnfe) {
+ // expected
+ }
+ }
+
+
+/*
+ @Test
+ public void testHasGawkInstalled() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ exec(0, ["gawk", "--version"])
+ }
+*/
+
+/*
+ @Test
+ public void testHasXargsInstalled() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ exec(0, ["xargs", "--version"])
+ }
+*/
+
+ @Test
+ public void testExecNonexistentBinary() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ def commands = ["undefined-application", "--version"]
+ try {
+ exec(0, commands)
+ fail("expected an exception")
+ } catch (ServiceStateException e) {
+ if (!(e.cause instanceof FileNotFoundException)) {
+ throw e;
+ }
+ }
+ }
+ @Test
+ public void testExecNonexistentBinary2() throws Throwable {
+ assumeWindows()
+ assert !doesWindowsAppExist(["undefined-application", "--version"])
+ }
+
+ public assumeWindows() {
+ assume(Shell.WINDOWS, "not windows")
+ }
+
+ @Test
+ public void testEmitKillCommand() throws Throwable {
+
+ def result = killJavaProcesses("regionserver", 9)
+ // we know the exit code if there is no supported kill operation
+ assert kill_supported || result == -1
+ }
+
+ @Test
+ public void testHadoopHomeDefined() throws Throwable {
+ assumeWindows()
+ def hadoopHome = Shell.hadoopHome
+ log.info("HADOOP_HOME=$hadoopHome")
+ }
+
+ @Test
+ public void testHasWinutils() throws Throwable {
+ assumeWindows()
+ SliderUtils.maybeVerifyWinUtilsValid()
+ }
+
+ @Test
+ public void testExecWinutils() throws Throwable {
+ assumeWindows()
+ def winUtilsPath = Shell.winUtilsPath
+ assert winUtilsPath
+ File winUtils = new File(winUtilsPath)
+ log.debug("Winutils is at $winUtils)")
+
+ exec(0, [winUtilsPath, "systeminfo"])
+ }
+
+ @Test
+ public void testPath() throws Throwable {
+ String path = extractPath()
+ log.info("Path value = $path")
+ }
+
+ @Test
+ public void testFindJavac() throws Throwable {
+ String name = Shell.WINDOWS ? "javac.exe" : "javac"
+ assert locateExecutable(name)
+ }
+
+ @Test
+ public void testHadoopDLL() throws Throwable {
+ assumeWindows()
+ // split the path
+ File exepath = locateExecutable("HADOOP.DLL")
+ assert exepath
+ log.info "Hadoop DLL at: $exepath"
+ }
+
+ public File locateExecutable(String exe) {
+ File exepath = null
+ String path = extractPath()
+ String[] dirs = path.split(System.getProperty("path.separator"));
+ dirs.each { String dirname ->
+ File dir = new File(dirname)
+
+ File possible = new File(dir, exe)
+ if (possible.exists()) {
+ exepath = possible
+ }
+ }
+ return exepath
+ }
+
+ public String extractPath() {
+ String pathkey = "";
+
+ System.getenv().keySet().each { String it ->
+ if (it.toLowerCase(Locale.ENGLISH).equals("path")) {
+ pathkey = it;
+ }
+ }
+ assert pathkey
+ log.info("Path env variable is $pathkey")
+ def path = System.getenv(pathkey)
+ return path
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index fe3bef7..431c49f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -18,8 +18,11 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.registry.server.services.MicroZookeeperServiceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.zk.ZKIntegration
import org.apache.slider.test.KeysForTests
@@ -27,36 +30,47 @@
import org.apache.zookeeper.CreateMode
import org.apache.zookeeper.ZooDefs
import org.apache.zookeeper.data.Stat
+import org.junit.After
import org.junit.Before
import org.junit.Test
@Slf4j
-
+@CompileStatic
class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTests {
+
+ // as the static compiler doesn't resolve consistently
+ public static final String USER = KeysForTests.USERNAME
+ private ZKIntegration zki
+
@Before
void createCluster() {
- Configuration conf = getConfiguration()
- createMicroZKCluster(conf)
+ Configuration conf = configuration
+ def name = methodName.methodName
+ File zkdir = new File("target/zk/${name}")
+ FileUtil.fullyDelete(zkdir);
+ conf.set(MicroZookeeperServiceKeys.KEY_ZKSERVICE_DIR, zkdir.absolutePath)
+ createMicroZKCluster("-"+ name, conf)
}
- @Test
- public void testIntegrationCreate() throws Throwable {
- assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "cluster1", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
- Stat stat = zki.stat(userPath)
- assert stat != null
- log.info("User path $userPath has stat $stat")
+ @After
+ void closeZKI() {
+ zki?.close()
+ zki = null;
+ }
+
+ public ZKIntegration initZKI() {
+ zki = createZKIntegrationInstance(
+ getZKBinding(), methodName.methodName, true, false, 5000)
+ return zki
}
@Test
public void testListUserClustersWithoutAnyClusters() throws Throwable {
assertHasZKCluster()
-
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
- List<String> clusters = zki.clusters
+ initZKI()
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
+ List<String> clusters = this.zki.clusters
assert clusters.empty
}
@@ -64,8 +78,8 @@
public void testListUserClustersWithOneCluster() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ initZKI()
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
String fullPath = zki.createPath(userPath, "/cluster-",
ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL_SEQUENTIAL)
@@ -77,8 +91,8 @@
@Test
public void testListUserClustersWithTwoCluster() throws Throwable {
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ initZKI()
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
String c1 = createEphemeralChild(zki, userPath)
log.info("Ephemeral path $c1")
String c2 = createEphemeralChild(zki, userPath)
@@ -94,25 +108,25 @@
MockSliderClient client = new MockSliderClient()
String path = client.createZookeeperNode("cl1", true)
- ZKIntegration zki = client.getLastZKIntegration()
+ zki = client.lastZKIntegration
- String zkPath = ZKIntegration.mkClusterPath(USERNAME, "cl1")
- assert zkPath == "/services/slider/users/" + USERNAME + "/cl1", "zkPath must be as expected"
+ String zkPath = ZKIntegration.mkClusterPath(USER, "cl1")
+ assert zkPath == "/services/slider/users/" + USER + "/cl1", "zkPath must be as expected"
assert path == zkPath
assert zki == null, "ZKIntegration should be null."
zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, 5000);
- assert false == zki.exists(zkPath), "zkPath should not exist"
+ assert !zki.exists(zkPath)
path = client.createZookeeperNode("cl1", false)
- zki = client.getLastZKIntegration()
- assert zkPath == "/services/slider/users/" + USERNAME + "/cl1", "zkPath must be as expected"
+ zki = client.lastZKIntegration
+ assert zki
+ assert zkPath == "/services/slider/users/" + USER + "/cl1", "zkPath must be as expected"
assert path == zkPath
- assert true == zki.exists(zkPath), "zkPath must exist"
+ assert zki.exists(zkPath)
zki.createPath(zkPath, "/cn", ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT)
- assert true == zki.exists(zkPath + "/cn"), "zkPath with child node must exist"
+ assert zki.exists(zkPath + "/cn")
client.deleteZookeeperNode("cl1")
- assert false == zki.exists(zkPath), "zkPath must not exist"
-
+ assert !zki.exists(zkPath)
}
public String createEphemeralChild(ZKIntegration zki, String userPath) {
@@ -126,19 +140,18 @@
@Override
public String getUsername() {
- return USERNAME
+ return USER
}
@Override
protected ZKIntegration getZkClient(String clusterName, String user) {
- zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, 5000)
+ zki = createZKIntegrationInstance(getZKBinding(), clusterName, true, false, 5000)
return zki;
}
@Override
public synchronized Configuration getConfig() {
- Configuration conf = new Configuration();
- return conf;
+ new Configuration();
}
public ZKIntegration getLastZKIntegration() {
diff --git a/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy b/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy
index b655be8..156ae71 100644
--- a/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy
@@ -19,10 +19,11 @@
package org.apache.slider.core.conf
import groovy.util.logging.Slf4j
+import static org.apache.slider.api.InternalKeys.*
import org.junit.Assert
import org.junit.Test
-import static org.apache.slider.core.conf.ExampleConfResources.overridden
+import static org.apache.slider.core.conf.ExampleConfResources.*
/**
* Test
@@ -89,4 +90,23 @@
assert worker["timeout"] == "1000"
}
+
+ @Test
+ public void testTimeIntervalLoading() throws Throwable {
+
+ def orig = ExampleConfResources.loadResource(internal)
+
+ MapOperations internals = new MapOperations(orig.global)
+ def s = internals.getOptionInt(
+ CHAOS_MONKEY_INTERVAL + MapOperations.SECONDS,
+ 0)
+ assert s == 60
+ long monkeyInterval = internals.getTimeRange(
+ CHAOS_MONKEY_INTERVAL,
+ DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS,
+ DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
+ DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
+ 0);
+ assert monkeyInterval == 60;
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index 6dee64f..5bf1c1f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -26,6 +26,7 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.client.SliderClient
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.Before
@@ -47,6 +48,17 @@
@Rule
public TemporaryFolder folder = new TemporaryFolder();
+ /**
+ * Server side test: validate system env before launch
+ */
+ public static void assumeValidServerEnv() {
+ try {
+ SliderUtils.validateSliderServerEnvironment(log, true)
+ } catch (Exception e) {
+ skip(e.toString())
+ }
+ }
+
public String app_def_pkg_path;
@Before
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestUtils.groovy
index 989919f..54c2fe7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestUtils.groovy
@@ -38,7 +38,7 @@
Register register = new Register();
register.setResponseId(-1);
register.setTimestamp(System.currentTimeMillis());
- register.setHostname("dummyHost");
+ register.setLabel("dummyHost");
return register;
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index e248ec3..887ca89 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -22,11 +22,20 @@
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.exceptions.YarnException
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.core.build.InstanceBuilder
+import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.MapOperations
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.launch.LaunchedApplication
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.LockAcquireFailedException
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus
import org.apache.slider.server.services.security.CertificateManager
@@ -49,6 +58,7 @@
@CompileStatic
@Slf4j
class TestAgentAMManagementWS extends AgentTestBase {
+ private static String password;
public static final String AGENT_URI = "ws/v1/slider/agents/";
final static Logger logger = LoggerFactory.getLogger(TestAgentAMManagementWS.class)
@@ -82,12 +92,12 @@
void setup() {
super.setup()
MapOperations compOperations = new MapOperations();
- compOperations.put(SliderKeys.KEYSTORE_LOCATION, "/tmp/work/security/keystore.p12");
- SecurityUtils.initializeSecurityParameters(compOperations);
+ compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, "/tmp/work/security/keystore.p12");
+ SecurityUtils.initializeSecurityParameters(compOperations, true);
CertificateManager certificateManager = new CertificateManager();
- certificateManager.initRootCert(compOperations);
+ certificateManager.initialize(compOperations);
String keystoreFile = SecurityUtils.getSecurityDir() + File.separator + SliderKeys.KEYSTORE_FILE_NAME;
- String password = SecurityUtils.getKeystorePass();
+ password = SecurityUtils.getKeystorePass();
System.setProperty("javax.net.ssl.trustStore", keystoreFile);
System.setProperty("javax.net.ssl.trustStorePassword", password);
System.setProperty("javax.net.ssl.trustStoreType", "PKCS12");
@@ -113,54 +123,91 @@
assert app_def_path.exists()
assert agt_ver_path.exists()
assert agt_conf_path.exists()
- ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
- roles,
- [
- ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
- ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
- ],
- true, true,
- true)
- SliderClient sliderClient = launcher.service
- def report = waitForClusterLive(sliderClient)
- def trackingUrl = report.trackingUrl
- log.info("tracking URL is $trackingUrl")
- def agent_url = trackingUrl + AGENT_URI
+ try {
+ sliderClientClassName = TestSliderClient.name
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ roles,
+ [
+ ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
+ ],
+ true, true,
+ true)
+ SliderClient sliderClient = launcher.service
+ def report = waitForClusterLive(sliderClient)
+ def trackingUrl = report.trackingUrl
+ log.info("tracking URL is $trackingUrl")
+ def agent_url = trackingUrl + AGENT_URI
-
- def status = dumpClusterStatus(sliderClient, "agent AM")
- def liveURL = status.getInfo(StatusKeys.INFO_AM_AGENT_URL)
- if (liveURL) {
- agent_url = liveURL + AGENT_URI
+
+ def status = dumpClusterStatus(sliderClient, "agent AM")
+ def liveURL = status.getInfo(StatusKeys.INFO_AM_AGENT_OPS_URL)
+ if (liveURL) {
+ agent_url = liveURL + AGENT_URI
+ }
+
+ log.info("Agent is $agent_url")
+ log.info("stacks is ${liveURL}stacks")
+ log.info("conf is ${liveURL}conf")
+
+
+ def sleeptime = 10
+ log.info "sleeping for $sleeptime seconds"
+ Thread.sleep(sleeptime * 1000)
+
+
+ String page = fetchWebPageWithoutError(agent_url);
+ log.info(page);
+
+ //WS get
+ Client client = createTestClient();
+
+
+ WebResource webResource = client.resource(agent_url + "test/register");
+ RegistrationResponse response = webResource.type(MediaType.APPLICATION_JSON)
+ .post(
+ RegistrationResponse.class,
+ createDummyJSONRegister());
+
+ //TODO: assert failure as actual agent is not started. This test only starts the AM.
+ assert RegistrationStatus.FAILED == response.getResponseStatus();
+ } finally {
+ sliderClientClassName = DEFAULT_SLIDER_CLIENT
}
- log.info("Agent is $agent_url")
- log.info("stacks is ${liveURL}stacks")
- log.info("conf is ${liveURL}conf")
-
-
- def sleeptime = 10
- log.info "sleeping for $sleeptime seconds"
- Thread.sleep(sleeptime * 1000)
-
-
- String page = fetchWebPageWithoutError(agent_url);
- log.info(page);
-
- //WS get
- Client client = createTestClient();
-
-
- WebResource webResource = client.resource(agent_url + "test/register");
- RegistrationResponse response = webResource.type(MediaType.APPLICATION_JSON)
- .post(
- RegistrationResponse.class,
- createDummyJSONRegister());
-
- //TODO: assert failure as actual agent is not started. This test only starts the AM.
- assert RegistrationStatus.FAILED == response.getResponseStatus();
-
}
+
+ static class TestSliderClient extends SliderClient {
+ @Override
+ protected void persistInstanceDefinition(boolean overwrite,
+ Path appconfdir,
+ InstanceBuilder builder)
+ throws IOException, SliderException, LockAcquireFailedException {
+ AggregateConf conf = builder.getInstanceDescription()
+ MapOperations component = conf.getAppConfOperations().getComponent("slider-appmaster")
+ component.put(
+ "ssl.server.keystore.location",
+ "/tmp/work/security/keystore.p12")
+ component.put("ssl.server.keystore.password", password)
+ super.persistInstanceDefinition(overwrite, appconfdir, builder)
+ }
+
+ @Override
+ LaunchedApplication launchApplication(String clustername,
+ Path clusterDirectory,
+ AggregateConf instanceDefinition,
+ boolean debugAM)
+ throws YarnException, IOException {
+ MapOperations component = instanceDefinition.getAppConfOperations().getComponent("slider-appmaster")
+ component.put(
+ "ssl.server.keystore.location",
+ "/tmp/work/security/keystore.p12")
+ component.put("ssl.server.keystore.password", password)
+ return super.launchApplication(clustername, clusterDirectory, instanceDefinition, debugAM)
+ }
+ }
+
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index a29c8cb..2eb39e3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -23,7 +23,11 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ResourceKeys
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.core.exceptions.BadClusterStateException
import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Before
import org.junit.Test
import static org.apache.slider.common.params.Arguments.*
@@ -36,6 +40,28 @@
@Slf4j
class TestAgentEcho extends AgentTestBase {
+ File slider_core
+ String echo_py
+ File echo_py_path
+ File app_def_path
+ String agt_ver
+ File agt_ver_path
+ String agt_conf
+ File agt_conf_path
+
+ @Before
+ public void setupArtifacts() {
+ slider_core = new File(new File(".").absoluteFile, "src/test/python");
+ echo_py = "echo.py"
+ echo_py_path = new File(slider_core, echo_py)
+ app_def_path = new File(app_def_pkg_path)
+ agt_ver = "version"
+ agt_ver_path = new File(slider_core, agt_ver)
+ agt_conf = "agent.ini"
+ agt_conf_path = new File(slider_core, agt_conf)
+
+ }
+
@Override
void checkTestAssumptions(YarnConfiguration conf) {
@@ -43,6 +69,8 @@
@Test
public void testEchoOperation() throws Throwable {
+ assumeValidServerEnv()
+
String clustername = createMiniCluster("",
configuration,
1,
@@ -51,14 +79,6 @@
true,
false)
- File slider_core = new File(new File(".").absoluteFile, "src/test/python");
- String echo_py = "echo.py"
- File echo_py_path = new File(slider_core, echo_py)
- File app_def_path = new File(app_def_pkg_path)
- String agt_ver = "version"
- File agt_ver_path = new File(slider_core, agt_ver)
- String agt_conf = "agent.ini"
- File agt_conf_path = new File(slider_core, agt_conf)
assert echo_py_path.exists()
assert app_def_path.exists()
assert agt_ver_path.exists()
@@ -72,12 +92,14 @@
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
- ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
+ ARG_DEFINE,
+ SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false"
],
true, true,
true)
@@ -85,9 +107,23 @@
waitForRoleCount(sliderClient, roles, AGENT_CLUSTER_STARTUP_TIME)
//sleep a bit
- sleep(20000)
+ sleep(5000)
//expect the role count to be the same
waitForRoleCount(sliderClient, roles, 1000)
+ // flex size
+ // while running, flex it with no changes
+ sliderClient.flex(clustername, [(role): 2]);
+ sleep(1000)
+ waitForRoleCount(sliderClient, roles, 1000)
+
+ // flex to an illegal value
+ try {
+ sliderClient.flex(clustername, [(role): -1]);
+ fail("expected an exception")
+ } catch (BadClusterStateException e) {
+ assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "negative")
+ }
+
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index 99f7f49..264d260 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -21,9 +21,11 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.InternalKeys
import org.apache.slider.api.ResourceKeys
import org.apache.slider.api.RoleKeys
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderKeys
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.exceptions.BadConfigException
@@ -53,17 +55,32 @@
return new File(app_def_pkg_path);
}
+ private String getAppDefURI() {
+ appDef.toURI().toString()
+ }
+
private File getBadAppDef() {
return bad_app_def_path;
}
+ private String getBadAppDefURI() {
+ badAppDef.toURI().toString()
+ }
+
private File getAgentConf() {
return agt_conf_path;
}
+ private String getAgentConfURI() {
+ agentConf.toURI().toString()
+ }
+
private File getAgentImg() {
return new File(app_def_pkg_path);
}
+ private String getAgentImgURI() {
+ agentImg.toURI().toString()
+ }
@Test
@@ -78,12 +95,12 @@
true,
false)
buildAgentCluster("test_build_basic_agent_node_only",
- [(ROLE_NODE): 5],
+ [(ROLE_NODE): 1],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
ARG_PACKAGE, ".",
- ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
ARG_OPTION, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_COMP_OPT, ROLE_NODE, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_RES_COMP_OPT, ROLE_NODE, ResourceKeys.COMPONENT_PRIORITY, "1",
@@ -95,15 +112,15 @@
def rs = "hbase-rs"
ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
[
- (ROLE_NODE): 5,
+ (ROLE_NODE): 1,
(master): 1,
(rs): 5
],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
ARG_OPTION, PACKAGE_PATH, ".",
- ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
ARG_COMP_OPT, master, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_COMP_OPT, rs, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_RES_COMP_OPT, master, ResourceKeys.COMPONENT_PRIORITY, "2",
@@ -137,7 +154,7 @@
def name2 = clustername + "-2"
buildAgentCluster(name2,
[
- (ROLE_NODE): 5,
+ (ROLE_NODE): 2,
"role3": 1,
"newnode": 5
],
@@ -153,6 +170,26 @@
} catch (BadConfigException expected) {
}
+ try {
+ launcher = buildAgentCluster(clustername + "-10",
+ [
+ (ROLE_NODE): 4,
+ ],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_OPTION, PACKAGE_PATH, ".",
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
+ ARG_COMP_OPT, ROLE_NODE, SCRIPT_PATH, "agent/scripts/agent.py",
+ ARG_RES_COMP_OPT, ROLE_NODE, ResourceKeys.COMPONENT_PRIORITY, "1",
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(ROLE_NODE, "too many instances")
+ } catch (BadConfigException expected) {
+ assert expected.message.contains("Expected minimum is 1 and maximum is 2")
+ assert expected.message.contains("Component echo, yarn.component.instances value 4 out of range.")
+ }
//duplicate priorities
try {
def name3 = clustername + "-3"
@@ -184,8 +221,8 @@
(rs): 5
],
[
- ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
ARG_PACKAGE, ".",
ARG_COMP_OPT, SliderKeys.COMPONENT_AM, RoleKeys.JVM_OPTS, jvmopts,
ARG_COMP_OPT, master, RoleKeys.JVM_OPTS, jvmopts,
@@ -211,18 +248,71 @@
def name5 = clustername + "-5"
buildAgentCluster(name5,
[
- "role": 1,
+ "hbase-rs": 1,
],
[
- ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
ARG_PACKAGE, ".",
- ARG_RES_COMP_OPT, "role", ResourceKeys.COMPONENT_PRIORITY, "3",
+ ARG_RES_COMP_OPT, "hbase-rs", ResourceKeys.COMPONENT_PRIORITY, "3",
],
true, false,
false)
}
-
+
+ @Test
+ public void testLabelExpressionArgs() throws Throwable {
+ String clustername = createMiniCluster(
+ "",
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ try {
+ buildAgentCluster(clustername,
+ [:],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_RESOURCES, TEST_FILES + "good/resources_with_label.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
+ ],
+ true, false,
+ false)
+ } catch (BadConfigException exception) {
+ log.error(
+ "Build operation should not have failed with exception : \n$exception")
+ fail("Build operation should not fail")
+ }
+
+ AggregateConf instanceDefinition = loadInstanceDefinition(clustername)
+ def opt = instanceDefinition.getResourceOperations().getComponentOpt(
+ "echo",
+ ResourceKeys.YARN_LABEL_EXPRESSION,
+ null)
+ assert opt == null, "Expect null"
+
+ opt = instanceDefinition.getResourceOperations().getComponentOpt(
+ "hbase-master",
+ ResourceKeys.YARN_LABEL_EXPRESSION,
+ null)
+ assert opt == "", "Expect empty string"
+
+ opt = instanceDefinition.getResourceOperations().getComponentOpt(
+ "hbase-rs",
+ ResourceKeys.YARN_LABEL_EXPRESSION,
+ null)
+ assert opt == "coquelicot && amaranth", "Expect colors you have not heard of"
+
+ def label = instanceDefinition.getInternalOperations().get(
+ InternalKeys.INTERNAL_QUEUE)
+ assert label == null, "Default queue expected"
+ }
+
@Test
public void testUpdateBasicAgent() throws Throwable {
@@ -239,15 +329,15 @@
def rs = "hbase-rs"
ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
[
- (ROLE_NODE): 5,
+ (ROLE_NODE): 2,
(master): 1,
(rs): 5
],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
ARG_OPTION, PACKAGE_PATH, ".",
- ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
ARG_COMP_OPT, master, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_COMP_OPT, rs, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_RES_COMP_OPT, master, ResourceKeys.COMPONENT_PRIORITY, "2",
@@ -278,15 +368,15 @@
// change master priority and rs instances through update action
ServiceLauncher<SliderClient> launcher2 = updateAgentCluster(clustername,
[
- (ROLE_NODE): 5,
+ (ROLE_NODE): 2,
(master): 1,
(rs): 6
],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
ARG_OPTION, PACKAGE_PATH, ".",
- ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
ARG_COMP_OPT, master, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_COMP_OPT, rs, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_RES_COMP_OPT, master, ResourceKeys.COMPONENT_PRIORITY, "4",
@@ -342,7 +432,7 @@
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
ARG_PACKAGE, ".",
- ARG_OPTION, APP_DEF, "file://" + appDef.absolutePath,
+ ARG_OPTION, APP_DEF, appDefURI,
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
],
@@ -354,7 +444,74 @@
fail("Build operation should not fail")
}
}
-
+
+ @Test
+ public void testBadAgentArgs_Unknown_Component() throws Throwable {
+ String clustername = createMiniCluster(
+ "",
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ try {
+ def badArgs1 = "test_bad_agent_unk_comp"
+ buildAgentCluster(clustername,
+ [:],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_RESOURCES, TEST_FILES + "bad/resources-3.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(badArgs1, "bad component type node")
+ } catch (BadConfigException expected) {
+ assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Component node is not a member of application")
+ }
+ }
+
+ @Test
+ public void testSubmitToSpecificQueue() throws Throwable {
+ String clustername = createMiniCluster(
+ "",
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ try {
+ buildAgentCluster(clustername,
+ [:],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_RESOURCES, TEST_FILES + "good/resources.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json",
+ ARG_QUEUE, "labeled"
+ ],
+ true, false,
+ false)
+ } catch (BadConfigException exception) {
+ log.error(
+ "Build operation should not have failed with exception : \n$exception")
+ fail("Build operation should not fail")
+ }
+
+ AggregateConf instanceDefinition = loadInstanceDefinition(clustername)
+ def label = instanceDefinition.getInternalOperations().get(
+ InternalKeys.INTERNAL_QUEUE)
+ assert label == "labeled", "Expect labeled as the queue"
+ }
+
@Test
public void testBadAgentArgs() throws Throwable {
String clustername = createMiniCluster(
@@ -367,33 +524,14 @@
false)
try {
- def badArgs1 = "test_bad_agent_args-1"
- buildAgentCluster(badArgs1,
- [:],
- [
- ARG_OPTION, CONTROLLER_URL, "http://localhost",
- ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
- ARG_RESOURCES, TEST_FILES + "good/resources.json",
- ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
- ],
- true, false,
- false)
- failWithBuildSucceeding(badArgs1, "missing package home or image path")
- } catch (BadConfigException expected) {
- log.info("Expected failure.", expected)
- assert expected.message.contains("Either agent package path agent.package.root or image root internal.application.image.path must be provided")
- }
-
- try {
def badArgs1 = "test_bad_agent_args-2"
buildAgentCluster(badArgs1,
[:],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
- ARG_IMAGE, "file://" + getAgentImg().absolutePath,
- ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_IMAGE, agentImgURI,
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
],
@@ -411,7 +549,7 @@
[:],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
ARG_PACKAGE, ".",
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
@@ -430,9 +568,9 @@
[:],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
ARG_PACKAGE, ".",
- ARG_OPTION, APP_DEF, "file://" + getBadAppDef().absolutePath,
+ ARG_OPTION, APP_DEF, badAppDefURI,
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
],
@@ -460,8 +598,8 @@
[:],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
- ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, appDefURI,
+ ARG_OPTION, AGENT_CONF, agentConfURI,
ARG_PACKAGE, ".",
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy
new file mode 100644
index 0000000..5f95e0b
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.registry
+
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.slider.core.registry.SliderRegistryUtils
+import org.apache.slider.test.SliderTestUtils
+import org.junit.Assert
+import org.junit.Test
+
+class TestRegistryPaths {
+
+ @Test
+ public void testHomedirKerberos() throws Throwable {
+ def home = RegistryUtils.homePathForUser("hbase@HADOOP.APACHE.ORG")
+ try {
+ Assert.assertEquals("/users/hbase", home)
+ } catch (AssertionError e) {
+ SliderTestUtils.skip("homedir filtering not yet in hadoop registry module")
+ }
+ }
+
+ @Test
+ public void testHomedirKerberosHost() throws Throwable {
+ def home = RegistryUtils.homePathForUser("hbase/localhost@HADOOP.APACHE.ORG")
+ try {
+ Assert.assertEquals("/users/hbase", home)
+ } catch (AssertionError e) {
+ SliderTestUtils.skip("homedir filtering not yet in hadoop registry module")
+ }
+ }
+
+ @Test
+ public void testRegistryPathForInstance() throws Throwable {
+ def path = SliderRegistryUtils.registryPathForInstance("instance")
+ assert path.endsWith("/instance")
+ }
+
+ @Test
+ public void testPathResolution() throws Throwable {
+ def home = RegistryUtils.homePathForCurrentUser()
+ assert home == SliderRegistryUtils.resolvePath("~")
+ assert (home +"/") == SliderRegistryUtils.resolvePath("~/")
+ assert (home +"/something") == SliderRegistryUtils.resolvePath("~/something")
+ assert ("~unresolved") == SliderRegistryUtils.resolvePath("~unresolved")
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy
deleted file mode 100644
index f750f84..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.registry.curator
-
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.core.registry.retrieve.RegistryRetriever
-import org.apache.slider.server.services.curator.CuratorHelper
-import org.apache.slider.server.services.curator.RegistryBinderService
-import org.apache.slider.core.registry.info.RegistryNaming
-import org.apache.slider.test.MicroZKCluster
-import org.apache.slider.test.SliderTestUtils
-import org.junit.After
-import org.junit.Before
-import org.junit.Test
-
-class TestLocalRegistry {
- MicroZKCluster miniZK
- RegistryBinderService<ServiceInstanceData> registryBinder
-
- @Before
- void setup() {
- miniZK = new MicroZKCluster()
- miniZK.createCluster()
-
- registryBinder = createRegistry()
- }
-
- def RegistryBinderService<ServiceInstanceData> createRegistry(
- ) {
- def conf = new YarnConfiguration()
- CuratorHelper curatorHelper =
- new CuratorHelper(conf, miniZK.zkBindingString);
-
- def registry
- registry = curatorHelper.createRegistryBinderService("/services");
- registry.init(conf)
- registry.start()
- return registry
- }
-
- @After
- void teardown() {
- registryBinder?.stop()
- miniZK?.close()
- }
-
-
- @Test
- public void testRegisterAndQuery() throws Throwable {
- registryBinder.register(SliderKeys.APP_TYPE, "instance3",
- new URL("http", "localhost", 80, "/"),
- null)
- def instance = registryBinder.queryForInstance(
- SliderKeys.APP_TYPE,
- "instance3")
- assert instance != null
- }
-
- @Test
- public void testRegisterAndList() throws Throwable {
- registryBinder.register(SliderKeys.APP_TYPE, "instance3",
- new URL("http", "localhost", 80, "/"),
- null)
- registryBinder.register(SliderKeys.APP_TYPE, "instance2",
- new URL("http", "localhost", 8090, "/"),
- null)
- def instances = registryBinder.instanceIDs(SliderKeys.APP_TYPE)
- assert instances.size() ==2
- def instance = registryBinder.queryForInstance(
- SliderKeys.APP_TYPE,
- "instance3")
- assert instance != null
- }
-
- @Test
- public void testMultipleRegistryBinders() throws Throwable {
- registryBinder.register(SliderKeys.APP_TYPE, "instance3",
- new URL("http", "localhost", 80, "/"),
- null)
- registryBinder.register(SliderKeys.APP_TYPE, "instance2",
- new URL("http", "localhost", 8090, "/"),
- null)
- RegistryBinderService<ServiceInstanceData> registry2 = createRegistry()
- RegistryBinderService<ServiceInstanceData> registry3 = createRegistry()
- try {
- def instances = registry3.instanceIDs(SliderKeys.APP_TYPE)
- assert instances.size() == 2
- def instance = registryBinder.queryForInstance(
- SliderKeys.APP_TYPE,
- "instance3")
- assert instance.id == "instance3"
- assert instance.name == SliderKeys.APP_TYPE
- } finally {
- registry3.stop()
- registry2.stop()
-
- }
- }
-
- @Test
- public void testNamingPolicy() throws Throwable {
-
- String hobbitName = RegistryNaming.createRegistryServiceType("hobbiton",
- "bilbo",
- SliderKeys.APP_TYPE);
- String hobbitId =
- RegistryNaming.createRegistryName(
- "hobbiton",
- "bilbo",
- SliderKeys.APP_TYPE,
- 1);
- String mordorName = RegistryNaming.createRegistryServiceType("mordor",
- "bilbo",
- SliderKeys.APP_TYPE);
- String mordorId =
- RegistryNaming.createRegistryName(
- "mordor",
- "bilbo",
- SliderKeys.APP_TYPE,
- 1);
-
- // service have same name
- assert hobbitName == mordorName;
- assert mordorId != hobbitId;
- registryBinder.register(mordorName, mordorId,
- new URL("http", "localhost", 8090, "/"),
- new ServiceInstanceData())
- registryBinder.register(hobbitName, hobbitId,
- new URL("http", "localhost", 80, "/mordor"),
- new ServiceInstanceData())
- def mordorInstance = registryBinder.queryForInstance(
- mordorName,
- mordorId)
- assert mordorInstance.port == 8090
-
- def instances = registryBinder.listInstances(SliderKeys.APP_TYPE);
- SliderTestUtils.dumpRegistryInstances(instances)
- assert instances.size() == 2
-
- // now set up a registry retriever
- RegistryRetriever retriever = new RegistryRetriever()
-
- }
-
-}
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
deleted file mode 100644
index 1a1e5aa..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.registry.curator
-
-import com.sun.jersey.api.client.Client
-import com.sun.jersey.api.client.ClientResponse
-import com.sun.jersey.api.client.UniformInterfaceException
-import com.sun.jersey.api.client.WebResource
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.curator.x.discovery.ServiceType
-import org.apache.hadoop.security.UserGroupInformation
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.providers.agent.AgentTestBase
-import org.apache.slider.server.appmaster.web.rest.RestPaths
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.apache.slider.server.services.curator.CuratorServiceInstances
-import org.apache.slider.core.registry.info.RegistryNaming
-import org.junit.Test
-
-import javax.ws.rs.core.MediaType
-
-import static org.apache.slider.common.params.Arguments.ARG_OPTION
-import static org.apache.slider.providers.agent.AgentKeys.*
-import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient
-
-@CompileStatic
-@Slf4j
-class TestRegistryRestResources extends AgentTestBase {
-
- public static final String REGISTRY_URI = RestPaths.SLIDER_PATH_REGISTRY;
- public static final String WADL = "vnd.sun.wadl+xml"
- public static final String CLUSTERNAME = "testregistryws"
-
-
- private String id(String instanceName) {
-
- RegistryNaming.createRegistryName(
- instanceName,
- UserGroupInformation.getCurrentUser().getUserName(),
- SliderKeys.APP_TYPE,
- 1);
- }
-
-
- @Test
- public void testRestURIs() throws Throwable {
- def clustername = CLUSTERNAME
- createMiniCluster(
- clustername,
- configuration,
- 1,
- 1,
- 1,
- true,
- false)
- Map<String, Integer> roles = [:]
- File slider_core = new File(new File(".").absoluteFile, "src/test/python");
- File app_def_path = new File(app_def_pkg_path)
- String agt_ver = "version"
- File agt_ver_path = new File(slider_core, agt_ver)
- String agt_conf = "agent.ini"
- File agt_conf_path = new File(slider_core, agt_conf)
- assert app_def_path.exists()
- assert agt_ver_path.exists()
- assert agt_conf_path.exists()
- ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
- roles,
- [
- ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
- ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
- ],
- true, true,
- true)
- SliderClient sliderClient = launcher.service
- def report = waitForClusterLive(sliderClient)
- def trackingUrl = report.trackingUrl
- log.info("tracking URL is $trackingUrl")
- def registry_url = appendToURL(trackingUrl, REGISTRY_URI)
-
-
- def status = dumpClusterStatus(sliderClient, "agent AM")
- def liveURL = status.getInfo(StatusKeys.INFO_AM_WEB_URL)
- if (liveURL) {
- registry_url = appendToURL(liveURL, REGISTRY_URI)
-
- }
-
- log.info("Registry is $registry_url")
- log.info("stacks is ${liveURL}stacks")
- log.info("conf is ${liveURL}conf")
-
-
- //WS get
- Client client = createTestClient();
-
- WebResource webResource = client.resource(registry_url);
- List<String> serviceList = webResource.type(MediaType.APPLICATION_JSON)
- .get(List.class);
- log.info("service list: {}", serviceList)
- assert serviceList.contains(SliderKeys.APP_TYPE)
-
- // test the available GET URIs
- webResource = client.resource(
- appendToURL(registry_url, RestPaths.REGISTRY_SERVICE));
-
- ClientResponse response = webResource.type(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
- def responseStr = response.getEntity(String.class)
- log.info("response is " + responseStr)
-
- "{\"names\":[\"${SliderKeys.APP_TYPE}\"]}".equals(responseStr)
-
- webResource = client.resource(
- appendToURL(registry_url,
- "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}"));
- CuratorServiceInstances<ServiceInstanceData> services = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstances.class);
- assert services.services.size() == 1
- CuratorServiceInstance<ServiceInstanceData> service = services.services.get(0)
- validateService(service)
-
- webResource = client.resource(
- appendToURL(registry_url,
- "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}/"+id(
- clustername)));
- service = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstance.class);
- validateService(service)
-
- webResource = client.resource(
- appendToURL(
- registry_url, "${RestPaths.REGISTRY_ANYSERVICE}/${SliderKeys.APP_TYPE}"));
- service = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstance.class);
- validateService(service)
-
- // some negative tests...
- webResource = client.resource(
- appendToURL(registry_url, "${RestPaths.REGISTRY_SERVICE}/dummy"))
- services = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstances.class);
- assert services.services.size() == 0
-
- try {
- webResource = client.resource(appendToURL(registry_url,
- "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}/testregistryws99"));
-
- service = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstance.class);
- fail("should throw an exception for a 404 response....")
- } catch (UniformInterfaceException e) {
- assert e.response.getStatus() == 404
- }
-
- try {
- webResource = client.resource(
- appendToURL(registry_url, "${RestPaths.REGISTRY_ANYSERVICE}/dummy"));
-
- service = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstance.class);
- fail("should throw an exception for a 404 response....")
- } catch (UniformInterfaceException e) {
- assert e.response.getStatus() == 404
- }
- }
-
- private void validateService(CuratorServiceInstance service) {
- assert service.name.equals(SliderKeys.APP_TYPE)
- assert service.serviceType == ServiceType.DYNAMIC
- assert service.id.contains(CLUSTERNAME)
- }
-}
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy
deleted file mode 100644
index 6a57cd9..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.registry.curator
-
-import groovy.util.logging.Slf4j
-import org.apache.curator.x.discovery.ServiceInstance
-import org.apache.curator.x.discovery.ServiceInstanceBuilder
-import org.apache.curator.x.discovery.UriSpec
-import org.apache.slider.core.persist.JsonSerDeser
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.junit.Test
-
-/**
- * Here to identify why curator was complaining about deserializing
- * its own published artifacts.
- *
- * ... its a Jackson version thing
- */
-@Slf4j
-class TestServiceInstanceSerDeser {
-
- JsonSerDeser<ServiceInstance> serializer = new JsonSerDeser<>(
- ServiceInstance.class)
- JsonSerDeser<CuratorServiceInstance> deser = new JsonSerDeser<>(
- CuratorServiceInstance.class)
-
- @Test
- public void testDefault() throws Throwable {
- def builder = ServiceInstance.builder()
- builder.name("defined")
- buildAndRoundTrip("testDefault", builder)
- }
-
- @Test
- public void testEmpty() throws Throwable {
- def builder = ServiceInstance.builder()
- builder.address(null).id("").name("").port(0).uriSpec(null)
-
- buildAndRoundTrip("testEmpty", builder)
- }
-
- @Test
- public void testFilled() throws Throwable {
- def builder = ServiceInstance.builder()
-
- builder.id("service").name("name").port(0)
- .uriSpec(new UriSpec("http:{}:{}"))
-
-
- buildAndRoundTrip("testFilled", builder)
- }
-
- @Test
- public void testPayload() throws Throwable {
- def builder = ServiceInstance.builder()
- builder.address(null).id("testPayload").name("").port(0).uriSpec(null)
-
- ServiceInstanceData data = new ServiceInstanceData()
- data.serviceType = "testPayload"
- data.externalView.documentsURL = "http://documents"
- builder.payload(data)
-
- def instance = buildAndRoundTrip("", builder)
-
- def payload = instance.payload as ServiceInstanceData
- log.info("payload = $payload")
- assert payload.externalView.documentsURL == "http://documents"
- }
-
- @Test
- public void testHackedDeser() throws Throwable {
- def builder = ServiceInstance.builder()
- builder.address("localhost")
- builder.id("service").name("name").port(8080)
- .uriSpec(new UriSpec("http:{}:{}"))
- .sslPort(443)
- ServiceInstanceData data = new ServiceInstanceData()
- data.externalView.documentsURL = "http://documents"
-
- builder.payload(data)
- log.info("Test: testHackedDeser")
- String json = serialize(builder)
- CuratorServiceInstance<ServiceInstanceData> curatorInstance = deser.fromJson(json)
- log.info("resolved =$curatorInstance")
- def payload = curatorInstance.payload
- log.info("payload = $payload")
- assert payload.externalView.documentsURL == "http://documents"
-
- }
-
- def buildAndRoundTrip(String testname, ServiceInstanceBuilder builder) {
- log.info("Test: $testname")
- String json = serialize(builder)
-
- return serializer.fromJson(json)
- }
-
- def String serialize(ServiceInstanceBuilder builder) {
- ServiceInstance<ServiceInstanceData> instance = builder.build()
- def json = serializer.toJson(instance)
- log.info(json)
- return json
- }
-
-}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy
new file mode 100644
index 0000000..26a2c95
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.exceptions.YarnException
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionKillContainerArgs
+import org.apache.slider.core.build.InstanceBuilder
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.launch.LaunchedApplication
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.LockAcquireFailedException
+import org.apache.slider.providers.agent.AgentKeys
+import org.apache.slider.providers.agent.AgentTestBase
+import org.junit.Before
+import org.junit.Test
+
+import static org.apache.slider.common.params.Arguments.*
+import static org.apache.slider.providers.agent.AgentKeys.*
+
+/**
+ * Tests an echo command
+ */
+@Slf4j
+class TestDelayInContainerLaunch extends AgentTestBase {
+
+ File slider_core
+ String echo_py
+ File echo_py_path
+ File app_def_path
+ String agt_ver
+ File agt_ver_path
+ String agt_conf
+ File agt_conf_path
+
+ @Before
+ public void setupArtifacts() {
+ slider_core = new File(new File(".").absoluteFile, "src/test/python");
+ echo_py = "echo.py"
+ echo_py_path = new File(slider_core, echo_py)
+ app_def_path = new File(app_def_pkg_path)
+ agt_ver = "version"
+ agt_ver_path = new File(slider_core, agt_ver)
+ agt_conf = "agent.ini"
+ agt_conf_path = new File(slider_core, agt_conf)
+
+ }
+
+ @Override
+ void checkTestAssumptions(YarnConfiguration conf) {
+
+ }
+
+ @Test
+ public void testDelayInContainerLaunch() throws Throwable {
+ String clustername = createMiniCluster("",
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ assert echo_py_path.exists()
+ assert app_def_path.exists()
+ assert agt_ver_path.exists()
+ assert agt_conf_path.exists()
+
+ def role = "echo"
+ Map<String, Integer> roles = [
+ (role): 1,
+ ];
+ long delay = 30
+
+ DelayingSliderClient.delay = delay
+ sliderClientClassName = DelayingSliderClient.name
+ try {
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ roles,
+ [
+ ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
+ ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
+ ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
+ ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
+ ],
+ true, true,
+ true)
+ SliderClient sliderClient = launcher.service
+ waitForRoleCount(sliderClient, roles, AGENT_CLUSTER_STARTUP_TIME)
+
+ ClusterDescription status = sliderClient.clusterDescription
+ def workers = status.instances["echo"]
+ assert workers.size() == 1
+ def worker1 = workers[0]
+
+ // set the delay for 10 seconds more than start duration
+ ActionKillContainerArgs args = new ActionKillContainerArgs();
+ args.id = worker1
+ long start = System.currentTimeMillis()
+ assert 0 == sliderClient.actionKillContainer(clustername, args)
+ sleep(5000)
+ waitForRoleCount(sliderClient, roles, AGENT_CLUSTER_STARTUP_TIME)
+ long duration = System.currentTimeMillis() - start
+ assert duration/1000 >= delay
+
+ } finally {
+ sliderClientClassName = DEFAULT_SLIDER_CLIENT
+ }
+
+
+ }
+
+ static class DelayingSliderClient extends SliderClient {
+
+
+ static long delay
+ @Override
+ protected void persistInstanceDefinition(boolean overwrite,
+ Path appconfdir,
+ InstanceBuilder builder)
+ throws IOException, SliderException, LockAcquireFailedException {
+ AggregateConf conf = builder.getInstanceDescription()
+ conf.getAppConfOperations().getGlobalOptions().put(
+ AgentKeys.KEY_CONTAINER_LAUNCH_DELAY,
+ String.valueOf(delay))
+ super.persistInstanceDefinition(overwrite, appconfdir, builder)
+ }
+
+ @Override
+ LaunchedApplication launchApplication(String clustername,
+ Path clusterDirectory,
+ AggregateConf instanceDefinition,
+ boolean debugAM)
+ throws YarnException, IOException {
+ instanceDefinition.getAppConfOperations().getGlobalOptions().put(
+ AgentKeys.KEY_CONTAINER_LAUNCH_DELAY,
+ String.valueOf(delay))
+ return super.launchApplication(clustername, clusterDirectory, instanceDefinition, debugAM)
+ }
+
+ public static void setDelay (long aDelay) {
+ delay = aDelay
+ }
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
index 7e03e7b..a3a0025 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
@@ -191,7 +191,7 @@
queues.renewing("note", renewer)
assert queues.removeRenewingAction("note")
queues.stop()
- queues.waitForServiceToStop(10000)
+ assert queues.waitForServiceToStop(10000)
}
public class ActionNoteExecuted extends AsyncAction {
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
index 068b876..6368a3d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
@@ -23,14 +23,18 @@
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ResourceKeys
import org.apache.slider.core.conf.AggregateConf
-import org.apache.slider.core.conf.MapOperations
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.exceptions.TriggerClusterTeardownException
import org.apache.slider.server.appmaster.actions.ResetFailureWindow
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
/**
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
new file mode 100644
index 0000000..aa7bb11
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.core.exceptions.BadConfigException
+import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.providers.ProviderRole
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockAppState
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockAppStateDynamicHistory"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(8, 1)
+ }
+
+ @Override
+ void initApp() {
+ super.initApp()
+ appState = new MockAppState()
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+
+ def instance = factory.newInstanceDefinition(0,0,0)
+
+ appState.buildInstance(
+ instance,
+ new Configuration(),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null,
+ null, new SimpleReleaseSelector())
+ }
+
+
+ @Test
+ public void testDynamicRoleHistory() throws Throwable {
+
+ def dynamic = "dynamicRole"
+ int role_priority_8 = 8
+ int desired = 1
+ int placementPolicy = PlacementPolicy.DEFAULT
+ // snapshot and patch existing spec
+ def resources = ConfTreeOperations.fromInstance(
+ appState.resourcesSnapshot.confTree)
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): ""+desired,
+ (ResourceKeys.COMPONENT_PRIORITY) : "" +role_priority_8,
+ (ResourceKeys.COMPONENT_PLACEMENT_POLICY): "" + placementPolicy
+ ]
+
+ resources.components[dynamic] = opts
+
+
+ // write the definitions
+ def updates = appState.updateResourceDefinitions(resources.confTree);
+ assert updates.size() == 1
+ def updatedRole = updates[0]
+ assert updatedRole.placementPolicy == placementPolicy
+
+ // verify the new role was persisted
+ def snapshotDefinition = appState.resourcesSnapshot.getMandatoryComponent(
+ dynamic)
+ assert snapshotDefinition.getMandatoryOptionInt(
+ ResourceKeys.COMPONENT_PRIORITY) == role_priority_8
+
+ // now look at the role map
+ assert appState.roleMap[dynamic] != null
+ def mappedRole = appState.roleMap[dynamic]
+ assert mappedRole.id == role_priority_8
+
+ def priorityMap = appState.rolePriorityMap
+ assert priorityMap.size() == 4
+ ProviderRole dynamicProviderRole
+ assert (dynamicProviderRole = priorityMap[role_priority_8]) != null
+ assert dynamicProviderRole.id == role_priority_8
+
+ assert null != appState.roleStatusMap[role_priority_8]
+ def dynamicRoleStatus = appState.roleStatusMap[role_priority_8]
+ assert dynamicRoleStatus.desired == desired
+
+
+ // before allocating the nodes, fill up the capacity of some of the
+ // hosts
+ engine.allocator.nextIndex()
+
+ def targetNode = 2
+ assert targetNode == engine.allocator.nextIndex()
+ def targetHostname = engine.cluster.nodeAt(targetNode).hostname
+
+ // clock is set to a small value
+ appState.time = 100000
+
+ // allocate the nodes
+ def actions = appState.reviewRequestAndReleaseNodes()
+ assert actions.size() == 1
+ def action0 = (ContainerRequestOperation)actions[0]
+
+ def request = action0.request
+ assert !request.nodes
+
+ List<ContainerId> released = []
+ List<RoleInstance> allocations = submitOperations(actions, released)
+ processSubmissionOperations(allocations, [], released)
+ assert allocations.size() == 1
+ RoleInstance ri = allocations[0]
+
+ assert ri.role == dynamic
+ assert ri.roleId == role_priority_8
+ assert ri.host.host == targetHostname
+
+ // now look at the role history
+
+ def roleHistory = appState.roleHistory
+ def activeNodes = roleHistory.listActiveNodes(role_priority_8)
+ assert activeNodes.size() == 1
+ NodeInstance activeNode = activeNodes[0]
+ assert activeNode.get(role_priority_8)
+ def entry8 = activeNode.get(role_priority_8)
+ assert entry8.active == 1
+
+ assert activeNode.hostname == targetHostname
+
+ def activeNodeInstance = roleHistory.getOrCreateNodeInstance(ri.container)
+
+ assert activeNode == activeNodeInstance
+ def entry
+ assert (entry = activeNodeInstance.get(role_priority_8)) != null
+ assert entry.active
+ assert entry.live
+
+
+ // now trigger a termination event on that role
+
+ // increment time for a long-lived failure event
+ appState.time = appState.time + 100000
+
+ log.debug("Triggering failure")
+ def cid = ri.id
+ AppState.NodeCompletionResult result = appState.onCompletedNode(
+ containerStatus(cid, 1))
+ assert result.roleInstance == ri
+ assert result.containerFailed
+
+ roleHistory.dump();
+ // values should have changed
+ assert entry.failed == 1
+ assert !entry.startFailed
+ assert !entry.active
+ assert !entry.live
+
+
+ def nodesForRoleId = roleHistory.getNodesForRoleId(role_priority_8)
+ assert nodesForRoleId
+
+ // make sure new nodes will default to a different host in the engine
+ assert targetNode < engine.allocator.nextIndex()
+
+ actions = appState.reviewRequestAndReleaseNodes()
+ assert actions.size() == 1
+ def action1 = (ContainerRequestOperation) actions[0]
+ def request1 = action1.request
+ assert request1.nodes
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testRoleHistoryRoleAdditions() throws Throwable {
+ MockRoleHistory roleHistory = new MockRoleHistory([])
+ roleHistory.addNewProviderRole(new ProviderRole("one", 1))
+ roleHistory.addNewProviderRole(new ProviderRole("two", 1))
+ roleHistory.dump()
+ fail("should have raised an exception")
+ }
+
+
+ @Test(expected = BadConfigException.class)
+ public void testRoleHistoryRoleStartupConflict() throws Throwable {
+ MockRoleHistory roleHistory = new MockRoleHistory([
+ new ProviderRole("one", 1), new ProviderRole("two", 1)
+ ])
+ roleHistory.dump()
+ fail("should have raised an exception")
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index 136e1ea..83fb273 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -21,13 +21,18 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.ContainerPriority
+import org.apache.slider.server.appmaster.state.RoleHistoryUtils
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.junit.Test
@@ -39,6 +44,10 @@
@Slf4j
class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
implements MockRoles {
+ private static final String ROLE4 = "4"
+ private static final String ROLE5 = "5"
+ private static final int ID4 = 4
+ private static final int ID5 = 5
@Override
String getTestName() {
@@ -52,25 +61,33 @@
*/
@Override
MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 4)
+ return new MockYarnEngine(8, 2)
}
@Override
void initApp() {
super.initApp()
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
-
def instance = factory.newInstanceDefinition(0,0,0)
def opts = [
+ (ResourceKeys.COMPONENT_PRIORITY): ROLE4,
(ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "4",
]
- instance.resourceOperations.components["dynamic"]= opts
-
-
+
+ instance.resourceOperations.components[ROLE4]= opts
+
+ def opts5 = [
+ (ResourceKeys.COMPONENT_PRIORITY) : ROLE5,
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PLACEMENT_POLICY):
+ Integer.toString(PlacementPolicy.STRICT),
+ ]
+
+ instance.resourceOperations.components[ROLE5]= opts5
+
appState.buildInstance(
instance,
new Configuration(),
@@ -85,10 +102,149 @@
@Test
public void testAllocateReleaseRealloc() throws Throwable {
- List<RoleInstance> instances = createAndStartNodes()
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ createAndStartNodes()
+ appState.reviewRequestAndReleaseNodes()
appState.getRoleHistory().dump();
+ }
+
+ /**
+ * Find all allocations for a specific role
+ * @param role role Id/priority
+ * @param actions source list
+ * @return found list
+ */
+ List<ContainerRequestOperation> findAllocationsForRole(int role,
+ List<AbstractRMOperation> actions) {
+ List <ContainerRequestOperation > results = []
+ actions.each { AbstractRMOperation operation ->
+ if (operation instanceof ContainerRequestOperation) {
+ def req = (ContainerRequestOperation) operation;
+ def reqrole = ContainerPriority.extractRole(req.request.priority)
+ if (role == reqrole) {
+ results << req
+ }
+ }
+ }
+ return results
+ }
+
+ @Test
+ public void testStrictPlacementInitialRequest() throws Throwable {
+ log.info("Initial engine state = $engine")
+ List<AbstractRMOperation> actions = appState.reviewRequestAndReleaseNodes()
+ assert actions.size() == 2
+
+ // neither have locality at this point
+ assertRelaxLocalityFlag(ID4, null, true, actions)
+ assertRelaxLocalityFlag(ID5, null, true, actions)
+ }
+
+
+ @Test
+ public void testPolicyPropagation() throws Throwable {
+ assert !(appState.lookupRoleStatus(ROLE4).placementPolicy & PlacementPolicy.STRICT)
+ assert (appState.lookupRoleStatus(ROLE5).placementPolicy & PlacementPolicy.STRICT)
+
+ }
+
+ @Test
+ public void testLaxPlacementSecondRequestRole4() throws Throwable {
+ log.info("Initial engine state = $engine")
+ def role4 = appState.lookupRoleStatus(ROLE4)
+ def role5 = appState.lookupRoleStatus(ROLE5)
+ role4.desired = 1
+ role5.desired = 0
+
+ def instances = createStartAndStopNodes([])
+ assert instances.size() == 1
+
+ def instanceA = instances.find { RoleInstance instance ->
+ instance.roleId = ID4
+ }
+ assert instanceA
+ def hostname = RoleHistoryUtils.hostnameOf(instanceA.container)
+
+
+ log.info("Allocated engine state = $engine")
+ assert engine.containerCount() == 1
+
+ assert role4.actual == 1
+ // shrinking cluster
+
+ role4.desired = 0
+ appState.lookupRoleStatus(ROLE4).desired = 0
+ def completionResults = []
+ def containersToRelease = []
+ instances = createStartAndStopNodes(completionResults)
+ assert engine.containerCount() == 0
+ assert completionResults.size() == 1
+
+ // expanding: expect hostnames now
+ role4.desired = 1
+ def actions = appState.reviewRequestAndReleaseNodes()
+ assert actions.size() == 1
+
+ assertRelaxLocalityFlag(ID4, "", true, actions)
+ ContainerRequestOperation cro = (ContainerRequestOperation) actions[0]
+ def nodes = cro.request.nodes
+ assert nodes.size() == 1
+ assert hostname == nodes[0]
+ }
+
+ @Test
+ public void testStrictPlacementSecondRequestRole5() throws Throwable {
+ log.info("Initial engine state = $engine")
+ def role4 = appState.lookupRoleStatus(ROLE4)
+ def role5 = appState.lookupRoleStatus(ROLE5)
+ role4.desired = 0
+ role5.desired = 1
+
+ def instances = createStartAndStopNodes([])
+ assert instances.size() == 1
+
+ def instanceA = instances.find { RoleInstance instance ->
+ instance.roleId = ID5
+ }
+ assert instanceA
+ def hostname = RoleHistoryUtils.hostnameOf(instanceA.container)
+
+
+
+ log.info("Allocated engine state = $engine")
+ assert engine.containerCount() == 1
+
+ assert role5.actual == 1
+ // shrinking cluster
+
+ role5.desired = 0
+ def completionResults = []
+ def containersToRelease = []
+ instances = createStartAndStopNodes(completionResults)
+ assert engine.containerCount() == 0
+ assert completionResults.size() == 1
+ assert role5.actual == 0
+
+ role5.desired = 1
+ def actions = appState.reviewRequestAndReleaseNodes()
+ assert actions.size() == 1
+ assertRelaxLocalityFlag(ID5, "", false, actions)
+ ContainerRequestOperation cro = (ContainerRequestOperation) actions[0]
+ def nodes = cro.request.nodes
+ assert nodes.size() == 1
+ assert hostname == nodes[0]
}
-
+
+ public void assertRelaxLocalityFlag(
+ int id,
+ String expectedHost,
+ boolean expectedRelaxFlag,
+ List<AbstractRMOperation> actions) {
+ def requests
+ requests = findAllocationsForRole(id, actions)
+ assert requests.size() == 1
+ def req = requests[0]
+ assert expectedRelaxFlag == req.request.relaxLocality
+ }
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index 5c9dce9..53299dd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -26,10 +26,9 @@
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@@ -60,7 +59,7 @@
@Override
void initApp() {
super.initApp()
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
def instance = factory.newInstanceDefinition(0, 0, 0)
@@ -70,7 +69,7 @@
(ResourceKeys.COMPONENT_PRIORITY): "6",
]
- instance.resourceOperations.components["dynamic"] = opts
+ instance.resourceOperations.components["dynamic-6"] = opts
appState.buildInstance(instance,
@@ -97,7 +96,7 @@
(ResourceKeys.COMPONENT_PRIORITY): "7",
]
- cd.components["role4"] = opts
+ cd.components["dynamicAdd7"] = opts
appState.updateResourceDefinitions(cd.confTree);
createAndStartNodes();
dumpClusterDescription("updated CD", appState.getClusterStatus())
@@ -118,12 +117,17 @@
(ResourceKeys.COMPONENT_PRIORITY): "6",
]
- cd.components["role4"] = opts
+ cd.components["conflictingPriority"] = opts
try {
appState.updateResourceDefinitions(cd.confTree);
- dumpClusterDescription("updated CD", appState.getClusterStatus())
- fail("Expected an exception")
+
+ def status = appState.getClusterStatus()
+ dumpClusterDescription("updated CD", status)
+ fail("Expected an exception, got $status")
} catch (BadConfigException expected) {
+ log.info("Expected: {}", expected)
+ log.debug("Expected: {}", expected, expected)
+ // expected
}
}
@@ -148,24 +152,24 @@
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "7",
+ (ResourceKeys.COMPONENT_PRIORITY): "9",
]
- cd.components["role4"] = opts
+ cd.components["HistorySaveFlexLoad"] = opts
appState.updateResourceDefinitions(cd.confTree);
createAndStartNodes();
historyWriter.read(fs, history, appState.roleHistory)
}
@Test
- public void testHistoryFlexSaveLoad() throws Throwable {
+ public void testHistoryFlexSaveResetLoad() throws Throwable {
def cd = init()
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "7",
+ (ResourceKeys.COMPONENT_PRIORITY): "10",
]
- cd.components["role4"] = opts
+ cd.components["HistoryFlexSaveLoad"] = opts
appState.updateResourceDefinitions(cd.confTree);
createAndStartNodes();
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
@@ -174,7 +178,7 @@
//now reset the app state
def historyWorkDir2 = new File("target/history" + testName + "-0002")
def historyPath2 = new Path(historyWorkDir2.toURI())
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
@@ -184,7 +188,13 @@
fs,
historyPath2,
null, null, new SimpleReleaseSelector())
- historyWriter.read(fs, history, appState.roleHistory)
+ // on this read there won't be the right number of roles
+ try {
+ historyWriter.read(fs, history, appState.roleHistory)
+ fail("expected an exception")
+ } catch (IOException e) {
+ assert e.toString().contains("Number of roles")
+ }
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
index a7bf068..1db500b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -20,6 +20,7 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -112,5 +113,28 @@
}
+ @Test
+ public void testFlexNegative() throws Throwable {
+ int r0 = 6
+ int r1 = 0
+ int r2 = 0
+ role0Status.desired = r0
+ role1Status.desired = r1
+ role2Status.desired = r2
+ List<RoleInstance> instances = createAndStartNodes()
+
+ int clusterSize = r0 + r1 + r2
+ assert instances.size() == clusterSize
+ log.info("shrinking cluster")
+ role0Status.desired = -2
+ List<AppState.NodeCompletionResult> completionResults = []
+ try {
+ createStartAndStopNodes(completionResults)
+ fail("expected an exception")
+ } catch (TriggerClusterTeardownException e) {
+ }
+
+ }
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
index f8e852e..ee5eead 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -25,11 +25,14 @@
import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.CancelRequestOperation
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.operations.RMOperationHandler
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
@@ -63,32 +66,181 @@
public void testMockAddOp() throws Throwable {
role0Status.desired = 1
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
+ assertListLength(ops, 1)
ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
int priority = operation.request.priority.priority
assert extractRole(priority) == MockFactory.PROVIDER_ROLE0.id
RMOperationHandler handler = new MockRMOperationHandler()
handler.execute(ops)
- //tell the container its been allocated
AbstractRMOperation op = handler.operations[0]
assert op instanceof ContainerRequestOperation
}
+ /**
+ * Test of a flex up and down op which verifies that outstanding
+ * requests are cancelled first.
+ * <ol>
+ * <li>request 5 nodes, assert 5 request made</li>
+ * <li>allocate 1 of them</li>
+ * <li>flex cluster size to 3</li>
+ * <li>assert this generates 2 cancel requests</li>
+ * </ol>
+ */
+ @Test
+ public void testRequestThenCancelOps() throws Throwable {
+ def role0 = role0Status
+ role0.desired = 5
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assertListLength(ops, 5)
+ // now 5 outstanding requests.
+ assert role0.requested == 5
+
+ // allocate one
+ role0.incActual()
+ role0.decRequested()
+ assert role0.requested == 4
+
+
+ // flex cluster to 3
+ role0.desired = 3
+ ops = appState.reviewRequestAndReleaseNodes()
+
+ // expect a cancel operation from review
+ assertListLength(ops, 1)
+ assert ops[0] instanceof CancelRequestOperation
+ RMOperationHandler handler = new MockRMOperationHandler()
+ handler.availableToCancel = 4;
+ handler.execute(ops)
+ assert handler.availableToCancel == 2
+ assert role0.requested == 2
+
+ // flex down one more
+ role0.desired = 2
+ ops = appState.reviewRequestAndReleaseNodes()
+ assertListLength(ops, 1)
+ assert ops[0] instanceof CancelRequestOperation
+ handler.execute(ops)
+ assert handler.availableToCancel == 1
+ assert role0.requested == 1
+
+ }
+
+ @Test
+ public void testCancelNoActualContainers() throws Throwable {
+ def role0 = role0Status
+ role0.desired = 5
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assertListLength(ops, 5)
+ // now 5 outstanding requests.
+ assert role0.requested == 5
+ role0.desired = 0
+ ops = appState.reviewRequestAndReleaseNodes()
+ assertListLength(ops, 1)
+ CancelRequestOperation cancel = ops[0] as CancelRequestOperation
+ assert cancel.count == 5
+ }
+
+
+ @Test
+ public void testFlexDownOutstandingRequests() throws Throwable {
+ // engine only has two nodes, so > 2 will be outstanding
+ engine = new MockYarnEngine(1, 2)
+ List<AbstractRMOperation> ops
+ // role: desired = 2, requested = 1, actual=1
+ def role0 = role0Status
+ role0.desired = 4
+ createAndSubmitNodes()
+
+ assert role0.requested == 2
+ assert role0.actual == 2
+ // there are now two outstanding, two actual
+ // Release 3 and verify that the two
+ // cancellations were combined with a release
+ role0.desired = 1;
+ assert role0.delta == -3
+ ops = appState.reviewRequestAndReleaseNodes()
+ assertListLength(ops, 2)
+ assert role0.requested == 0
+ assert role0.releasing == 1
+ }
+
+ @Test
+ public void testCancelAllOutstandingRequests() throws Throwable {
+
+ // role: desired = 2, requested = 1, actual=1
+ def role0 = role0Status
+ role0.desired = 2
+ role0.incRequested()
+ role0.incRequested()
+ List<AbstractRMOperation> ops
+
+ // there are now two outstanding, two actual
+ // Release 3 and verify that the two
+ // cancellations were combined with a release
+ role0.desired = 0;
+ ops = appState.reviewRequestAndReleaseNodes()
+ assertListLength(ops, 1)
+ CancelRequestOperation cancel = ops[0] as CancelRequestOperation
+ assert cancel.getCount() == 2
+ }
+
+
+ @Test
+ public void testFlexUpOutstandingRequests() throws Throwable {
+
+ // role: desired = 2, requested = 1, actual=1
+ def role0 = role0Status
+ role0.desired = 2
+ role0.incActual();
+ role0.incRequested()
+
+ List<AbstractRMOperation> ops
+
+ // flex up 2 nodes, yet expect only one node to be requested,
+ // as the outstanding request is taken into account
+ role0.desired = 4;
+ role0.incRequested()
+
+ assert role0.actual == 1;
+ assert role0.requested == 2;
+ assert role0.actualAndRequested == 3;
+ assert role0.delta == 1
+ ops = appState.reviewRequestAndReleaseNodes()
+ assertListLength(ops, 1)
+ assert ops[0] instanceof ContainerRequestOperation
+ assert role0.requested == 3
+ }
+
+ @Test
+ public void testFlexUpNoSpace() throws Throwable {
+ // engine only has two nodes, so > 2 will be outstanding
+ engine = new MockYarnEngine(1, 2)
+ List<AbstractRMOperation> ops
+ // role: desired = 2, requested = 1, actual=1
+ def role0 = role0Status
+ role0.desired = 4
+ createAndSubmitNodes()
+
+ assert role0.requested == 2
+ assert role0.actual == 2
+ role0.desired = 8;
+ assert role0.delta == 4
+ createAndSubmitNodes()
+ assert role0.requested == 6
+ }
+
+
@Test
public void testAllocateReleaseOp() throws Throwable {
role0Status.desired = 1
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- AMRMClient.ContainerRequest request = operation.request
- Container cont = engine.allocateContainer(request)
- List<Container> allocated = [cont]
- List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> operations = []
- appState.onContainersAllocated(allocated, assignments, operations)
- assert operations.size() == 0
- assert assignments.size() == 1
+ def (ArrayList<ContainerAssignment> assignments, Container cont, AMRMClient.ContainerRequest request) = satisfyContainerRequest(
+ operation)
+ assertListLength(ops, 1)
+ assertListLength(assignments, 1)
ContainerAssignment assigned = assignments[0]
Container target = assigned.container
assert target.id == cont.id
@@ -104,13 +256,23 @@
//now release it by changing the role status
role0Status.desired = 0
ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
+ assertListLength(ops, 1)
assert ops[0] instanceof ContainerReleaseOperation
ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0]
assert release.containerId == cont.id
}
+ public List satisfyContainerRequest(ContainerRequestOperation operation) {
+ AMRMClient.ContainerRequest request = operation.request
+ Container cont = engine.allocateContainer(request)
+ List<Container> allocated = [cont]
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> operations = []
+ appState.onContainersAllocated(allocated, assignments, operations)
+ return [assignments, cont, request]
+ }
+
@Test
public void testComplexAllocation() throws Throwable {
role0Status.desired = 1
@@ -121,8 +283,8 @@
List<ContainerAssignment> assignments = [];
List<AbstractRMOperation> releases = []
appState.onContainersAllocated(allocations, assignments, releases)
- assert releases.size() == 0
- assert assignments.size() == 4
+ assertListLength(releases, 0)
+ assertListLength(assignments, 4)
assignments.each { ContainerAssignment assigned ->
Container target = assigned.container
RoleInstance ri = roleInstance(assigned)
@@ -136,7 +298,7 @@
assert engine.containerCount() == 4;
role1Status.desired = 0
ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 3
+ assertListLength(ops, 3)
allocations = engine.execute(ops)
assert engine.containerCount() == 1;
@@ -154,7 +316,7 @@
List<ContainerAssignment> assignments = [];
List<AbstractRMOperation> releases = []
appState.onContainersAllocated(allocations, assignments, releases)
- assert assignments.size() == 1
+ assertListLength(assignments, 1)
ContainerAssignment assigned = assignments[0]
Container target = assigned.container
RoleInstance ri = roleInstance(assigned)
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index c2783f3..b48a683 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -24,10 +24,14 @@
import org.apache.hadoop.yarn.api.records.Container
import org.apache.slider.api.StatusKeys
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.junit.Test
/**
@@ -67,7 +71,7 @@
NodeMap nodemap = appState.roleHistory.cloneNodemap()
// now destroy the app state
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
//and rebuild
appState.buildInstance(
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
index 17ebc31..e9de390 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
@@ -27,7 +27,9 @@
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.RoleHistoryUtils
+import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
@@ -54,6 +56,10 @@
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
AMRMClient.ContainerRequest request = operation.request
+ assert request.relaxLocality
+ assert request.nodes == null
+ assert request.racks == null
+
Container allocated = engine.allocateContainer(request)
List<ContainerAssignment> assignments = [];
List<AbstractRMOperation> operations = []
@@ -95,6 +101,7 @@
AMRMClient.ContainerRequest request2 = operation.request
assert request2 != null
assert request2.nodes[0] == containerHostname
+ assert request2.relaxLocality
engine.execute(ops)
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index 340e72d..dbb70fa 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -18,9 +18,14 @@
package org.apache.slider.server.appmaster.model.history
+import java.util.List;
+
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority
import org.apache.hadoop.yarn.api.records.Resource
import org.apache.hadoop.yarn.client.api.AMRMClient
@@ -365,4 +370,54 @@
MockContainer c2 = (MockContainer) sortedContainers[1]
assert c2.priority.getPriority() == 1
}
+
+ @Test
+ public void testNodeUpdated() throws Throwable {
+ describe("fail a node")
+
+ int role = 0
+ ProviderRole provRole = new ProviderRole(roleName, role)
+ RoleStatus roleStatus = new RoleStatus(provRole)
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestNode(roleStatus, resource);
+
+ String hostname = request.getNodes()[0]
+ assert hostname == age3Active0.hostname
+
+ // build a container
+ MockContainer container = factory.newContainer()
+ container.nodeId = new MockNodeId(hostname, 0)
+ container.priority = request.getPriority()
+ roleHistory.onContainerAssigned(container);
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname)
+ NodeEntry roleEntry = allocated.get(role)
+ assert roleEntry.starting == 1
+ assert !roleEntry.available
+ RoleInstance ri = new RoleInstance(container);
+ // start it
+ roleHistory.onContainerStartSubmitted(container, ri)
+ roleHistory.onContainerStarted(container)
+
+ int startSize = nodemap.size()
+
+ // now send a list of updated (failed) nodes event
+ List<NodeReport> nodesUpdated = new ArrayList<NodeReport>();
+ NodeId nodeId = NodeId.newInstance(hostname, 0)
+ NodeReport nodeReport = NodeReport.newInstance(nodeId, NodeState.LOST, null, null, null, null, 1, null, 0)
+ nodesUpdated.add(nodeReport)
+ roleHistory.onNodesUpdated(nodesUpdated)
+
+ nodemap = roleHistory.cloneNodemap()
+ int endSize = nodemap.size()
+ if (startSize == 0) {
+ assert endSize == 0
+ } else {
+ assert startSize - endSize == 1
+ }
+ assert nodemap.get(hostname) == null
+ List<String> failedNodes = roleHistory.cloneFailedNodes()
+ assert failedNodes.contains(hostname)
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index 4242ba1..5575076 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -117,9 +117,9 @@
assert loadedNE2.lastUsed == savetime
assert rh2.thawedDataTime == savetime
- // now thaw it
+ // now start it
rh2.buildAvailableNodeLists();
- describe("thawing")
+ describe("starting")
rh2.dump();
List<NodeInstance> available0 = rh2.cloneAvailableList(0)
assert available0.size() == 1
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index 0a2ba60..8f577e5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -77,7 +77,8 @@
assert age3Active0 == ni
AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
roleStatus,
- resource)
+ resource,
+ "")
List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
assertListEquals([age2Active0], a2)
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
index 639c632..a027098 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
@@ -51,11 +51,11 @@
MockContainer allocate(AMRMClient.ContainerRequest request) {
MockYarnCluster.MockYarnClusterNode node = null
MockYarnCluster.MockYarnClusterContainer allocated = null
- if (request.nodes != null) {
+ if (request.nodes) {
for (String host : request.nodes) {
node = cluster.lookup(host)
allocated = node.allocate()
- if (allocated != null) {
+ if (allocated) {
break
}
}
@@ -64,7 +64,7 @@
if (allocated) {
return createContainerRecord(request, allocated, node)
} else {
- if (request.relaxLocality || request.nodes.isEmpty()) {
+ if (request.relaxLocality || request.nodes.empty) {
// fallback to anywhere
return allocateRandom(request)
} else {
@@ -117,7 +117,7 @@
return container;
}
- private int nextIndex() {
+ public int nextIndex() {
rollingIndex = (rollingIndex + 1) % cluster.clusterSize;
return rollingIndex;
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index f96a238..6c83c55 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -33,7 +33,12 @@
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.RoleStatus
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.apache.slider.test.SliderTestBase
import org.junit.Before
@@ -43,12 +48,14 @@
public static final int RM_MAX_RAM = 4096
public static final int RM_MAX_CORES = 64
MockFactory factory = new MockFactory()
- AppState appState
+ MockAppState appState
MockYarnEngine engine
protected HadoopFS fs
protected SliderFileSystem sliderFileSystem
protected File historyWorkDir
protected Path historyPath;
+ protected MockApplicationId applicationId;
+ protected MockApplicationAttemptId applicationAttemptId;
@Override
void setup() {
@@ -75,12 +82,16 @@
YarnConfiguration conf = SliderUtils.createConfiguration()
+ applicationId = new MockApplicationId(id: 1, clusterTimestamp: 0)
+ applicationAttemptId = new MockApplicationAttemptId(
+ applicationId: applicationId,
+ attemptId: 1)
fs = HadoopFS.get(new URI("file:///"), conf)
historyWorkDir = new File("target/history", historyDirName)
historyPath = new Path(historyWorkDir.toURI())
fs.delete(historyPath, true)
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
appState.buildInstance(
buildInstanceDefinition(),
@@ -130,7 +141,7 @@
Container target = assigned.container
RoleInstance ri = new RoleInstance(target)
ri.roleId = assigned.role.priority
- ri.role = assigned.role
+ ri.role = assigned.role.name
return ri
}
@@ -166,7 +177,7 @@
*/
public ContainerStatus containerStatus(ContainerId cid) {
ContainerStatus status = containerStatus(cid,
- LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN)
+ LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN)
return status
}
@@ -187,6 +198,7 @@
return createStartAndStopNodes([])
}
+
/**
* Create, Start and stop nodes
* @param completionResults List filled in with the status on all completed nodes
@@ -196,7 +208,22 @@
List<AppState.NodeCompletionResult> completionResults) {
List<ContainerId> released = []
List<RoleInstance> instances = createAndSubmitNodes(released)
+ processSubmissionOperations(instances, completionResults, released)
+ return instances
+ }
+
+ /**
+ * Process the start/stop operations from
+ * @param instances
+ * @param completionResults
+ * @param released
+ */
+ public void processSubmissionOperations(
+ List<RoleInstance> instances,
+ List<AppState.NodeCompletionResult> completionResults,
+ List<ContainerId> released) {
for (RoleInstance instance : instances) {
+ log.debug("Started ${instance.role} on ${instance.id} ")
assert appState.onNodeManagerContainerStarted(instance.containerId)
}
releaseContainers(completionResults,
@@ -205,7 +232,6 @@
"released",
0
)
- return instances
}
/**
@@ -249,6 +275,19 @@
public List<RoleInstance> createAndSubmitNodes(
List<ContainerId> released) {
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ return submitOperations(ops, released)
+ }
+
+ /**
+ * Process the RM operations and send <code>onContainersAllocated</code>
+ * events to the app state
+ * @param ops
+ * @param released
+ * @return
+ */
+ public List<RoleInstance> submitOperations(
+ List<AbstractRMOperation> ops,
+ List<ContainerId> released) {
List<Container> allocatedContainers = engine.execute(ops, released)
List<ContainerAssignment> assignments = [];
List<AbstractRMOperation> operations = []
@@ -259,12 +298,24 @@
RoleInstance ri = roleInstance(assigned)
instances << ri
//tell the app it arrived
+ log.debug("Start submitted ${ri.role} on ${container.id} ")
appState.containerStartSubmitted(container, ri);
}
return instances
}
/**
+ * Add the AM to the app state
+ */
+ protected void addAppMastertoAppState() {
+ appState.buildAppMasterNode(
+ new MockContainerId(applicationAttemptId, 999999L),
+ "appmaster",
+ 0,
+ null)
+ }
+
+ /**
* Extract the list of container IDs from the list of role instances
* @param instances instance list
* @param role role to look up
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
index 25c957e..e683587 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
@@ -17,13 +17,38 @@
package org.apache.slider.server.appmaster.model.mock
+import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.state.AbstractRecordFactory
import org.apache.slider.server.appmaster.state.AppState
+/**
+ * Extended app state that makes more things public
+ */
class MockAppState extends AppState {
public MockAppState(AbstractRecordFactory recordFactory) {
super(recordFactory);
}
+ long time = 0;
+
+ /**
+ * Instance with a mock record factory
+ */
+ public MockAppState() {
+ super(new MockRecordFactory());
+ }
+
+ public Map<String, ProviderRole> getRoleMap() {
+ return super.roleMap;
+ }
+
+ /**
+ * Current time. if the <code>time</code> field
+ * is set, that value is returned
+ * @return the current time.
+ */
+ protected long now() {
+ return time ?: System.currentTimeMillis();
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.groovy
index 3cb0a63..9cba4e2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.groovy
@@ -26,6 +26,14 @@
ApplicationId applicationId
int attemptId
+ MockApplicationAttemptId() {
+ }
+
+ MockApplicationAttemptId(ApplicationId applicationId, int attemptId) {
+ this.applicationId = applicationId
+ this.attemptId = attemptId
+ }
+
@Override
ApplicationId getApplicationId() {
return applicationId
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationId.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationId.groovy
index d7a63e1..85c0f07 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationId.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationId.groovy
@@ -25,6 +25,13 @@
private int id;
private long clusterTimestamp
+ MockApplicationId() {
+ }
+
+ MockApplicationId(int id) {
+ this.id = id
+ }
+
@Override
int getId() {
return id;
@@ -41,7 +48,7 @@
}
@Override
- public void setClusterTimestamp(long clusterTimestamp) {
+ public void setClusterTimestamp(long clusterTimestamp) {
this.clusterTimestamp = clusterTimestamp
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
index 3eba7c4..9f5d939 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
@@ -18,7 +18,12 @@
package org.apache.slider.server.appmaster.model.mock
-import org.apache.hadoop.yarn.api.records.*
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.Priority
+import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.hadoop.yarn.api.records.Token
class MockContainer extends Container {
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerId.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerId.groovy
index 8985f3a..c2984b7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerId.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerId.groovy
@@ -23,28 +23,51 @@
class MockContainerId extends ContainerId implements Cloneable {
- int id;
+ private static final defaultAppAttemptId = new MockApplicationAttemptId(
+ new MockApplicationId(1), 1)
+
+ long containerId;
ApplicationAttemptId applicationAttemptId;
MockContainerId() {
}
-
- MockContainerId(int id) {
- setId(id);
+
+ /**
+ * Sets up a default app Attempt ID
+ * @param containerId
+ */
+ @Deprecated
+ MockContainerId(long containerId) {
+ this.containerId = containerId;
+ this.applicationAttemptId = defaultAppAttemptId;
+ }
+
+ MockContainerId(ApplicationAttemptId applicationAttemptId, long containerId) {
+ this.containerId = containerId;
+ this.applicationAttemptId = applicationAttemptId
}
MockContainerId(ContainerId that) {
- id = that.id
+ containerId = that.containerId
applicationAttemptId = that.applicationAttemptId
}
int getId() {
- return id
+ return (int) containerId
}
+ // TODO: Temporarily adding it back
void setId(int id) {
- this.id = id
+ containerId = (long) id;
+ }
+
+ long getContainerId() {
+ return this.containerId;
+ }
+
+ void setContainerId(long id) {
+ this.containerId = id
}
ApplicationAttemptId getApplicationAttemptId() {
@@ -62,7 +85,7 @@
@Override
public String toString() {
- return "mockcontainer_" + id;
+ return "mockcontainer_" + containerId;
}
@Override
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index 52472f0..311c049 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -35,7 +35,7 @@
*/
//@CompileStatic
@Slf4j
-class MockFactory implements MockRoles {
+class MockFactory implements MockRoles {
public static final ProviderRole PROVIDER_ROLE0 = new ProviderRole(
MockRoles.ROLE0,
@@ -70,7 +70,7 @@
MockContainerId newContainerId(ApplicationAttemptId attemptId) {
MockContainerId cid = new MockContainerId()
- cid.id = containerIdCount++
+ cid.containerId = containerIdCount++
cid.applicationAttemptId = attemptId;
return cid;
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 6db1ac5..7efcd96 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -18,14 +18,15 @@
package org.apache.slider.server.appmaster.model.mock
-import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.service.LifecycleEvent
+import org.apache.hadoop.service.Service.STATE
import org.apache.hadoop.service.ServiceStateChangeListener
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.Priority
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.tools.SliderFileSystem
@@ -34,7 +35,7 @@
import org.apache.slider.core.exceptions.BadCommandArgumentsException
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.launch.ContainerLauncher
-import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.providers.ProviderCompleted
import org.apache.slider.providers.ProviderRole
import org.apache.slider.providers.ProviderService
import org.apache.slider.server.appmaster.actions.QueueAccess
@@ -47,9 +48,7 @@
import org.apache.slider.server.appmaster.web.rest.agent.Register
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus
-import org.apache.slider.server.services.registry.RegistryViewForProviders
-import org.apache.slider.providers.ProviderCompleted
-import org.apache.hadoop.service.Service.STATE
+import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders
class MockProviderService implements ProviderService {
@@ -208,13 +207,17 @@
@Override
void bind(
StateAccessForProviders stateAccessor,
- RegistryViewForProviders registry,
QueueAccess queueAccess,
List<Container> liveContainers) {
}
@Override
+ void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
+
+ }
+
+ @Override
AgentRestOperations getAgentRestOperations() {
return new AgentRestOperations() {
@Override
@@ -240,10 +243,10 @@
@Override
void applyInitialRegistryDefinitions(
- URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData registryInstanceData)
- throws MalformedURLException, IOException {
+ URL amWebURI,
+ URL agentOpsURI,
+ URL agentStatusURI,
+ ServiceRecord serviceRecord) throws IOException {
}
@@ -267,6 +270,14 @@
}
@Override
+ int cancelContainerRequests(
+ Priority priority1,
+ Priority priority2,
+ int count) {
+ return 0
+ }
+
+ @Override
void rebuildContainerDetails(List<Container> liveContainers, String applicationId,
Map<Integer, ProviderRole> roleProviderMap) {
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
index 0fdba6b..297c597 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
@@ -20,6 +20,7 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.Priority
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
@@ -30,6 +31,10 @@
class MockRMOperationHandler extends RMOperationHandler {
public List<AbstractRMOperation> operations = [];
int requests, releases;
+ // number available to cancel
+ int availableToCancel = 0;
+ // count of cancelled values. This must be explicitly set
+ int cancelled
@Override
public void releaseAssignedContainer(ContainerId containerId) {
@@ -45,9 +50,20 @@
requests++;
}
- /**
- * clear the history
- */
+ @Override
+ int cancelContainerRequests(
+ Priority priority1,
+ Priority priority2,
+ int count) {
+ int releaseable = Math.min(count, availableToCancel)
+ availableToCancel -= releaseable;
+ cancelled += releaseable;
+ return releaseable;
+ }
+
+/**
+ * clear the history
+ */
public void clear() {
operations.clear()
releases = 0;
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.groovy
new file mode 100644
index 0000000..d70fca9
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.groovy
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.fs.FileAlreadyExistsException
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException
+import org.apache.hadoop.fs.PathNotFoundException
+import org.apache.hadoop.registry.client.api.RegistryOperations
+import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException
+import org.apache.hadoop.registry.client.exceptions.NoRecordException
+import org.apache.hadoop.registry.client.types.RegistryPathStatus
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.hadoop.service.AbstractService
+
+/**
+ * Simple stub registry for when one is needed for its API, but the operations
+ * are not actually required
+ */
+class MockRegistryOperations extends AbstractService implements RegistryOperations{
+
+ MockRegistryOperations() {
+ super("mock")
+ }
+
+ @Override
+ boolean mknode(String path, boolean createParents)
+ throws PathNotFoundException, InvalidPathnameException, IOException {
+ return true
+ }
+
+ @Override
+ void bind(String path, ServiceRecord record, int flags) throws
+ PathNotFoundException,
+ FileAlreadyExistsException,
+ InvalidPathnameException,
+ IOException {
+
+ }
+
+ @Override
+ ServiceRecord resolve(String path) throws
+ PathNotFoundException,
+ NoRecordException,
+ InvalidRecordException,
+ IOException {
+ throw new PathNotFoundException(path);
+ }
+
+ @Override
+ RegistryPathStatus stat(String path)
+ throws PathNotFoundException, InvalidPathnameException, IOException {
+ throw new PathNotFoundException(path);
+ }
+
+ @Override
+ boolean exists(String path) throws IOException {
+ return false
+ }
+
+ @Override
+ List<String> list(String path)
+ throws PathNotFoundException, InvalidPathnameException, IOException {
+ throw new PathNotFoundException(path);
+ }
+
+ @Override
+ void delete(String path, boolean recursive) throws
+ PathNotFoundException,
+ PathIsNotEmptyDirectoryException,
+ InvalidPathnameException,
+ IOException {
+
+ }
+
+ @Override
+ boolean addWriteAccessor(String id, String pass) throws IOException {
+ return true
+ }
+
+ @Override
+ void clearWriteAccessors() {
+
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
similarity index 64%
copy from slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
copy to slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
index ee24dc1..c521697 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
@@ -16,11 +16,20 @@
* limitations under the License.
*/
-package org.apache.slider.server.services.registry;
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.slider.core.exceptions.BadConfigException
+import org.apache.slider.providers.ProviderRole
+import org.apache.slider.server.appmaster.state.RoleHistory
/**
- * These constants are unique to the slider registry service itself
+ * subclass to enable access to some of the protected methods
*/
-public class RegistryServiceConstants {
- public static final int INSTANCE_REFRESH_MS = 1000;
+class MockRoleHistory extends RoleHistory {
+
+ MockRoleHistory(List<ProviderRole> providerRoles) throws BadConfigException {
+ super(providerRoles)
+ }
+
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index eac5b6c..6056e3a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -56,6 +56,12 @@
build();
}
+ @Override
+ String toString() {
+ return "MockYarnCluster size=$clusterSize, capacity=${totalClusterCapacity()},"+
+ " in use=${containersInUse()}"
+ }
+
/**
* Build the cluster.
*/
@@ -90,7 +96,9 @@
*/
MockYarnClusterContainer release(ContainerId cid) {
int host = extractHost(cid.id)
- return nodeAt(host).release(cid.id)
+ def inUse = nodeAt(host).release(cid.id)
+ log.debug("Released $cid inuse=$inUse")
+ return inUse
}
int containersInUse() {
@@ -159,11 +167,16 @@
containers = new MockYarnClusterContainer[size];
for (int i = 0; i < size; i++) {
int cid = makeCid(index, i);
- MockContainerId mci = new MockContainerId(id: cid)
+ MockContainerId mci = new MockContainerId(containerId: cid)
containers[i] = new MockYarnClusterContainer(mci)
}
}
+ /**
+ * Look up a container
+ * @param containerId
+ * @return
+ */
public MockYarnClusterContainer lookup(int containerId) {
return containers[extractContainer(containerId)]
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index f405188..04466c6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -50,6 +50,11 @@
attemptId: 1,
)
+ @Override
+ String toString() {
+ return "MockYarnEngine $cluster + pending=${pending.size()}"
+ }
+
int containerCount() {
return cluster.containersInUse();
}
@@ -109,6 +114,7 @@
} else {
ContainerRequestOperation req = (ContainerRequestOperation) op
Container container = allocateContainer(req.request)
+ log.info("allocated container $container for $req")
if (container != null) {
allocation.add(container)
} else {
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
index c789011..e4a42fc 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
@@ -21,6 +21,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.InternalKeys
import org.apache.slider.server.appmaster.actions.ActionHalt
import org.apache.slider.server.appmaster.actions.ActionKillContainer
import org.apache.slider.server.appmaster.actions.QueueService
@@ -35,6 +36,8 @@
import org.junit.Before
import org.junit.Test
+import java.util.concurrent.TimeUnit
+
@CompileStatic
@Slf4j
class TestMockMonkey extends BaseMockAppStateTest {
@@ -43,14 +46,15 @@
* This queue service is NOT started; tests need to poll the queue
* rather than expect them to execute
*/
- QueueService queues = new QueueService();
- ChaosMonkeyService monkey = new ChaosMonkeyService(metricRegistry,
- queues)
+ QueueService queues
+ ChaosMonkeyService monkey
@Before
public void init() {
def configuration = new YarnConfiguration()
+ queues = new QueueService();
queues.init(configuration)
+ monkey = new ChaosMonkeyService(metricRegistry, queues)
monkey.init(configuration)
}
@@ -60,22 +64,50 @@
monkey.stop()
}
-
@Test
public void testMonkeyPlay() throws Throwable {
ChaosCounter counter = new ChaosCounter()
- monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100)
-
+ monkey.addTarget("target", counter, InternalKeys.PROBABILITY_PERCENT_100)
+ assert 1 == monkey.targetCount;
monkey.play()
assert counter.count == 1
}
@Test
+ public void testMonkeySchedule() throws Throwable {
+ ChaosCounter counter = new ChaosCounter()
+ assert 0 == monkey.targetCount;
+ monkey.addTarget("target", counter, InternalKeys.PROBABILITY_PERCENT_100)
+ assert 1 == monkey.targetCount;
+ assert monkey.schedule(0, 1, TimeUnit.SECONDS)
+ assert 1 == queues.scheduledActions.size()
+ }
+
+ @Test
+ public void testMonkeyDoesntAddProb0Actions() throws Throwable {
+ ChaosCounter counter = new ChaosCounter()
+ monkey.addTarget("target", counter, 0)
+ assert 0 == monkey.targetCount;
+ monkey.play()
+ assert counter.count == 0
+ }
+
+
+ @Test
+ public void testMonkeyScheduleProb0Actions() throws Throwable {
+ ChaosCounter counter = new ChaosCounter()
+ monkey.addTarget("target", counter, 0)
+ assert !monkey.schedule(0, 1, TimeUnit.SECONDS)
+ assert 0 == queues.scheduledActions.size()
+ }
+
+
+ @Test
public void testMonkeyPlaySometimes() throws Throwable {
ChaosCounter counter = new ChaosCounter()
ChaosCounter counter2 = new ChaosCounter()
- monkey.addTarget("target1", counter, ChaosMonkeyService.PERCENT_1 * 50)
- monkey.addTarget("target2", counter2, ChaosMonkeyService.PERCENT_1 * 25)
+ monkey.addTarget("target1", counter, InternalKeys.PROBABILITY_PERCENT_1 * 50)
+ monkey.addTarget("target2", counter2, InternalKeys.PROBABILITY_PERCENT_1 * 25)
for (int i = 0; i < 100; i++) {
monkey.play()
@@ -111,6 +143,20 @@
assert queues.scheduledActions.size() == 0
}
+
+ @Test
+ public void testContainerKillerIgnoresAM() throws Throwable {
+
+ addAppMastertoAppState()
+ assert 1 == appState.liveNodes.size()
+
+ def chaos = new ChaosKillContainer(appState,
+ queues,
+ new MockRMOperationHandler())
+ chaos.chaosAction();
+ assert queues.scheduledActions.size() == 0
+ }
+
@Test
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
new file mode 100644
index 0000000..4b60ead
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.security
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.conf.MapOperations
+import org.apache.slider.core.exceptions.SliderException;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class SecurityConfigurationTest {
+ final shouldFail = new GroovyTestCase().&shouldFail
+
+ @Test
+ public void testValidLocalConfiguration() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+
+ @Test
+ public void testValidDistributedConfiguration() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+
+ @Test
+ public void testMissingPrincipalNoLoginWithDistributedConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster") {
+ @Override
+ protected UserGroupInformation getLoginUser() throws IOException {
+ return null
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMissingPrincipalNoLoginWithLocalConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster") {
+ @Override
+ protected UserGroupInformation getLoginUser() throws IOException {
+ return null
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testBothKeytabMechanismsConfigured() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+ compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+ }
+
+ @Test
+ public void testNoKeytabMechanismConfigured() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+ }
+
+ @Test
+ public void testMissingPrincipalButLoginWithDistributedConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+
+ @Test
+ public void testMissingPrincipalButLoginWithLocalConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+
+ @Test
+ public void testKeypathLocationOnceLocalized() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+
+ assert new File(SliderKeys.KEYTAB_DIR, "some.keytab").getAbsolutePath() ==
+ securityConfiguration.getKeytabFile(aggregateConf).getAbsolutePath()
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index c1732e6..1964497 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -23,6 +23,7 @@
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.util.Shell
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
@@ -65,14 +66,15 @@
assert app_def_path.exists()
assert agt_ver_path.exists()
assert agt_conf_path.exists()
+
ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
roles,
[
ARG_PROVIDER, "org.apache.slider.server.appmaster.web.rest.publisher.TestSliderProviderFactory",
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
- ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path)
],
true, true,
true)
@@ -116,7 +118,13 @@
Map<String,String> val = webResource.type(MediaType.APPLICATION_JSON).get(Map.class);
assert "val1".equals(val.get("prop1"))
- // some negative tests...
+ // testing security_enabled auto-setting feature (SLIDER-392)
+ webResource = client.resource(sliderConfigset +
+ "global/site.global.security_enabled");
+ val = webResource.type(MediaType.APPLICATION_JSON).get(Map.class);
+ assert "false".equals(val.get("site.global.security_enabled"))
+
+ // some negative tests...
webResource = client.resource(appendToURL(sliderConfigset,
"foobar-site"));
@@ -133,13 +141,14 @@
Set uris = webResource.type(MediaType.APPLICATION_JSON)
.get(Set.class)
assert uris.size() > 0
- log.info("Classpath URIs: {}", uris)
- // check for some expected classpath elements
- assert uris.any {it =~ /curator-x-discovery/}
- assert uris.any {it =~ /hadoop-yarn-api/}
- assert uris.any {it =~ /hadoop-hdfs/}
- // and a negative test...
- assert !uris.any {it =~ /foo-bar/}
+ if (!Shell.WINDOWS) {
+ log.info("Classpath URIs: {}", uris)
+ // check for some expected classpath elements
+ assert uris.any {it =~ /hadoop-yarn-api/}
+ assert uris.any {it =~ /hadoop-hdfs/}
+ // and a negative test...
+ assert !uris.any {it =~ /foo-bar/}
+ }
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
similarity index 74%
rename from slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
rename to slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
index ee24dc1..5f987c2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
@@ -16,11 +16,13 @@
* limitations under the License.
*/
-package org.apache.slider.server.services.registry;
+package org.apache.slider.server.appmaster.web.rest.registry
-/**
- * These constants are unique to the slider registry service itself
- */
-public class RegistryServiceConstants {
- public static final int INSTANCE_REFRESH_MS = 1000;
-}
+import org.apache.hadoop.registry.client.binding.JsonSerDeser
+
+class PathEntryMarshalling
+ extends JsonSerDeser<PathEntryResource> {
+ public PathEntryMarshalling() {
+ super(PathEntryResource.class);
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
new file mode 100644
index 0000000..2a25012
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.registry
+
+import groovy.transform.CompileStatic
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
+import org.junit.Ignore
+import org.junit.Test
+
+/**
+ * This test exists because Jersey appears to behave "oddly"
+ * when it comes to marshalling JSON, and some of the REST resources
+ * appear to have trouble.
+ *
+ * This test tries to isolate it
+ */
+@CompileStatic
+class TestRegistryRestMarshalling {
+
+ @Test
+ @Ignore("SLIDER-531")
+ public void testDeser() throws Throwable {
+ PathEntryMarshalling pem = new PathEntryMarshalling();
+ def unmarshalled = pem.fromResource(
+ "/org/apache/slider/server/appmaster/web/rest/registry/sample.json")
+
+ def serviceRecord = unmarshalled.service
+ assert serviceRecord
+ assert serviceRecord[YarnRegistryAttributes.YARN_ID] != null
+ assert serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] != ""
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
new file mode 100644
index 0000000..b65c57e
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.registry
+
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.ClientResponse
+import com.sun.jersey.api.client.UniformInterfaceException
+import com.sun.jersey.api.client.WebResource
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.registry.client.api.RegistryConstants
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.agent.AgentTestBase
+import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.junit.Ignore
+import org.junit.Test
+
+import javax.ws.rs.core.MediaType
+
+import static org.apache.slider.common.params.Arguments.ARG_OPTION
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+import static org.apache.slider.providers.agent.AgentKeys.*
+import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient
+
+@CompileStatic
+@Slf4j
+class TestRegistryRestResources extends AgentTestBase {
+
+ public static final String REGISTRY_URI = RestPaths.SLIDER_PATH_REGISTRY;
+ public static final String WADL = "vnd.sun.wadl+xml"
+ public static final String CLUSTERNAME = "test-registry-rest-resources"
+
+
+ @Test
+ @Ignore("SLIDER-531")
+ public void testRestURIs() throws Throwable {
+
+ def clustername = CLUSTERNAME
+ createMiniCluster(
+ clustername,
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+ Map<String, Integer> roles = [:]
+ File slider_core = new File(new File(".").absoluteFile, "src/test/python");
+ File app_def_path = new File(app_def_pkg_path)
+ String agt_ver = "version"
+ File agt_ver_path = new File(slider_core, agt_ver)
+ String agt_conf = "agent.ini"
+ File agt_conf_path = new File(slider_core, agt_conf)
+ assert app_def_path.exists()
+ assert agt_ver_path.exists()
+ assert agt_conf_path.exists()
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ roles,
+ [
+ ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
+ ],
+ true, true,
+ true)
+ SliderClient sliderClient = launcher.service
+ def report = waitForClusterLive(sliderClient)
+ def trackingUrl = report.trackingUrl
+ log.info("tracking URL is $trackingUrl")
+ def registry_url = appendToURL(trackingUrl, REGISTRY_URI)
+
+
+ def status = dumpClusterStatus(sliderClient, "agent AM")
+ def liveURL = status.getInfo(StatusKeys.INFO_AM_WEB_URL)
+ if (liveURL) {
+ registry_url = appendToURL(liveURL, REGISTRY_URI)
+
+ }
+
+ log.info("Registry is $registry_url")
+ log.info("stacks is ${liveURL}stacks")
+ log.info("conf is ${liveURL}conf")
+
+ //WS get
+ Client client = createTestClient();
+
+ WebResource webResource = client.resource(registry_url);
+
+ def jsonType = webResource.type(MediaType.APPLICATION_JSON)
+ PathEntryResource entryResource = jsonType
+ .get(PathEntryResource.class);
+ processResponse("/", jsonType.get(ClientResponse.class))
+ assert entryResource.nodes != null;
+ assert entryResource.service == null;
+
+ // test the available GET URIs
+ def userhome = RegistryUtils.homePathForCurrentUser()
+
+ def userServicesURL = appendToURL(registry_url,
+ userhome + RegistryConstants.PATH_USER_SERVICES)
+ webResource = client.resource(userServicesURL);
+
+
+ def jsonBuilder = webResource.type(MediaType.APPLICATION_JSON)
+ ClientResponse response = jsonBuilder.get(ClientResponse.class);
+ def responseStr = processResponse(userServicesURL, response)
+
+ assert responseStr.contains("\"nodes\"")
+ assert responseStr.contains(SliderKeys.APP_TYPE)
+
+ entryResource = jsonBuilder.get(PathEntryResource.class)
+ assert entryResource.nodes.size() == 1;
+ assert entryResource.service == null;
+
+
+ def userServicesSlider = appendToURL(userServicesURL, SliderKeys.APP_TYPE)
+ webResource = client.resource(
+ userServicesSlider);
+ jsonBuilder = webResource.type(MediaType.APPLICATION_JSON);
+ response = jsonBuilder.get(ClientResponse.class);
+ processResponse(userServicesURL, response)
+ entryResource = jsonBuilder.get(PathEntryResource.class)
+ assert entryResource.nodes.size() == 1;
+ assert entryResource.service == null;
+
+ def servicePath = entryResource.nodes[0]
+
+ // now work with a real instances
+
+ def instanceURL = appendToURL(userServicesSlider, clustername)
+ assert instanceURL.endsWith(servicePath)
+
+ webResource = client.resource(instanceURL);
+
+ // work with it via direct Jackson unmarshalling
+ responseStr = processResponse(instanceURL, webResource)
+ PathEntryMarshalling pem = new PathEntryMarshalling();
+ def unmarshalled = pem.fromJson(responseStr)
+ def r1 = unmarshalled.service
+ assert r1
+ assert r1[YarnRegistryAttributes.YARN_ID] != null
+ assert r1[YarnRegistryAttributes.YARN_PERSISTENCE] != ""
+
+
+ // and via the web resource AP
+ jsonBuilder = webResource.type(MediaType.APPLICATION_JSON);
+ entryResource = jsonBuilder.get(PathEntryResource.class);
+
+ def serviceRecord = entryResource.service
+ assert serviceRecord != null;
+ assert serviceRecord[YarnRegistryAttributes.YARN_ID] != null
+ def externalEndpoints = serviceRecord.external;
+ assert externalEndpoints.size() > 0
+
+ def am_ipc_protocol = AM_IPC_PROTOCOL
+ def epr = serviceRecord.getExternalEndpoint(am_ipc_protocol)
+ assert null != epr;
+
+ assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API)
+ assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
+ // internals
+ assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API)
+ assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API)
+
+ // negative tests...
+ try {
+ webResource = client.resource(
+ appendToURL(registry_url, "/users/no-such-user"));
+ def clientResponse = webResource.get(ClientResponse.class)
+ assert 404 == clientResponse.status
+
+ def body = processResponse(userServicesURL, webResource)
+ jsonBuilder = webResource.type(MediaType.APPLICATION_JSON);
+ entryResource = jsonBuilder.get(PathEntryResource.class);
+
+
+ fail("should throw an exception for a 404 response, got " + body)
+ } catch (UniformInterfaceException e) {
+ assert e.response.status == 404
+ }
+ }
+
+ public String processResponse(String asURL, WebResource response) {
+ def jsonBuilder = response.type(MediaType.APPLICATION_JSON)
+ def clientResponse = jsonBuilder.get(ClientResponse.class);
+ return processResponse(asURL, clientResponse)
+ }
+
+ public String processResponse(String asURL, ClientResponse response) {
+ def responseStr = response.getEntity(String.class)
+ log.info(asURL + " ==>\n " + responseStr)
+ responseStr
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
index 1c98e94..c49abec 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
@@ -52,7 +52,12 @@
appState)
ProviderService providerService = new MockProviderService();
- WebAppApiImpl inst = new WebAppApiImpl(clusterProto, providerAppState, providerService, null);
+ WebAppApiImpl inst = new WebAppApiImpl(
+ clusterProto,
+ providerAppState,
+ providerService,
+ null,
+ null);
Injector injector = Guice.createInjector(new AbstractModule() {
@Override
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy
index f871b07..9292285 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy
@@ -32,7 +32,6 @@
import org.apache.slider.api.SliderClusterProtocol
import org.apache.slider.providers.ProviderService
import org.apache.slider.server.appmaster.model.mock.*
-import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.web.WebAppApi
@@ -45,7 +44,7 @@
@Slf4j
@CompileStatic
-public class TestContainerStatsBlock {
+public class TestContainerStatsBlock extends BaseMockAppStateTest {
private ContainerStatsBlock statsBlock;
@@ -55,13 +54,17 @@
@Before
public void setup() {
SliderClusterProtocol clusterProto = new MockSliderClusterProtocol();
- AppState appState = new MockAppState(new MockRecordFactory());
ProviderService providerService = new MockProviderService();
ProviderAppState providerAppState = new ProviderAppState(
"undefined",
appState)
- WebAppApiImpl inst = new WebAppApiImpl(clusterProto, providerAppState, providerService, null);
+ WebAppApiImpl inst = new WebAppApiImpl(
+ clusterProto,
+ providerAppState,
+ providerService,
+ null,
+ null);
Injector injector = Guice.createInjector(new AbstractModule() {
@Override
@@ -73,20 +76,23 @@
statsBlock = injector.getInstance(ContainerStatsBlock.class);
cont1 = new MockContainer();
- cont1.id = new MockContainerId();
- ((MockContainerId) cont1.id).setId(0);
+
+ cont1.id = mockContainerId(0);
cont1.nodeId = new MockNodeId();
cont1.priority = Priority.newInstance(1);
cont1.resource = new MockResource();
cont2 = new MockContainer();
- cont2.id = new MockContainerId();
- ((MockContainerId) cont2.id).setId(1);
+ cont2.id = mockContainerId(1);
cont2.nodeId = new MockNodeId();
cont2.priority = Priority.newInstance(1);
cont2.resource = new MockResource();
}
+ public MockContainerId mockContainerId(int count) {
+ new MockContainerId(applicationAttemptId, count)
+ }
+
@Test
public void testGetContainerInstances() {
List<RoleInstance> roles = [
@@ -179,9 +185,9 @@
@Test
public void testClusterNodeNameComparator() {
- ClusterNode n1 = new ClusterNode(new MockContainerId(1)),
- n2 = new ClusterNode(new MockContainerId(2)),
- n3 = new ClusterNode(new MockContainerId(3));
+ ClusterNode n1 = new ClusterNode(mockContainerId(1)),
+ n2 = new ClusterNode(mockContainerId(2)),
+ n3 = new ClusterNode(mockContainerId(3));
List<ClusterNode> nodes = new ArrayList<ClusterNode>();
nodes.add(n2);
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
index 6b46591..d232ecb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
@@ -27,7 +27,6 @@
import org.apache.slider.api.SliderClusterProtocol
import org.apache.slider.providers.ProviderService
import org.apache.slider.server.appmaster.model.mock.*
-import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.web.WebAppApi
import org.apache.slider.server.appmaster.web.WebAppApiImpl
@@ -36,7 +35,7 @@
@Slf4j
@CompileStatic
-public class TestIndexBlock {
+public class TestIndexBlock extends BaseMockAppStateTest {
private IndexBlock indexBlock;
@@ -44,14 +43,20 @@
@Before
public void setup() {
+ super.setup()
+ assert appState
SliderClusterProtocol clusterProto = new MockSliderClusterProtocol();
- AppState appState = new MockAppState(new MockRecordFactory());
ProviderService providerService = new MockProviderService();
ProviderAppState providerAppState = new ProviderAppState(
"undefined",
appState)
- WebAppApiImpl inst = new WebAppApiImpl(clusterProto, providerAppState, providerService, null);
+ WebAppApiImpl inst = new WebAppApiImpl(
+ clusterProto,
+ providerAppState,
+ providerService,
+ null,
+ null);
Injector injector = Guice.createInjector(new AbstractModule() {
@Override
@@ -63,15 +68,13 @@
indexBlock = injector.getInstance(IndexBlock.class);
cont1 = new MockContainer();
- cont1.id = new MockContainerId();
- ((MockContainerId) cont1.id).setId(0);
+ cont1.id = new MockContainerId(applicationAttemptId, 0);
cont1.nodeId = new MockNodeId();
cont1.priority = Priority.newInstance(1);
cont1.resource = new MockResource();
cont2 = new MockContainer();
- cont2.id = new MockContainerId();
- ((MockContainerId) cont2.id).setId(1);
+ cont2.id = new MockContainerId(applicationAttemptId, 1);
cont2.nodeId = new MockNodeId();
cont2.priority = Priority.newInstance(1);
cont2.resource = new MockResource();
@@ -79,6 +82,20 @@
@Test
public void testIndex() {
+ def role0 = role0Status
+ def role1 = role1Status
+ role0.desired = 8
+ role0.incActual()
+ role0.incActual()
+ role0.incActual()
+ role0.incActual()
+ role0.incActual()
+ role1.incRequested()
+ role1.incRequested()
+ role1.incRequested()
+ role0.noteFailed(false, "")
+ role0.noteFailed(true, "")
+
StringWriter sw = new StringWriter(64);
PrintWriter pw = new PrintWriter(sw);
@@ -86,7 +103,23 @@
int level = hamlet.nestLevel();
indexBlock.doIndex(hamlet, "accumulo");
+
+ def body = sw.toString()
+ log.info(body)
+ assertEquals(body, level, hamlet.nestLevel())
+ // verify role data came out
+ assert body.contains("role0")
+ //
+ assert body.contains("8")
+ assert body.contains("5")
+ assert body.contains("3")
+ assert body.contains("2")
+ assert body.contains("1")
- assert level == hamlet.nestLevel();
+ assert body.contains("role1")
+ assert body.contains("role2")
+ // verify that the sorting took place
+ assert body.indexOf("role0") < body.indexOf("role1")
+ assert body.indexOf("role1") < body.indexOf("role2")
}
}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy b/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy
new file mode 100644
index 0000000..3de3c67
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.security
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.RawLocalFileSystem
+import org.apache.hadoop.hdfs.DFSConfigKeys
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier
+import org.apache.hadoop.io.Text
+import org.apache.hadoop.security.Credentials
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.security.token.SecretManager
+import org.apache.hadoop.security.token.Token
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager
+import org.apache.hadoop.service.ServiceOperations
+import org.apache.hadoop.util.Time
+import org.apache.slider.common.tools.CoreFileSystem
+import org.apache.slider.server.appmaster.actions.ActionStopQueue
+import org.apache.slider.server.appmaster.actions.QueueExecutor
+import org.apache.slider.server.appmaster.actions.QueueService
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicLong
+
+@Slf4j
+//@CompileStatic
+class TestFsDelegationTokenManager {
+
+ QueueService queues;
+ FsDelegationTokenManager tokenManager;
+ Configuration conf;
+ UserGroupInformation currentUser;
+
+
+ @Before
+ void setup() {
+ queues = new QueueService();
+
+ conf = new Configuration()
+ conf.set(
+ DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION,
+ "TOKEN")
+ conf.setLong(
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+ 1000)
+ queues.init(conf)
+ queues.start();
+
+ HadoopFS fs = new TestFileSystem()
+
+ CoreFileSystem coreFileSystem = new CoreFileSystem(fs, conf)
+
+ String[] groups = new String[1];
+ groups[0] = 'testGroup1'
+
+ currentUser = UserGroupInformation.createUserForTesting("test", groups)
+ UserGroupInformation.setLoginUser(currentUser)
+
+ tokenManager = new FsDelegationTokenManager(queues) {
+ @Override
+ protected int getRenewingLimit() {
+ return 5
+ }
+
+ @Override
+ protected org.apache.hadoop.fs.FileSystem getRemoteFileSystemForRenewal(Configuration config) throws IOException {
+ return new TestFileSystem();
+ }
+
+ @Override
+ protected String getRenewingActionName() {
+ return "TEST RENEW"
+ }
+ }
+
+ }
+
+ public static class DummySecretManager extends
+ AbstractDelegationTokenSecretManager<DelegationTokenIdentifier> {
+
+ public DummySecretManager(long delegationKeyUpdateInterval,
+ long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
+ long delegationTokenRemoverScanInterval) {
+ super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
+ delegationTokenRenewInterval, delegationTokenRemoverScanInterval);
+ }
+
+ @Override
+ public DelegationTokenIdentifier createIdentifier() {
+ return null;
+ }
+
+ @Override
+ public byte[] createPassword(DelegationTokenIdentifier dtId) {
+ return new byte[1];
+ }
+ }
+
+ public class TestFileSystem extends RawLocalFileSystem {
+ int sequenceNum = 0;
+ SecretManager<DelegationTokenIdentifier> mgr =
+ new DummySecretManager(0, 0, 0, 0);
+ @Override
+ Token<DelegationTokenIdentifier> getDelegationToken(String renewer) throws IOException {
+ return new TestToken(getIdentifier(), mgr);
+ }
+
+ @Override
+ Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException {
+ Token[] tokens = new Token[1]
+ tokens[0] = new TestToken(getIdentifier(), mgr)
+ return tokens
+ }
+
+ private DelegationTokenIdentifier getIdentifier() {
+ def user = new Text(currentUser.getUserName())
+ def id = new DelegationTokenIdentifier(user, user, user)
+ id.setSequenceNumber(sequenceNum++)
+ id.setMaxDate(Time.now() + 10000)
+
+ return id
+ }
+ }
+
+ public class TestToken extends Token<DelegationTokenIdentifier> {
+ static long maxCount = 0;
+ private final AtomicLong renewCount = new AtomicLong()
+ private final AtomicLong totalCount = new AtomicLong()
+ public final AtomicBoolean expired = new AtomicBoolean(false);
+ public final AtomicBoolean cancelled = new AtomicBoolean(false);
+
+ TestToken(DelegationTokenIdentifier id, SecretManager<DelegationTokenIdentifier> mgr) {
+ super(id, mgr)
+ }
+
+ @Override
+ Text getService() {
+ return new Text("HDFS")
+ }
+
+ @Override
+ long renew(Configuration conf) throws IOException, InterruptedException {
+ totalCount.getAndIncrement();
+ if (maxCount > 0 && renewCount.getAndIncrement() > maxCount) {
+ renewCount.set(0L)
+ expired.set(true)
+ throw new IOException("Expired")
+ }
+
+
+ return Time.now() + 1000;
+ }
+
+ @Override
+ void cancel(Configuration conf) throws IOException, InterruptedException {
+ cancelled.set(true)
+ }
+ }
+
+ @After
+ void destroyService() {
+ ServiceOperations.stop(queues);
+ }
+
+ public void runQueuesToCompletion() {
+ new Thread(queues).start();
+ QueueExecutor ex = new QueueExecutor(queues)
+ ex.run();
+ }
+
+ public void runQueuesButNotToCompletion() {
+ new Thread(queues).start();
+ QueueExecutor ex = new QueueExecutor(queues)
+ new Thread(ex).start();
+ Thread.sleep(1000)
+ tokenManager.cancelDelegationToken(conf)
+ }
+
+ @Test
+ public void testRenew() throws Throwable {
+ tokenManager.acquireDelegationToken(conf)
+ def stop = new ActionStopQueue(10, TimeUnit.SECONDS)
+ queues.schedule(stop);
+ runQueuesToCompletion()
+
+ TestToken token = (TestToken) currentUser.getTokens()[0]
+ assert token.totalCount.get() > 4
+ }
+
+ @Test
+ public void testCancel() throws Throwable {
+ tokenManager.acquireDelegationToken(conf)
+ def stop = new ActionStopQueue(10, TimeUnit.SECONDS)
+ queues.schedule(stop);
+ runQueuesButNotToCompletion()
+
+ TestToken token = (TestToken) currentUser.getTokens()[0]
+ assert token.cancelled.get()
+ assert queues.lookupRenewingAction("TEST RENEW") == null
+ }
+
+
+ @Test
+ public void testRenewPastExpiry() throws Throwable {
+ try {
+ TestToken.maxCount = 3L
+ tokenManager.acquireDelegationToken(conf)
+ TestToken origToken = currentUser.getTokens()[0]
+ def stop = new ActionStopQueue(10, TimeUnit.SECONDS)
+ queues.schedule(stop);
+ runQueuesToCompletion()
+
+ TestToken token = (TestToken) currentUser.getTokens()[0]
+ assert token != null
+ assert token != origToken
+ assert origToken.getService().equals(token.getService())
+ assert origToken.totalCount.get() > 4
+ assert origToken.expired.get()
+ } finally {
+ TestToken.maxCount = 0
+ }
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
index cb1d9b5..7affb44 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
@@ -21,18 +21,20 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.registry.client.api.RegistryOperations
+import org.apache.hadoop.registry.client.impl.zk.RegistryOperationsService
+import org.apache.hadoop.registry.server.services.MicroZookeeperService
import org.apache.slider.common.tools.SliderUtils
-import org.apache.slider.core.zk.MiniZooKeeperCluster
@Slf4j
@CompileStatic
class MicroZKCluster implements Closeable {
public static final String HOSTS = "127.0.0.1"
- MiniZooKeeperCluster zkCluster
+ MicroZookeeperService zkService
String zkBindingString
- Configuration conf
- int port
+ final Configuration conf
+ public RegistryOperations registryOperations
MicroZKCluster() {
this(SliderUtils.createConfiguration())
@@ -42,21 +44,30 @@
this.conf = conf
}
- void createCluster() {
- zkCluster = new MiniZooKeeperCluster(1)
- zkCluster.init(conf)
- zkCluster.start()
- zkBindingString = zkCluster.zkQuorum
+ void createCluster(String name) {
+ zkService = new MicroZookeeperService(name)
+
+ zkService.init(conf)
+ zkService.start()
+ zkBindingString = zkService.connectionString
log.info("Created $this")
+ registryOperations = new RegistryOperationsService(
+ "registry",
+ zkService)
+ registryOperations.init(conf)
+ registryOperations.start()
}
@Override
void close() throws IOException {
- zkCluster?.stop()
+ registryOperations?.stop()
+ zkService?.stop()
}
@Override
String toString() {
return "Micro ZK cluster as $zkBindingString"
}
+
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java b/slider-core/src/test/groovy/org/apache/slider/test/Outcome.groovy
similarity index 61%
copy from slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
copy to slider-core/src/test/groovy/org/apache/slider/test/Outcome.groovy
index ee24dc1..d7e027f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
+++ b/slider-core/src/test/groovy/org/apache/slider/test/Outcome.groovy
@@ -16,11 +16,32 @@
* limitations under the License.
*/
-package org.apache.slider.server.services.registry;
+package org.apache.slider.test
/**
- * These constants are unique to the slider registry service itself
+ * Outcome for probes
*/
-public class RegistryServiceConstants {
- public static final int INSTANCE_REFRESH_MS = 1000;
+
+class Outcome {
+
+ public final String name;
+
+ private Outcome(String name) {
+ this.name = name
+ }
+
+ static Outcome Success = new Outcome(
+ "Success")
+ static Outcome Retry = new Outcome("Retry")
+ static Outcome Fail = new Outcome("Fail")
+
+ /**
+ * build from a bool, where false is mapped to retry
+ * @param b boolean
+ * @return an outcome
+ */
+ static Outcome fromBool(boolean b) {
+ return b ? Success : Retry;
+ }
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
index 28b484f..e0798a4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
@@ -27,7 +27,6 @@
import org.junit.Rule
import org.junit.rules.TestName
-import java.nio.file.Files
/**
* Base class for unit tests as well as ones starting mini clusters
@@ -54,10 +53,8 @@
@Before
public void setup() {
+ sliderClientClassName = DEFAULT_SLIDER_CLIENT
FileUtil.fullyDelete(new File(SliderXMLConfKeysForTesting.TEST_SECURITY_DIR))
}
-
-
-
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 3fc3e55..3688644 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -28,8 +28,11 @@
import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.service.ServiceStateException
+import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.RoleKeys
@@ -43,13 +46,13 @@
import org.apache.slider.core.exceptions.WaitTimeoutException
import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfigSet
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.server.services.curator.CuratorServiceInstance
+import org.apache.slider.server.services.workflow.ForkedProcessService
import org.junit.Assert
import org.junit.Assume
+import java.util.concurrent.TimeoutException
+
import static Arguments.ARG_OPTION
/**
@@ -65,6 +68,8 @@
@Slf4j
@CompileStatic
class SliderTestUtils extends Assert {
+ public static final String DEFAULT_SLIDER_CLIENT = SliderClient.class.name
+ static String sliderClientClassName = DEFAULT_SLIDER_CLIENT
public static void describe(String s) {
log.info("");
@@ -79,26 +84,51 @@
}
public static void skip(String message) {
- log.warn("Skipping test: " + message)
+ log.warn("Skipping test: {}", message)
Assume.assumeTrue(message, false);
}
public static void assume(boolean condition, String message) {
if (!condition) {
- log.warn("Skipping test: " + message)
- Assume.assumeTrue(message, false);
+ skip(message)
}
}
-
+ /**
+ * Equality size for a list
+ * @param left
+ * @param right
+ */
public static void assertListEquals(List left, List right) {
- assert left.size() == right.size();
+ String lval = collectionToString(left)
+ String rval = collectionToString(right)
+ String text = "comparing $lval to $rval"
+ assertEquals(text, left.size(), right.size())
for (int i = 0; i < left.size(); i++) {
- assert left[0] == right[0]
+ assertEquals(text, left[i], right[i])
}
}
/**
+ * Assert a list has a given length
+ * @param list list
+ * @param size size to have
+ */
+ public static void assertListLength(List list, int size) {
+ String lval = collectionToString(list)
+ assertEquals(lval, size, list.size())
+ }
+
+ /**
+ * Stringify a collection with [ ] at either end
+ * @param collection collection
+ * @return string value
+ */
+ public static String collectionToString(List collection) {
+ return "[" + SliderUtils.join(collection, ", ", false) + "]"
+ }
+
+ /**
* Assume that a string option is set and not equal to ""
* @param conf configuration file
* @param key key to look for
@@ -108,8 +138,7 @@
skip("Configuration key $key not set")
}
}
-
-
+
/**
* assert that a string option is set and not equal to ""
* @param conf configuration file
@@ -118,8 +147,6 @@
public static void assertStringOptionSet(Configuration conf, String key) {
getRequiredConfOption(conf, key)
}
-
-
/**
* Assume that a boolean option is set and true.
@@ -140,8 +167,8 @@
*/
public static void assumeBoolOption(
Configuration conf, String key, boolean defval) {
- assume(conf.getBoolean(key, defval),
- "Configuration key $key is false")
+ assume(conf.getBoolean(key, defval),
+ "Configuration key $key is false")
}
/**
@@ -167,11 +194,33 @@
}
/**
+ * skip a test on windows
+ */
+ public static void skipOnWindows() {
+ if (Shell.WINDOWS) {
+ skip("Not supported on windows")
+ }
+ }
+
+ /**
+ * Assert that any needed libraries being present. On Unix none are needed;
+ * on windows they must be present
+ */
+ public static void assertNativeLibrariesPresent() {
+ String errorText = SliderUtils.checkForRequiredNativeLibraries()
+ if (errorText != "") {
+ fail(errorText)
+ }
+ }
+
+ /**
* Wait for the cluster live; fail if it isn't within the (standard) timeout
* @param sliderClient client
* @return the app report of the live cluster
*/
- public static ApplicationReport waitForClusterLive(SliderClient sliderClient,int goLiveTime) {
+ public static ApplicationReport waitForClusterLive(
+ SliderClient sliderClient,
+ int goLiveTime) {
ApplicationReport report = sliderClient.monitorAppToRunning(
new Duration(goLiveTime));
assertNotNull(
@@ -287,7 +336,7 @@
duration.finish();
describe("$operation: role count not met after $duration: $details")
log.info(prettyPrint(status.toJsonString()))
- fail("$operation: role counts not met after $duration: " +
+ fail("$operation: role counts not met after $duration: " +
details.toString() +
" in \n$status ")
}
@@ -305,35 +354,42 @@
* @throws IOException
* @throws SliderException
*/
- public static boolean spinForClusterStartup(SliderClient client, long spintime,
+ public static boolean spinForClusterStartup(
+ SliderClient client,
+ long spintime,
String role)
- throws WaitTimeoutException, IOException, SliderException {
+ throws WaitTimeoutException, IOException, SliderException {
int state = client.waitForRoleInstanceLive(role, spintime);
return state == ClusterDescription.STATE_LIVE;
}
- public static ClusterDescription dumpClusterStatus(SliderClient client, String text) {
+ public static ClusterDescription dumpClusterStatus(
+ SliderClient client,
+ String text) {
ClusterDescription status = client.clusterDescription;
dumpClusterDescription(text, status)
return status;
}
- public static List<ClusterNode> listNodesInRole(SliderClient client, String role) {
+ public static List<ClusterNode> listNodesInRole(
+ SliderClient client,
+ String role) {
return client.listClusterNodesInRole(role)
}
- public static void dumpClusterDescription(String text, ClusterDescription status) {
+ public static void dumpClusterDescription(
+ String text,
+ ClusterDescription status) {
describe(text)
log.info(prettyPrint(status.toJsonString()))
}
-
+
public static void dumpClusterDescription(String text, AggregateConf status) {
describe(text)
log.info(status.toString())
}
-
/**
* Fetch the current site config from the Slider AM, from the
* <code>clientProperties</code> field of the ClusterDescription
@@ -359,15 +415,15 @@
public static String GET(URL url) {
return fetchWebPageWithoutError(url.toString())
}
-
+
public static String GET(URL url, String path) {
return GET(url.toString(), path)
}
-
+
public static String GET(String base, String path) {
String s = appendToURL(base, path)
return GET(s)
-
+
}
def static String GET(String s) {
@@ -398,17 +454,17 @@
int resultCode
try {
resultCode = httpclient.executeMethod(get);
- if (resultCode!=200) {
+ if (resultCode != 200) {
log.warn("Result code of $resultCode")
}
} catch (IOException e) {
- log.error("Failed on $url: $e",e)
+ log.error("Failed on $url: $e", e)
throw e;
}
String body = get.responseBodyAsString;
return body;
}
-
+
/**
* Fetches a web page asserting that the response code is between 200 and 400.
* Will error on 400 and 500 series response codes and let 200 and 300 through.
@@ -417,13 +473,13 @@
*/
public static String fetchWebPageWithoutError(String url) {
assert null != url
-
+
log.info("Fetching HTTP content at " + url);
-
+
def client = new HttpClient(new MultiThreadedHttpConnectionManager());
client.httpConnectionManager.params.connectionTimeout = 10000;
GetMethod get = new GetMethod(url);
-
+
get.followRedirects = true;
int resultCode = client.executeMethod(get);
@@ -444,6 +500,113 @@
assert 0 == service.serviceExitCode;
}
+ public static void assertContainersLive(ClusterDescription clusterDescription,
+ String component, int expected) {
+ log.info("Asserting component $component expected count $expected}",)
+ int actual = extractLiveContainerCount(clusterDescription, component)
+ if (expected != actual) {
+ log.warn(
+ "$component actual=$actual, expected $expected in \n$clusterDescription")
+ }
+ assert expected == actual
+ }
+
+ /**
+ * Robust extraction of live container count
+ * @param clusterDescription status
+ * @param component component to resolve
+ * @return the number of containers live.
+ */
+ public static int extractLiveContainerCount(
+ ClusterDescription clusterDescription,
+ String component) {
+ def instances = clusterDescription?.instances?.get(component)
+ int actual = instances != null ? instances.size() : 0
+ return actual
+ }
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param status code
+ * @param commands
+ * @return the process
+ */
+ public static ForkedProcessService exec(int status, List<String> commands) {
+ ForkedProcessService process = exec(commands)
+
+ def exitCode = process.exitCode
+ assert exitCode != null
+ assert status == exitCode
+ return process
+ }
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param commands
+ * @return
+ */
+ public static ForkedProcessService exec(List<String> commands) {
+ ForkedProcessService process;
+ process = new ForkedProcessService(
+ commands[0],
+ [:],
+ commands);
+ process.init(new Configuration());
+ process.start();
+ int timeoutMillis = 5000
+ if (!process.waitForServiceToStop(timeoutMillis)) {
+ throw new TimeoutException(
+ "Process did not stop in " + timeoutMillis + "mS");
+ }
+ process
+ }
+
+ public static boolean doesWindowsAppExist(List<String> commands) {
+ try {
+ exec(0, commands)
+ return true;
+ } catch (ServiceStateException e) {
+ if (!(e.cause instanceof FileNotFoundException)) {
+ throw e;
+ }
+ return false;
+ }
+ }
+
+ /**
+ * Execute a closure, assert it fails with a given exit code and text
+ * @param exitCode exit code
+ * @param text text (can be "")
+ * @param action action
+ * @return
+ */
+ def assertFailsWithException(int exitCode,
+ String text,
+ Closure action) {
+ try {
+ action()
+ fail("Operation was expected to fail —but it succeeded")
+ } catch (ServiceLaunchException e) {
+ assertExceptionDetails(e, exitCode, text)
+ }
+ }
+
+ /**
+ * Execute a closure, assert it fails with a given exit code and text
+ * @param exitCode exit code
+ * @param text text (can be "")
+ * @param action action
+ * @return
+ */
+ def assertFailsWithExceptionClass(Class clazz,
+ String text,
+ Closure action) {
+ try {
+ action()
+ fail("Operation was expected to fail —but it succeeded")
+ } catch (Exception e) {
+ assertExceptionDetails(e, clazz, text)
+ }
+ }
+
/**
* Make an assertion about the exit code of an exception
* @param ex exception
@@ -462,11 +625,28 @@
}
if (text) {
if (!(ex.toString().contains(text))) {
- log.warn("String match failed in $ex", ex)
+ log.warn("String match for \"${text}\"failed in $ex", ex)
assert ex.toString().contains(text);
}
}
}
+ /**
+ * Make an assertion about the exit code of an exception
+ * @param ex exception
+ * @param exitCode exit code
+ * @param text error text to look for in the exception
+ */
+ static void assertExceptionDetails(
+ Exception ex,
+ Class clazz,
+ String text = "") {
+ if (ex.class != clazz) {
+ throw ex;
+ }
+ if (text && !(ex.toString().contains(text))) {
+ throw ex;
+ }
+ }
/**
* Launch the slider client with the specific args; no validation
@@ -479,7 +659,7 @@
Configuration conf,
List args) {
ServiceLauncher<SliderClient> serviceLauncher =
- new ServiceLauncher<SliderClient>(SliderClient.name);
+ new ServiceLauncher<SliderClient>(sliderClientClassName);
log.debug("slider ${SliderUtils.join(args, " ", false)}")
serviceLauncher.launchService(conf,
@@ -489,8 +669,8 @@
}
public static ServiceLauncher launch(Class serviceClass,
- Configuration conf,
- List<Object> args) throws
+ Configuration conf,
+ List<Object> args) throws
Throwable {
ServiceLauncher serviceLauncher =
new ServiceLauncher(serviceClass.name);
@@ -503,18 +683,21 @@
}
public static Throwable launchExpectingException(Class serviceClass,
- Configuration conf,
- String expectedText,
- List args)
- throws Throwable {
+ Configuration conf,
+ String expectedText,
+ List args)
+ throws Throwable {
try {
ServiceLauncher launch = launch(serviceClass, conf, args);
- throw new AssertionError("Expected an exception with text containing " + expectedText
- + " -but the service completed with exit code "
- + launch.serviceExitCode);
+ throw new AssertionError(
+ "Expected an exception with text containing " + expectedText
+ + " -but the service completed with exit code "
+ + launch.serviceExitCode);
} catch (Throwable thrown) {
if (expectedText && !thrown.toString().contains(expectedText)) {
//not the right exception -rethrow
+ log.warn("Caught Exception did not contain expected text" +
+ "\"" + expectedText + "\"")
throw thrown;
}
return thrown;
@@ -545,10 +728,10 @@
List<String> extraArgs,
YarnConfiguration conf,
String option) {
-
+
conf.getTrimmed(option);
extraArgs << ARG_OPTION << option << getRequiredConfOption(conf, option)
-
+
}
/**
@@ -558,7 +741,7 @@
* @throws IOException on File IO problems
*/
public static void assertIsDirectory(HadoopFS fs,
- Path path) throws IOException {
+ Path path) throws IOException {
FileStatus fileStatus = fs.getFileStatus(path);
assertIsDirectory(fileStatus);
}
@@ -569,7 +752,7 @@
*/
public static void assertIsDirectory(FileStatus fileStatus) {
assertTrue("Should be a dir -but isn't: " + fileStatus,
- fileStatus.isDirectory());
+ fileStatus.isDirectory());
}
/**
@@ -587,8 +770,10 @@
Path path) throws IOException {
if (!fileSystem.exists(path)) {
//failure, report it
- fail(message + ": not found \"" + path + "\" in " + path.getParent() + "-" +
- ls(fileSystem, path.getParent()));
+ fail(
+ message + ": not found \"" + path + "\" in " + path.getParent() +
+ "-" +
+ ls(fileSystem, path.getParent()));
}
}
@@ -622,8 +807,8 @@
* @throws IOException IO probles
*/
public static void assertListStatusFinds(HadoopFS fs,
- Path dir,
- Path subdir) throws IOException {
+ Path dir,
+ Path subdir) throws IOException {
FileStatus[] stats = fs.listStatus(dir);
boolean found = false;
StringBuilder builder = new StringBuilder();
@@ -634,8 +819,8 @@
}
}
assertTrue("Path " + subdir
- + " not found in directory " + dir + ":" + builder,
- found);
+ + " not found in directory " + dir + ":" + builder,
+ found);
}
/**
@@ -684,19 +869,15 @@
}
public static void dumpRegistryInstances(
- List<CuratorServiceInstance<ServiceInstanceData>> instances) {
+ Map<String, ServiceRecord> instances) {
describe "service registry slider instances"
- JsonSerDeser<ServiceInstanceData> serDeser = new JsonSerDeser<>(
- ServiceInstanceData)
-
- instances.each { CuratorServiceInstance<ServiceInstanceData> svc ->
- ServiceInstanceData payload = svc.payload
- def json = serDeser.toJson(payload)
- log.info("service $svc payload=\n$json")
+ instances.each { Map.Entry<String, ServiceRecord> it ->
+ log.info(" $it.key : $it.value")
}
describe "end list service registry slider instances"
}
+
public static void dumpRegistryInstanceIDs(List<String> instanceIds) {
describe "service registry instance IDs"
dumpCollection(instanceIds)
@@ -707,9 +888,20 @@
dumpCollection(entries)
}
- def static void dumpCollection(Collection<String> entries) {
+ def static void dumpCollection(Collection entries) {
log.info("number of entries: ${entries.size()}")
- entries.each { String it -> log.info(it) }
+ entries.each { log.info(it.toString()) }
+ }
+
+ def static void dumpArray(Object[] entries) {
+ log.info("number of entries: ${entries.length}")
+ entries.each { log.info(it.toString()) }
+ }
+
+ public static void dumpMap(Map map) {
+ map.entrySet().each { Map.Entry it ->
+ log.info("\"${it.key.toString()}\": \"${it.value.toString()}\"")
+ }
}
/**
@@ -721,7 +913,10 @@
* @param defVal
* @return
*/
- public static int getTimeOptionMillis(Configuration conf, String key, int defValMillis) {
+ public static int getTimeOptionMillis(
+ Configuration conf,
+ String key,
+ int defValMillis) {
int val = conf.getInt(key, 0)
val = Integer.getInteger(key, val)
int time = 1000 * val
@@ -737,4 +932,95 @@
log.info "$key -- ${config.description}"
}
}
+
+ /**
+ * Convert a file to a URI suitable for use in an argument
+ * @param file file
+ * @return a URI string valid on all platforms
+ */
+ public String toURIArg(File file) {
+ file.absoluteFile.toURI().toString()
+ }
+
+ /**
+ * Assert a file exists; fails with a listing of the parent dir
+ * @param text text for front of message
+ * @param file file to look for
+ * @throws FileNotFoundException
+ */
+ public void assertFileExists(String text, File file) {
+ if (!file.exists()) {
+ def parent = file.parentFile
+ def files = parent.list()
+ StringBuilder builder = new StringBuilder()
+ builder.append("${parent.absolutePath}:\n")
+ files.each { String name -> builder.append(" $name\n") }
+ throw new FileNotFoundException("$text: $file not found in $builder")
+ }
+ }
+
+ /**
+ * Repeat a probe until it succeeds, if it does not execute a failure
+ * closure then raise an exception with the supplied message
+ * @param probe probe
+ * @param timeout time in millis before giving up
+ * @param sleepDur sleep between failing attempts
+ * @param args map of arguments to the probe
+ * @param failIfUnsuccessful if the probe fails after all the attempts
+ * —should it raise an exception
+ * @param failureMessage message to include in exception raised
+ * @param failureHandler closure to invoke prior to the failure being raised
+ */
+ protected void repeatUntilSuccess(
+ String action,
+ Closure probe,
+ int timeout,
+ int sleepDur,
+ Map args,
+ boolean failIfUnsuccessful,
+ String failureMessage,
+ Closure failureHandler) {
+ log.debug("Probe $action timelimit $timeout")
+ if (timeout < 1000) {
+ fail("Timeout $timeout too low: milliseconds are expected, not seconds")
+ }
+ int attemptCount = 0
+ boolean succeeded = false;
+ boolean completed = false;
+ Duration duration = new Duration(timeout)
+ duration.start();
+ while (!completed) {
+ Outcome outcome = (Outcome) probe(args)
+ if (outcome.equals(Outcome.Success)) {
+ // success
+ log.debug("Success after $attemptCount attempt(s)")
+ succeeded = true;
+ completed = true;
+ } else if (outcome.equals(Outcome.Retry)) {
+ // failed but retry possible
+ attemptCount++;
+ completed = duration.limitExceeded
+ if (!completed) {
+ log.debug("Attempt $attemptCount failed")
+ sleep(sleepDur)
+ }
+ } else if (outcome.equals(Outcome.Fail)) {
+ // fast fail
+ log.debug("Fast fail of probe")
+ completed = true;
+ }
+ }
+ if (!succeeded) {
+ if (duration.limitExceeded) {
+ log.info("probe timed out after $timeout and $attemptCount attempts")
+ }
+ if (failureHandler) {
+ failureHandler()
+ }
+ if (failIfUnsuccessful) {
+ fail(failureMessage)
+ }
+ }
+ }
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/TestAssertions.groovy b/slider-core/src/test/groovy/org/apache/slider/test/TestAssertions.groovy
new file mode 100644
index 0000000..abfbe65
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/test/TestAssertions.groovy
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.test
+
+import org.apache.slider.api.ClusterDescription
+import org.junit.Test
+
+/**
+ * Test for some of the command test base operations
+ */
+class TestAssertions {
+ @Test
+ public void testNoInstances() throws Throwable {
+ ClusterDescription clusterDescription = new ClusterDescription();
+ clusterDescription.instances = null
+ SliderTestUtils.assertContainersLive(clusterDescription, "example", 0);
+ }
+
+ @Test
+ public void testEmptyInstances() throws Throwable {
+ ClusterDescription clusterDescription = new ClusterDescription();
+ SliderTestUtils.assertContainersLive(clusterDescription, "example", 0);
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index b6f863b..326c217 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -18,8 +18,8 @@
package org.apache.slider.test
-import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.commons.io.FileUtils
import org.apache.commons.logging.Log
import org.apache.commons.logging.LogFactory
import org.apache.hadoop.conf.Configuration
@@ -49,22 +49,18 @@
import org.apache.slider.core.main.ServiceLauncherBaseTest
import org.apache.slider.server.appmaster.SliderAppMaster
import org.junit.After
-import org.junit.Assert
-import org.junit.Before
import org.junit.BeforeClass
import org.junit.Rule
-import org.junit.rules.TestName
import org.junit.rules.Timeout
import static org.apache.slider.test.KeysForTests.*
-import static org.apache.slider.common.SliderKeys.*;
import static org.apache.slider.common.SliderXMLConfKeysForTesting.*;
/**
* Base class for mini cluster tests -creates a field for the
* mini yarn cluster
*/
-@CompileStatic
+//@CompileStatic
@Slf4j
public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
/**
@@ -88,11 +84,16 @@
public static final YarnConfiguration SLIDER_CONFIG = SliderUtils.createConfiguration();
+
+ public static boolean kill_supported;
+
static {
SLIDER_CONFIG.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, 1)
SLIDER_CONFIG.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100)
SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false)
SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false)
+ SLIDER_CONFIG.setBoolean(SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED,
+ true)
SLIDER_CONFIG.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1)
}
@@ -129,12 +130,14 @@
KEY_TEST_TIMEOUT,
DEFAULT_TEST_TIMEOUT_SECONDS * 1000)
)
+
+ /**
+ * Clent side test: validate system env before launch
+ */
@BeforeClass
- public static void checkWindowsSupport() {
- if (Shell.WINDOWS) {
- assertNotNull("winutils.exe not found", Shell.WINUTILS)
- }
- }
+ public static void checkClientEnv() {
+ SliderUtils.validateSliderClientEnvironment(null)
+ }
protected String buildClustername(String clustername) {
return clustername ?: createClusterName()
@@ -194,11 +197,64 @@
protected void addToTeardown(SliderClient client) {
clustersToTeardown << client;
}
+
protected void addToTeardown(ServiceLauncher<SliderClient> launcher) {
- SliderClient sliderClient = launcher.service
- if (sliderClient) addToTeardown(sliderClient)
+ SliderClient sliderClient = launcher?.service
+ if (sliderClient) {
+ addToTeardown(sliderClient)
+ }
}
+ /**
+ * Work out if kill is supported
+ */
+ @BeforeClass
+ public static void checkKillSupport() {
+ kill_supported = !Shell.WINDOWS
+ }
+
+ /**
+ * Kill any java process with the given grep pattern
+ * @param grepString string to grep for
+ */
+ public int killJavaProcesses(String grepString, int signal) {
+
+ def commandString
+ if (!Shell.WINDOWS) {
+ GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal"
+ log.info("Command command = $killCommand")
+
+ commandString = ["bash", "-c", killCommand]
+ } else {
+ // windows
+ if (!kill_supported) {
+ return -1;
+ }
+
+ /*
+ "jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
+ */
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID\""
+ commandString = ["CMD", "/C", killCommand]
+ }
+ Process command = commandString.execute()
+ def exitCode = command.waitFor()
+
+ log.info(command.in.text)
+ log.error(command.err.text)
+ return exitCode
+ }
+
+ /**
+ * Kill all processes which match one of the list of grepstrings
+ * @param greps
+ * @param signal
+ */
+ public void killJavaProcesses(List<String> greps, int signal) {
+ for (String grep : greps) {
+ killJavaProcesses(grep, signal)
+ }
+ }
protected YarnConfiguration getConfiguration() {
return SLIDER_CONFIG;
@@ -210,7 +266,7 @@
public void stopRunningClusters() {
clustersToTeardown.each { SliderClient client ->
try {
- maybeStopCluster(client, "", "Teardown at end of test case");
+ maybeStopCluster(client, "", "Teardown at end of test case", true);
} catch (Exception e) {
log.warn("While stopping cluster " + e, e);
}
@@ -240,11 +296,16 @@
int numLocalDirs,
int numLogDirs,
boolean startHDFS) {
+ assertNativeLibrariesPresent();
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
SliderUtils.patchConfiguration(conf)
name = buildClustername(name)
- miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs)
+ miniCluster = new MiniYARNCluster(
+ name,
+ noOfNodeManagers,
+ numLocalDirs,
+ numLogDirs)
miniCluster.init(conf)
miniCluster.start();
if (startHDFS) {
@@ -271,6 +332,8 @@
public static MiniDFSCluster buildMiniHDFSCluster(
String name,
YarnConfiguration conf) {
+ assertNativeLibrariesPresent();
+
File baseDir = new File("./target/hdfs/$name").absoluteFile;
//use file: to rm it recursively
FileUtil.fullyDelete(baseDir)
@@ -314,35 +377,14 @@
/**
- * Kill all Slider Services. That i
+ * Kill all Slider Services.
* @param signal
*/
- public void killAM(int signal) {
+ public int killAM(int signal) {
killJavaProcesses(SliderAppMaster.SERVICE_CLASSNAME_SHORT, signal)
}
/**
- * Kill any java process with the given grep pattern
- * @param grepString string to grep for
- */
- public void killJavaProcesses(String grepString, int signal) {
-
- GString bashCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal"
- log.info("Bash command = $bashCommand" )
- Process bash = ["bash", "-c", bashCommand].execute()
- bash.waitFor()
-
- log.info(bash.in.text)
- log.error(bash.err.text)
- }
-
- public void killJavaProcesses(List<String> greps, int signal) {
- for (String grep : greps) {
- killJavaProcesses(grep,signal)
- }
- }
-
- /**
* List any java process with the given grep pattern
* @param grepString string to grep for
*/
@@ -360,9 +402,8 @@
public YarnConfiguration getTestConfiguration() {
YarnConfiguration conf = getConfiguration()
-
conf.addResource(SLIDER_TEST_XML)
- return conf
+ return conf;
}
protected String getRMAddr() {
@@ -448,11 +489,13 @@
def config = miniCluster.config
if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) {
HadoopFS dfs = HadoopFS.get(new URI(fsDefaultName), config)
- Path clusterDir = new SliderFileSystem(dfs, config).buildClusterDirPath(clustername)
- log.info("deleting customer data at $clusterDir")
+
+ def sliderFileSystem = new SliderFileSystem(dfs, config)
+ Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername)
+ log.info("deleting instance data at $clusterDir")
//this is a safety check to stop us doing something stupid like deleting /
assert clusterDir.toString().contains("/.slider/")
- dfs.delete(clusterDir, true)
+ rigorousDelete(sliderFileSystem, clusterDir, 60000)
}
@@ -500,6 +543,58 @@
}
/**
+ * Delete with some pauses and backoff; designed to handle slow delete
+ * operation in windows
+ * @param dfs
+ * @param path
+ */
+ public void rigorousDelete(
+ SliderFileSystem sliderFileSystem,
+ Path path, long timeout) {
+
+ if (path.toUri().scheme == "file") {
+ File dir = new File(path.toUri().getPath());
+ rigorousDelete(dir, timeout)
+ } else {
+ Duration duration = new Duration(timeout)
+ duration.start()
+ HadoopFS dfs = sliderFileSystem.fileSystem
+ boolean deleted = false;
+ while (!deleted && !duration.limitExceeded) {
+ dfs.delete(path, true)
+ deleted = !dfs.exists(path)
+ if (!deleted) {
+ sleep(1000)
+ }
+ }
+ }
+ sliderFileSystem.verifyDirectoryNonexistent(path)
+ }
+
+ /**
+ * Delete with some pauses and backoff; designed to handle slow delete
+ * operation in windows
+ * @param dir dir to delete
+ * @param timeout timeout in millis
+ */
+ public void rigorousDelete(File dir, long timeout) {
+ Duration duration = new Duration(timeout)
+ duration.start()
+ boolean deleted = false;
+ while (!deleted && !duration.limitExceeded) {
+ FileUtils.deleteQuietly(dir)
+ deleted = !dir.exists()
+ if (!deleted) {
+ sleep(1000)
+ }
+ }
+ if (!deleted) {
+ // noisy delete raises an IOE
+ FileUtils.deleteDirectory(dir)
+ }
+ }
+
+ /**
* Add arguments to launch Slider with.
*
* Extra arguments are added after standard arguments and before roles.
@@ -668,12 +763,18 @@
return resourceConfDir.absoluteFile.toURI().toString()
}
-
+ /**
+ * Log an application report
+ * @param report
+ */
public void logReport(ApplicationReport report) {
log.info(SliderUtils.reportToString(report));
}
-
+ /**
+ * Log a list of application reports
+ * @param apps
+ */
public void logApplications(List<ApplicationReport> apps) {
apps.each { ApplicationReport r -> logReport(r) }
}
@@ -691,6 +792,7 @@
* force kill the application after waiting for
* it to shut down cleanly
* @param client client to talk to
+ * @return the final application report
*/
public ApplicationReport waitForAppToFinish(SliderClient client) {
@@ -698,6 +800,13 @@
return waitForAppToFinish(client, waitTime)
}
+ /**
+ * force kill the application after waiting for
+ * it to shut down cleanly
+ * @param client client to talk to
+ * @param waitTime time in milliseconds to wait
+ * @return the final application report
+ */
public static ApplicationReport waitForAppToFinish(
SliderClient client,
int waitTime) {
@@ -714,6 +823,7 @@
}
nodes.each { ClusterNode node -> log.info(node.toString())}
client.forceKillApplication("timed out waiting for application to complete");
+ report = client.applicationReport
}
return report;
}
@@ -726,15 +836,17 @@
* @return the exit code
*/
public int clusterActionFreeze(SliderClient sliderClient, String clustername,
- String message = "action freeze") {
- log.info("Freezing cluster $clustername: $message")
+ String message = "action stop",
+ boolean force = false) {
+ log.info("Stopping cluster $clustername: $message")
ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
freezeArgs.waittime = CLUSTER_STOP_TIME
freezeArgs.message = message
+ freezeArgs.force = force
int exitCode = sliderClient.actionFreeze(clustername,
freezeArgs);
if (exitCode != 0) {
- log.warn("Cluster freeze failed with error code $exitCode")
+ log.warn("Cluster stop failed with error code $exitCode")
}
return exitCode
}
@@ -749,14 +861,15 @@
public int maybeStopCluster(
SliderClient sliderClient,
String clustername,
- String message) {
+ String message,
+ boolean force = false) {
if (sliderClient != null) {
if (!clustername) {
clustername = sliderClient.deployedClusterName;
}
//only stop a cluster that exists
if (clustername) {
- return clusterActionFreeze(sliderClient, clustername, message);
+ return clusterActionFreeze(sliderClient, clustername, message, force);
}
}
return 0;
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
index 0259fb7..f6b13a4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
@@ -23,14 +23,12 @@
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.IOUtils
import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.slider.core.zk.BlockingZKWatcher
import org.apache.slider.core.zk.ZKIntegration
import java.util.concurrent.atomic.AtomicBoolean
-import static org.apache.slider.common.SliderKeys.*;
-import static org.apache.slider.common.SliderXMLConfKeysForTesting.*;
import static org.apache.slider.test.KeysForTests.*;
/**
@@ -86,13 +84,13 @@
}
/**
- * Create and start a minicluster
+ * Create and start a minicluster with ZK
* @param name cluster/test name
* @param conf configuration to use
* @param noOfNodeManagers #of NMs
* @param numLocalDirs #of local dirs
* @param numLogDirs #of log dirs
- * @param startZK create a ZK micro cluster
+ * @param startZK create a ZK micro cluster *THIS IS IGNORED*
* @param startHDFS create an HDFS mini cluster
*/
protected String createMiniCluster(String name,
@@ -102,12 +100,12 @@
int numLogDirs,
boolean startZK,
boolean startHDFS) {
- name = createMiniCluster(name, conf, noOfNodeManagers, numLocalDirs, numLogDirs,
+ createMicroZKCluster("-${name?:methodName.methodName}", conf)
+ conf.setBoolean(RegistryConstants.KEY_REGISTRY_ENABLED, true)
+ conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, ZKBinding)
+ name = super.createMiniCluster(name, conf, noOfNodeManagers, numLocalDirs, numLogDirs,
startHDFS)
- if (startZK) {
- createMicroZKCluster(conf)
- }
return name
}
@@ -138,16 +136,16 @@
return createMiniCluster("", conf, noOfNodeManagers, 1, 1, startZK, false)
}
- public void createMicroZKCluster(Configuration conf) {
+ public void createMicroZKCluster(String name, Configuration conf) {
microZKCluster = new MicroZKCluster(new Configuration(conf))
- microZKCluster.createCluster();
+ microZKCluster.createCluster(name);
}
void assertHasZKCluster() {
assert microZKCluster != null
}
- protected String getZKBinding() {
+ public String getZKBinding() {
if (!microZKCluster) {
return "localhost:1"
} else {
@@ -155,21 +153,13 @@
}
}
- protected int getZKPort() {
- return microZKCluster ? microZKCluster.port : 2181;
- }
-
- protected String getZKHosts() {
- return MicroZKCluster.HOSTS;
- }
-
/**
* CLI args include all the ZK bindings needed
* @return
*/
protected List<String> getExtraCLIArgs() {
[
- "-D", define(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, ZKBinding)
+ "-D", define(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, ZKBinding)
]
}
}
diff --git a/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
index be850da..c1e0940 100644
--- a/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
+++ b/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
@@ -19,6 +19,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl;
import org.apache.slider.tools.TestUtility;
import org.junit.Assert;
import org.junit.Rule;
@@ -28,6 +31,8 @@
import org.slf4j.LoggerFactory;
import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
/** Test slider util methods. */
public class TestSliderUtils {
@@ -68,4 +73,64 @@
Assert.assertEquals(SliderUtils.truncate("1234567890", 10), "1234567890");
Assert.assertEquals(SliderUtils.truncate("", 10), "");
}
+
+ @Test
+ public void testApplicationReportComparison() {
+ List<ApplicationReport> instances = getApplicationReports();
+
+ SliderUtils.sortApplicationsByMostRecent(instances);
+
+ Assert.assertEquals(1000, instances.get(0).getStartTime());
+ Assert.assertEquals(1000, instances.get(1).getStartTime());
+ Assert.assertEquals(1000, instances.get(2).getStartTime());
+ Assert.assertEquals(1000, instances.get(3).getStartTime());
+
+ instances = getApplicationReports();
+
+ SliderUtils.sortApplicationReport(instances);
+ Assert.assertEquals(1000, instances.get(0).getStartTime());
+ Assert.assertEquals(1000, instances.get(1).getStartTime());
+ Assert.assertEquals(1000, instances.get(2).getStartTime());
+ Assert.assertEquals(1000, instances.get(3).getStartTime());
+
+ Assert.assertTrue(instances.get(0).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
+ instances.get(0).getYarnApplicationState() == YarnApplicationState.RUNNING);
+ Assert.assertTrue(instances.get(1).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
+ instances.get(1).getYarnApplicationState() == YarnApplicationState.RUNNING);
+ Assert.assertTrue(instances.get(2).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
+ instances.get(2).getYarnApplicationState() == YarnApplicationState.RUNNING);
+ Assert.assertTrue(instances.get(3).getYarnApplicationState() == YarnApplicationState.KILLED);
+ }
+
+ private List<ApplicationReport> getApplicationReports() {
+ List<ApplicationReport> instances = new ArrayList<ApplicationReport>();
+ instances.add(getApplicationReport(1000, 0, "app1", YarnApplicationState.ACCEPTED));
+ instances.add(getApplicationReport(900, 998, "app1", YarnApplicationState.KILLED));
+ instances.add(getApplicationReport(900, 998, "app2", YarnApplicationState.FAILED));
+ instances.add(getApplicationReport(1000, 0, "app2", YarnApplicationState.RUNNING));
+ instances.add(getApplicationReport(800, 837, "app3", YarnApplicationState.FINISHED));
+ instances.add(getApplicationReport(1000, 0, "app3", YarnApplicationState.RUNNING));
+ instances.add(getApplicationReport(900, 998, "app3", YarnApplicationState.KILLED));
+ instances.add(getApplicationReport(800, 837, "app4", YarnApplicationState.FINISHED));
+ instances.add(getApplicationReport(1000, 1050, "app4", YarnApplicationState.KILLED));
+ instances.add(getApplicationReport(900, 998, "app4", YarnApplicationState.FINISHED));
+
+ Assert.assertEquals("app1", instances.get(0).getApplicationType());
+ Assert.assertEquals("app1", instances.get(1).getApplicationType());
+ Assert.assertEquals("app2", instances.get(2).getApplicationType());
+ Assert.assertEquals("app2", instances.get(3).getApplicationType());
+ return instances;
+ }
+
+ private ApplicationReportPBImpl getApplicationReport(long startTime,
+ long finishTime,
+ String name,
+ YarnApplicationState state) {
+ ApplicationReportPBImpl ar = new ApplicationReportPBImpl();
+ ar.setFinishTime(finishTime);
+ ar.setStartTime(startTime);
+ ar.setApplicationType(name);
+ ar.setYarnApplicationState(state);
+ return ar;
+ }
}
diff --git a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
new file mode 100644
index 0000000..1267098
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.common.SliderKeys;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAppMasterLauncher {
+ SliderYarnClientImpl mockYarnClient;
+ YarnClientApplication yarnClientApp;
+ ApplicationSubmissionContext appSubmissionContext;
+ Set<String> tags = Collections.emptySet();
+
+ @Before
+ public void initialize() throws Exception {
+ mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class);
+ yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class);
+ appSubmissionContext = EasyMock
+ .createNiceMock(ApplicationSubmissionContext.class);
+ EasyMock.expect(yarnClientApp.getApplicationSubmissionContext())
+ .andReturn(appSubmissionContext).once();
+ EasyMock.expect(mockYarnClient.createApplication())
+ .andReturn(yarnClientApp).once();
+ }
+
+ @Test
+ public void testExtractLogAggregationContext() throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS,
+ " | slider*.txt |agent.out| |");
+ options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
+ "command*.json| agent.log* | ");
+
+ EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
+ AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1",
+ SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options,
+ tags);
+
+ // Verify the include/exclude patterns
+ String expectedInclude = "slider*.txt|agent.out";
+ Assert.assertEquals(expectedInclude,
+ appMasterLauncher.logAggregationContext.getIncludePattern());
+
+ String expectedExclude = "command*.json|agent.log*";
+ Assert.assertEquals(expectedExclude,
+ appMasterLauncher.logAggregationContext.getExcludePattern());
+
+ EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+ }
+
+ @Test
+ public void testExtractLogAggregationContextEmptyIncludePattern()
+ throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " ");
+ options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
+ "command*.json| agent.log* | ");
+
+ EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
+ AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1",
+ SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options,
+ tags);
+
+ // Verify the include/exclude patterns
+ String expectedInclude = "";
+ Assert.assertEquals(expectedInclude,
+ appMasterLauncher.logAggregationContext.getIncludePattern());
+
+ String expectedExclude = "command*.json|agent.log*";
+ Assert.assertEquals(expectedExclude,
+ appMasterLauncher.logAggregationContext.getExcludePattern());
+
+ EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+ }
+
+}
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
index 4cb35aa..0bea8fa 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
@@ -67,7 +67,7 @@
AggregateConf instanceDefinition = new AggregateConf();
try {
- provider.validateInstanceDefinition(instanceDefinition);
+ provider.validateInstanceDefinition(instanceDefinition, null);
Assert.assertFalse("Should fail with BadConfigException", true);
} catch (BadConfigException e) {
log.info(e.toString());
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
new file mode 100644
index 0000000..0ff613a
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.InternalKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.providers.ProviderUtils;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+
+/**
+ *
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(ProviderUtils.class)
+public class TestAgentClientProvider2 {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAgentClientProvider2.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ @Test
+ public void testPrepareAMAndConfigForLaunch() throws Exception {
+ AgentClientProvider provider = new AgentClientProvider(null);
+ SliderFileSystem sfs = PowerMock.createMock(SliderFileSystem.class);
+ FileSystem fs = PowerMock.createMock(FileSystem.class);
+ Configuration serviceConf = PowerMock.createMock(Configuration.class);
+ PowerMock.mockStatic(ProviderUtils.class);
+
+ expect(sfs.getFileSystem()).andReturn(fs);
+ expect(fs.mkdirs(anyObject(Path.class))).andReturn(true);
+ expect(ProviderUtils.addAgentTar(
+ anyObject(), anyObject(String.class), anyObject(SliderFileSystem.class), anyObject(Path.class))).
+ andReturn(true);
+
+ AggregateConf instanceDefinition = new AggregateConf();
+ ConfTree tree = new ConfTree();
+ tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
+ instanceDefinition.setInternal(tree);
+
+ PowerMock.replay(sfs, fs, serviceConf, ProviderUtils.class);
+
+ provider.prepareAMAndConfigForLaunch(
+ sfs, serviceConf, null, instanceDefinition, null,
+ null, null, null, null, false);
+
+ Assert.assertTrue(tree.global.containsKey(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH));
+ tree.global.remove(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+
+ // Verify that slider-agent.tar.gz got added
+ Path tempPath = new Path(".", "temp");
+ provider.prepareAMAndConfigForLaunch(
+ sfs, serviceConf, null, instanceDefinition, null,
+ null, null, null, tempPath, false);
+ PowerMock.verify(sfs, fs, ProviderUtils.class);
+ Assert.assertTrue(tree.global.containsKey(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH));
+ }
+}
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 6ed950f..2c977f4 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -22,16 +22,22 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
+import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
@@ -40,19 +46,31 @@
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.ExportEntry;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
+import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
+import org.apache.slider.providers.agent.application.metadata.CommandScript;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.ComponentExport;
+import org.apache.slider.providers.agent.application.metadata.ConfigFile;
+import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
import org.apache.slider.providers.agent.application.metadata.Export;
import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
+import org.apache.slider.providers.agent.application.metadata.PropertyInfo;
+import org.apache.slider.server.appmaster.model.mock.MockRegistryOperations;
+import org.apache.slider.server.appmaster.model.mock.MockApplicationAttemptId;
+import org.apache.slider.server.appmaster.model.mock.MockApplicationId;
import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.apache.slider.server.appmaster.model.mock.MockFileSystem;
import org.apache.slider.server.appmaster.model.mock.MockNodeId;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.CommandReport;
import org.apache.slider.server.appmaster.web.rest.agent.ComponentStatus;
import org.apache.slider.server.appmaster.web.rest.agent.ExecutionCommand;
@@ -61,6 +79,7 @@
import org.apache.slider.server.appmaster.web.rest.agent.Register;
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
+import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
@@ -79,13 +98,14 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
-import static org.easymock.EasyMock.anyBoolean;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyCollection;
import static org.mockito.Matchers.anyMap;
@@ -121,6 +141,10 @@
+ " <name>Master_Status</name>\n"
+ " <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>\n"
+ " </export>\n"
+ + " <export>\n"
+ + " <name>Comp_Endpoint</name>\n"
+ + " <value>http://${HBASE_REGIONSERVER_HOST}:${site.global.listen_port}</value>\n"
+ + " </export>\n"
+ " </exports>\n"
+ " </exportGroup>\n"
+ " </exportGroups>\n"
@@ -150,6 +174,7 @@
+ " <publishConfig>true</publishConfig>\n"
+ " <autoStartOnFailure>true</autoStartOnFailure>\n"
+ " <appExports>QuickLinks-JMX_Endpoint,QuickLinks-Master_Status</appExports>\n"
+ + " <compExports>QuickLinks-Comp_Endpoint</compExports>\n"
+ " <minInstanceCount>1</minInstanceCount>\n"
+ " <maxInstanceCount>2</maxInstanceCount>\n"
+ " <commandScript>\n"
@@ -167,6 +192,7 @@
+ " <script>scripts/hbase_regionserver.py</script>\n"
+ " <scriptType>PYTHON</scriptType>\n"
+ " </commandScript>\n"
+ + " <compExports>QuickLinks-Comp_Endpoint</compExports>\n"
+ " <componentExports>\n"
+ " <componentExport>\n"
+ " <name>PropertyA</name>\n"
@@ -190,6 +216,18 @@
+ " </packages>\n"
+ " </osSpecific>\n"
+ " </osSpecifics>\n"
+ + " <configFiles>\n"
+ + " <configFile>\n"
+ + " <type>xml</type>\n"
+ + " <fileName>hbase-site.xml</fileName>\n"
+ + " <dictionaryName>hbase-site</dictionaryName>\n"
+ + " </configFile>\n"
+ + " <configFile>\n"
+ + " <type>env</type>\n"
+ + " <fileName>hbase-env.sh</fileName>\n"
+ + " <dictionaryName>hbase-env</dictionaryName>\n"
+ + " </configFile>\n"
+ + " </configFiles>\n"
+ " </application>\n"
+ "</metainfo>";
private static final String metainfo_2_str = "<metainfo>\n"
@@ -234,7 +272,7 @@
ConfTree tree = new ConfTree();
tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
- AgentProviderService aps = new AgentProviderService();
+ AgentProviderService aps = createAgentProviderService(new Configuration());
ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
AggregateConf instanceDefinition = new AggregateConf();
@@ -252,9 +290,10 @@
MapOperations resourceComponent = new MapOperations();
MapOperations appComponent = new MapOperations();
Path containerTmpDirPath = new Path(".", "test");
- FileSystem mockFs = new MockFileSystem();
+ FileSystem mockFs = createNiceMock(FileSystem.class);
+ expect(mockFs.exists(anyObject(Path.class))).andReturn(true);
expect(sliderFileSystem.getFileSystem())
- .andReturn(new FilterFileSystem(mockFs)).anyTimes();
+ .andReturn(mockFs).anyTimes();
expect(sliderFileSystem.createAmResource(anyObject(Path.class),
anyObject(LocalResourceType.class)))
.andReturn(createNiceMock(LocalResource.class)).anyTimes();
@@ -264,14 +303,16 @@
AgentProviderService mockAps = Mockito.spy(aps);
doReturn(access).when(mockAps).getAmState();
- doReturn("scripts/hbase_master.py").when(mockAps).getScriptPathFromMetainfo(anyString());
+ CommandScript cs = new CommandScript();
+ cs.setScript("scripts/hbase_master.py");
+ doReturn(cs).when(mockAps).getScriptPathFromMetainfo(anyString());
Metainfo metainfo = new Metainfo();
metainfo.setApplication(new Application());
doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());
Configuration conf = new Configuration();
- conf.set(SliderXmlConfKeys.REGISTRY_PATH,
- SliderXmlConfKeys.DEFAULT_REGISTRY_PATH);
+ conf.set(RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
try {
doReturn(true).when(mockAps).isMaster(anyString());
@@ -279,7 +320,8 @@
eq("HBASE_MASTER"),
eq("mockcontainer_1"),
any(HeartBeatResponse.class),
- eq("scripts/hbase_master.py"));
+ eq("scripts/hbase_master.py"),
+ eq(600L));
doReturn(conf).when(mockAps).getConfig();
} catch (SliderException e) {
}
@@ -290,6 +332,12 @@
anyString(),
anyMap()
);
+
+ doNothing().when(mockAps).publishFolderPaths(anyMap(),
+ anyString(),
+ anyString(),
+ anyString()
+ );
expect(access.isApplicationLive()).andReturn(true).anyTimes();
ClusterDescription desc = new ClusterDescription();
desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
@@ -302,7 +350,7 @@
treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf);
expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
- replay(access, ctx, container, sliderFileSystem);
+ replay(access, ctx, container, sliderFileSystem, mockFs);
try {
mockAps.buildContainerLaunchContext(launcher,
@@ -323,10 +371,150 @@
Register reg = new Register();
reg.setResponseId(0);
- reg.setHostname("mockcontainer_1___HBASE_MASTER");
- Map<String,String> ports = new HashMap();
+ reg.setLabel("mockcontainer_1___HBASE_MASTER");
+ Map<String,String> ports = new HashMap<String, String>();
ports.put("a","100");
reg.setAllocatedPorts(ports);
+ Map<String, String> folders = new HashMap<String, String>();
+ folders.put("F1", "F2");
+ reg.setLogFolders(folders);
+ RegistrationResponse resp = mockAps.handleRegistration(reg);
+ Assert.assertEquals(0, resp.getResponseId());
+ Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
+
+ Mockito.verify(mockAps, Mockito.times(1)).processAllocatedPorts(
+ anyString(),
+ anyString(),
+ anyString(),
+ anyMap()
+ );
+
+ Mockito.verify(mockAps, Mockito.times(1)).publishFolderPaths(
+ anyMap(),
+ anyString(),
+ anyString(),
+ anyString()
+ );
+
+ HeartBeat hb = new HeartBeat();
+ hb.setResponseId(1);
+ hb.setHostname("mockcontainer_1___HBASE_MASTER");
+ HeartBeatResponse hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(2, hbr.getResponseId());
+ }
+
+
+ private AggregateConf prepareConfForAgentStateTests() {
+ ConfTree tree = new ConfTree();
+ tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
+
+ AggregateConf instanceDefinition = new AggregateConf();
+ instanceDefinition.setInternal(tree);
+ instanceDefinition.setAppConf(tree);
+ instanceDefinition.getAppConfOperations().getGlobalOptions()
+ .put(AgentKeys.APP_DEF, ".");
+ instanceDefinition.getAppConfOperations().getGlobalOptions()
+ .put(AgentKeys.AGENT_CONF, ".");
+ instanceDefinition.getAppConfOperations().getGlobalOptions()
+ .put(AgentKeys.AGENT_VERSION, ".");
+ return instanceDefinition;
+ }
+
+ private AgentProviderService prepareProviderServiceForAgentStateTests()
+ throws IOException {
+ ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
+ Container container = createNiceMock(Container.class);
+ String role = "HBASE_MASTER";
+ SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = new MockFileSystem();
+ expect(sliderFileSystem.getFileSystem()).andReturn(
+ new FilterFileSystem(mockFs)).anyTimes();
+ expect(
+ sliderFileSystem.createAmResource(anyObject(Path.class),
+ anyObject(LocalResourceType.class))).andReturn(
+ createNiceMock(LocalResource.class)).anyTimes();
+ expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes();
+ expect(container.getNodeId()).andReturn(new MockNodeId("localhost"))
+ .anyTimes();
+ expect(container.getPriority()).andReturn(Priority.newInstance(1));
+
+ StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
+ Configuration conf = new Configuration();
+
+ AgentProviderService aps = createAgentProviderService(conf);
+ AgentProviderService mockAps = Mockito.spy(aps);
+
+ doReturn(access).when(mockAps).getAmState();
+ CommandScript cs = new CommandScript();
+ cs.setScript("scripts/hbase_master.py");
+ doReturn(cs).when(mockAps)
+ .getScriptPathFromMetainfo(anyString());
+ Metainfo metainfo = new Metainfo();
+ Application application = new Application();
+ metainfo.setApplication(application);
+ doReturn(metainfo).when(mockAps).getApplicationMetainfo(
+ any(SliderFileSystem.class), anyString());
+ doReturn(metainfo).when(mockAps).getMetainfo();
+
+
+
+ try {
+ doReturn(true).when(mockAps).isMaster(anyString());
+ doNothing().when(mockAps).addInstallCommand(eq("HBASE_MASTER"),
+ eq("mockcontainer_1"), any(HeartBeatResponse.class),
+ eq("scripts/hbase_master.py"), eq(600L));
+ doReturn(conf).when(mockAps).getConfig();
+ } catch (SliderException e) {
+ }
+
+ doNothing().when(mockAps).processAllocatedPorts(anyString(), anyString(),
+ anyString(), anyMap());
+ expect(access.isApplicationLive()).andReturn(true).anyTimes();
+ ClusterDescription desc = new ClusterDescription();
+ desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
+ desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE");
+ expect(access.getClusterStatus()).andReturn(desc).anyTimes();
+
+ AggregateConf aggConf = new AggregateConf();
+ ConfTreeOperations treeOps = aggConf.getAppConfOperations();
+ treeOps.getOrAddComponent("HBASE_MASTER")
+ .put(AgentKeys.WAIT_HEARTBEAT, "0");
+ treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
+ treeOps.set("java_home", "/usr/jdk7/");
+ treeOps.set("site.fs.defaultFS", "hdfs://c6409.ambari.apache.org:8020");
+ treeOps.set(InternalKeys.INTERNAL_DATA_DIR_PATH, "hdfs://c6409.ambari.apache.org:8020/user/yarn/.slider/cluster/cl1/data");
+ expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf);
+ expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
+ expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes();
+ replay(access, ctx, container, sliderFileSystem);
+
+ List<Container> containers = new ArrayList<Container>();
+ containers.add(container);
+ Map<Integer, ProviderRole> providerRoleMap = new HashMap<Integer, ProviderRole>();
+ ProviderRole providerRole = new ProviderRole(role, 1);
+ providerRoleMap.put(1, providerRole);
+ mockAps.rebuildContainerDetails(containers, "mockcontainer_1",
+ providerRoleMap);
+ return mockAps;
+ }
+
+ @Test
+ public void testAgentStateStarted() throws IOException, SliderException {
+ AggregateConf instanceDefinition = prepareConfForAgentStateTests();
+ AgentProviderService mockAps = prepareProviderServiceForAgentStateTests();
+ Register reg = new Register();
+ reg.setResponseId(0);
+ reg.setLabel("mockcontainer_1___HBASE_MASTER");
+ Map<String,String> ports = new HashMap<String,String>();
+ ports.put("a","100");
+ reg.setAllocatedPorts(ports);
+
+ // Simulating agent in STARTED state
+ reg.setActualState(State.STARTED);
+
+ mockAps.initializeApplicationConfiguration(instanceDefinition,
+ null);
+
RegistrationResponse resp = mockAps.handleRegistration(reg);
Assert.assertEquals(0, resp.getResponseId());
Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
@@ -343,11 +531,95 @@
hb.setHostname("mockcontainer_1___HBASE_MASTER");
HeartBeatResponse hbr = mockAps.handleHeartBeat(hb);
Assert.assertEquals(2, hbr.getResponseId());
+ Assert.assertNotNull(
+ "Status command from AM cannot be null when agent's actualState "
+ + "is set to STARTED during registration", hbr.getStatusCommands());
+ Assert.assertTrue(
+ "Status command from AM cannot be empty when agent's actualState "
+ + "is set to STARTED during registration", hbr.getStatusCommands()
+ .size() > 0);
+ Assert.assertEquals(
+ "AM should directly send a STATUS request if agent's actualState is "
+ + "set to STARTED during registration",
+ AgentCommandType.STATUS_COMMAND, hbr.getStatusCommands().get(0)
+ .getCommandType());
+ Assert.assertEquals(
+ "AM should directly request for CONFIG if agent's actualState is "
+ + "set to STARTED during registration",
+ "GET_CONFIG", hbr.getStatusCommands().get(0)
+ .getRoleCommand());
+ Assert.assertFalse("AM cannot ask agent to restart", hbr.isRestartAgent());
+ }
+
+ @Test
+ public void testAgentStateInstalled() throws IOException, SliderException {
+ AggregateConf instanceDefinition = prepareConfForAgentStateTests();
+ AgentProviderService mockAps = prepareProviderServiceForAgentStateTests();
+
+ Metainfo metainfo = new Metainfo();
+ Application application = new Application();
+ CommandOrder cmdOrder = new CommandOrder();
+ cmdOrder.setCommand("HBASE_MASTER-START");
+ cmdOrder.setRequires("HBASE_MASTER-INSTALLED");
+ application.addCommandOrder(cmdOrder);
+ metainfo.setApplication(application);
+ doReturn(metainfo).when(mockAps).getApplicationMetainfo(
+ any(SliderFileSystem.class), anyString());
+ doReturn(metainfo).when(mockAps).getMetainfo();
+ doNothing().when(mockAps).addRoleRelatedTokens(anyMap());
+
+ Register reg = new Register();
+ reg.setResponseId(0);
+ reg.setLabel("mockcontainer_1___HBASE_MASTER");
+ Map<String,String> ports = new HashMap<String,String>();
+ ports.put("a","100");
+ reg.setAllocatedPorts(ports);
+
+ // Simulating agent in INSTALLED state
+ reg.setActualState(State.INSTALLED);
+
+ mockAps.initializeApplicationConfiguration(instanceDefinition,
+ null);
+
+ RegistrationResponse resp = mockAps.handleRegistration(reg);
+ Assert.assertEquals(0, resp.getResponseId());
+ Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
+
+ Mockito.verify(mockAps, Mockito.times(1)).processAllocatedPorts(
+ anyString(),
+ anyString(),
+ anyString(),
+ anyMap()
+ );
+
+ HeartBeat hb = new HeartBeat();
+ hb.setResponseId(1);
+ hb.setHostname("mockcontainer_1___HBASE_MASTER");
+ HeartBeatResponse hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(2, hbr.getResponseId());
+ Assert.assertNotNull(
+ "Execution command from AM cannot be null when agent's actualState "
+ + "is set to INSTALLED during registration", hbr.getExecutionCommands());
+ Assert.assertTrue(
+ "Execution command from AM cannot be empty when agent's actualState "
+ + "is set to INSTALLED during registration", hbr.getExecutionCommands()
+ .size() > 0);
+ Assert.assertEquals(
+ "AM should send an EXECUTION command if agent's actualState is "
+ + "set to INSTALLED during registration",
+ AgentCommandType.EXECUTION_COMMAND, hbr.getExecutionCommands().get(0)
+ .getCommandType());
+ Assert.assertEquals(
+ "AM should request for START if agent's actualState is "
+ + "set to INSTALLED during registration",
+ "START", hbr.getExecutionCommands().get(0)
+ .getRoleCommand());
+ Assert.assertFalse("AM cannot ask agent to restart", hbr.isRestartAgent());
}
@Test
public void testRoleHostMapping() throws Exception {
- AgentProviderService aps = new AgentProviderService();
+ AgentProviderService aps = createAgentProviderService(new Configuration());
StateAccessForProviders appState = new ProviderAppState("undefined", null) {
@Override
public ClusterDescription getClusterStatus() {
@@ -398,7 +670,7 @@
public void testComponentSpecificPublishes() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
- AgentProviderService aps = new AgentProviderService();
+ AgentProviderService aps = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps);
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
doReturn(metainfo).when(mockAps).getMetainfo();
@@ -433,12 +705,98 @@
}
}
+
+ @Test
+ public void testComponentSpecificPublishes2() throws Exception {
+ InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
+ Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ AgentProviderService aps = createAgentProviderService(new Configuration());
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
+ doReturn(metainfo).when(mockAps).getMetainfo();
+ StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
+ doReturn(access).when(mockAps).getAmState();
+ PublishedExportsSet pubExpSet = new PublishedExportsSet();
+ expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes();
+ replay(access);
+
+ Map<String, String> ports = new HashMap<String, String>();
+ ports.put("global.listen_port", "10010");
+ mockAps.processAndPublishComponentSpecificExports(ports,
+ "mockcontainer_1",
+ "host1",
+ "HBASE_REGIONSERVER");
+ ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
+ forClass(Collection.class);
+ ArgumentCaptor<String> publishNameCaptor = ArgumentCaptor.
+ forClass(String.class);
+ Mockito.verify(mockAps, Mockito.times(1)).publishApplicationInstanceData(
+ anyString(),
+ publishNameCaptor.capture(),
+ entriesCaptor.capture());
+
+ PublishedExports pubExports = pubExpSet.get("QuickLinks".toLowerCase());
+ Assert.assertEquals(1, pubExports.entries.size());
+ Assert.assertEquals("QuickLinks", pubExports.description);
+ List<ExportEntry> expEntries = pubExports.entries.get("Comp_Endpoint");
+ Assert.assertEquals(1, expEntries.size());
+ Assert.assertEquals("mockcontainer_1", expEntries.get(0).getContainerId());
+ Assert.assertEquals("component", expEntries.get(0).getLevel());
+ Assert.assertEquals("1", expEntries.get(0).getTag());
+ Assert.assertEquals("http://host1:10010", expEntries.get(0).getValue());
+ Assert.assertNotNull(expEntries.get(0).getUpdatedTime());
+ Assert.assertNull(expEntries.get(0).getValidUntil());
+
+ assert entriesCaptor.getAllValues().size() == 1;
+ for (Collection coll : entriesCaptor.getAllValues()) {
+ Set<Map.Entry<String, String>> entrySet = (Set<Map.Entry<String, String>>) coll;
+ for (Map.Entry entry : entrySet) {
+ log.info("{}:{}", entry.getKey(), entry.getValue().toString());
+ if (entry.getKey().equals("Comp_Endpoint")) {
+ assert entry.getValue().toString().equals("http://host1:10010");
+ }
+ }
+ }
+ assert publishNameCaptor.getAllValues().size() == 1;
+ for (String coll : publishNameCaptor.getAllValues()) {
+ assert coll.equals("QuickLinks");
+ }
+
+ mockAps.notifyContainerCompleted(new MockContainerId(1));
+ pubExports = pubExpSet.get("QuickLinks".toLowerCase());
+ Assert.assertEquals(1, pubExports.entries.size());
+ Assert.assertEquals("QuickLinks", pubExports.description);
+ expEntries = pubExports.entries.get("Comp_Endpoint");
+ Assert.assertEquals(0, expEntries.size());
+
+ mockAps.notifyContainerCompleted(new MockContainerId(1));
+ mockAps.notifyContainerCompleted(new MockContainerId(2));
+
+ mockAps.processAndPublishComponentSpecificExports(ports,
+ "mockcontainer_1",
+ "host1",
+ "HBASE_REGIONSERVER");
+ mockAps.processAndPublishComponentSpecificExports(ports,
+ "mockcontainer_2",
+ "host1",
+ "HBASE_REGIONSERVER");
+ pubExports = pubExpSet.get("QuickLinks".toLowerCase());
+ Assert.assertEquals(1, pubExports.entries.size());
+ Assert.assertEquals("QuickLinks", pubExports.description);
+ expEntries = pubExports.entries.get("Comp_Endpoint");
+ Assert.assertEquals(2, expEntries.size());
+
+ mockAps.notifyContainerCompleted(new MockContainerId(2));
+ expEntries = pubExports.entries.get("Comp_Endpoint");
+ Assert.assertEquals(1, expEntries.size());
+ }
+
@Test
public void testProcessConfig() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
Assert.assertNotNull(metainfo.getApplication());
- AgentProviderService aps = new AgentProviderService();
+ AgentProviderService aps = createAgentProviderService(new Configuration());
HeartBeat hb = new HeartBeat();
ComponentStatus status = new ComponentStatus();
status.setClusterName("test");
@@ -467,6 +825,11 @@
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
doReturn(metainfo).when(mockAps).getMetainfo();
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
+ StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
+ doReturn(access).when(mockAps).getAmState();
+ PublishedExportsSet pubExpSet = new PublishedExportsSet();
+ expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes();
+ replay(access);
mockAps.publishConfigAndExportGroups(hb, componentStatus, "HBASE_MASTER");
Assert.assertTrue(componentStatus.getConfigReported());
@@ -487,15 +850,30 @@
}
}
- Map<String, String> exports = mockAps.getCurrentExports("QuickLinks");
+ Map<String, List<ExportEntry>> exports = mockAps.getCurrentExports("QuickLinks");
Assert.assertEquals(2, exports.size());
- Assert.assertEquals(exports.get("JMX_Endpoint"), "http://HOST1:60012/jmx");
+ Assert.assertEquals(exports.get("JMX_Endpoint").get(0).getValue(), "http://HOST1:60012/jmx");
mockAps.publishConfigAndExportGroups(hb, componentStatus, "HBASE_REST");
Mockito.verify(mockAps, Mockito.times(3)).publishApplicationInstanceData(
anyString(),
anyString(),
entriesCaptor.capture());
+ PublishedExports pubExports = pubExpSet.get("QuickLinks".toLowerCase());
+ Assert.assertEquals(2, pubExports.entries.size());
+ Assert.assertEquals("QuickLinks", pubExports.description);
+ List<ExportEntry> expEntries = pubExports.entries.get("JMX_Endpoint");
+ Assert.assertEquals(1, expEntries.size());
+ Assert.assertEquals(null, expEntries.get(0).getContainerId());
+ Assert.assertEquals("application", expEntries.get(0).getLevel());
+ Assert.assertEquals(null, expEntries.get(0).getTag());
+ Assert.assertEquals("http://HOST1:60012/jmx", expEntries.get(0).getValue());
+ Assert.assertNull(expEntries.get(0).getValidUntil());
+
+ expEntries = pubExports.entries.get("Master_Status");
+ Assert.assertEquals(1, expEntries.size());
+ expEntries = pubExports.entries.get("JMX_Endpoint");
+ Assert.assertEquals("http://HOST1:60012/jmx", expEntries.get(0).getValue());
}
@Test
@@ -520,6 +898,7 @@
Assert.assertEquals(component.getCategory(), "MASTER");
Assert.assertEquals(component.getComponentExports().size(), 0);
Assert.assertEquals(component.getAppExports(), "QuickLinks-JMX_Endpoint,QuickLinks-Master_Status");
+ Assert.assertEquals(component.getCompExports(), "QuickLinks-Comp_Endpoint");
found++;
}
if (component.getName().equals("HBASE_REGIONSERVER")) {
@@ -546,7 +925,7 @@
List<ExportGroup> egs = application.getExportGroups();
ExportGroup eg = egs.get(0);
Assert.assertEquals(eg.getName(), "QuickLinks");
- Assert.assertEquals(eg.getExports().size(), 2);
+ Assert.assertEquals(eg.getExports().size(), 3);
found = 0;
for (Export export : eg.getExports()) {
@@ -578,11 +957,28 @@
}
Assert.assertEquals(found, 2);
- AgentProviderService aps = new AgentProviderService();
+ List<ConfigFile> configFiles = application.getConfigFiles();
+ Assert.assertEquals(configFiles.size(), 2);
+ found = 0;
+ for (ConfigFile configFile : configFiles) {
+ if (configFile.getDictionaryName().equals("hbase-site")) {
+ Assert.assertEquals("hbase-site.xml", configFile.getFileName());
+ Assert.assertEquals("xml", configFile.getType());
+ found++;
+ }
+ if (configFile.getDictionaryName().equals("hbase-env")) {
+ Assert.assertEquals("hbase-env.sh", configFile.getFileName());
+ Assert.assertEquals("env", configFile.getType());
+ found++;
+ }
+ }
+ Assert.assertEquals("Two config dependencies must be found.", found, 2);
+
+ AgentProviderService aps = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps);
doReturn(metainfo).when(mockAps).getMetainfo();
- String scriptPath = mockAps.getScriptPathFromMetainfo("HBASE_MASTER");
- Assert.assertEquals(scriptPath, "scripts/hbase_master.py");
+ CommandScript script = mockAps.getScriptPathFromMetainfo("HBASE_MASTER");
+ Assert.assertEquals(script.getScript(), "scripts/hbase_master.py");
String metainfo_1_str_bad = "<metainfo>\n"
+ " <schemaVersion>2.0</schemaVersion>\n"
@@ -607,7 +1003,7 @@
String role_hm = "HBASE_MASTER";
String role_hrs = "HBASE_REGIONSERVER";
- AgentProviderService aps1 = new AgentProviderService();
+ AgentProviderService aps1 = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps1);
doReturn(metainfo).when(mockAps).getMetainfo();
@@ -636,7 +1032,11 @@
ConfTree tree = new ConfTree();
tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
- AgentProviderService aps = new AgentProviderService();
+ Configuration conf = new Configuration();
+ AgentProviderService aps = createAgentProviderService(conf);
+ YarnRegistryViewForProviders registryViewForProviders = aps.getYarnRegistry();
+ assertNotNull(registryViewForProviders);
+
ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
AggregateConf instanceDefinition = new AggregateConf();
@@ -651,13 +1051,15 @@
String role_hrs = "HBASE_REGIONSERVER";
SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
ContainerLauncher launcher = createNiceMock(ContainerLauncher.class);
+ ContainerLauncher launcher2 = createNiceMock(ContainerLauncher.class);
Path generatedConfPath = new Path(".", "test");
MapOperations resourceComponent = new MapOperations();
MapOperations appComponent = new MapOperations();
Path containerTmpDirPath = new Path(".", "test");
- FileSystem mockFs = new MockFileSystem();
+ FilterFileSystem mockFs = createNiceMock(FilterFileSystem.class);
expect(sliderFileSystem.getFileSystem())
- .andReturn(new FilterFileSystem(mockFs)).anyTimes();
+ .andReturn(mockFs).anyTimes();
+ expect(mockFs.exists(anyObject(Path.class))).andReturn(true);
expect(sliderFileSystem.createAmResource(anyObject(Path.class),
anyObject(LocalResourceType.class)))
.andReturn(createNiceMock(LocalResource.class)).anyTimes();
@@ -668,10 +1070,9 @@
AgentProviderService mockAps = Mockito.spy(aps);
doReturn(access).when(mockAps).getAmState();
doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());
+ doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).
+ initializeDefaultConfigs(any(SliderFileSystem.class), anyString(), any(Metainfo.class));
- Configuration conf = new Configuration();
- conf.set(SliderXmlConfKeys.REGISTRY_PATH,
- SliderXmlConfKeys.DEFAULT_REGISTRY_PATH);
try {
doReturn(true).when(mockAps).isMaster(anyString());
@@ -679,26 +1080,31 @@
anyString(),
anyString(),
any(HeartBeatResponse.class),
- anyString());
+ anyString(),
+ Mockito.anyLong());
doNothing().when(mockAps).addStartCommand(
anyString(),
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ Mockito.anyLong(),
Matchers.anyBoolean());
doNothing().when(mockAps).addGetConfigCommand(
anyString(),
anyString(),
any(HeartBeatResponse.class));
- doNothing().when(mockAps).publishApplicationInstanceData(
+ doNothing().when(mockAps).publishFolderPaths(
+ anyMap(),
anyString(),
anyString(),
- anyCollection());
+ anyString());
doReturn(conf).when(mockAps).getConfig();
} catch (SliderException e) {
}
+ PublishedExportsSet pubExpSet = new PublishedExportsSet();
expect(access.isApplicationLive()).andReturn(true).anyTimes();
+ expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes();
ClusterDescription desc = new ClusterDescription();
desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE");
@@ -711,7 +1117,8 @@
treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf).anyTimes();
expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
- replay(access, ctx, container, sliderFileSystem);
+ doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
+ replay(access, ctx, container, sliderFileSystem, mockFs);
// build two containers
try {
@@ -725,7 +1132,7 @@
appComponent,
containerTmpDirPath);
- mockAps.buildContainerLaunchContext(launcher,
+ mockAps.buildContainerLaunchContext(launcher2,
instanceDefinition,
container,
role_hrs,
@@ -738,14 +1145,14 @@
// Both containers register
Register reg = new Register();
reg.setResponseId(0);
- reg.setHostname("mockcontainer_1___HBASE_MASTER");
+ reg.setLabel("mockcontainer_1___HBASE_MASTER");
RegistrationResponse resp = mockAps.handleRegistration(reg);
Assert.assertEquals(0, resp.getResponseId());
Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
reg = new Register();
reg.setResponseId(0);
- reg.setHostname("mockcontainer_1___HBASE_REGIONSERVER");
+ reg.setLabel("mockcontainer_1___HBASE_REGIONSERVER");
resp = mockAps.handleRegistration(reg);
Assert.assertEquals(0, resp.getResponseId());
Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
@@ -759,7 +1166,8 @@
Mockito.verify(mockAps, Mockito.times(1)).addInstallCommand(anyString(),
anyString(),
any(HeartBeatResponse.class),
- anyString());
+ anyString(),
+ Mockito.anyLong());
hb = new HeartBeat();
hb.setResponseId(1);
@@ -769,7 +1177,8 @@
Mockito.verify(mockAps, Mockito.times(2)).addInstallCommand(anyString(),
anyString(),
any(HeartBeatResponse.class),
- anyString());
+ anyString(),
+ Mockito.anyLong());
// RS succeeds install but does not start
hb = new HeartBeat();
hb.setResponseId(2);
@@ -788,6 +1197,7 @@
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ Mockito.anyLong(),
Matchers.anyBoolean());
// RS still does not start
hb = new HeartBeat();
@@ -799,12 +1209,14 @@
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ Mockito.anyLong(),
Matchers.anyBoolean());
// MASTER succeeds install and issues start
hb = new HeartBeat();
hb.setResponseId(2);
hb.setHostname("mockcontainer_1___HBASE_MASTER");
+ hb.setFqdn("host1");
cr = new CommandReport();
cr.setRole("HBASE_MASTER");
cr.setRoleCommand("INSTALL");
@@ -819,6 +1231,7 @@
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ Mockito.anyLong(),
Matchers.anyBoolean());
Map<String, String> allocatedPorts = mockAps.getAllocatedPorts();
Assert.assertTrue(allocatedPorts != null);
@@ -835,6 +1248,7 @@
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ Mockito.anyLong(),
Matchers.anyBoolean());
// MASTER succeeds start
hb = new HeartBeat();
@@ -860,6 +1274,7 @@
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ Mockito.anyLong(),
Matchers.anyBoolean());
// JDK7
} catch (SliderException he) {
@@ -868,15 +1283,80 @@
log.warn(he.getMessage());
}
- Mockito.verify(mockAps, Mockito.times(1)).publishApplicationInstanceData(
+ Mockito.verify(mockAps, Mockito.times(1)).publishFolderPaths(
+ anyMap(),
anyString(),
anyString(),
- anyCollection());
+ anyString());
+ }
+
+ protected AgentProviderService createAgentProviderService(Configuration conf) throws
+ IOException {
+ AgentProviderService aps = new AgentProviderService();
+ YarnRegistryViewForProviders registryViewForProviders =
+ createYarnRegistryViewForProviders(conf);
+ aps.bindToYarnRegistry(registryViewForProviders);
+ return aps;
+ }
+
+ protected YarnRegistryViewForProviders createYarnRegistryViewForProviders(
+ Configuration conf) throws IOException {
+ conf.set(SliderXmlConfKeys.REGISTRY_PATH,
+ SliderXmlConfKeys.DEFAULT_REGISTRY_PATH);
+
+ RegistryOperations registryOperations = new MockRegistryOperations();
+ registryOperations.init(conf);
+ YarnRegistryViewForProviders registryViewForProviders =
+ new YarnRegistryViewForProviders(registryOperations,
+ "hbase",
+ SliderKeys.APP_TYPE,
+ "hbase1",
+ new MockApplicationAttemptId(new MockApplicationId(1), 1));
+ registryViewForProviders.registerSelf(new ServiceRecord(), true);
+ return registryViewForProviders;
}
@Test
- public void testNotifyContainerCompleted() {
- AgentProviderService aps = new AgentProviderService();
+ public void testPublishFolderPaths() throws IOException {
+ AgentProviderService aps = createAgentProviderService(new Configuration());
+ StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doReturn(access).when(mockAps).getAmState();
+ PublishedExportsSet pubExpSet = new PublishedExportsSet();
+ expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes();
+ replay(access);
+
+ Map<String, String> folders = new HashMap<String, String>();
+ folders.put("AGENT_LOG_ROOT", "aFolder");
+ folders.put("AGENT_WORK_ROOT", "folderB");
+ mockAps.publishFolderPaths(folders, "cid", "role", "fqdn");
+
+ PublishedExports exports = pubExpSet.get("container_log_dirs");
+ Assert.assertEquals(1, exports.entries.size());
+ List<ExportEntry> expEntries = exports.entries.get("role");
+ Assert.assertEquals(1, expEntries.size());
+ Assert.assertEquals("cid", expEntries.get(0).getContainerId());
+ Assert.assertEquals("component", expEntries.get(0).getLevel());
+ Assert.assertEquals("role", expEntries.get(0).getTag());
+ Assert.assertEquals("fqdn:aFolder", expEntries.get(0).getValue());
+ Assert.assertNull(expEntries.get(0).getValidUntil());
+ Assert.assertEquals(null, expEntries.get(0).getValidUntil());
+
+ exports = pubExpSet.get("container_work_dirs");
+ Assert.assertEquals(1, exports.entries.size());
+ expEntries = exports.entries.get("role");
+ Assert.assertEquals(1, expEntries.size());
+ Assert.assertEquals("cid", expEntries.get(0).getContainerId());
+ Assert.assertEquals("component", expEntries.get(0).getLevel());
+ Assert.assertEquals("role", expEntries.get(0).getTag());
+ Assert.assertEquals("fqdn:folderB", expEntries.get(0).getValue());
+ Assert.assertNull(expEntries.get(0).getValidUntil());
+ Assert.assertEquals(null, expEntries.get(0).getValidUntil());
+ }
+
+ @Test
+ public void testNotifyContainerCompleted() throws IOException {
+ AgentProviderService aps = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps);
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
@@ -884,6 +1364,7 @@
String id = cid.toString();
ContainerId cid2 = new MockContainerId(2);
mockAps.getAllocatedPorts().put("a", "100");
+ mockAps.getAllocatedPorts(id).put("a", "100");
mockAps.getAllocatedPorts(id).put("b", "101");
mockAps.getAllocatedPorts("cid2").put("c", "102");
@@ -900,13 +1381,13 @@
Assert.assertNotNull(mockAps.getComponentStatuses().get("cid2_HM"));
Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
- Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 1);
+ Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 2);
Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
// Make the call
mockAps.notifyContainerCompleted(new MockContainerId(1));
- Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
+ Assert.assertEquals(mockAps.getAllocatedPorts().size(), 0);
Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 0);
Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
@@ -921,7 +1402,7 @@
public void testAddInstallCommand() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
- AgentProviderService aps = new AgentProviderService();
+ AgentProviderService aps = createAgentProviderService(new Configuration());
HeartBeatResponse hbr = new HeartBeatResponse();
StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
@@ -942,6 +1423,7 @@
doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString());
doReturn(metainfo).when(mockAps).getMetainfo();
+ doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).getDefaultConfigs();
Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
Map<String, ClusterNode> container = new HashMap<String, ClusterNode>();
@@ -953,18 +1435,25 @@
replay(access);
- mockAps.addInstallCommand("HBASE_MASTER", "cid1", hbr, "");
+ mockAps.addInstallCommand("HBASE_MASTER", "cid1", hbr, "", 0);
ExecutionCommand cmd = hbr.getExecutionCommands().get(0);
String pkgs = cmd.getHostLevelParams().get(AgentKeys.PACKAGE_LIST);
Assert.assertEquals("[{\"type\":\"tarball\",\"name\":\"files/hbase-0.96.1-hadoop2-bin.tar.gz\"}]", pkgs);
Assert.assertEquals("java_home", cmd.getHostLevelParams().get(AgentKeys.JAVA_HOME));
Assert.assertEquals("cid1", cmd.getHostLevelParams().get("container_id"));
Assert.assertEquals(Command.INSTALL.toString(), cmd.getRoleCommand());
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_log_dir"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_pid_dir"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_install_dir"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_input_conf_dir"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_container_id"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("pid_file"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_root"));
}
@Test
public void testAddStartCommand() throws Exception {
- AgentProviderService aps = new AgentProviderService();
+ AgentProviderService aps = createAgentProviderService(new Configuration());
HeartBeatResponse hbr = new HeartBeatResponse();
StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
@@ -978,17 +1467,36 @@
treeOps.set("site.fs.defaultFS", "hdfs://HOST1:8020/");
treeOps.set("internal.data.dir.path", "hdfs://HOST1:8020/database");
treeOps.set(OptionKeys.ZOOKEEPER_HOSTS, "HOST1");
- treeOps.set("config_types", "hbase-site");
treeOps.getGlobalOptions().put("site.hbase-site.a.port", "${HBASE_MASTER.ALLOCATED_PORT}");
treeOps.getGlobalOptions().put("site.hbase-site.b.port", "${HBASE_MASTER.ALLOCATED_PORT}");
- treeOps.getGlobalOptions().put("site.hbase-site.random.port", "${HBASE_MASTER.ALLOCATED_PORT}{DO_NOT_PROPAGATE}");
+ treeOps.getGlobalOptions().put("site.hbase-site.random.port", "${HBASE_MASTER.ALLOCATED_PORT}{PER_CONTAINER}");
treeOps.getGlobalOptions().put("site.hbase-site.random2.port", "${HBASE_MASTER.ALLOCATED_PORT}");
+ Map<String, DefaultConfig> defaultConfigMap = new HashMap<String, DefaultConfig>();
+ DefaultConfig defaultConfig = new DefaultConfig();
+ PropertyInfo propertyInfo1 = new PropertyInfo();
+ propertyInfo1.setName("defaultA");
+ propertyInfo1.setValue("Avalue");
+ defaultConfig.addPropertyInfo(propertyInfo1);
+ propertyInfo1 = new PropertyInfo();
+ propertyInfo1.setName("defaultB");
+ propertyInfo1.setValue("");
+ defaultConfig.addPropertyInfo(propertyInfo1);
+ defaultConfigMap.put("hbase-site", defaultConfig);
+
expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes();
expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
expect(access.isApplicationLive()).andReturn(true).anyTimes();
doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString());
+ doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs();
+ List<String> configurations = new ArrayList<String>();
+ configurations.add("hbase-site");
+ configurations.add("global");
+ List<String> sysConfigurations = new ArrayList<String>();
+ configurations.add("core-site");
+ doReturn(configurations).when(mockAps).getApplicationConfigurationTypes();
+ doReturn(sysConfigurations).when(mockAps).getSystemConfigurationsRequested(any(ConfTreeOperations.class));
Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
Map<String, ClusterNode> container = new HashMap<String, ClusterNode>();
@@ -1007,15 +1515,67 @@
replay(access);
- mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", Boolean.FALSE);
+ mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", 0, Boolean.FALSE);
Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("hbase-site"));
+ Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("core-site"));
Map<String, String> hbaseSiteConf = hbr.getExecutionCommands().get(0).getConfigurations().get("hbase-site");
Assert.assertTrue(hbaseSiteConf.containsKey("a.port"));
Assert.assertEquals("10023", hbaseSiteConf.get("a.port"));
Assert.assertEquals("10024", hbaseSiteConf.get("b.port"));
Assert.assertEquals("10025", hbaseSiteConf.get("random.port"));
assertEquals("${HBASE_MASTER.ALLOCATED_PORT}",
- hbaseSiteConf.get("random2.port"));
+ hbaseSiteConf.get("random2.port"));
+ ExecutionCommand cmd = hbr.getExecutionCommands().get(0);
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_log_dir"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_pid_dir"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_install_dir"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_input_conf_dir"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_container_id"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("pid_file"));
+ Assert.assertTrue(cmd.getConfigurations().get("global").containsKey("app_root"));
+ Assert.assertTrue(cmd.getConfigurations().get("hbase-site").containsKey("defaultA"));
+ Assert.assertFalse(cmd.getConfigurations().get("hbase-site").containsKey("defaultB"));
+ }
+
+ @Test
+ public void testParameterParsing() throws IOException {
+ AgentProviderService aps = createAgentProviderService(new Configuration());
+ AggregateConf aggConf = new AggregateConf();
+ ConfTreeOperations treeOps = aggConf.getAppConfOperations();
+ treeOps.getGlobalOptions().put(AgentKeys.SYSTEM_CONFIGS, "core-site,yarn-site, core-site ");
+ List<String> configs = aps.getSystemConfigurationsRequested(treeOps);
+ Assert.assertEquals(2, configs.size());
+ Assert.assertTrue(configs.contains("core-site"));
+ Assert.assertFalse(configs.contains("bore-site"));
+ }
+
+ @Test
+ public void testDereferenceAllConfig() throws IOException {
+ AgentProviderService aps = createAgentProviderService(new Configuration());
+ Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>();
+ Map<String, String> cfg1 = new HashMap<String, String>();
+ cfg1.put("a1", "${@//site/cfg-2/A1}");
+ cfg1.put("b1", "22");
+ cfg1.put("c1", "33");
+ cfg1.put("d1", "${@//site/cfg1/c1}AA");
+ Map<String, String> cfg2 = new HashMap<String, String>();
+ cfg2.put("A1", "11");
+ cfg2.put("B1", "${@//site/cfg-2/A1},${@//site/cfg-2/A1},AA,${@//site/cfg1/c1}");
+ cfg2.put("C1", "DD${@//site/cfg1/c1}");
+ cfg2.put("D1", "${14}");
+
+ allConfigs.put("cfg1", cfg1);
+ allConfigs.put("cfg-2", cfg2);
+ aps.dereferenceAllConfigs(allConfigs);
+ Assert.assertEquals("11", cfg1.get("a1"));
+ Assert.assertEquals("22", cfg1.get("b1"));
+ Assert.assertEquals("33", cfg1.get("c1"));
+ Assert.assertEquals("33AA", cfg1.get("d1"));
+
+ Assert.assertEquals("11", cfg2.get("A1"));
+ Assert.assertEquals("11,11,AA,33", cfg2.get("B1"));
+ Assert.assertEquals("DD33", cfg2.get("C1"));
+ Assert.assertEquals("${14}", cfg2.get("D1"));
}
}
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java
new file mode 100644
index 0000000..7b38ee3
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class TestComponentTagProvider {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestComponentTagProvider.class);
+
+ @Test
+ public void testTagProvider() throws Exception {
+ ComponentTagProvider ctp = new ComponentTagProvider();
+ Assert.assertEquals("", ctp.getTag(null, null));
+ Assert.assertEquals("", ctp.getTag(null, "cid"));
+ Assert.assertEquals("", ctp.getTag("comp1", null));
+
+ Assert.assertEquals("1", ctp.getTag("comp1", "cid1"));
+ Assert.assertEquals("2", ctp.getTag("comp1", "cid2"));
+ Assert.assertEquals("3", ctp.getTag("comp1", "cid3"));
+ ctp.releaseTag("comp1", "cid2");
+ Assert.assertEquals("2", ctp.getTag("comp1", "cid22"));
+
+ ctp.releaseTag("comp1", "cid4");
+ ctp.recordAssignedTag("comp1", "cid5", "5");
+ Assert.assertEquals("4", ctp.getTag("comp1", "cid4"));
+ Assert.assertEquals("4", ctp.getTag("comp1", "cid4"));
+ Assert.assertEquals("6", ctp.getTag("comp1", "cid6"));
+
+ ctp.recordAssignedTag("comp1", "cid55", "5");
+ Assert.assertEquals("5", ctp.getTag("comp1", "cid55"));
+
+ ctp.recordAssignedTag("comp2", "cidb3", "3");
+ Assert.assertEquals("1", ctp.getTag("comp2", "cidb1"));
+ Assert.assertEquals("2", ctp.getTag("comp2", "cidb2"));
+ Assert.assertEquals("4", ctp.getTag("comp2", "cidb4"));
+
+ ctp.recordAssignedTag("comp2", "cidb5", "six");
+ ctp.recordAssignedTag("comp2", "cidb5", "-55");
+ ctp.recordAssignedTag("comp2", "cidb5", "tags");
+ ctp.recordAssignedTag("comp2", "cidb5", null);
+ ctp.recordAssignedTag("comp2", "cidb5", "");
+ ctp.recordAssignedTag("comp2", "cidb5", "5");
+ Assert.assertEquals("6", ctp.getTag("comp2", "cidb6"));
+
+ ctp.recordAssignedTag("comp2", null, "5");
+ ctp.recordAssignedTag(null, null, "5");
+ ctp.releaseTag("comp1", null);
+ ctp.releaseTag(null, "cid4");
+ ctp.releaseTag(null, null);
+ }
+
+ @Test
+ public void testTagProviderWithThread() throws Exception {
+ ComponentTagProvider ctp = new ComponentTagProvider();
+ Thread thread = new Thread(new Taggged(ctp));
+ Thread thread2 = new Thread(new Taggged(ctp));
+ Thread thread3 = new Thread(new Taggged(ctp));
+ thread.start();
+ thread2.start();
+ thread3.start();
+ ctp.getTag("comp1", "cid50");
+ thread.join();
+ thread2.join();
+ thread3.join();
+ Assert.assertEquals("101", ctp.getTag("comp1", "cid101"));
+ }
+
+ public class Taggged implements Runnable {
+ private final ComponentTagProvider ctp;
+
+ public Taggged(ComponentTagProvider ctp) {
+ this.ctp = ctp;
+ }
+
+ public void run() {
+ for (int i = 0; i < 100; i++) {
+ String containerId = "cid" + (i + 1);
+ this.ctp.getTag("comp1", containerId);
+ }
+ for (int i = 0; i < 100; i++) {
+ String containerId = "cid" + (i + 1);
+ this.ctp.getTag("comp1", containerId);
+ }
+ for (int i = 0; i < 100; i += 2) {
+ String containerId = "cid" + (i + 1);
+ this.ctp.releaseTag("comp1", containerId);
+ }
+ for (int i = 0; i < 100; i += 2) {
+ String containerId = "cid" + (i + 1);
+ this.ctp.getTag("comp1", containerId);
+ }
+ }
+ }
+}
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java
new file mode 100644
index 0000000..3aa44a1
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class TestConfigParser {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestConfigParser.class);
+ private static final String config_1_str = "<configuration>\n"
+ + " <property>\n"
+ + " <name>security.client.protocol.acl</name>\n"
+ + " <value>*</value>\n"
+ + " <description>ACL for HRegionInterface protocol implementations (ie. \n"
+ + " clients talking to HRegionServers)\n"
+ + " The ACL is a comma-separated list of user and group names. The user and \n"
+ + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
+ + " A special value of \"*\" means all users are allowed.</description>\n"
+ + " </property>\n"
+ + "\n"
+ + " <property>\n"
+ + " <name>security.admin.protocol.acl</name>\n"
+ + " <value>*</value>\n"
+ + " <description>ACL for HMasterInterface protocol implementation (ie. \n"
+ + " clients talking to HMaster for admin operations).\n"
+ + " The ACL is a comma-separated list of user and group names. The user and \n"
+ + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
+ + " A special value of \"*\" means all users are allowed.</description>\n"
+ + " </property>\n"
+ + "\n"
+ + " <property>\n"
+ + " <name>security.masterregion.protocol.acl</name>\n"
+ + " <value>*</value>\n"
+ + " <description>ACL for HMasterRegionInterface protocol implementations\n"
+ + " (for HRegionServers communicating with HMaster)\n"
+ + " The ACL is a comma-separated list of user and group names. The user and \n"
+ + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
+ + " A special value of \"*\" means all users are allowed.</description>\n"
+ + " </property>\n"
+ + " <property>\n"
+ + " <name>emptyVal</name>\n"
+ + " <value></value>\n"
+ + " <description>non-empty-desc</description>\n"
+ + " </property>\n"
+ + " <property>\n"
+ + " <name>emptyDesc</name>\n"
+ + " <value></value>\n"
+ + " <description></description>\n"
+ + " </property>\n"
+ + " <property>\n"
+ + " <name>noDesc</name>\n"
+ + " <value></value>\n"
+ + " </property>\n"
+ + "</configuration>";
+
+ @Test
+ public void testParse() throws IOException {
+
+ InputStream config_1 = new ByteArrayInputStream(config_1_str.getBytes());
+ DefaultConfig config = new DefaultConfigParser().parse(config_1);
+ Assert.assertNotNull(config);
+ Assert.assertNotNull(config.getPropertyInfos());
+ Assert.assertEquals(6, config.getPropertyInfos().size());
+ for (PropertyInfo pInfo : config.getPropertyInfos()) {
+ if (pInfo.getName().equals("security.client.protocol.acl")) {
+ Assert.assertEquals("*", pInfo.getValue());
+ Assert.assertTrue(pInfo.getDescription().startsWith("ACL for HRegionInterface "));
+ }
+ if (pInfo.getName().equals("emptyVal")) {
+ Assert.assertEquals("", pInfo.getValue());
+ Assert.assertEquals("non-empty-desc", pInfo.getDescription());
+ }
+ if (pInfo.getName().equals("emptyDesc")) {
+ Assert.assertEquals("", pInfo.getValue());
+ Assert.assertEquals("", pInfo.getDescription());
+ }
+ if (pInfo.getName().equals("noDesc")) {
+ Assert.assertEquals("", pInfo.getValue());
+ Assert.assertNull(pInfo.getDescription());
+ }
+ }
+ }
+}
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
index 98f0afb..1177e9d 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
@@ -16,13 +16,19 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.providers.agent.AgentProviderService;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.List;
+
+import static org.mockito.Mockito.doReturn;
/**
*
@@ -59,5 +65,6 @@
}
}
assert found;
+ Assert.assertEquals(0, application.getConfigFiles().size());
}
}
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/TestSliderAmFilter.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/TestSliderAmFilter.java
deleted file mode 100644
index 00e193d..0000000
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/TestSliderAmFilter.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web;
-
-import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
-import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter;
-import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletContext;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.ServletResponseWrapper;
-import javax.servlet.http.Cookie;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.servlet.http.HttpServletResponseWrapper;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
-import static junit.framework.Assert.fail;
-
-/**
- * Test AmIpFilter. Requests to a no declared hosts should has way through
- * proxy. Another requests can be filtered with (without) user name.
- *
- */
-public class TestSliderAmFilter {
-
- private String proxyHost = "localhost";
- private String proxyUri = "http://bogus";
- private String doFilterRequest;
- private AmIpServletRequestWrapper servletWrapper;
- private String wsContextRoot = "/ws";
-
- private class TestAmIpFilter extends AmIpFilter {
-
- private Set<String> proxyAddresses = null;
-
- protected Set<String> getProxyAddresses() {
- if (proxyAddresses == null) {
- proxyAddresses = new HashSet<String>();
- }
- proxyAddresses.add(proxyHost);
- return proxyAddresses;
- }
- }
-
- private static class DummyFilterConfig implements FilterConfig {
- final Map<String, String> map;
-
- DummyFilterConfig(Map<String, String> map) {
- this.map = map;
- }
-
- @Override
- public String getFilterName() {
- return "dummy";
- }
-
- @Override
- public String getInitParameter(String arg0) {
- return map.get(arg0);
- }
-
- @Override
- public Enumeration<String> getInitParameterNames() {
- return Collections.enumeration(map.keySet());
- }
-
- @Override
- public ServletContext getServletContext() {
- return null;
- }
- }
-
- /**
- * Test AmIpFilter
- */
- @Test(timeout = 1000)
- public void testFilter() throws Exception {
- Map<String, String> params = new HashMap<String, String>();
- params.put(SliderAmIpFilter.PROXY_HOST, proxyHost);
- params.put(SliderAmIpFilter.PROXY_URI_BASE, proxyUri);
- params.put(SliderAmIpFilter.WS_CONTEXT_ROOT, wsContextRoot);
- FilterConfig config = new DummyFilterConfig(params);
-
- // dummy filter
- FilterChain chain = new FilterChain() {
- @Override
- public void doFilter(ServletRequest servletRequest,
- ServletResponse servletResponse) throws IOException, ServletException {
- doFilterRequest = servletRequest.getClass().getName();
- if (servletRequest instanceof AmIpServletRequestWrapper) {
- servletWrapper = (AmIpServletRequestWrapper) servletRequest;
-
- }
- }
- };
- SliderAmIpFilter testFilter = new SliderAmIpFilter();
- testFilter.init(config);
-
- HttpServletResponse mockResponse = Mockito.mock(HttpServletResponse.class);
- HttpServletResponseForTest response =
- new HttpServletResponseForTest(mockResponse);
- // Test request should implements HttpServletRequest
-
- ServletRequest failRequest = Mockito.mock(ServletRequest.class);
- try {
- testFilter.doFilter(failRequest, response, chain);
- fail();
- } catch (ServletException e) {
- assertEquals("This filter only works for HTTP/HTTPS", e.getMessage());
- }
-
- // request with HttpServletRequest
- HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
- Mockito.when(request.getRemoteAddr()).thenReturn("redirect");
- Mockito.when(request.getRequestURI()).thenReturn("/ws/v1/slider/agent");
- testFilter.doFilter(request, response, chain);
- // address "redirect" is not in host list
- assertEquals(0, response.getRedirect().length());
- Mockito.when(request.getRequestURI()).thenReturn("/redirect");
- testFilter.doFilter(request, response, chain);
- assertEquals("http://bogus/redirect", response.getRedirect());
- // "127.0.0.1" contains in host list. Without cookie
- Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1");
- testFilter.doFilter(request, response, chain);
-
- assertTrue(doFilterRequest
- .contains("javax.servlet.http.HttpServletRequest"));
- // cookie added
- Cookie[] cookies = new Cookie[1];
- cookies[0] = new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user");
-
- Mockito.when(request.getCookies()).thenReturn(cookies);
- testFilter.doFilter(request, response, chain);
-
- assertEquals(
- "org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper",
- doFilterRequest);
- // request contains principal from cookie
- assertEquals("user", servletWrapper.getUserPrincipal().getName());
- assertEquals("user", servletWrapper.getRemoteUser());
- assertFalse(servletWrapper.isUserInRole(""));
-
- }
-
- private class HttpServletResponseForTest extends HttpServletResponseWrapper {
- String redirectLocation = "";
-
- public HttpServletResponseForTest(HttpServletResponse response) {
- super(response);
- }
-
- public String getRedirect() {
- return redirectLocation;
- }
-
- @Override
- public void sendRedirect(String location) throws IOException {
- redirectLocation = location;
- }
-
- @Override
- public String encodeRedirectURL(String url) {
- return url;
- }
-
- }
-
-}
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index 7e2ab3c..faec5d8 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -25,8 +25,10 @@
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
import junit.framework.Assert;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderUtils;
@@ -52,14 +54,13 @@
import javax.ws.rs.core.MediaType;
import java.io.File;
-import java.io.IOException;
import java.net.URI;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
+//import java.nio.file.FileVisitResult;
+//import java.nio.file.Files;
+//import java.nio.file.Path;
+//import java.nio.file.Paths;
+//import java.nio.file.SimpleFileVisitor;
+//import java.nio.file.attribute.BasicFileAttributes;
import static org.junit.Assert.assertEquals;
@@ -79,10 +80,10 @@
}
});
- SecurityUtils.initializeSecurityParameters(new MapOperations());
- MapOperations compOperations = new MapOperations();
+ MapOperations configMap = new MapOperations();
+ SecurityUtils.initializeSecurityParameters(configMap, true);
CertificateManager certificateManager = new CertificateManager();
- certificateManager.initRootCert(compOperations);
+ certificateManager.initialize(configMap);
String keystoreFile = SecurityUtils.getSecurityDir() + File.separator + SliderKeys.KEYSTORE_FILE_NAME;
String password = SecurityUtils.getKeystorePass();
System.setProperty("javax.net.ssl.trustStore", keystoreFile);
@@ -137,7 +138,7 @@
appState);
slider = new WebAppApiImpl(new MockSliderClusterProtocol(), providerAppState,
- new MockProviderService(), null);
+ new MockProviderService(), null, null);
MapOperations compOperations = new MapOperations();
@@ -152,7 +153,7 @@
@After
public void tearDown () throws Exception {
- webApp.stop();
+ IOUtils.closeStream(webApp);
webApp = null;
}
@@ -204,7 +205,7 @@
Register register = new Register();
register.setResponseId(-1);
register.setTimestamp(System.currentTimeMillis());
- register.setHostname("dummyHost");
+ register.setLabel("dummyHost");
return register;
}
@@ -218,22 +219,23 @@
@AfterClass
public static void tearDownClass() throws Exception{
- Path directory = Paths.get(SecurityUtils.getSecurityDir());
- Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
- throws IOException {
- Files.delete(file);
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult postVisitDirectory(Path dir, IOException exc)
- throws IOException {
- Files.delete(dir);
- return FileVisitResult.CONTINUE;
- }
-
- });
+ FileUtils.deleteDirectory(new File(SecurityUtils.getSecurityDir()));
+// Path directory = Paths.get(SecurityUtils.getSecurityDir());
+// Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
+// @Override
+// public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+// throws IOException {
+// Files.delete(file);
+// return FileVisitResult.CONTINUE;
+// }
+//
+// @Override
+// public FileVisitResult postVisitDirectory(Path dir, IOException exc)
+// throws IOException {
+// Files.delete(dir);
+// return FileVisitResult.CONTINUE;
+// }
+//
+// });
}
}
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index 91aa2b0..20889ac 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -191,7 +191,7 @@
appState);
slider = new WebAppApiImpl(new MockSliderClusterProtocol(), providerAppState,
- new MockProviderService(), null);
+ new MockProviderService(), null, null);
bind(SliderJacksonJaxbJsonProvider.class);
bind(MockSliderAMWebServices.class);
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index eb368e3..7fceac7 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -17,10 +17,10 @@
package org.apache.slider.server.appmaster.web.rest.publisher;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.providers.agent.AgentProviderService;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.services.registry.RegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,17 +42,19 @@
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg,
QueueAccess queueAccess,
List<Container> liveContainers) {
- super.bind(stateAccessor, reg, queueAccess, liveContainers);
+ super.bind(stateAccessor, queueAccess, liveContainers);
Map<String,String> dummyProps = new HashMap<String, String>();
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
log.info("publishing dummy-site.xml with values {}", dummyProps);
publishApplicationInstanceData("dummy-site", "dummy configuration",
dummyProps.entrySet());
-
+ // publishing global config for testing purposes
+ publishApplicationInstanceData("global", "global configuration",
+ stateAccessor.getAppConfSnapshot()
+ .getGlobalOptions().entrySet());
}
}
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java b/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
index 6d2d051..97a3f74 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
@@ -17,6 +17,7 @@
package org.apache.slider.server.services.security;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.conf.MapOperations;
import org.junit.Assert;
import org.junit.Before;
@@ -40,9 +41,9 @@
MapOperations compOperations = new MapOperations();
secDir = new File(workDir.getRoot(), SliderKeys.SECURITY_DIR);
File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
- compOperations.put(SliderKeys.KEYSTORE_LOCATION,
+ compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION,
keystoreFile.getAbsolutePath());
- certMan.initRootCert(compOperations);
+ certMan.initialize(compOperations);
}
@Test
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/EndOfServiceWaiter.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/EndOfServiceWaiter.java
deleted file mode 100644
index 5e6df3b..0000000
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/EndOfServiceWaiter.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-import org.junit.Assert;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Wait for a service to stop
- */
-public class EndOfServiceWaiter implements ServiceStateChangeListener {
-
- private final AtomicBoolean finished = new AtomicBoolean(false);
-
- public EndOfServiceWaiter(Service svc) {
- svc.registerServiceListener(this);
- }
-
- public synchronized void waitForServiceToStop(long timeout) throws
- InterruptedException {
- if (!finished.get()) {
- wait(timeout);
- }
- Assert.assertTrue("Service did not finish in time period",
- finished.get());
- }
-
- @Override
- public synchronized void stateChanged(Service service) {
- if (service.isInState(Service.STATE.STOPPED)) {
- finished.set(true);
- notify();
- }
- }
-
-
-}
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
index 45fdc86..4a19417 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
@@ -18,8 +18,11 @@
package org.apache.slider.server.services.workflow;
+import org.apache.hadoop.util.Shell;
+
import java.io.File;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
/**
@@ -37,10 +40,12 @@
* @return commands
*/
public List<String> ls(File dir) {
- List<String> commands = new ArrayList<String>(5);
- commands.add("ls");
- commands.add("-1");
- commands.add(dir.getAbsolutePath());
+ List<String> commands;
+ if (!Shell.WINDOWS) {
+ commands = Arrays.asList("ls","-1", dir.getAbsolutePath());
+ } else {
+ commands = Arrays.asList("cmd", "/c", "dir", dir.getAbsolutePath());
+ }
return commands;
}
@@ -61,8 +66,12 @@
* @return commands
*/
public List<String> env() {
- List<String> commands = new ArrayList<String>(1);
- commands.add("env");
+ List<String> commands;
+ if (!Shell.WINDOWS) {
+ commands = Arrays.asList("env");
+ } else {
+ commands = Arrays.asList("cmd", "/c", "set");
+ }
return commands;
}
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
index 668bcca..0eb2b78 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
@@ -18,6 +18,7 @@
package org.apache.slider.server.services.workflow;
+import org.apache.slider.test.SliderTestUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -33,11 +34,9 @@
*/
public class TestLongLivedProcess extends WorkflowServiceTestBase implements
LongLivedProcessLifecycleEvent {
- private static final Logger
- log = LoggerFactory.getLogger(TestLongLivedProcess.class);
+ private static final Logger log = LoggerFactory.getLogger(TestLongLivedProcess.class);
- private static final Logger
- processLog =
+ private static final Logger processLog =
LoggerFactory.getLogger("org.apache.hadoop.services.workflow.Process");
@@ -71,12 +70,11 @@
assertTrue("process stop callback not received", stopped);
assertFalse(process.isRunning());
assertEquals(0, process.getExitCode().intValue());
-
- assertStringInOutput("test-classes", getFinalOutput());
}
@Test
public void testExitCodes() throws Throwable {
+ SliderTestUtils.skipOnWindows();
initProcess(commandFactory.exitFalse());
process.start();
@@ -95,6 +93,7 @@
@Test
public void testEcho() throws Throwable {
+ SliderTestUtils.skipOnWindows();
String echoText = "hello, world";
initProcess(commandFactory.echo(echoText));
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
index 638547f..39516b7 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
@@ -37,7 +37,7 @@
@Test
public void testNullClose() throws Throwable {
- ClosingService<OpenClose> svc = new ClosingService<OpenClose>(null);
+ ClosingService<OpenClose> svc = new ClosingService<OpenClose>("", null);
svc.init(new Configuration());
svc.start();
assertNull(svc.getCloseable());
@@ -81,7 +81,7 @@
@Test
public void testCloseSelf() throws Throwable {
ClosingService<ClosingService> svc =
- new ClosingService<ClosingService>(null);
+ new ClosingService<ClosingService>("");
svc.setCloseable(svc);
svc.stop();
}
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
index 6d08156..bc1476a 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.ServiceOperations;
+import org.apache.slider.test.SliderTestUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -31,6 +32,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeoutException;
/**
* Test the long lived process by executing a command that works and a command
@@ -67,20 +69,22 @@
initProcess(commandFactory.ls(testDir));
exec();
assertFalse(process.isProcessRunning());
- assertEquals(0, process.getExitCode());
-
- assertStringInOutput("test-classes", getFinalOutput());
+ Integer exitCode = process.getExitCode();
+ assertNotNull("null exit code", exitCode);
+ assertEquals(0, exitCode.intValue());
// assert that the service did not fail
assertNull(process.getFailureCause());
}
@Test
public void testExitCodes() throws Throwable {
+ SliderTestUtils.skipOnWindows();
initProcess(commandFactory.exitFalse());
exec();
assertFalse(process.isProcessRunning());
- int exitCode = process.getExitCode();
+ Integer exitCode = process.getExitCode();
+ assertNotNull("null exit code", exitCode);
assertTrue(exitCode != 0);
int corrected = process.getExitCodeSignCorrected();
assertEquals(1, corrected);
@@ -91,12 +95,13 @@
@Test
public void testEcho() throws Throwable {
-
+ SliderTestUtils.skipOnWindows();
String echoText = "hello, world";
initProcess(commandFactory.echo(echoText));
exec();
-
- assertEquals(0, process.getExitCode());
+ Integer exitCode = process.getExitCode();
+ assertNotNull("null exit code", exitCode);
+ assertEquals(0, exitCode.intValue());
assertStringInOutput(echoText, getFinalOutput());
}
@@ -109,8 +114,9 @@
env.put(var, val);
initProcess(commandFactory.env());
exec();
-
- assertEquals(0, process.getExitCode());
+ Integer exitCode = process.getExitCode();
+ assertNotNull("null exit code", exitCode);
+ assertEquals(0, exitCode.intValue());
assertStringInOutput(val, getFinalOutput());
}
@@ -131,11 +137,10 @@
return process;
}
- public void exec() throws InterruptedException {
+ public void exec() throws InterruptedException, TimeoutException {
assertNotNull(process);
- EndOfServiceWaiter waiter = new EndOfServiceWaiter(process);
process.start();
- waiter.waitForServiceToStop(5000);
+ assert process.waitForServiceToStop(5000);
}
}
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
index 3049d8f..f38bd9d 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
@@ -29,6 +29,7 @@
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Locale;
import java.util.concurrent.Callable;
/**
@@ -122,7 +123,7 @@
boolean found = false;
StringBuilder builder = new StringBuilder();
for (String s : output) {
- builder.append(s).append('\n');
+ builder.append(s.toLowerCase(Locale.ENGLISH)).append('\n');
if (s.contains(text)) {
found = true;
break;
diff --git a/slider-core/src/test/python/agent/main.py b/slider-core/src/test/python/agent/main.py
index e50642d..116d179 100755
--- a/slider-core/src/test/python/agent/main.py
+++ b/slider-core/src/test/python/agent/main.py
@@ -27,7 +27,10 @@
def main():
print "Executing echo"
- print 'Argument List: {0}'.format(str(sys.argv))
+ try:
+ print 'Argument List: {0}'.format(str(sys.argv))
+ except AttributeError:
+ pass
parser = OptionParser()
parser.add_option("--log", dest="log_folder", help="log destination")
diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml
index 09b314e..cf4afe1 100644
--- a/slider-core/src/test/python/metainfo.xml
+++ b/slider-core/src/test/python/metainfo.xml
@@ -27,6 +27,27 @@
<minHadoopVersion>2.1.0</minHadoopVersion>
<components>
<component>
+ <name>hbase-rs</name>
+ <category>MASTER</category>
+ <minInstanceCount>0</minInstanceCount>
+ <commandScript>
+ <script>echo.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ <component>
+ <name>hbase-master</name>
+ <category>MASTER</category>
+ <minInstanceCount>0</minInstanceCount>
+ <maxInstanceCount>2</maxInstanceCount>
+ <commandScript>
+ <script>echo.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ <component>
<name>echo</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
diff --git a/slider-core/src/test/resources/example-slider-test.xml b/slider-core/src/test/resources/example-slider-test.xml
index a752cfd..abf42f9 100644
--- a/slider-core/src/test/resources/example-slider-test.xml
+++ b/slider-core/src/test/resources/example-slider-test.xml
@@ -40,13 +40,13 @@
<property>
<name>slider.test.thaw.wait.seconds</name>
- <description>Time to wait for a thaw to work</description>
+ <description>Time to wait for a start to work</description>
<value>60</value>
</property>
<property>
<name>slider.test.freeze.wait.seconds</name>
- <description>Time to wait for a freeze to work</description>
+ <description>Time to wait for a stop to work</description>
<value>60</value>
</property>
diff --git a/slider-core/src/test/resources/log4j.properties b/slider-core/src/test/resources/log4j.properties
index c1a524d..2fe1e49 100644
--- a/slider-core/src/test/resources/log4j.properties
+++ b/slider-core/src/test/resources/log4j.properties
@@ -29,8 +29,7 @@
log4j.logger.org.apache.slider=DEBUG
log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-
-
+log4j.logger.org.apache.hadoop.yarn.registry=DEBUG
#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
diff --git a/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml b/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml
index 3d24f96..cb8eab2 100644
--- a/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml
+++ b/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml
@@ -87,9 +87,12 @@
</osSpecific>
</osSpecifics>
- <configuration-dependencies>
- <config-type>storm-site</config-type>
- <config-type>global</config-type>
- </configuration-dependencies>
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>storm-site.xml</fileName>
+ <dictionaryName>storm-site</dictionaryName>
+ </configFile>
+ </configFiles>
</application>
</metainfo>
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration_tokenized.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration_tokenized.json
new file mode 100644
index 0000000..b902469
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration_tokenized.json
@@ -0,0 +1,27 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://${CLUSTER_NAME}:8020",
+ "site.fs.default.name": "hdfs://${CLUSTER_NAME}:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "site.hbase.user_name": "${USER}",
+ "site.hbase.another.user": "${USER_NAME}"
+ },
+ "components": {
+
+ "worker": {
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "jvm.heapsize": "512M"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
index 8617d1f..4c782fb 100644
--- a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
@@ -7,7 +7,10 @@
"global": {
"application.name": "small_cluster",
"application.type": "hbase",
- "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar"
+ "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar",
+ "internal.chaos.monkey.probability.amlaunchfailure": "10000",
+ "internal.chaos.monkey.interval.seconds": "60",
+ "internal.chaos.monkey.enabled": "true"
},
"components": {
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json
new file mode 100644
index 0000000..625b10e
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json
@@ -0,0 +1,13 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ },
+ "components": {
+ "node": {
+ "yarn.memory": "256",
+ "yarn.component.instances": "2",
+ "yarn.role.priority":"1"
+ }
+ }
+}
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json
index f6111a0..83cdb99 100644
--- a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json
@@ -4,9 +4,9 @@
"global": {
},
"components": {
- "node": {
+ "echo": {
"yarn.memory": "256",
- "yarn.component.instances": "5",
+ "yarn.component.instances": "2",
"yarn.role.priority":"1"
}
}
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources_with_label.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources_with_label.json
new file mode 100644
index 0000000..a2ce107
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources_with_label.json
@@ -0,0 +1,25 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ },
+ "components": {
+ "echo": {
+ "yarn.memory": "256",
+ "yarn.component.instances": "1",
+ "yarn.role.priority":"1"
+ },
+ "hbase-master": {
+ "yarn.memory": "257",
+ "yarn.component.instances": "1",
+ "yarn.role.priority":"2",
+ "yarn.label.expression":""
+ },
+ "hbase-rs": {
+ "yarn.memory": "258",
+ "yarn.component.instances": "0",
+ "yarn.role.priority":"4",
+ "yarn.label.expression":"coquelicot && amaranth"
+ }
+ }
+}
diff --git a/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json b/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
new file mode 100644
index 0000000..bc6429c
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
@@ -0,0 +1,9 @@
+{
+ "nodes": ["/users/example/services/org-apache-slider/test-registry-rest-resources/components"], "service": {
+ "description": "Slider Application Master",
+ "yarn:id": "application_1411664296263_0001",
+ "yarn:persistence": 1,
+ "external": [],
+ "internal": []
+}
+}
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index f4b1261..b5ef62e 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
</parent>
<properties>
<work.dir>package-tmp</work.dir>
@@ -119,34 +119,15 @@
<slider.bin.dir>../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
<test.app.pkg.dir>../app-packages/command-logger/slider-pkg/target</test.app.pkg.dir>
<test.app.pkg.file>apache-slider-command-logger.zip</test.app.pkg.file>
+ <test.app.pkg.name>CMD_LOGGER</test.app.pkg.name>
<test.app.resource>../slider-core/src/test/app_packages/test_command_log/resources.json</test.app.resource>
<test.app.template>../slider-core/src/test/app_packages/test_command_log/appConfig.json</test.app.template>
+ <vagrant.current.working.dir></vagrant.current.working.dir>
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>**/httpfs-signature.secret</exclude>
- <exclude>**/regionservers</exclude>
- </excludes>
- </configuration>
- </plugin>
-
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
@@ -191,8 +172,6 @@
<reporting>
<plugins>
-
-
</plugins>
</reporting>
@@ -309,7 +288,11 @@
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <version>0.1.51</version>
+ </dependency>
</dependencies>
@@ -345,7 +328,34 @@
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
+ <profile>
+ <id>rat</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/httpfs-signature.secret</exclude>
+ <exclude>**/regionservers</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/BuildPidPath.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/BuildPidPath.groovy
new file mode 100644
index 0000000..3daa2f4
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/BuildPidPath.groovy
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.chaos.remote
+
+import org.apache.hadoop.conf.Configured
+
+/**
+ *
+ */
+class BuildPidPath extends Configured {
+
+ public static final String PID_DIR = "/var/run/hadoop"
+
+
+ String buildPidName(String user, String service) {
+ "${user}-${service}"
+ }
+
+ File buildPidPath(String pidName) {
+ new File(PID_DIR, "hadoop-${pidName}.pid")
+ }
+
+ /**
+ * Find the Pid string value of a hadoop process
+ * @param user user ID
+ * @param service
+ * @return the pid as a string or null for none found. If the string
+ * is empty it means the pid file was empty -there's no validation of
+ * it.
+ */
+ String findPid(String pidName) {
+ File pidFile = buildPidPath(pidName)
+ if (!pidFile.exists()) {
+ return null
+ } else {
+ return pidFile.text.trim()
+ }
+ }
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/Clustat.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/Clustat.groovy
new file mode 100644
index 0000000..742aa2e
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/Clustat.groovy
@@ -0,0 +1,64 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.chaos.remote
+
+/**
+ * Parses RHAT HA Clustat files pulled back over the network
+ */
+class Clustat {
+
+ Node xml;
+
+ Clustat(String source) {
+ xml = new XmlParser().parseText(source)
+ }
+
+/**
+ * ser
+ * @param service something like service:NameNodeService
+ * @return the owner
+ */
+ Node serviceGroup(String service) {
+ NodeList groups = xml.groups
+ def found = groups.group.findAll { it.@name == service}
+ if (found.empty) {
+ return null
+ }
+ Node serviceGroup = found[0]
+ return serviceGroup
+ }
+
+ String hostRunningService(String service) {
+ Node serviceGroup = serviceGroup("service:NameNodeService")
+ return serviceGroup?.@owner
+ }
+
+ /**
+ * Get all the nodes in the cluster
+ * @return the XML nodes element; every node is an elt underneath with @name==server
+ */
+ NodeList clusterNodes() {
+ return xml.nodes.node
+ }
+
+ Node clusterNode(String name) {
+ def all = clusterNodes() findAll { it.@name == name}
+ return all.size() > 0 ? all[0] : null
+ }
+
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/JschToCommonsLog.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/JschToCommonsLog.groovy
new file mode 100644
index 0000000..49eb856
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/JschToCommonsLog.groovy
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.chaos.remote
+
+import com.jcraft.jsch.Logger
+import org.apache.commons.logging.Log;
+
+/**
+ * relay from jsch to commons logging.
+ * JSCH is very chatty, so it is downgraded to debug on its info ops
+ */
+
+
+class JschToCommonsLog implements Logger {
+
+
+ final Log log
+
+ JschToCommonsLog(Log log) {
+ this.log = log
+ }
+
+ @Override
+ void log(int level, String message) {
+ switch (level) {
+ case FATAL:
+ log.fatal(message);
+ break;
+ case ERROR:
+ log.error(message);
+ break;
+ case WARN:
+ log.info(message);
+ break;
+ case INFO:
+ case DEBUG:
+ if (message.contains("Caught an exception")) {
+ log.debug(message, new Exception('here'))
+ }
+ log.debug(message);
+ break;
+ default:
+ log.info(message);
+ break;
+ }
+ }
+
+ @Override
+ boolean isEnabled(int level) {
+ switch (level) {
+ case FATAL:
+ return log.isFatalEnabled()
+ case ERROR:
+ return log.isErrorEnabled()
+ case WARN:
+ return log.isInfoEnabled()
+ case INFO:
+ case DEBUG:
+ return log.isDebugEnabled()
+ }
+ return false
+ }
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteDaemonOperations.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteDaemonOperations.groovy
new file mode 100644
index 0000000..946fbf7
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteDaemonOperations.groovy
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.chaos.remote
+
+/**
+ * Operations to perform on a remote init.d service by ssh-ing in and issuing
+ * the command as root.
+ * Use the full name, such as "hadoop-namenode"
+ */
+class RemoteDaemonOperations {
+
+ public static final int TIMEOUT = 0
+ RemoteServer server
+ String name;
+
+ RemoteDaemonOperations(RemoteServer server, String name) {
+ this.server = server
+ this.name = name
+ }
+
+ def start() {
+ exec("start")
+ }
+
+ def stop() {
+ exec("stop")
+ }
+
+ def restart() {
+ exec("restart")
+ }
+
+ def status() {
+ exec("status")
+ }
+
+ /**
+ *
+ * @param action action to exec
+ * @return ( rv , out )
+ */
+ List exec(String action) {
+ server.exec([["service", name, action], "exit"])
+ }
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteServer.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteServer.groovy
new file mode 100644
index 0000000..5dc9dab
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteServer.groovy
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.chaos.remote
+
+import com.jcraft.jsch.JSch
+import com.jcraft.jsch.Session
+import com.jcraft.jsch.UIKeyboardInteractive
+import com.jcraft.jsch.UserInfo
+import groovy.util.logging.Commons
+import org.apache.commons.logging.LogFactory
+
+/**
+ * A remote server you can SSH to
+ */
+@Commons
+class RemoteServer implements UserInfo, UIKeyboardInteractive {
+
+
+ boolean publicKeyAuth = true
+ File publicKeyFile
+ File knownHosts
+ String username
+ String password
+ String host
+ int port = 22
+ int timeout = 60000
+ int connectTimeout = 10000
+ private JSch jsch
+
+ @Override
+ String getPassphrase() {
+ return password
+ }
+
+ @Override
+ boolean promptPassword(String message) {
+ log.debug("Password prompt: $message")
+ return false
+ }
+
+ @Override
+ boolean promptPassphrase(String message) {
+ log.debug("Passphrase prompt: $message")
+ return false
+ }
+
+ /**
+ * This is for the "do you trust this host" message. We do
+ * @param message
+ * @return
+ */
+ @Override
+ boolean promptYesNo(String message) {
+ log.debug("YesNo prompt: $message")
+ return true
+ }
+
+ @Override
+ String[] promptKeyboardInteractive(String destination, String name, String instruction, String[] prompt, boolean[] echo) {
+ return new String[0]
+ }
+
+ @Override
+ void showMessage(String message) {
+ log.info(message)
+ }
+
+ private synchronized JSch createJschInstance() {
+ if (jsch) {
+ return jsch
+ }
+ jsch = new JSch()
+ //do not try and convert a groovy shortcut as overloaded methods then cause confusion
+ jsch.setLogger(new JschToCommonsLog(LogFactory.getLog("org.apache.chaos.remote.RemoteServer.ssh")))
+ if (publicKeyAuth) {
+ assert publicKeyFile
+ assert publicKeyFile.exists() && publicKeyFile.canRead()
+ jsch.addIdentity(publicKeyFile.toString(), (String) password);
+ } else {
+ assert password != null
+ }
+ if (knownHosts) {
+ assert knownHosts.exists() && knownHosts.canRead()
+ jsch.setKnownHosts(knownHosts.toString())
+ }
+ jsch
+ }
+
+ public JSch getJschInstance() {
+ return jsch ?: createJschInstance()
+ }
+
+ Session connect(JSch jsch, String host, int port) {
+ assert host
+ Session session = jsch.getSession(username, host, port)
+ session.userInfo = this
+ if (!publicKeyAuth) {
+ session.setPassword(password)
+ }
+ session.timeout = timeout
+ session.connect(connectTimeout)
+ session
+ }
+
+ /**
+ * Connect to the (host, port)
+ * @return a new session
+ */
+ Session connect() {
+ return connect(jschInstance, host, port)
+ }
+
+ /**
+ * Connect to the server and issue the commands, one per line. Nest arguments
+ * in inner lists for better structure
+ * @param args multi-dimensional list
+ * @return ( return value , output )
+ */
+ List exec(List args) {
+ def session = connect()
+ SshCommands cmds = new SshCommands(session)
+ return cmds.exec(args)
+ }
+
+ List command(String command) {
+ def session = connect()
+ SshCommands cmds = new SshCommands(session)
+ def (status, text) = cmds.command([command])
+ log.info("$status : $text")
+ [status, text]
+ }
+
+ /**
+ * Connect to a host and kill a process there
+ * @param pidFile the PID file
+ * @param signal the signal
+ * @return the outcome of the command
+ */
+ List kill(int signal, String pidFile) {
+ return command("kill -$signal `cat $pidFile`");
+ }
+
+ Clustat clustat() {
+ def (status, text) = command("clustat -x")
+ if (status != 0) {
+ throw new IOException("Clustat command failed [$status]: $text")
+ }
+ Clustat clustat = new Clustat(text)
+ return clustat
+ }
+
+ void waitForServerLive(int timeout) {
+ long endtime = System.currentTimeMillis() + timeout
+ boolean live = false;
+
+ while (!live && System.currentTimeMillis() < endtime) {
+ try {
+ command("true")
+ live = true
+ break
+ } catch (IOException ignored) {
+ Thread.sleep(1000)
+ }
+
+ }
+
+ }
+
+ @Override
+ String toString() {
+ return "Remote server user=$username @ host=$host"
+ }
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/SshCommands.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/SshCommands.groovy
new file mode 100644
index 0000000..334475d
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/SshCommands.groovy
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.chaos.remote
+
+import com.jcraft.jsch.ChannelExec
+import com.jcraft.jsch.ChannelShell
+import com.jcraft.jsch.Session
+import groovy.util.logging.Commons
+import org.apache.tools.ant.util.KeepAliveOutputStream
+import org.apache.tools.ant.util.TeeOutputStream
+
+/**
+ * Send lists strings of commands
+ */
+@Commons
+class SshCommands {
+
+ private final Session session
+ private volatile boolean stop = false
+ private volatile Thread thread
+ long commandsTimeout = 10000
+
+ SshCommands(Session session) {
+ this.session = session
+ }
+
+ private synchronized void begin() {
+ thread = Thread.currentThread()
+ Thread.start("ssh command waker") {
+ sleep(commandsTimeout);
+ stop()
+ }
+ }
+
+ private synchronized end() {
+ thread = null
+ stop = false
+ }
+
+ public synchronized void stop() {
+ if (!stop) {
+ stop = true;
+ if (thread) {
+ thread.interrupt()
+ }
+ }
+ }
+
+ /**
+ * Execute a list of commands, return the exit status. Output goes to the
+ * output stream
+ * @param args , one per line -all converted to strings first. If the list
+ * element is itself a collection, it is flattened then joined with spaces. This
+ * can be used to build up more complex commands.
+ * @return the result of the execution -the last response from the command
+ */
+ List exec(List args) {
+ assert session && session.connected
+ List<String> commandSet = args.collect() {
+ (it instanceof Collection) ? (it.flatten().join(" ")) : it.toString()
+ }
+ String commandLine = commandSet.join("\n")
+ log.info("commands: $commandLine")
+ ChannelShell channel = null
+ try {
+ begin()
+ channel = (ChannelShell) session.openChannel("shell")
+
+ //connect triggers a linkup and the streaming
+ channel.connect();
+ assert channel.connected
+ channel.pty = true
+ OutputStream toServer = channel.outputStream
+ Reader fromServer = new BufferedReader(
+ new InputStreamReader(channel.inputStream, "UTF-8"));
+ toServer.write(commandLine.bytes)
+ toServer.flush()
+ StringBuilder builder = new StringBuilder()
+
+ String line
+ while (!channel.closed && !stop) {
+ line = fromServer.readLine()
+ StringBuilder append = builder.append(line)
+ log.debug(line);
+ }
+
+ int status = channel.exitStatus
+ if (Thread.interrupted()) {
+ log.debug("Interrupted while waiting for the end of the SSH stream")
+ throw new InterruptedIOException()
+ }
+ return [status, builder.toString()]
+ } finally {
+ end()
+ boolean interrupted = Thread.interrupted()
+ channel?.disconnect()
+ }
+ }
+ /**
+ * Execute a single of commands, return the exit status. Output goes to the
+ * output stream
+ * @param args -a list that is flattened then joined with spaces. This
+ * can be used to build up more complex commands.
+ * @return the result of the execution -the response from the command
+ */
+ List command(String arg) {
+ return command([arg])
+ }
+
+ List command(List args) {
+ assert session && session.connected
+ session.timeout = commandsTimeout
+ String commandLine = args.join(" ")
+ log.info("commands: $commandLine")
+ ChannelExec channel = null
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ TeeOutputStream tee =
+ new TeeOutputStream(out,
+ KeepAliveOutputStream.wrapSystemOut());
+
+ try {
+ begin()
+ channel = (ChannelExec) session.openChannel("exec")
+
+
+ channel.setCommand(commandLine);
+ channel.setOutputStream(tee);
+ channel.setExtOutputStream(tee);
+ channel.pty = true
+
+ //connect triggers a linkup and the streaming
+ channel.connect();
+ assert channel.connected
+
+ Thread thread = Thread.start() {
+ while (!channel.isClosed()) {
+ if (thread == null) {
+ return;
+ }
+ try {
+ sleep(500);
+ } catch (Exception e) {
+ log.debug("In channel thread $e", e)
+ // ignored
+ }
+ }
+ }
+
+ thread.join(commandsTimeout);
+ if (thread.isAlive()) {
+ // ran out of time
+ thread = null;
+ throw new IOException("Timeout executing command " + commandLine);
+ }
+
+ String textResult = out.toString("UTF-8")
+ int status = channel.exitStatus
+ if (Thread.interrupted()) {
+ log.debug("Interrupted while waiting for the end of the SSH stream")
+ throw new InterruptedIOException()
+ }
+ return [status, textResult]
+ } finally {
+ end()
+ channel?.disconnect()
+ }
+ }
+
+/* List exec(List args, int timeout) {
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream()
+ int rv = exec(outputStream, args, timeout)
+ String outstr = outputStream.toString("UTF-8")
+ [rv, outstr]
+}*/
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy
index e0b87f7..9f0c4d7 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy
@@ -137,7 +137,6 @@
def fs = HadoopFS.get(path.toUri(), conf)
assert fs.exists(path)
-
}
@Test
@@ -160,6 +159,9 @@
@Test
public void testSecuritySettingsValid() throws Throwable {
Configuration conf = loadSliderConf();
+ if (SliderUtils.isHadoopClusterSecure(conf)) {
+ UserGroupInformation.setLoginUser(null)
+ }
if (SliderUtils.maybeInitSecurity(conf)) {
log.info("Security enabled")
SliderUtils.forceLogin()
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 0a0ac16..668a264 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -20,6 +20,8 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -35,16 +37,21 @@
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
public static final boolean AGENTTESTS_ENABLED
+ public static final boolean AGENTTESTS_QUEUE_LABELED_DEFINED
+ public static final boolean AGENTTESTS_LABELS_RED_BLUE_DEFINED
+ public static final boolean AGENTTESTS_AM_FAILURES_ENABLED
private static String TEST_APP_PKG_DIR_PROP = "test.app.pkg.dir"
private static String TEST_APP_PKG_FILE_PROP = "test.app.pkg.file"
+ private static String TEST_APP_PKG_NAME_PROP = "test.app.pkg.name"
private static String TEST_APP_RESOURCE = "test.app.resource"
private static String TEST_APP_TEMPLATE = "test.app.template"
- protected String APP_RESOURCE = sysprop(TEST_APP_RESOURCE)
- protected String APP_TEMPLATE = sysprop(TEST_APP_TEMPLATE)
+ protected String APP_RESOURCE = getAppResource()
+ protected String APP_TEMPLATE = getAppTemplate()
public static final String TEST_APP_PKG_DIR = sysprop(TEST_APP_PKG_DIR_PROP)
public static final String TEST_APP_PKG_FILE = sysprop(TEST_APP_PKG_FILE_PROP)
+ public static final String TEST_APP_PKG_NAME = sysprop(TEST_APP_PKG_NAME_PROP)
protected static Path agentTarballPath;
@@ -55,6 +62,21 @@
static {
AGENTTESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, false)
+ AGENTTESTS_QUEUE_LABELED_DEFINED =
+ SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_QUEUE_LABELED_DEFINED, false)
+ AGENTTESTS_LABELS_RED_BLUE_DEFINED =
+ SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_LABELS_RED_BLUE_DEFINED, false)
+ AGENTTESTS_AM_FAILURES_ENABLED =
+ SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_AM_FAILURES_ENABLED,
+ AGENTTESTS_ENABLED)
+ }
+
+ protected String getAppResource() {
+ return sysprop(TEST_APP_RESOURCE)
+ }
+
+ protected String getAppTemplate() {
+ return sysprop(TEST_APP_TEMPLATE)
}
@Rule
@@ -64,103 +86,66 @@
assume(AGENTTESTS_ENABLED, "Agent tests disabled")
}
+ public static void assumeQueueNamedLabelDefined() {
+ assume(AGENTTESTS_QUEUE_LABELED_DEFINED, "Custom queue named labeled is not defined")
+ }
+
+ public static void assumeLabelsRedAndBlueAdded() {
+ assume(AGENTTESTS_LABELS_RED_BLUE_DEFINED, "Custom node labels not defined")
+ }
+
+ public static void assumeAmFailureTestsEnabled() {
+ assume(AGENTTESTS_AM_FAILURES_ENABLED, "AM failure tests are disabled")
+ }
+
@BeforeClass
public static void setupAgent() {
assumeAgentTestsEnabled()
-
+ def uploader = new FileUploader(SLIDER_CONFIG,
+ UserGroupInformation.currentUser)
+ uploader.mkHomeDir();
}
@Before
- public void uploadAgentTarball() {
- def agentUploads = new AgentUploads(SLIDER_CONFIG)
- (agentTarballPath, agtIniPath) =
- agentUploads.uploadAgentFiles(SLIDER_TAR_DIRECTORY, false)
- }
-
-
- @Before
public void setupApplicationPackage() {
try {
- AgentUploads agentUploads = new AgentUploads(SLIDER_CONFIG)
- agentUploads.uploader.mkHomeDir()
-
- appPkgPath = new Path(clusterFS.homeDirectory, TEST_APP_PKG_FILE)
- if (clusterFS.exists(appPkgPath)) {
- clusterFS.delete(appPkgPath, false)
- log.info "Existing app pkg deleted from $appPkgPath"
- }
-
File zipFileName = new File(TEST_APP_PKG_DIR, TEST_APP_PKG_FILE).canonicalFile
- agentUploads.uploader.copyIfOutOfDate(zipFileName, appPkgPath, false)
- assume(clusterFS.exists(appPkgPath), "App pkg not uploaded to $appPkgPath")
- log.info "App pkg uploaded at $appPkgPath"
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_INSTALL_PACKAGE,
+ ARG_NAME, TEST_APP_PKG_NAME,
+ ARG_PACKAGE, zipFileName.absolutePath,
+ ARG_REPLACE_PKG
+ ])
+ logShell(shell)
+ log.info "App pkg uploaded at home directory .slider/package/$TEST_APP_PKG_NAME/$TEST_APP_PKG_FILE"
} catch (Exception e) {
setup_failed = true
- fail("Setup failed "+e)
+ throw e;
}
}
- public static void logShell(SliderShell shell) {
- for (String str in shell.out) {
- log.info str
- }
- }
-
- public static void assertComponentCount(String component, int count, SliderShell shell) {
- log.info("Asserting component count.")
- String entry = findLineEntry(shell, ["instances", component] as String[])
- log.info(entry)
- assert entry != null
- int instanceCount = 0
- int index = entry.indexOf("container_")
- while (index != -1) {
- instanceCount++;
- index = entry.indexOf("container_", index + 1)
- }
-
- assert instanceCount == count, 'Instance count for component did not match expected. Parsed: ' + entry
- }
-
public static String findLineEntry(SliderShell shell, String[] locaters) {
- int index = 0;
- for (String str in shell.out) {
- if (str.contains("\"" + locaters[index] + "\"")) {
- if (locaters.size() == index + 1) {
- return str;
- } else {
- index++;
- }
- }
- }
+ return shell.findLineEntry(locaters)
+ }
- return null;
+ public static boolean containsString(SliderShell shell, String lookThisUp, int n = 1) {
+ return shell.outputContains(lookThisUp, n)
}
public static String findLineEntryValue(SliderShell shell, String[] locaters) {
- String line = findLineEntry(shell, locaters);
-
- if (line != null) {
- log.info("Parsing {} for value.", line)
- int dividerIndex = line.indexOf(":");
- if (dividerIndex > 0) {
- String value = line.substring(dividerIndex + 1).trim()
- if (value.endsWith(",")) {
- value = value.subSequence(0, value.length() - 1)
- }
- return value;
- }
- }
- return null;
+ return shell.findLineEntryValue(locaters)
}
public static void addDir(File dirObj, ZipOutputStream zipFile, String prefix) {
- dirObj.eachFile() { file ->
+ dirObj.eachFile() {File file ->
if (file.directory) {
addDir(file, zipFile, prefix + file.name + File.separator)
} else {
log.info("Adding to zip - " + prefix + file.getName())
zipFile.putNextEntry(new ZipEntry(prefix + file.getName()))
- file.eachByte(1024) { buffer, len -> zipFile.write(buffer, 0, len) }
+ file.eachByte(1024) {
+ byte[] buffer, int len -> zipFile.write(buffer, 0, len) }
zipFile.closeEntry()
}
}
@@ -172,19 +157,25 @@
return
}
- log.info "Cleaning app instance, if exists, by name " + applicationName
+ describe "Teardown app instance " + applicationName
+ // forced freeze with wait
teardown(applicationName)
-
- // sleep till the instance is frozen
- sleep(1000 * 3)
-
SliderShell shell = slider([
ACTION_DESTROY,
applicationName])
if (shell.ret != 0 && shell.ret != EXIT_UNKNOWN_INSTANCE) {
logShell(shell)
- assert fail("Old cluster either should not exist or should get destroyed.")
+ assert fail("Old cluster either should not exist or should get destroyed; destroy exit code = ${shell.ret}")
}
}
+
+ /**
+ * Assert that the application is running (i.e in state
+ * {@link YarnApplicationState#RUNNING})
+ * @param appId application ID
+ */
+ def assertAppRunning(String appId) {
+ assertInYarnState(appId, YarnApplicationState.RUNNING)
+ }
}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 278bd2b..1cb245a 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -22,16 +22,25 @@
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hdfs.HdfsConfiguration
+import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.hadoop.util.ExitUtil
+import org.apache.hadoop.util.Shell
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.common.tools.ConfigHelper
+import org.apache.slider.core.launch.SerializedApplicationReport
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.client.SliderClient
+import org.apache.slider.core.persist.ApplicationReportSerDeser
import org.apache.slider.test.SliderTestUtils
+import org.apache.slider.test.Outcome;
+
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Rule
@@ -39,6 +48,7 @@
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import static org.apache.slider.common.SliderExitCodes.*
+import static org.apache.slider.core.main.LauncherExitCodes.*
import static org.apache.slider.funtest.framework.FuntestProperties.*
import static org.apache.slider.common.params.Arguments.*
import static org.apache.slider.common.params.SliderActions.*
@@ -56,6 +66,9 @@
public static final File SLIDER_SCRIPT = new File(
SLIDER_TAR_DIRECTORY,
BIN_SLIDER).canonicalFile
+ public static final File SLIDER_SCRIPT_PYTHON = new File(
+ SLIDER_TAR_DIRECTORY,
+ BIN_SLIDER_PYTHON).canonicalFile
public static final File SLIDER_CONF_DIRECTORY = new File(
SLIDER_CONF_DIR).canonicalFile
public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
@@ -67,11 +80,31 @@
public static final int SLIDER_TEST_TIMEOUT
public static final String YARN_RAM_REQUEST
-
+ /**
+ * Keytab for secure cluster
+ */
+ public static final String TEST_AM_KEYTAB
+ static File keytabFile
+ /**
+ * shell-escaped ~ symbol. On windows this does
+ * not need to be escaped
+ */
+ public static final String TILDE
+ public static final int CONTAINER_LAUNCH_TIMEOUT = 90000
+ public static final int PROBE_SLEEP_TIME = 4000
+ public static final int REGISTRY_STARTUP_TIMEOUT = 60000
+ public static final String E_LAUNCH_FAIL = 'Application did not start'
+
+ /*
+ Static initializer for test configurations. If this code throws exceptions
+ (which it may) the class will not be instantiable.
+ */
static {
- SLIDER_CONFIG = ConfLoader.loadSliderConf(SLIDER_CONF_XML);
+ new HdfsConfiguration()
+ ConfigHelper.registerDeprecatedConfigItems();
+ SLIDER_CONFIG = ConfLoader.loadSliderConf(SLIDER_CONF_XML, true);
THAW_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
KEY_TEST_THAW_WAIT_TIME,
1000 * DEFAULT_THAW_WAIT_TIME_SECONDS)
@@ -82,10 +115,16 @@
KEY_TEST_TIMEOUT,
1000 * DEFAULT_TEST_TIMEOUT_SECONDS)
- YARN_RAM_REQUEST = SLIDER_CONFIG.get(
+ YARN_RAM_REQUEST = SLIDER_CONFIG.getTrimmed(
KEY_TEST_YARN_RAM_REQUEST,
DEFAULT_YARN_RAM_REQUEST)
+
+ TEST_AM_KEYTAB = SLIDER_CONFIG.getTrimmed(
+ KEY_TEST_AM_KEYTAB)
+
+
+ TILDE = Shell.WINDOWS? "~" : "\\~"
}
@Rule
@@ -95,21 +134,55 @@
@BeforeClass
public static void setupTestBase() {
Configuration conf = loadSliderConf();
+
+ SliderShell.confDir = SLIDER_CONF_DIRECTORY
+
+ // choose python script if on windows or the launch key recommends it
+ //
+ boolean python = SLIDER_CONFIG.getBoolean(KEY_LAUNCH_PYTHON, false)
+ SliderShell.scriptFile =
+ (SliderShell.windows || python) ? SLIDER_SCRIPT_PYTHON : SLIDER_SCRIPT
+
+ //set the property of the configuration directory
+ def path = SLIDER_CONF_DIRECTORY.absolutePath
+ SLIDER_CONFIG.set(ENV_SLIDER_CONF_DIR, path)
+ // locate any hadoop conf dir
+ def hadoopConf = SLIDER_CONFIG.getTrimmed(ENV_HADOOP_CONF_DIR)
+ if (hadoopConf) {
+ File hadoopConfDir = new File(hadoopConf).canonicalFile
+ // propagate the value to the client config
+ SliderShell.setEnv(ENV_HADOOP_CONF_DIR, hadoopConfDir.absolutePath)
+ }
+
if (SliderUtils.maybeInitSecurity(conf)) {
log.debug("Security enabled")
SliderUtils.forceLogin()
+ // now look for the security key
+/*
+ if (!TEST_AM_KEYTAB) {
+ fail("Security keytab is not defined in $KEY_TEST_AM_KEYTAB")
+ }
+ keytabFile = new File(TEST_AM_KEYTAB)
+ if (!keytabFile.exists()) {
+ throw new FileNotFoundException("Security keytab ${keytabFile} " +
+ " defined in $KEY_TEST_AM_KEYTAB")
+ }
+*/
+
} else {
log.info "Security is off"
}
- SliderShell.confDir = SLIDER_CONF_DIRECTORY
- SliderShell.script = SLIDER_SCRIPT
+
log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " +
"and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
+ }
+ public static void logShell(SliderShell shell) {
+ shell.dumpOutput();
}
/**
- * give our thread a name
+ * give the test thread a name
*/
@Before
public void nameThread() {
@@ -117,8 +190,28 @@
}
/**
- * Add a jar to the slider classpath
- * @param clazz
+ * Add a configuration file at a given path
+ * @param dir directory containing the file
+ * @param filename filename
+ * @return true if the file was found
+ * @throws IOException loading problems (other than a missing file)
+ */
+ public static boolean maybeAddConfFile(File dir, String filename) throws IOException {
+ File confFile = new File(dir, filename)
+ if (confFile.isFile()) {
+ ConfigHelper.addConfigurationFile(SLIDER_CONFIG, confFile, true)
+ log.debug("Loaded $confFile")
+ return true;
+ } else {
+ log.debug("Did not find $confFile —skipping load")
+ return false;
+ }
+ }
+
+ /**
+ * Add a jar to the slider classpath by looking up a class and determining
+ * its containing JAR
+ * @param clazz class inside the JAR
*/
public static void addExtraJar(Class clazz) {
def jar = SliderUtils.findContainingJarOrFail(clazz)
@@ -129,6 +222,12 @@
}
}
+ /**
+ * Resolve a system property, throwing an exception if it is not present
+ * @param key property name
+ * @return the value
+ * @throws RuntimeException if the property is not set
+ */
public static String sysprop(String key) {
def property = System.getProperty(key)
if (!property) {
@@ -138,6 +237,22 @@
}
/**
+ * Print to system out
+ * @param string
+ */
+ static void println(String s) {
+ System.out.println(s)
+ }
+
+ /**
+ * Print a newline to system out
+ * @param string
+ */
+ static void println() {
+ System.out.println()
+ }
+
+ /**
* Exec any slider command
* @param conf
* @param commands
@@ -164,7 +279,7 @@
* @return
*/
public static Configuration loadSliderConf() {
- Configuration conf = ConfLoader.loadSliderConf(SLIDER_CONF_XML)
+ Configuration conf = ConfLoader.loadSliderConf(SLIDER_CONF_XML, true)
return conf
}
@@ -220,7 +335,7 @@
}
/**
- * Freeze cluster: no exit code checking
+ * Stop cluster: no exit code checking
* @param name
* @param args
* @return
@@ -230,20 +345,7 @@
}
static SliderShell freezeForce(String name) {
- freeze(name, [ARG_FORCE])
- }
-
- static SliderShell getConf(String name) {
- slider([
- ACTION_GETCONF, name
- ])
- }
-
- static SliderShell getConf(int result, String name) {
- slider(result,
- [
- ACTION_GETCONF, name
- ])
+ freeze(name, [ARG_FORCE, ARG_WAIT, "10000"])
}
static SliderShell killContainer(String name, String containerID) {
@@ -265,14 +367,26 @@
slider(cmd)
}
- static SliderShell list(int result, String name) {
- List<String> cmd = [
- ACTION_LIST
- ]
- if (name != null) {
- cmd << name
+ static SliderShell lookup(int result, String id, File out) {
+ assert id
+ def commands = [ACTION_LOOKUP, ARG_ID, id]
+ if (out) {
+ commands += [ARG_OUTPUT, out.absolutePath]
}
- slider(result, cmd)
+ slider(result, commands)
+ }
+
+ static SliderShell lookup(String id, File out) {
+ assert id
+ def commands = [ACTION_LOOKUP, ARG_ID, id]
+ if (out) {
+ commands += [ARG_OUTPUT, out.absolutePath]
+ }
+ slider(commands)
+ }
+
+ static SliderShell list(int result, Collection<String> commands =[]) {
+ slider(result, [ACTION_LIST] + commands )
}
static SliderShell status(String name) {
@@ -305,6 +419,12 @@
slider(0, [ACTION_THAW, name] + args)
}
+ static SliderShell resolve(int result, Collection<String> commands) {
+ slider(result,
+ [ACTION_RESOLVE] + commands
+ )
+ }
+
static SliderShell registry(int result, Collection<String> commands) {
slider(result,
[ACTION_REGISTRY] + commands
@@ -337,6 +457,7 @@
* @param cluster
*/
static void setupCluster(String cluster) {
+ describe "setting up $cluster"
ensureClusterDestroyed(cluster)
}
@@ -350,7 +471,7 @@
}
/**
- * Assert the exit code is that the cluster is unknown
+ * Assert the exit code is that the cluster is 0
* @param shell shell
*/
public static void assertSuccess(SliderShell shell) {
@@ -375,6 +496,26 @@
}
/**
+ * Assert that the stdout/stderr streams of the shell contain the string
+ * to look for.
+ * If the assertion does not hold, the output is logged before
+ * the assertion is thrown
+ * @param shell
+ * @param lookThisUp
+ * @param n number of times (default = 1)
+ */
+ public static void assertOutputContains(
+ SliderShell shell,
+ String lookThisUp,
+ int n = 1) {
+ if (!shell.outputContains(lookThisUp)) {
+ log.error("Missing $lookThisUp from:")
+ shell.dumpOutput()
+ fail("Missing $lookThisUp from:\n$shell.out\n$shell.err" )
+ }
+ }
+
+ /**
* Create a connection to the cluster by execing the status command
*
* @param clustername
@@ -423,7 +564,7 @@
List<String> argsList = [action, clustername]
argsList << ARG_ZKHOSTS <<
- SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM)
+ SLIDER_CONFIG.getTrimmed(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
if (blockUntilRunning) {
@@ -473,6 +614,162 @@
clusterOps)
}
+ /**
+ * Create a templated slider app.
+ * <p>
+ * If the extraArgs list does not contain a --wait parm then a wait
+ * duration of THAW_WAIT_TIME will be added to the launch args.
+ * @param name name
+ * @param appTemplate application template
+ * @param resourceTemplate resource template
+ * @param extraArgs list of extra arguments to the command
+ * @param launchReportFile optional file to save the AM launch report to
+ * @return the shell
+ */
+ public SliderShell createTemplatedSliderApplication(
+ String name,
+ String appTemplate,
+ String resourceTemplate,
+ List<String> extraArgs = [],
+ File launchReportFile = null) {
+
+ if (!launchReportFile) {
+ launchReportFile = createTempJsonFile()
+ }
+ // delete any previous copy of the file
+ launchReportFile.delete();
+
+ List<String> commands = [
+ ACTION_CREATE, name,
+ ARG_TEMPLATE, appTemplate,
+ ARG_RESOURCES, resourceTemplate,
+ ARG_OUTPUT, launchReportFile.absolutePath
+ ]
+
+ if (!extraArgs.contains(ARG_WAIT)) {
+ commands << ARG_WAIT << Integer.toString(THAW_WAIT_TIME)
+ }
+
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME));
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR));
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH));
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL));
+ commands.addAll(extraArgs)
+ SliderShell shell = new SliderShell(commands)
+ if (0 != shell.execute()) {
+ // app has failed.
+
+ // grab the app report of the last known instance of this app
+ // which may not be there if it was a config failure; may be out of date
+ // from a previous run
+ log.error("Launch failed with exit code ${shell.ret}")
+ shell.dumpOutput()
+
+ // now grab that app report if it is there
+ def appReport = maybeLookupFromLaunchReport(launchReportFile)
+ String extraText = ""
+ if (appReport) {
+ log.error("Application report:\n$appReport")
+ extraText = appReport.diagnostics
+ }
+
+ fail("Application Launch Failure, exit code ${shell.ret}\n${extraText}")
+ }
+ return shell
+ }
+
+ /**
+ * Create a temp JSON file. After coming up with the name, the file
+ * is deleted
+ * @return the filename
+ */
+ public static File createTempJsonFile() {
+ return tmpFile(".json")
+ }
+
+ public static File tmpFile(String suffix) {
+ File reportFile = File.createTempFile(
+ "launch",
+ suffix,
+ new File("target"))
+ reportFile.delete()
+ return reportFile
+ }
+
+ /**
+ * If the option is not null/empty, add the command and the option
+ * @param args arg list being built up
+ * @param command command to add option
+ * @param option option to probe and use
+ * @return the (possibly extended) list
+ */
+ public static List<String> maybeAddCommandOption(
+ List<String> args, List<String> commands, String option) {
+ if ( SliderUtils.isSet(option)) {
+ args.addAll(commands)
+ args << option
+ }
+ return args
+ }
+
+ public static SerializedApplicationReport maybeLoadAppReport(File reportFile) {
+ if (reportFile.exists() && reportFile.length()> 0) {
+ ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
+ def report = serDeser.fromFile(reportFile)
+ return report
+ }
+ return null;
+ }
+
+ public static SerializedApplicationReport loadAppReport(File reportFile) {
+ if (reportFile.exists() && reportFile.length()> 0) {
+ ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
+ def report = serDeser.fromFile(reportFile)
+ return report
+ } else {
+ throw new FileNotFoundException(reportFile.absolutePath)
+ }
+ }
+
+ public static SerializedApplicationReport maybeLookupFromLaunchReport(File launchReport) {
+ def report = maybeLoadAppReport(launchReport)
+ if (report) {
+ return lookupApplication(report.applicationId)
+ } else {
+ return null
+ }
+ }
+
+ /**
+ * Lookup an application, return null if loading failed
+ * @param id application ID
+ * @return an application report or null
+ */
+ public static SerializedApplicationReport lookupApplication(String id) {
+ File reportFile = createTempJsonFile();
+ try {
+ def shell = lookup(id, reportFile)
+ if (shell.ret == 0) {
+ return maybeLoadAppReport(reportFile)
+ } else {
+ log.warn("Lookup operation failed with ${shell.ret}")
+ shell.dumpOutput()
+ return null
+ }
+ } finally {
+ reportFile.delete()
+ }
+ }
+
+
public Path buildClusterPath(String clustername) {
return new Path(
clusterFS.homeDirectory,
@@ -491,69 +788,481 @@
ARG_MESSAGE, "suicide"
])
+ sleep(5000)
+ ensureApplicationIsUp(cluster)
+ return sliderClient.clusterDescription
+ }
+ /**
+ * Kill an AM and await restrt
+ * @param sliderClient
+ * @param application
+ * @param appId
+ * @return
+ */
+ public void killAmAndWaitForRestart(String application, String appId) {
- def sleeptime = SLIDER_CONFIG.getInt(KEY_AM_RESTART_SLEEP_TIME,
- DEFAULT_AM_RESTART_SLEEP_TIME)
- sleep(sleeptime)
- ClusterDescription status
+ assert application
+ slider(0, [
+ ACTION_AM_SUICIDE, application,
+ ARG_EXITCODE, "1",
+ ARG_WAIT, "500",
+ ARG_MESSAGE, "suicide"
+ ])
+ // app gets accepted
+ log.info "awaiting app to enter ACCEPTED state"
+ awaitYarnApplicationAccepted(appId)
+ // app goes live
+ log.info "awaiting app to enter RUNNING state"
+ ensureYarnApplicationIsUp(appId)
+ }
+
+ /**
+ * Spinning operation to perform a registry call
+ * @param application application
+ */
+ protected void ensureRegistryCallSucceeds(String application) {
+ repeatUntilSuccess("registry",
+ this.&isRegistryAccessible,
+ REGISTRY_STARTUP_TIMEOUT,
+ PROBE_SLEEP_TIME,
+ [application: application],
+ true,
+ "Application registry is not accessible after $REGISTRY_STARTUP_TIMEOUT") {
+ describe "Not able to access registry after after $REGISTRY_STARTUP_TIMEOUT"
+ exists(application, true).dumpOutput()
+ SliderShell shell = registry(0, [
+ ARG_NAME,
+ application,
+ ARG_LISTEXP
+ ])
+ }
+ }
+
+ /**
+ * wait for an application to come up
+ * @param application
+ */
+ protected void ensureApplicationIsUp(String application) {
+ repeatUntilSuccess("await application up",
+ this.&isApplicationRunning,
+ instanceLaunchTime,
+ PROBE_SLEEP_TIME,
+ [application: application],
+ true,
+ E_LAUNCH_FAIL) {
+ describe "final state of app that tests say is not up"
+ exists(application, true).dumpOutput()
+ }
+ }
+
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map containing <code>"application"</code>
+ * @return probe outcome
+ */
+ protected Outcome isRegistryAccessible(Map<String, String> args) {
+ String applicationName = args['application'];
+ SliderShell shell = slider(
+ [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ applicationName,
+ ARG_LISTEXP
+ ])
+ if (EXIT_SUCCESS != shell.execute()) {
+ logShell(shell)
+ }
+ return Outcome.fromBool(EXIT_SUCCESS == shell.execute())
+ }
+
+ /**
+ * Probe for an application running; uses <code>exists</code> operation
+ * @param args argument map containing <code>"application"</code>
+ * @return
+ */
+ protected Outcome isApplicationRunning(Map<String, String> args) {
+ String applicationName = args['application'];
+ return Outcome.fromBool(isApplicationUp(applicationName))
+ }
+
+ /**
+ * Use <code>exists</code> operation to probe for an application being up
+ * @param applicationName app name
+ * @return true if it s running
+ */
+ protected boolean isApplicationUp(String applicationName) {
+ return isApplicationInState(
+ applicationName,
+ YarnApplicationState.RUNNING
+ );
+ }
+
+ /**
+ * is an application in a desired yarn state. Uses the <code>exists</code>
+ * CLI operation
+ * @param yarnState
+ * @param applicationName
+ * @return
+ */
+ public static boolean isApplicationInState(
+ String applicationName,
+ YarnApplicationState yarnState) {
+ SliderShell shell = slider(
+ [ACTION_EXISTS, applicationName, ARG_STATE, yarnState.toString()])
+ return shell.ret == 0
+ }
+
+ /**
+ * Probe callback for is the the app running or not
+ * @param args map where 'applicationId' must m
+ * @return
+ */
+
+ protected static Outcome isYarnApplicationRunning(Map<String, String> args) {
+ String applicationId = args['applicationId'];
+ return isYarnApplicationInState(applicationId,
+ YarnApplicationState.RUNNING, true)
+ }
+
+ /**
+ * Probe callback for is the the app running or not
+ * @param args map where 'applicationId' must m
+ * @return
+ */
+ protected static Outcome isYarnApplicationInExactState(Map<String, String> args) {
+ String applicationId = args['applicationId'];
+ String state = args['state']
+ def desired = YarnApplicationState.valueOf(state)
+ return isYarnApplicationInState(applicationId, desired, false)
+ }
+
+ /**
+ * is a yarn application in a desired yarn state
+ * @param yarnState
+ * @param applicationName
+ * @return an outcome indicating whether the app is at the state, on its way
+ * or has gone past
+ */
+ public static Outcome isYarnApplicationRunning(
+ String applicationId) {
+ return isYarnApplicationInState(applicationId,
+ YarnApplicationState.RUNNING, true)
+ }
+
+ /**
+ * Probe for a YARN application being in a given state
+ * @param applicationId app id
+ * @param yarnStat desired state
+ * @return success for a match, retry if state below desired, and fail if
+ * above it
+ */
+ public static Outcome isYarnApplicationInState(
+ String applicationId, YarnApplicationState yarnState, boolean failfast) {
+ YarnApplicationState appState = lookupYarnAppState(applicationId)
+ if (yarnState == appState) {
+ return Outcome.Success;
+ }
+
+ if (failfast && appState.ordinal() > yarnState.ordinal()) {
+ log.debug("App state $appState past desired state $yarnState: failing")
+ // app has passed beyond hope
+ return Outcome.Fail
+ }
+ return Outcome.Retry
+ }
+
+ /**
+ * Look up the YARN application by ID, get its application record
+ * @param applicationId the application ID
+ * @return the application state
+ */
+ public static YarnApplicationState lookupYarnAppState(String applicationId) {
+ def sar = lookupApplication(applicationId)
+ assert sar != null;
+ YarnApplicationState appState = YarnApplicationState.valueOf(sar.state)
+ return appState
+ }
+
+ /**
+ * Assert an application is in a given state; fail if not
+ * @param applicationId appId
+ * @param expectedState expected state
+ */
+ public static void assertInYarnState(String applicationId,
+ YarnApplicationState expectedState) {
+ def applicationReport = lookupApplication(applicationId)
+ assert expectedState.toString() == applicationReport.state
+ }
+
+ /**
+ * Wait for the YARN app to come up. This will fail fast
+ * @param launchReportFile launch time file containing app id
+ * @return the app ID
+ */
+ protected String ensureYarnApplicationIsUp(File launchReportFile) {
+ def id = loadAppReport(launchReportFile).applicationId
+ ensureYarnApplicationIsUp(id)
+ return id;
+ }
+
+ /**
+ * Wait for the YARN app to come up. This will fail fast
+ * @param applicationId
+ */
+ protected void ensureYarnApplicationIsUp(String applicationId) {
+ repeatUntilSuccess("await yarn application Running",
+ this.&isYarnApplicationRunning,
+ instanceLaunchTime,
+ PROBE_SLEEP_TIME,
+ [applicationId: applicationId],
+ true,
+ E_LAUNCH_FAIL) {
+ describe "final state of application"
+ def sar = lookupApplication(applicationId)
+
+ def message = E_LAUNCH_FAIL + "\n$sar"
+ log.error(message)
+ fail(message)
+ }
+ }
+
+ /**
+ * Wait for the YARN app to come up. This will fail fast
+ * @param applicationId
+ */
+ protected void awaitYarnApplicationAccepted(String applicationId) {
+ repeatUntilSuccess("Await Yarn App Accepted",
+ this.&isYarnApplicationInExactState,
+ instanceLaunchTime,
+ 1000,
+ [applicationId: applicationId,
+ state: YarnApplicationState.ACCEPTED.toString()],
+ true,
+ "application never reached accepted state") {
+ describe "app did not enter accepted"
+ def sar = lookupApplication(applicationId)
+
+ def message = 'Application did not enter ACCEPTED state' + "\n$sar"
+ log.error(message)
+ fail(message)
+ }
+ }
+ /**
+ * Get the expected launch time. Default is the configuration option
+ * {@link FuntestProperties#KEY_TEST_INSTANCE_LAUNCH_TIME} and
+ * default value {@link FuntestProperties#KEY_TEST_INSTANCE_LAUNCH_TIME}
+ * @return
+ */
+ public int getInstanceLaunchTime() {
+ return SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
+ DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS) * 1000;
+ }
+
+ public String getInfoAmWebUrl(String applicationName) {
+ ClusterDescription cd = execStatus(applicationName);
+ String urlString = cd.getInfo("info.am.web.url");
+ return urlString;
+ }
+
+ public ClusterDescription execStatus(String application) {
+ ClusterDescription cd
+ File statusFile = File.createTempFile("status", ".json")
try {
- // am should have restarted it by now
- // cluster is live
- exists(0, cluster, true)
+ slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ application,
+ ARG_OUTPUT, statusFile.absolutePath
+ ])
- status = sliderClient.clusterDescription
- } catch (SliderException e) {
- if (e.exitCode == EXIT_BAD_STATE) {
- log.error(
- "Property $YarnConfiguration.RM_AM_MAX_ATTEMPTS may be too low")
- }
- throw e;
- }
- return status
- }
-
- protected void ensureApplicationIsUp(String clusterName) {
- repeatUntilTrue(this.&isApplicationUp, 15, 1000 * 3, ['arg1': clusterName],
- true, 'Application did not start, aborting test.')
- }
-
- protected boolean isApplicationUp(Map<String, String> args) {
- String applicationName = args['arg1'];
- return isApplicationInState("RUNNING", applicationName);
- }
-
- public static boolean isApplicationInState(String text, String applicationName) {
- boolean exists = false
- SliderShell shell = slider(0,
- [
- ACTION_LIST,
- applicationName])
- for (String str in shell.out) {
- if (str.contains(text)) {
- exists = true
- }
- }
-
- return exists
- }
-
- protected void repeatUntilTrue(Closure c, int maxAttempts, int sleepDur, Map args,
- boolean failIfUnsuccessful = false, String message = "") {
- int attemptCount = 0
- while (attemptCount < maxAttempts) {
- if (c(args)) {
- break
- };
- attemptCount++;
-
- if (failIfUnsuccessful) {
- assert attemptCount != maxAttempts, message
- }
-
- sleep(sleepDur)
+ assert statusFile.exists()
+ return ClusterDescription.fromFile(statusFile)
+ } finally {
+ statusFile.delete()
}
}
+ public int queryRequestedCount(String application, String role) {
+ ClusterDescription cd = execStatus(application)
+
+ if (cd.statistics.size() == 0) {
+ log.debug("No statistics entries")
+ }
+
+ if (!cd.statistics[role]) {
+ log.debug("No stats for role $role")
+ return 0;
+ }
+ def statsForRole = cd.statistics[role]
+
+ def requested = statsForRole[StatusKeys.STATISTICS_CONTAINERS_REQUESTED]
+ assert null != statsForRole[StatusKeys.STATISTICS_CONTAINERS_REQUESTED]
+ return requested
+ }
+
+ /**
+ * Probe: has the requested container count of a specific role been reached?
+ * @param args map with: "application", "role", "limit"
+ * @return success on a match, retry if not
+ */
+ Outcome hasRequestedContainerCountReached(Map<String, String> args) {
+ String application = args['application']
+ String role = args['role']
+ int expectedCount = args['limit'].toInteger();
+
+ int requestedCount = queryRequestedCount(application, role)
+ log.debug("requested $role count = $requestedCount; expected=$expectedCount")
+ return Outcome.fromBool(requestedCount >= expectedCount)
+ }
+
+ void expectContainerRequestedCountReached(String application, String role, int limit,
+ int container_launch_timeout) {
+
+ repeatUntilSuccess(
+ "await container count",
+ this.&hasRequestedContainerCountReached,
+ container_launch_timeout,
+ PROBE_SLEEP_TIME,
+ [limit : Integer.toString(limit),
+ role : role,
+ application: application],
+ true,
+ "countainer count not reached") {
+ int requestedCount = queryRequestedCount(application, role)
+
+ def message = "expected count of $role = $limit not reached: $requestedCount" +
+ " after $container_launch_timeout mS"
+ describe message
+ ClusterDescription cd = execStatus(application);
+ log.info("Parsed status \n$cd")
+ fail(message)
+ }
+ }
+
+ public ClusterDescription assertContainersLive(String clustername,
+ String component,
+ int count) {
+ ClusterDescription cd = execStatus(clustername)
+ assertContainersLive(cd, component, count)
+ return cd;
+ }
+
+ /**
+ * Outcome checker for the live container count
+ * @param args argument map, must contain "application", "component" and "live"
+ * @return
+ */
+ Outcome hasLiveContainerCountReached(Map<String, String> args) {
+ assert args['application']
+ assert args['component']
+ assert args['live']
+ String application = args['application']
+ String component = args['component']
+ int expectedCount = args['live'].toInteger();
+ ClusterDescription cd = execStatus(application)
+ def actual = extractLiveContainerCount(cd, component)
+ log.debug(
+ "live $component count = $actual; expected=$expectedCount")
+ return Outcome.fromBool(actual >= expectedCount)
+ }
+
+ /**
+ * Wait for the live container count to be reached
+ * @param application application name
+ * @param component component name
+ * @param expected expected count
+ * @param container_launch_timeout launch timeout
+ */
+ void expectLiveContainerCountReached(
+ String application,
+ String component,
+ int expected,
+ int container_launch_timeout) {
+
+ repeatUntilSuccess(
+ "await live container count",
+ this.&hasLiveContainerCountReached,
+ container_launch_timeout,
+ PROBE_SLEEP_TIME,
+ [live : Integer.toString(expected),
+ component : component,
+ application: application],
+ true,
+ "countainer count not reached") {
+ describe "container count not reached"
+ assertContainersLive(application, component, expected)
+ }
+ }
+
+ /**
+ * Spin for <code>REGISTRY_STARTUP_TIMEOUT</code> waiting
+ * for the output of the registry command to contain the specified
+ * values
+ * @param outfile file to save the output to
+ * @param commands commands to execute
+ * @param match text to match on
+ */
+ public void awaitRegistryOutfileContains(
+ File outfile,
+ List<String> commands,
+ String match) {
+ def errorText = "failed to find $match in output. Check your hadoop versions and the agent logs"
+ repeatUntilSuccess("registry",
+ this.&generatedFileContains,
+ REGISTRY_STARTUP_TIMEOUT,
+ PROBE_SLEEP_TIME,
+ [
+ text : match,
+ filename: outfile.absolutePath,
+ command : commands
+ ],
+ true,
+ errorText) {
+ slider(0, commands).dumpOutput()
+ if (!outfile.length()) {
+ log.warn("No exported entries.\n" +
+ "Is your application using a consistent version of Hadoop? ")
+ }
+ fail(errorText + "\n" + outfile.text)
+ }
+ }
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map containing <code>"application"</code>
+ * @return probe outcome
+ */
+ protected Outcome commandOutputContains(Map args) {
+ String text = args['text'];
+ List<String> command = (List < String >)args['command']
+ SliderShell shell = slider(0, command)
+ return Outcome.fromBool(shell.outputContains(text))
+ }
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map containing <code>"application"</code>
+ * @return probe outcome
+ */
+ protected Outcome commandSucceeds(Map args) {
+ List<String> command = (List<String>) args['command']
+ SliderShell shell = slider(command)
+ return Outcome.fromBool(shell.ret == 0)
+ }
+ /**
+ * Is the registry accessible for an application?
+ * @param args argument map
+ * @return probe outcome
+ */
+ protected Outcome generatedFileContains(Map args) {
+ List<String> command = (List<String>) args['command']
+ String text = args['text'];
+ String filename = args['filename'];
+ File f = new File(filename)
+ f.delete()
+ SliderShell shell = slider(0, command)
+ shell.dumpOutput()
+ assert f.exists()
+ return Outcome.fromBool(f.text.contains(text))
+ }
}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy
index 447ea87..dcc699d 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy
@@ -18,17 +18,69 @@
package org.apache.slider.funtest.framework
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.tools.ConfigHelper
+import static org.apache.slider.funtest.framework.FuntestProperties.*
+
+@Slf4j
public class ConfLoader {
- public static YarnConfiguration loadSliderConf(File confFile) {
+ public static YarnConfiguration loadSliderConf(File confFile, boolean loadHadoopConfDir = true) {
URI confURI = confFile.toURI();
YarnConfiguration conf = new YarnConfiguration()
def confXmlUrl = confURI.toURL()
conf.addResource(confXmlUrl)
- conf.set(FuntestProperties.KEY_TEST_CONF_XML, confXmlUrl.toString())
- conf.set(FuntestProperties.KEY_TEST_CONF_DIR, CommandTestBase.SLIDER_CONF_DIRECTORY.absolutePath)
+ conf.set(KEY_TEST_CONF_XML, confXmlUrl.toString())
+
+ def sliderConfDir = confFile.parent
+ conf.set(KEY_TEST_CONF_DIR, sliderConfDir)
+ conf.set(ENV_SLIDER_CONF_DIR, sliderConfDir)
+ ConfigHelper.addEnvironmentVariables(conf, ENV_PREFIX)
+
+ if (loadHadoopConfDir) {
+ def hadoopConf = conf.getTrimmed(ENV_HADOOP_CONF_DIR)
+ if (hadoopConf) {
+ File hadoopConfDir = new File(hadoopConf).canonicalFile
+ def hadoopConfDirPath = hadoopConfDir.absolutePath
+ if (!hadoopConfDir.directory) {
+ throw new FileNotFoundException(hadoopConfDirPath)
+ }
+ log.debug("$ENV_HADOOP_CONF_DIR=$hadoopConfDirPath —loading")
+ // a conf dir has been set. Load the values locally
+ maybeAddConfFile(conf, hadoopConfDir, CORE_SITE_XML)
+ maybeAddConfFile(conf, hadoopConfDir, HDFS_SITE_XML)
+ maybeAddConfFile(conf, hadoopConfDir, YARN_SITE_XML)
+ // now need to force in the original again because otherwise
+ // the -site.xml values will overwrite the -client ones
+ ConfigHelper.addConfigurationFile(conf, confFile, true)
+ }
+
+ }
+
return conf
}
+
+ /**
+ * Add a configuration file at a given path
+ * @param dir directory containing the file
+ * @param filename filename
+ * @return true if the file was found
+ * @throws IOException loading problems (other than a missing file)
+ */
+ public static boolean maybeAddConfFile(Configuration conf, File dir, String filename)
+ throws IOException {
+ File confFile = new File(dir, filename)
+ if (confFile.file) {
+ ConfigHelper.addConfigurationFile(conf, confFile, true)
+ log.info("Loaded ${confFile.absolutePath}")
+ return true;
+ } else {
+ log.info("Did not find ${confFile.absolutePath} —skipping load")
+ return false;
+ }
+ }
+
}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
index 921adbf..70f119e 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -145,6 +145,7 @@
public void attemptToCreateHomeDir(String username, Path home) {
def privilegedFS = getFileSystemAsUserName(username)
+ log.info "Creating home dir $home as user ${user.userName} group ${user.primaryGroupName}"
privilegedFS.mkdirs(home, new FsPermission((short) 00755))
privilegedFS.setOwner(home, user.userName, user.primaryGroupName)
}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 1096dfa..92a7a95 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -50,16 +50,33 @@
String CLIENT_CONFIG_FILENAME = SliderKeys.CLIENT_RESOURCE
- String ENV_CONF_DIR = "SLIDER_CONF_DIR"
+ String ENV_SLIDER_CONF_DIR = "SLIDER_CONF_DIR"
+ String ENV_HADOOP_CONF_DIR = "HADOOP_CONF_DIR"
String ENV_SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
-
+
String SCRIPT_NAME = "slider"
String KEY_TEST_CONF_XML = "slider.test.conf.xml"
String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
String BIN_SLIDER = "bin/slider"
+ String BIN_SLIDER_PYTHON = "bin/slider.py"
String AGENT_INI = "agent.ini"
String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI
String AGENT_TAR_FILENAME = "slider-agent.tar.gz"
String AGENT_SLIDER_GZ_IN_SLIDER_TAR = "agent/" + AGENT_TAR_FILENAME
+
+
+ String KEY_TEST_INSTANCE_LAUNCH_TIME =
+ "slider.test.instance.launch.wait.seconds";
+ int DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS = 60 * 3;
+
+ String ENV_PREFIX = "env."
+ String CORE_SITE_XML = "core-site.xml"
+ String HDFS_SITE_XML = "hdfs-site.xml"
+ String YARN_SITE_XML = "yarn-site.xml"
+
+ /**
+ * Flag to indicate that the .py script should be launched: {@value}
+ */
+ String KEY_LAUNCH_PYTHON = "slider.test.launch.python"
}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 804e791..ae40d6a 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -27,9 +27,10 @@
class SliderShell extends Shell {
private static final Logger log = LoggerFactory.getLogger(SliderShell.class);
-
+ private static final Logger LOG = log;
public static final String BASH = '/bin/bash -s'
+ public static final String CMD = 'cmd'
/**
* Configuration directory, shared across all instances. Not marked as volatile,
@@ -37,10 +38,17 @@
*/
public static File confDir;
- public static File script;
+ public static File scriptFile;
+
+ public File shellScript;
public static final List<String> slider_classpath_extra = []
+ /**
+ * Environment varaibles
+ */
+ protected static final Map<String, String> environment = [:]
+
final String command
/**
@@ -48,10 +56,11 @@
* @param commands
*/
SliderShell(Collection<String> commands) {
- super(BASH)
+ super(org.apache.hadoop.util.Shell.WINDOWS ? CMD : BASH)
assert confDir != null;
- assert script != null;
- command = script.absolutePath + " " + commands.join(" ")
+ assert scriptFile != null;
+ shellScript = scriptFile;
+ command = scriptFile.absolutePath + " " + commands.join(" ")
}
/**
@@ -59,25 +68,80 @@
* @return the script exit code
*/
int execute() {
- String confDirCmd = env(FuntestProperties.ENV_CONF_DIR, confDir)
log.info(command)
- List<String> commandLine = [
- confDirCmd,
- ]
+ setEnv(FuntestProperties.ENV_SLIDER_CONF_DIR, confDir)
if (!slider_classpath_extra.empty) {
- commandLine << env(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA,
- SliderUtils.join(slider_classpath_extra, ":", false))
+ setEnv(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA,
+ SliderUtils.join(slider_classpath_extra,
+ pathElementSeparator,
+ false))
}
+ List<String> commandLine = buildEnvCommands()
+
commandLine << command
+ if (org.apache.hadoop.util.Shell.WINDOWS) {
+ // Ensure the errorlevel returned by last call is set for the invoking shell
+ commandLine << "@echo ERRORLEVEL=%ERRORLEVEL%"
+ commandLine << "@exit %ERRORLEVEL%"
+ }
String script = commandLine.join("\n")
log.debug(script)
- super.exec(script);
+ exec(script);
signCorrectReturnCode()
return ret;
}
- String env(String var, Object val) {
- return "export " + var + "=${val.toString()};"
+ public String getPathElementSeparator() {
+ File.pathSeparator
+ }
+
+ public static boolean isWindows() {
+ return org.apache.hadoop.util.Shell.WINDOWS
+ }
+
+ /**
+ * Set an environment variable
+ * @param var variable name
+ * @param val value
+ */
+ public static void setEnv(String var, Object val) {
+ environment[var] = val.toString()
+ }
+
+ /**
+ * Get an environment variable
+ * @param var variable name
+ * @return the value or null
+ */
+ public static String getEnv(String var) {
+ return environment[var]
+ }
+
+ /**
+ * Build up a list of environment variable setters from the
+ * env variables
+ * @return a list of commands to set up the env on the target system.
+ */
+ public static List<String> buildEnvCommands() {
+ List<String> commands = []
+ environment.each { String var, String val ->
+ commands << env(var, val)
+ }
+ return commands
+ }
+
+ /**
+ * Add an environment variable
+ * @param var variable
+ * @param val value (which will be stringified)
+ * @return an env variable command
+ */
+ static String env(String var, Object val) {
+ if (isWindows()) {
+ return "set " + var + "=${val.toString()}"
+ } else {
+ return "export " + var + "=${val.toString()};"
+ }
}
/**
@@ -131,26 +195,163 @@
log.error(toString())
log.error("return code = ${signCorrectReturnCode()}")
if (out.size() != 0) {
- log.info("\n<stdout>\n${out.join('\n')}\n</stdout>");
+ log.info("\n<stdout>\n${stdoutHistory}\n</stdout>");
}
if (err.size() != 0) {
- log.error("\n<stderr>\n${err.join('\n')}\n</stderr>");
+ log.error("\n<stderr>\n${stdErrHistory}\n</stderr>");
}
}
-
+
+ /**
+ * Get the stderr history
+ * @return the history
+ */
+ public String getStdErrHistory() {
+ return err.join('\n')
+ }
+
+ /**
+ * Get the stdout history
+ * @return the history
+ */
+ public String getStdoutHistory() {
+ return out.join('\n')
+ }
+
/**
* Assert the shell exited with a given error code
* if not the output is printed and an assertion is raised
* @param errorCode expected error code
*/
- public void assertExitCode(int errorCode) {
+ public void assertExitCode(int errorCode, String extra="") {
if (this.ret != errorCode) {
dumpOutput()
throw new SliderException(ret,
- "Expected exit code of command ${command} : ${errorCode} - actual=${ret}")
-
+ "Expected exit code of command ${command} : ${errorCode} - actual=${ret} $extra")
}
}
-
+ /**
+ * Execute shell script consisting of as many Strings as we have arguments,
+ * NOTE: individual strings are concatenated into a single script as though
+ * they were delimited with new line character. All quoting rules are exactly
+ * what one would expect in standalone shell script.
+ *
+ * After executing the script its return code can be accessed as getRet(),
+ * stdout as getOut() and stderr as getErr(). The script itself can be accessed
+ * as getScript()
+ * WARNING: it isn't thread safe
+ * @param args shell script split into multiple Strings
+ * @return Shell object for chaining
+ */
+ Shell exec(Object... args) {
+ Process proc = "$shell".execute()
+ script = args.join("\n")
+ ByteArrayOutputStream baosErr = new ByteArrayOutputStream(4096);
+ ByteArrayOutputStream baosOut = new ByteArrayOutputStream(4096);
+ proc.consumeProcessOutput(baosOut, baosErr)
+
+ Thread.start {
+ def writer = new PrintWriter(new BufferedOutputStream(proc.out))
+ writer.println(script)
+ writer.flush()
+ writer.close()
+ }
+
+ proc.waitFor()
+ ret = proc.exitValue()
+
+ out = streamToLines(baosOut)
+ err = streamToLines(baosErr)
+
+ if (LOG.isTraceEnabled()) {
+ if (ret != 0) {
+ LOG.trace("return: $ret");
+ }
+ if (out.size() != 0) {
+ LOG.trace("\n<stdout>\n${out.join('\n')}\n</stdout>");
+ }
+
+ if (err.size() != 0) {
+ LOG.trace("\n<stderr>\n${err.join('\n')}\n</stderr>");
+ }
+ }
+ return this
}
+
+ /**
+ * Convert a stream to lines in an array
+ * @param out output stream
+ * @return the list of entries
+ */
+ protected List<String> streamToLines(ByteArrayOutputStream out) {
+ if (out.size() != 0) {
+ return out.toString().split('\n');
+ } else {
+ return [];
+ }
+ }
+
+ public String findLineEntry(String[] locaters) {
+ int index = 0;
+ def output = out +"\n"+ err
+ for (String str in output) {
+ if (str.contains("\"" + locaters[index] + "\"")) {
+ if (locaters.size() == index + 1) {
+ return str;
+ } else {
+ index++;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ public boolean outputContains(
+ String lookThisUp,
+ int n = 1) {
+ int count = 0
+ def output = out + "\n" + err
+ for (String str in output) {
+ int subCount = countString(str, lookThisUp)
+ count = count + subCount
+ if (count == n) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static int countString(String str, String search) {
+ int count = 0
+ if (SliderUtils.isUnset(str) || SliderUtils.isUnset(search)) {
+ return count
+ }
+
+ int index = str.indexOf(search, 0)
+ while (index >= 0) {
+ ++count
+ index = str.indexOf(search, index + 1)
+ }
+ return count
+ }
+
+ public findLineEntryValue(String[] locaters) {
+ String line = findLineEntry(locaters);
+
+ if (line != null) {
+ log.info("Parsing {} for value.", line)
+ int dividerIndex = line.indexOf(":");
+ if (dividerIndex > 0) {
+ String value = line.substring(dividerIndex + 1).trim()
+ if (value.endsWith(",")) {
+ value = value.subSequence(0, value.length() - 1)
+ }
+ return value;
+ }
+ }
+ return null;
+ }
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
index 9b8fe6f..9826e97 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
@@ -21,6 +21,7 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.hadoop.net.NetUtils
+import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.client.SliderYarnClientImpl
import org.apache.slider.common.SliderXmlConfKeys
@@ -52,7 +53,8 @@
@Test
public void testZKBinding() throws Throwable {
- def quorum = SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM)
+ def quorum = SLIDER_CONFIG.getTrimmed(
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
assert quorum
def tuples = ZookeeperUtils.splitToHostsAndPortsStrictly(quorum);
tuples.each {
@@ -63,13 +65,17 @@
@Test
public void testRMTelnet() throws Throwable {
- def rmAddr = SLIDER_CONFIG.getSocketAddr(YarnConfiguration.RM_ADDRESS, "", 0)
- telnet(rmAddr.hostName, rmAddr.port)
+ def isHaEnabled = SLIDER_CONFIG.getTrimmed(YarnConfiguration.RM_HA_ENABLED)
+ // Telnet test is not straight forward for HA setup and is not required
+ // as long as RM binding is tested
+ if (!isHaEnabled) {
+ def rmAddr = SLIDER_CONFIG.getSocketAddr(YarnConfiguration.RM_ADDRESS, "", 0)
+ telnet(rmAddr.hostName, rmAddr.port)
+ }
}
@Test
public void testRMBinding() throws Throwable {
- testRMTelnet()
SliderYarnClientImpl yarnClient = new SliderYarnClientImpl()
try {
SLIDER_CONFIG.setInt("ipc.client.connect.retry.interval",100)
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
new file mode 100644
index 0000000..728920e
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.basic
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup
+import org.apache.slider.test.SliderTestUtils
+import org.junit.Test
+
+/**
+ * Simple tests to verify that the build has been set up: if these
+ * fail then the arguments to the test run are incomplete.
+ *
+ * This deliberately doesn't depend on CommandTestBase,
+ * so that individual tests fail with more diagnostics
+ * than the @BeforeClass failing
+ */
+@Slf4j
+class SyspropsIT extends SliderTestUtils{
+
+
+ @Test
+ public void testDumpSysprops() throws Throwable {
+ def sysprops = System.properties
+ TreeSet<String> sorted = new TreeSet<String>();
+ sysprops.keys().each { String it -> sorted.add(it)}
+ sorted.each { String key ->
+ log.info("$key=\"${sysprops[key]}\"")
+ }
+
+ }
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy
new file mode 100644
index 0000000..2974b10
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.commands
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class CommandEnvironmentIT extends CommandTestBase {
+
+ public static final String TESTPROPERTY_SET_IN_FUNTEST = "testpropertySetInFuntest"
+ public static final String TEST_PROPERTY_VALUE = "TestPropertyValue"
+ File originalScript
+
+ @Before
+ public void cacheScript() {
+ originalScript = SliderShell.scriptFile
+ }
+
+ @After
+ public void restoreScript() {
+ SliderShell.scriptFile = originalScript
+ }
+
+ @Test
+ public void testJVMOptionPassdownBash() throws Throwable {
+ SliderShell shell = diagnostics(false)
+ assertOutputContains(shell, TESTPROPERTY_SET_IN_FUNTEST, 2)
+ assertOutputContains(shell, TEST_PROPERTY_VALUE, 2)
+ }
+
+ @Test
+ public void testJVMOptionPassdownPython() throws Throwable {
+ SliderShell shell = diagnostics(true)
+ assertOutputContains(shell, TESTPROPERTY_SET_IN_FUNTEST, 2)
+ assertOutputContains(shell, TEST_PROPERTY_VALUE, 2)
+ }
+
+ @Test
+ public void testLibdirPython() throws Throwable {
+ SliderShell shell = diagnostics(true)
+ assertOutputContains(shell, SliderKeys.PROPERTY_LIB_DIR)
+ }
+
+ @Test
+ public void testLibdirBash() throws Throwable {
+ SliderShell shell = diagnostics(false)
+ assertOutputContains(shell, SliderKeys.PROPERTY_LIB_DIR)
+ }
+
+
+ @Test
+ public void testConfdirPython() throws Throwable {
+ SliderShell shell = diagnostics(true)
+ assertOutputContains(shell, SliderKeys.PROPERTY_CONF_DIR)
+ }
+
+ @Test
+ public void testConfdirBash() throws Throwable {
+ SliderShell shell = diagnostics(false)
+ assertOutputContains(shell, SliderKeys.PROPERTY_CONF_DIR)
+ }
+
+ public SliderShell diagnostics(boolean python) {
+ if (python) {
+ SliderShell.scriptFile = SLIDER_SCRIPT_PYTHON;
+ } else {
+ assume(!SliderShell.windows, "skip bash test on windows")
+ SliderShell.scriptFile = SLIDER_SCRIPT;
+ }
+ SliderShell shell = new SliderShell([
+ SliderActions.ACTION_DIAGNOSTICS,
+ Arguments.ARG_CLIENT,
+ Arguments.ARG_VERBOSE
+ ])
+ shell.setEnv(SliderKeys.SLIDER_JVM_OPTS,
+ "-D${TESTPROPERTY_SET_IN_FUNTEST}=${TEST_PROPERTY_VALUE}")
+ shell.execute(0)
+ return shell
+ }
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
new file mode 100644
index 0000000..73912e6
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.commands
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class CommandExitCodesIT extends CommandTestBase {
+
+ @Test
+ public void testHelp() throws Throwable {
+ slider(0, ["help"])
+ }
+
+ @Test
+ public void testStopWithoutCluster() throws Throwable {
+ slider(LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR, ["stop"]).dumpOutput()
+ }
+
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
new file mode 100644
index 0000000..da58dd2
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.commands
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class DiagnosticsCommandIT extends CommandTestBase {
+
+ @Test
+ public void testClientDiagnostics() throws Throwable {
+ def shell = slider(0,
+ [
+ SliderActions.ACTION_DIAGNOSTICS,
+ Arguments.ARG_CLIENT,
+ Arguments.ARG_VERBOSE
+ ]
+ )
+ println(shell.stdoutHistory)
+ println()
+ println(shell.stdErrHistory)
+ }
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
index 20bac88..916117c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
@@ -20,6 +20,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.funtest.framework.CommandTestBase
import org.junit.BeforeClass
import org.junit.Test
@@ -30,7 +31,13 @@
@Test
public void testListAll() throws Throwable {
- assertSuccess(list(null))
+ assertSuccess(list(""))
+ }
+
+ @Test
+ public void testListAllLive() throws Throwable {
+ def shell = list("--live")
+ assert shell.ret == 0 || shell.ret == LauncherExitCodes.EXIT_FALSE
}
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ResolveCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ResolveCommandIT.groovy
new file mode 100644
index 0000000..04d5623
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ResolveCommandIT.groovy
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.commands
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.registry.client.api.RegistryConstants
+
+import static org.apache.slider.common.params.Arguments.*
+import static org.apache.slider.core.main.LauncherExitCodes.*;
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class ResolveCommandIT extends CommandTestBase {
+
+ @Test
+ public void testRegistryIsNotLocalhost() throws Throwable {
+ def quorum = SLIDER_CONFIG.get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
+ assert quorum != RegistryConstants.DEFAULT_REGISTRY_ZK_QUORUM;
+ }
+
+ @Test
+ public void testResolveRoot() throws Throwable {
+ resolve(0,
+ [ARG_LIST, ARG_PATH, "/"])
+ }
+
+ @Test
+ public void testResolveUnknownPath() throws Throwable {
+ resolve(EXIT_NOT_FOUND,
+ [ARG_LIST, ARG_PATH, "/undefined"])
+ }
+
+ @Test
+ public void testResolveRootServiceRecord() throws Throwable {
+ resolve(EXIT_NOT_FOUND,
+ [ARG_PATH, "/"])
+ }
+
+ @Test
+ public void testResolveHomeServiceRecord() throws Throwable {
+ resolve(EXIT_NOT_FOUND,
+ [ARG_PATH, "~"])
+ }
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy
index bf742c9..75d0634 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy
@@ -32,14 +32,12 @@
@Test
public void testVersion() throws Throwable {
- Shell shell = slider([SliderActions.ACTION_VERSION])
- assertSuccess(shell)
+ slider(0, [SliderActions.ACTION_VERSION])
}
@Test
public void testUsage() throws Throwable {
- SliderShell shell = slider(0, [SliderActions.ACTION_USAGE])
- assertSuccess(shell)
+ slider(0, [SliderActions.ACTION_HELP])
}
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/UnknownClusterOperationsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/UnknownClusterOperationsIT.groovy
index 39ae4dd..5515894 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/UnknownClusterOperationsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/UnknownClusterOperationsIT.groovy
@@ -21,12 +21,10 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.funtest.categories.FunctionalTests
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.CommandTestBase
import org.apache.slider.funtest.framework.SliderShell
-import org.junit.BeforeClass
import org.junit.Test
/**
@@ -34,7 +32,6 @@
*/
@CompileStatic
@Slf4j
-@org.junit.experimental.categories.Category(FunctionalTests)
public class UnknownClusterOperationsIT extends CommandTestBase {
public static final String UNKNOWN = "unknown_cluster"
@@ -92,9 +89,4 @@
assertUnknownCluster(status(UNKNOWN))
}
- @Test
- public void testGetConfUnknownCluster() throws Throwable {
- assertUnknownCluster(getConf(UNKNOWN))
- }
-
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
new file mode 100644
index 0000000..0544b22
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import com.jcraft.jsch.Session
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.chaos.remote.RemoteServer
+import org.apache.chaos.remote.SshCommands
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.BeforeClass;
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AMFailuresIT extends AgentCommandTestBase
+implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ private static String COMMAND_LOGGER = "COMMAND_LOGGER"
+ private static String APPLICATION_NAME = "am-failures-it"
+ public static final String TEST_REMOTE_SSH_KEY = "test.remote.ssh.key"
+ public static final String VAGRANT_CWD = "vagrant.current.working.dir"
+ File sshkey
+
+ @BeforeClass
+ public static void setupAMTests() {
+ assumeAmFailureTestsEnabled()
+ }
+
+ @After
+ public void destroyCluster() {
+ cleanup(APPLICATION_NAME)
+ }
+
+ @Test
+ public void testAMKilledWithStateAMStartedAgentsStarted() throws Throwable {
+ cleanup(APPLICATION_NAME)
+ File launchReportFile = createTempJsonFile();
+
+ SliderShell shell = createTemplatedSliderApplication(
+ APPLICATION_NAME, APP_TEMPLATE, APP_RESOURCE,
+ [],
+ launchReportFile)
+ logShell(shell)
+
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1,
+ CONTAINER_LAUNCH_TIMEOUT)
+
+ def cd = assertContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
+ def loggerInstances = cd.instances[COMMAND_LOGGER]
+ assert loggerInstances.size() == 1
+
+ def loggerStats = cd.statistics[COMMAND_LOGGER]
+
+ assert loggerStats["containers.requested"] == 1
+ assert loggerStats["containers.live"] == 1
+
+ // Now kill the AM
+ log.info("Killing AM now ...")
+ killAmAndWaitForRestart(APPLICATION_NAME, appId)
+
+ // There should be exactly 1 live logger container
+ def cd2 = assertContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
+
+ // No new containers should be requested for the agents
+ def restartedStats = cd2.statistics[COMMAND_LOGGER]
+ assert restartedStats["containers.live"] == 1
+
+ assert 0==restartedStats["containers.requested"],
+ 'No new agent containers should be requested'
+ assert lookupYarnAppState(appId) == YarnApplicationState.RUNNING
+ }
+
+ protected void killAMUsingVagrantShell() {
+ String hostname = SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS).split(":")[0]
+ assert hostname
+ String vagrantVmName = hostname.split("\\.")[0]
+
+ String vagrantCwd = sysprop(VAGRANT_CWD)
+ log.info("VAGRANT_CWD = {}", vagrantCwd)
+ File dirCheck = new File(vagrantCwd)
+ assert dirCheck.exists(), "Please set $VAGRANT_CWD to the directory which contains the Vagrantfile"
+
+ String cmd = "export VAGRANT_CWD=$vagrantCwd; /usr/bin/vagrant ssh -c \"sudo -u root runuser -l yarn " +
+ "-c \\\"ps -ef | grep SliderAppMaster | egrep -v 'grep|bash' | sed 's/^[a-z]* *\\([^ ]*\\) *.*/\\1/' " +
+ "| xargs kill -9\\\"\" $vagrantVmName 2>/dev/null"
+ log.info("Vagrant Shell Command = {}", cmd)
+
+ Shell vagrantShell = new Shell('/bin/bash -s')
+ vagrantShell.exec(cmd)
+ }
+
+ protected void killAMUsingJsch() {
+ String hostname = SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS).split(":")[0]
+ String user = UserGroupInformation.currentUser
+ assert hostname
+ assert user
+
+ bindSSHKey()
+ RemoteServer remoteServer = new RemoteServer(
+ host: hostname,
+ username: user,
+ publicKeyFile: sshkey)
+ Session session = remoteServer.connect()
+ SshCommands cmds = new SshCommands(session)
+// def (String rv, String out) = cmds.command([
+// "ps -ef | grep SliderAppMaster | egrep -v 'grep|bash' | xargs kill -9"
+// ])
+// assert rv == 0
+// log.info("Kill cmd output", out)
+ }
+
+ /**
+ * Bind to the SSH key -assert that the file actually exists
+ */
+ protected void bindSSHKey() {
+ sshkey = new File(sysprop("user.home"), ".ssh/id_rsa")
+ sshkey = new File(sysprop(TEST_REMOTE_SSH_KEY), sshkey.toString())
+ assert sshkey.exists()
+ }
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index 6b0f2bd..0bc4e39 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -20,6 +20,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
@@ -40,7 +41,7 @@
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
- static String CLUSTER = "test_agent_cluster_lifecycle"
+ static String CLUSTER = "test-agent-cluster-lifecycle"
static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
@@ -48,7 +49,6 @@
@Before
public void prepareCluster() {
setupCluster(CLUSTER)
- describe("Create a 0-role cluster, so testing AM start/stop")
}
@After
@@ -61,21 +61,26 @@
describe "Walk a 0-role cluster through its lifecycle"
+ // sanity check to verify the config is correct
+ assert clusterFS.uri.scheme != "file"
def clusterpath = buildClusterPath(CLUSTER)
assert !clusterFS.exists(clusterpath)
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, CLUSTER,
- ARG_IMAGE, agentTarballPath.toString(),
- ARG_TEMPLATE, APP_TEMPLATE,
- ARG_RESOURCES, APP_RESOURCE2
- ])
+ File launchReportFile = createTempJsonFile();
+ SliderShell shell = createTemplatedSliderApplication(CLUSTER,
+ APP_TEMPLATE,
+ APP_RESOURCE2,
+ [],
+ launchReportFile)
logShell(shell)
+ assert launchReportFile.exists()
+ assert launchReportFile.size() > 0
+ def launchReport = maybeLoadAppReport(launchReportFile)
+ assert launchReport;
- ensureApplicationIsUp(CLUSTER)
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
//at this point the cluster should exist.
assertPathExists(clusterFS, "Cluster parent directory does not exist", clusterpath.parent)
@@ -88,14 +93,20 @@
//destroy will fail in use
destroy(EXIT_APPLICATION_IN_USE, CLUSTER)
- //thaw will fail as cluster is in use
+ //start will fail as cluster is in use
thaw(EXIT_APPLICATION_IN_USE, CLUSTER)
//it's still there
exists(0, CLUSTER)
//listing the cluster will succeed
- list(0, CLUSTER)
+ list(0, [CLUSTER])
+
+ list(0, [""])
+ list(0, [CLUSTER, ARG_LIVE])
+ list(0, [CLUSTER, ARG_STATE, "running"])
+ list(0, [ARG_LIVE])
+ list(0, [ARG_STATE, "running"])
//simple status
status(0, CLUSTER)
@@ -105,7 +116,7 @@
try {
slider(0,
[
- SliderActions.ACTION_STATUS, CLUSTER,
+ ACTION_STATUS, CLUSTER,
ARG_OUTPUT, jsonStatus.canonicalPath
])
@@ -116,8 +127,6 @@
log.info(cd.toJsonString())
- getConf(0, CLUSTER)
-
//get a slider client against the cluster
SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
ClusterDescription cd2 = sliderClient.clusterDescription
@@ -125,25 +134,49 @@
log.info("Connected via Client {}", sliderClient.toString())
- //freeze
+ //stop
freeze(0, CLUSTER, [
ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
ARG_MESSAGE, "freeze-in-test-cluster-lifecycle"
])
describe " >>> Cluster is now frozen."
+
+ // should be in finished state, as this was a clean shutdown
+ assertInYarnState(appId, YarnApplicationState.FINISHED)
//cluster exists if you don't want it to be live
exists(0, CLUSTER, false)
//condition returns false if it is required to be live
exists(EXIT_FALSE, CLUSTER, true)
- //thaw then freeze the cluster
+ list( 0, [CLUSTER])
+ list( 0, [CLUSTER, ARG_STATE, "FINISHED"])
+ list(-1, [CLUSTER, ARG_LIVE])
+ list(-1, [CLUSTER, ARG_STATE, "running"])
+
+ list(-1, [ARG_LIVE])
+ list(-1, [ARG_STATE, "running"])
+ list( 0, [ARG_STATE, "FINISHED"])
+
+ def thawReport = createTempJsonFile()
+ //start then stop the cluster
thaw(CLUSTER,
[
ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
+ ARG_OUTPUT, thawReport.absolutePath,
])
+ def thawedAppId = ensureYarnApplicationIsUp(thawReport)
+
+
+ assertAppRunning(thawedAppId)
+
exists(0, CLUSTER)
describe " >>> Cluster is now thawed."
+ list(0, [CLUSTER, ARG_LIVE])
+ list(0, [CLUSTER, ARG_STATE, "running"])
+ list(0, [ARG_LIVE])
+ list(0, [ARG_STATE, "running"])
+ list(0, [CLUSTER, ARG_STATE, "FINISHED"])
freeze(0, CLUSTER,
[
@@ -152,28 +185,35 @@
ARG_MESSAGE, "forced-freeze-in-test"
])
- describe " >>> Cluster is now frozen - 2nd time."
+ describe " >>> Cluster is now force frozen - 2nd time."
+ // new instance should be in killed state
+ assertInYarnState(thawedAppId, YarnApplicationState.KILLED)
//cluster is no longer live
exists(0, CLUSTER, false)
//condition returns false if it is required to be live
exists(EXIT_FALSE, CLUSTER, true)
- //thaw with a restart count set to enable restart
+ //start with a restart count set to enable restart
describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
+
+ def thawReport2 = createTempJsonFile()
+ //start then stop the cluster
thaw(CLUSTER,
[
ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
- ARG_DEFINE, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=3"
+ ARG_DEFINE, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=3",
+ ARG_OUTPUT, thawReport2.absolutePath
])
-
+ def thawedAppId2 = ensureYarnApplicationIsUp(thawReport2)
describe " >>> Cluster is now thawed - 2nd time."
- ClusterDescription status = killAmAndWaitForRestart(sliderClient, CLUSTER)
describe " >>> Kill AM and wait for restart."
+ ClusterDescription status = killAmAndWaitForRestart(sliderClient, CLUSTER)
+ assertAppRunning(thawedAppId2)
def restarted = status.getInfo(
StatusKeys.INFO_CONTAINERS_AM_RESTART)
assert restarted != null
@@ -185,10 +225,14 @@
ARG_MESSAGE, "final-shutdown"
])
+ list(0, [CLUSTER, "--verbose", "--state", "FINISHED"]).dumpOutput()
+
destroy(0, CLUSTER)
//cluster now missing
exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+ list(0, [])
+ list(EXIT_UNKNOWN_INSTANCE, [CLUSTER])
} finally {
jsonStatus.delete()
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index 0ba48ba..39b5d6c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -47,58 +47,24 @@
@Test
public void testAgentFailHeartbeatingTwiceOnce() throws Throwable {
- if (!AGENTTESTS_ENABLED) {
- log.info "TESTS are not run."
- return
- }
-
+ assumeAgentTestsEnabled()
+
cleanup(APPLICATION_NAME)
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, APPLICATION_NAME,
- ARG_IMAGE, agentTarballPath.toString(),
- ARG_TEMPLATE, APP_TEMPLATE3,
- ARG_RESOURCES, APP_RESOURCE
- ])
+ File launchReportFile = createTempJsonFile();
+ SliderShell shell = createTemplatedSliderApplication(
+ APPLICATION_NAME, APP_TEMPLATE3, APP_RESOURCE,
+ [],
+ launchReportFile)
logShell(shell)
- ensureApplicationIsUp(APPLICATION_NAME)
-
- repeatUntilTrue(this.&hasContainerCountExceeded, 20, 1000 * 10, ['arg1': '3']);
-
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2,
+ CONTAINER_LAUNCH_TIMEOUT * 2)
sleep(1000 * 20)
-
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- assertComponentCount(COMMAND_LOGGER, 1, shell)
- String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- assert requested != null && requested.isInteger() && requested.toInteger() >= 3,
- 'At least 2 containers must be requested'
-
- assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
-
- assertSuccess(shell)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3,
+ CONTAINER_LAUNCH_TIMEOUT * 2)
+ assertAppRunning(appId)
}
-
- boolean hasContainerCountExceeded(Map<String, String> args) {
- int expectedCount = args['arg1'].toInteger();
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- //logShell(shell)
- String requested = findLineEntryValue(
- shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
- return true
- }
-
- return false
- }
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index a51c769..7d1be89 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -39,7 +39,6 @@
private static String APP_TEMPLATE2 =
"../slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json"
-
@After
public void destroyCluster() {
cleanup(APPLICATION_NAME)
@@ -48,57 +47,28 @@
@Test
public void testAgentFailRegistrationOnce() throws Throwable {
if (!AGENTTESTS_ENABLED) {
- log.info "TESTS are not run."
- return
+ skip("Agent tests are not run.")
}
cleanup(APPLICATION_NAME)
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, APPLICATION_NAME,
- ARG_IMAGE, agentTarballPath.toString(),
- ARG_TEMPLATE, APP_TEMPLATE2,
- ARG_RESOURCES, APP_RESOURCE
- ])
+ File launchReportFile = createTempJsonFile();
+ SliderShell shell = createTemplatedSliderApplication(
+ APPLICATION_NAME,
+ APP_TEMPLATE2,
+ APP_RESOURCE,
+ [],
+ launchReportFile)
logShell(shell)
- ensureApplicationIsUp(APPLICATION_NAME)
-
- repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '2']);
-
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2,
+ CONTAINER_LAUNCH_TIMEOUT)
sleep(1000 * 20)
-
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- assertComponentCount(COMMAND_LOGGER, 1, shell)
- String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- assert requested != null && requested.isInteger() && requested.toInteger() >= 2,
- 'At least 2 containers must be requested'
-
- assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
-
- assertSuccess(shell)
+ assertAppRunning(appId)
+ def cd = assertContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
+ assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 2
+ assertAppRunning(appId)
}
-
- boolean hasContainerCountExceeded(Map<String, String> args) {
- int expectedCount = args['arg1'].toInteger();
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
-
- //logShell(shell)
- String requested = findLineEntryValue(
- shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
- if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
- return true
- }
-
- return false
- }
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
new file mode 100644
index 0000000..51a9010
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.server.appmaster.SliderAppMaster
+
+import static org.apache.slider.api.InternalKeys.*
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AgentLaunchFailureIT extends AgentCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+
+ static String CLUSTER = "test-agent-launchfail"
+
+ static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(CLUSTER)
+ }
+
+ @After
+ public void destroyCluster() {
+ cleanup(CLUSTER)
+ }
+
+ @Test
+ public void testAgentLaunchFailure() throws Throwable {
+ describe("Create a failing cluster and validate failure logic")
+
+ // verify no cluster
+ assert 0 != exists(CLUSTER).ret
+
+ // create an AM which fails to launch
+ File launchReportFile = createTempJsonFile();
+ createTemplatedSliderApplication(CLUSTER,
+ APP_TEMPLATE,
+ APP_RESOURCE2,
+ [
+ ARG_OPTION, CHAOS_MONKEY_ENABLED, "true",
+ ARG_OPTION, CHAOS_MONKEY_INTERVAL_SECONDS, "60",
+ ARG_OPTION, CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE,
+ Integer.toString(PROBABILITY_PERCENT_100),
+ ARG_DEFINE, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=1",
+ ARG_WAIT, "0"
+ ],
+ launchReportFile)
+
+ assert launchReportFile.exists()
+ assert launchReportFile.size() > 0
+ def launchReport = maybeLoadAppReport(launchReportFile)
+ assert launchReport;
+ assert launchReport.applicationId;
+
+ // spin expecting failure
+ def appId = launchReport.applicationId
+ sleep(5000)
+ describe("Awaiting failure")
+ try {
+ ensureYarnApplicationIsUp(appId)
+ fail("application is up")
+ } catch (AssertionError e) {
+ if(!e.toString().contains(SliderAppMaster.E_TRIGGERED_LAUNCH_FAILURE)) {
+ throw e;
+ }
+ }
+ def sar = lookupApplication(appId)
+ log.info(sar.toString())
+ assert sar.diagnostics.contains(SliderAppMaster.E_TRIGGERED_LAUNCH_FAILURE)
+ }
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
new file mode 100644
index 0000000..4f9701c
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.types.Endpoint
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.test.Outcome
+
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AgentRegistryIT extends AgentCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+
+ static String CLUSTER = "test-agent-registry"
+
+ static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
+
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(CLUSTER)
+ }
+
+ @After
+ public void destroyCluster() {
+ cleanup(CLUSTER)
+ }
+
+ @Test
+ public void testAgentRegistry() throws Throwable {
+ describe("Create a 0-role cluster and make registry queries against it")
+ def clusterpath = buildClusterPath(CLUSTER)
+ File launchReportFile = createTempJsonFile();
+ SliderShell shell = createTemplatedSliderApplication(CLUSTER,
+ APP_TEMPLATE,
+ APP_RESOURCE2,
+ [],
+ launchReportFile)
+
+ logShell(shell)
+
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
+
+ //at this point the cluster should exist.
+ assertPathExists(
+ clusterFS,
+ "Cluster parent directory does not exist",
+ clusterpath.parent)
+
+ assertPathExists(clusterFS, "Cluster directory does not exist", clusterpath)
+
+ // resolve the ~ path
+
+ resolve(0, [ARG_LIST, ARG_PATH, "/"]).dumpOutput()
+ resolve(0, [ARG_LIST, ARG_PATH, "/users"]).dumpOutput()
+
+
+ def home = homepath()
+ resolve(0, [ARG_LIST, ARG_PATH, home]).dumpOutput()
+
+
+ String sliderApps = "${home}/services/${SliderKeys.APP_TYPE}"
+ resolve(0, [ARG_LIST, ARG_PATH, sliderApps]).dumpOutput()
+
+ // running app
+ String appPath = sliderApps +"/"+ CLUSTER
+ resolve(0, [ARG_LIST, ARG_PATH, appPath]).dumpOutput()
+
+ resolve(0, [ARG_PATH, appPath]).dumpOutput()
+ // and the service record
+ File serviceRecordFile = File.createTempFile("tempfile", ".json")
+ serviceRecordFile.deleteOnExit()
+ resolve(0, [ARG_PATH, appPath,
+ ARG_OUTPUT, serviceRecordFile.absolutePath])
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal()
+
+ ServiceRecord serviceRecord = marshal.fromFile(serviceRecordFile)
+/*
+ def ipcEndpoint = serviceRecord.external.find { Endpoint epr ->
+ epr.api == AM_IPC_PROTOCOL;
+ }
+ assert ipcEndpoint != null;
+*/
+ def endpoints = [:]
+ serviceRecord.external.each { Endpoint epr ->
+ endpoints[epr.api] = epr;
+ }
+ serviceRecord.internal.each { Endpoint epr ->
+ endpoints[epr.api] = epr;
+ }
+ assert endpoints[PUBLISHER_REST_API]
+ assert endpoints[REGISTRY_REST_API]
+ assert endpoints[AGENT_SECURE_REST_API]
+ assert endpoints[AGENT_ONEWAY_REST_API]
+
+ //stop
+ freeze(0, CLUSTER,
+ [
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "final-shutdown"
+ ])
+
+ assertInYarnState(appId, YarnApplicationState.FINISHED)
+ destroy(0, CLUSTER)
+
+ //cluster now missing
+ exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+
+
+ repeatUntilSuccess("probe for missing registry entry",
+ this.&probeForEntryMissing, 10000, 1000,
+ [path: appPath],
+ true,
+ "registry entry never deleted") {
+ // await the registry lookup to fail
+ resolve(EXIT_NOT_FOUND, [ARG_PATH, appPath])
+ }
+ }
+
+ /**
+ * Return the home registry path
+ * @return
+ */
+ public String homepath() {
+ return RegistryUtils.homePathForCurrentUser()
+ }
+
+
+ Outcome probeForEntryMissing(Map args) {
+ String path = args["path"]
+ def shell = slider([ACTION_RESOLVE, ARG_PATH, path])
+ return Outcome.fromBool(shell.ret == EXIT_NOT_FOUND)
+ }
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
new file mode 100644
index 0000000..0f940cf
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * For a quick demo of a slider app; this starts the apps through agent test but
+ * neglects to tear it down afterwards
+ */
+@CompileStatic
+@Slf4j
+public class AppsThroughAgentDemo extends AppsThroughAgentIT {
+
+ @Override
+ void destroyCluster() {
+ super.destroyCluster()
+ }
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 00a876a..8b15a0c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -27,6 +27,7 @@
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
import org.junit.After
+import org.junit.Before
import org.junit.Test
@CompileStatic
@@ -35,7 +36,16 @@
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
- private static String APPLICATION_NAME = "happy-path-with-flex"
+ private static String APPLICATION_NAME = "apps-through-agent"
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(APPLICATION_NAME)
+ }
+
+ public String getApplicationName() {
+ return APPLICATION_NAME
+ }
@After
public void destroyCluster() {
@@ -46,24 +56,23 @@
public void testCreateFlex() throws Throwable {
assumeAgentTestsEnabled()
- cleanup(APPLICATION_NAME)
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, APPLICATION_NAME,
- ARG_IMAGE, agentTarballPath.toString(),
- ARG_TEMPLATE, APP_TEMPLATE,
- ARG_RESOURCES, APP_RESOURCE
- ])
-
+ def application = APPLICATION_NAME
+ cleanup(application)
+ File launchReportFile = createTempJsonFile();
+ SliderShell shell = createTemplatedSliderApplication(application,
+ APP_TEMPLATE,
+ APP_RESOURCE,
+ [],
+ launchReportFile)
logShell(shell)
- ensureApplicationIsUp(APPLICATION_NAME)
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
//flex
slider(EXIT_SUCCESS,
[
ACTION_FLEX,
- APPLICATION_NAME,
+ application,
ARG_COMPONENT,
COMMAND_LOGGER,
"2"])
@@ -71,14 +80,70 @@
// sleep till the new instance starts
sleep(1000 * 10)
+ status(0, application)
+ expectLiveContainerCountReached(application, COMMAND_LOGGER, 2,
+ CONTAINER_LAUNCH_TIMEOUT)
+
+ String amWebUrl = getInfoAmWebUrl(application)
+ log.info("Dumping data from AM Web URL");
+ log.info(amWebUrl.toURL().text);
+
+ ensureRegistryCallSucceeds(application)
+ assertAppRunning(appId)
+ def outfile = tmpFile(".txt")
+
+ def commands = [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ application,
+ ARG_LISTEXP,
+ ARG_OUTPUT,
+ outfile.absolutePath
+ ]
+
+ awaitRegistryOutfileContains(outfile, commands, "container_log_dirs")
+ awaitRegistryOutfileContains(outfile, commands, "container_work_dirs")
+
+ // get log folders
shell = slider(EXIT_SUCCESS,
[
- ACTION_STATUS,
- APPLICATION_NAME])
+ ACTION_REGISTRY,
+ ARG_NAME,
+ application,
+ ARG_GETEXP,
+ "container_log_dirs"])
- assertComponentCount(COMMAND_LOGGER, 2, shell)
+ assertOutputContains(shell, '"tag" : "COMMAND_LOGGER"', 2)
+ assertOutputContains(shell, '"level" : "component"', 2)
- assertSuccess(shell)
- assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+ // get cl-site config
+
+ def getconf = [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ application,
+ ARG_GETCONF,
+ "cl-site",
+ ARG_FORMAT,
+ "json"]
+
+ def pattern = '"pattern.for.test.to.verify" : "verify this pattern"'
+
+ repeatUntilSuccess("registry",
+ this.&commandSucceeds,
+ REGISTRY_STARTUP_TIMEOUT,
+ PROBE_SLEEP_TIME,
+ [
+ text: "pattern",
+ command: getconf
+ ],
+ true,
+ "failed to find $pattern in output") {
+ slider(0, getconf)
+ assertOutputContains(shell, pattern)
+ }
+
+ assertAppRunning(appId)
}
+
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
new file mode 100644
index 0000000..4ef3905
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Test
+
+/**
+ * SETUP FOR THE TEST
+ * Create valid labels, red and blue [yarn rmadmin -addToClusterNodeLabels red,blue]
+ * Add nodes with label [yarn rmadmin -replaceLabelsOnNode host1,red,blue]
+ * Perform refresh queue [yarn rmadmin -refreshQueues]
+ *
+ * Create a queue with access to labels - these are changes to capacity scheduler configuration
+ * Add a queue in addition to default
+ * yarn.scheduler.capacity.root.queues=default,labeled
+ * Provide capacity, take out from default
+ * yarn.scheduler.capacity.root.labeled.capacity=80
+ * yarn.scheduler.capacity.root.default.capacity=20
+ * Provide standard queue specs
+ * yarn.scheduler.capacity.root.labeled.state=RUNNING
+ * yarn.scheduler.capacity.root.labeled.maximum-capacity=80
+ * Have queue access the label
+ * yarn.scheduler.capacity.root.labeled.accessible-node-labels=red,blue
+ * yarn.scheduler.capacity.root.labeled.accessible-node-labels.blue.capacity=100
+ * yarn.scheduler.capacity.root.labeled.accessible-node-labels.red.capacity=100
+ *
+ * After specifying the new configuration call refresh [yarn rmadmin -refreshQueues]
+ *
+ * See resources_queue_labels.json for label configuration required for the test
+ * Label expression for slider-appmaster is also the default for all containers
+ * if they do not specify own label expressions
+ * "yarn.label.expression":"red"
+ *
+ */
+@CompileStatic
+@Slf4j
+public class AppsThroughAgentQueueAndLabelsIT extends AgentCommandTestBase
+implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ private static String COMMAND_LOGGER = "COMMAND_LOGGER"
+ private static String APPLICATION_NAME = "happy-path-with-queue-labels"
+ private static String TARGET_QUEUE = "labeled"
+ private static String APP_RESOURCE4 =
+ "../slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json"
+
+ @After
+ public void destroyCluster() {
+ cleanup(APPLICATION_NAME)
+ }
+
+ @Test
+ public void testCreateWithQueueAndLabels() throws Throwable {
+ assumeAgentTestsEnabled()
+ assumeQueueNamedLabelDefined()
+ assumeLabelsRedAndBlueAdded()
+
+ cleanup(APPLICATION_NAME)
+ File launchReportFile = createTempJsonFile();
+ SliderShell shell = createTemplatedSliderApplication(
+ APPLICATION_NAME,
+ APP_TEMPLATE,
+ APP_RESOURCE4,
+ [ARG_QUEUE, TARGET_QUEUE],
+ launchReportFile)
+ logShell(shell)
+
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
+
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1,
+ CONTAINER_LAUNCH_TIMEOUT)
+ assertContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
+
+ //flex
+ slider(EXIT_SUCCESS,
+ [
+ ACTION_FLEX,
+ APPLICATION_NAME,
+ ARG_COMPONENT,
+ COMMAND_LOGGER,
+ "3"
+ ])
+
+ // spin till the flexed instance starts
+ ensureYarnApplicationIsUp(appId)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3,
+ CONTAINER_LAUNCH_TIMEOUT)
+
+
+ sleep(1000 * 20)
+ def cd = execStatus(APPLICATION_NAME)
+ assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
+ assertInYarnState(appId, YarnApplicationState.RUNNING)
+ }
+
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
index f8caac5..f03fb63 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
@@ -21,9 +21,9 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
@@ -38,7 +38,7 @@
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
- static String CLUSTER = "test_cluster_build_destroy"
+ static String CLUSTER = "test-cluster-build-destroy"
@BeforeClass
@@ -61,14 +61,13 @@
[
ACTION_BUILD,
CLUSTER,
- ARG_IMAGE, agentTarballPath.toString(),
ARG_ZKHOSTS,
- SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
+ SLIDER_CONFIG.get(
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
ARG_TEMPLATE, APP_TEMPLATE,
ARG_RESOURCES, APP_RESOURCE
])
-
assert clusterFS.exists(clusterDirPath)
//cluster exists if you don't want it to be live
exists(EXIT_SUCCESS, CLUSTER, false)
diff --git a/slider-funtest/src/test/manual/python/SliderTester.py b/slider-funtest/src/test/manual/python/SliderTester.py
index 40bdf2b..1dfd0b5 100644
--- a/slider-funtest/src/test/manual/python/SliderTester.py
+++ b/slider-funtest/src/test/manual/python/SliderTester.py
@@ -235,7 +235,7 @@
pass
def clean_up(self):
- (retcode, out, err) = self.run_slider_command(" ".join([self.slider_exec, "freeze", self.cluster_name]),
+ (retcode, out, err) = self.run_slider_command(" ".join([self.slider_exec, "stop", self.cluster_name]),
self.slider_user)
if retcode != 0:
raise Exception("Could not clean cluster. Out: " + out + " Err: " + err)
@@ -352,7 +352,7 @@
# Finalize resources and appConf json files
# Call create
# Validate existence of the app
- # Call freeze
+ # Call start
if __name__ == "__main__":
diff --git a/slider-funtest/src/test/resources/log4j.properties b/slider-funtest/src/test/resources/log4j.properties
index a552a55..65135ca 100644
--- a/slider-funtest/src/test/resources/log4j.properties
+++ b/slider-funtest/src/test/resources/log4j.properties
@@ -42,7 +42,7 @@
log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
log4j.logger.org.apache.hadoop.hdfs=WARN
-
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=FATAL
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
diff --git a/slider-providers/accumulo/accumulo-funtests/pom.xml b/slider-providers/accumulo/accumulo-funtests/pom.xml
index b6afd98..ec48f59 100644
--- a/slider-providers/accumulo/accumulo-funtests/pom.xml
+++ b/slider-providers/accumulo/accumulo-funtests/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
<relativePath>../../../</relativePath>
</parent>
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCIIT.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCIIT.groovy
index 4ec5ff1..bef5a03 100644
--- a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCIIT.groovy
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCIIT.groovy
@@ -25,6 +25,7 @@
import org.apache.accumulo.test.continuous.ContinuousVerify
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.Text
+import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.hadoop.util.ToolRunner
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.common.SliderXmlConfKeys
@@ -59,7 +60,8 @@
assert clustername
String currentUser = System.getProperty("user.name");
- String zookeepers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+ String zookeepers = SLIDER_CONFIG.get(
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM,
FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
ZooKeeperInstance inst = new ZooKeeperInstance(currentUser + "-" + clustername, zookeepers)
Connector conn = inst.getConnector("root", new PasswordToken(getPassword()))
diff --git a/slider-providers/accumulo/slider-accumulo-provider/pom.xml b/slider-providers/accumulo/slider-accumulo-provider/pom.xml
index 64b95e9..99d7b65 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/pom.xml
+++ b/slider-providers/accumulo/slider-accumulo-provider/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
<relativePath>../../../</relativePath>
</parent>
@@ -185,6 +185,12 @@
<dependency>
<groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-fate</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-test</artifactId>
<scope>test</scope>
</dependency>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
index 7f99573..955e4fb 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
@@ -272,7 +272,7 @@
//now, if there is an extra client conf, merge it in too
if (clientConfExtras != null) {
ConfigHelper.mergeConfigurations(siteConf, clientConfExtras,
- "Slider Client");
+ "Slider Client", true);
}
if (log.isDebugEnabled()) {
@@ -300,9 +300,9 @@
}
@Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
SliderException {
- super.validateInstanceDefinition(instanceDefinition);
+ super.validateInstanceDefinition(instanceDefinition, fs);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
index b8f4c00..c1bc9fe 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
@@ -100,7 +100,7 @@
@Override
public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
SliderException {
- clientProvider.validateInstanceDefinition(instanceDefinition);
+ clientProvider.validateInstanceDefinition(instanceDefinition, null);
}
@Override
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
index bf35207..2a87cf0 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
@@ -27,6 +27,7 @@
import org.apache.slider.api.ResourceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.junit.internal.AssumptionViolatedException
import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
import static org.apache.slider.providers.accumulo.AccumuloKeys.*
@@ -70,8 +71,12 @@
@Override
void teardown() {
super.teardown();
- if (teardownKillall) {
- killAllAccumuloProcesses();
+ if (teardownKillall && kill_supported) {
+ try {
+ killAllAccumuloProcesses();
+ } catch (AssumptionViolatedException e) {
+ log.info e.toString();
+ }
}
}
@@ -223,9 +228,7 @@
//now flex
describe(
"Flexing " + roleMapToString(flexTarget));
- boolean flexed = 0 == sliderClient.flex(clustername,
- flexTarget
- );
+ sliderClient.flex(clustername, flexTarget);
cd = waitForRoleCount(sliderClient, flexTarget,
accumulo_cluster_startup_to_live_time);
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
index 2333fdf..1f1b726 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
@@ -20,17 +20,13 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.accumulo.core.client.Connector
-import org.apache.accumulo.core.client.ZooKeeperInstance
-import org.apache.accumulo.core.client.security.tokens.PasswordToken
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -47,8 +43,6 @@
describe(" Create an accumulo cluster");
//make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
@@ -79,11 +73,6 @@
//verify that all is still there
waitForRoleCount(sliderClient, roles, 0)
- // Making the connector validates that the instance name is correct, we have the right zk,
- // and the proper username and password were created in accumulo
- ZooKeeperInstance instance = new ZooKeeperInstance(System.getProperty("user.name") + "-" + clustername, zki.getConnectionString());
- Connector c = instance.getConnector("root", new PasswordToken(password));
-
log.info("Finishing")
status = sliderClient.getClusterDescription(clustername)
log.info(prettyPrint(status.toJsonString()))
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
index 6da00fb..f55b73e 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
@@ -20,12 +20,12 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -41,9 +41,6 @@
configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
@@ -68,10 +65,12 @@
AccumuloKeys.MONITOR_PAGE_JSON)
log.info(page);
- log.info("Freezing")
- clusterActionFreeze(sliderClient, clustername, "freeze");
- waitForAppToFinish(sliderClient)
-
+ log.info("Stopping")
+ clusterActionFreeze(sliderClient, clustername, "stop");
+ def finishedAppReport = waitForAppToFinish(sliderClient)
+ assert finishedAppReport.finalApplicationStatus ==
+ FinalApplicationStatus.SUCCEEDED
+
//make sure the fetch fails
try {
page = fetchLocalPage(AccumuloConfigFileOptions.MONITOR_PORT_CLIENT_INT,
@@ -84,7 +83,9 @@
log.debug("expected exception", expected)
}
//force kill any accumulo processes
- killAllAccumuloProcesses()
+ if (kill_supported) {
+ killAllAccumuloProcesses()
+ }
sleepForAccumuloClusterLive();
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
index 8d5890c..ecb9062 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
@@ -21,13 +21,12 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -44,9 +43,6 @@
describe(" Create an accumulo cluster from an archive");
enableTestRunAgainstUploadedArchive();
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
index df0bbd9..997fa19 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
@@ -21,13 +21,12 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -46,9 +45,6 @@
//image mode
switchToImageDeploy = true
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
index d94eb36..586c4c0 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
@@ -21,12 +21,11 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -41,9 +40,6 @@
String clustername = createMiniCluster( "", configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
index 2ed50f1..804d29d 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
@@ -21,13 +21,12 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
import org.junit.Test
@CompileStatic
@@ -44,9 +43,6 @@
"", configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): master,
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
index 4596b12..7afcd0f 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
@@ -22,14 +22,13 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.server.appmaster.web.SliderAMWebApp
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.accumulo.AccumuloTestBase
-import org.apache.slider.core.zk.ZKIntegration
+import org.apache.slider.server.appmaster.web.SliderAMWebApp
import org.junit.Test
@CompileStatic
@@ -45,9 +44,6 @@
configuration, 1, 1, 1, true, false)
describe(" Create an accumulo cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
- log.info("ZK up at $zki");
//now launch the cluster
Map<String, Integer> roles = [
(AccumuloKeys.ROLE_MASTER): 1,
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
index dca0c6b..a07ddc9 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
@@ -20,14 +20,14 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.core.main.ServiceLaunchException
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
-import org.apache.slider.providers.accumulo.AccumuloKeys
-import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLaunchException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.accumulo.AccumuloKeys
import org.apache.slider.providers.accumulo.AccumuloTestBase
import org.junit.Test
diff --git a/slider-providers/hbase/hbase-funtests/pom.xml b/slider-providers/hbase/hbase-funtests/pom.xml
index 96c1632..dd55a93 100644
--- a/slider-providers/hbase/hbase-funtests/pom.xml
+++ b/slider-providers/hbase/hbase-funtests/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
<relativePath>../../../</relativePath>
</parent>
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
index 84e55f7..d520471 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
@@ -21,16 +21,16 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.tools.ConfigHelper
-import org.apache.slider.core.registry.info.RegistryNaming
+
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.providers.hbase.HBaseConfigFileOptions
import org.apache.slider.providers.hbase.HBaseTestUtils
@@ -66,7 +66,8 @@
@Before
public void prepareCluster() {
- String quorumServers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS)
+ String quorumServers = SLIDER_CONFIG.get(
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS)
ZooKeeper monitor = new ZooKeeper(quorumServers,
1000, new Watcher(){
@@ -184,10 +185,7 @@
[ARG_LIST, ARG_NAME, "cluster-with-no-name"])
// how to work out the current service name?
- def name = RegistryNaming.createRegistryName(clustername,
- System.getProperty("user.name"),
- SliderKeys.APP_TYPE,
- 1)
+ def name = clustername
registry([ARG_LIST, ARG_VERBOSE, ARG_NAME, name])
registry([ARG_LISTCONF, ARG_NAME, name])
@@ -213,7 +211,7 @@
getConfDir.mkdirs();
registry([ARG_GETCONF, yarn_site_config,
ARG_NAME, name,
- ARG_DEST, getConfDir.absolutePath])
+ ARG_OUTPUT, getConfDir.absolutePath])
File retrieved = new File(getConfDir, yarn_site_config +".xml")
def confFromFile = ConfigHelper.loadConfFromFile(retrieved)
assert confFromFile.get(YarnConfiguration.RM_ADDRESS)
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy
index 3a44e30..63672d6 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy
@@ -21,6 +21,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
@@ -62,7 +63,8 @@
CLUSTER,
ARG_PROVIDER, HBaseKeys.PROVIDER_HBASE,
ARG_ZKHOSTS,
- SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
+ SLIDER_CONFIG.get(
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
ARG_IMAGE,
SLIDER_CONFIG.get(KEY_TEST_HBASE_TAR),
ARG_CONFDIR,
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
index 63b5fb6..5ffee83 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
@@ -20,6 +20,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.slider.server.appmaster.PublishedArtifacts
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
@@ -80,14 +81,14 @@
destroy(EXIT_APPLICATION_IN_USE, CLUSTER)
- //thaw will fail as cluster is in use
+ //start will fail as cluster is in use
thaw(EXIT_APPLICATION_IN_USE, CLUSTER)
//it's still there
exists(0, CLUSTER)
//listing the cluster will succeed
- list(0, CLUSTER)
+ list(0, [CLUSTER])
//simple status
status(0, CLUSTER)
@@ -108,7 +109,9 @@
log.info(cd.toJsonString())
- getConf(0, CLUSTER)
+ def yarn_site_config = PublishedArtifacts.COMPLETE_CONFIG
+ registry([ARG_GETCONF, yarn_site_config,
+ ARG_NAME, CLUSTER])
//get a slider client against the cluster
SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
@@ -117,7 +120,7 @@
log.info("Connected via Client {}", sliderClient.toString())
- //freeze
+ //stop
def frozen = freeze(0, CLUSTER, [
ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
ARG_MESSAGE, "freeze-in-test-cluster-lifecycle"
@@ -131,7 +134,7 @@
exists(EXIT_FALSE, CLUSTER, true)
- // thaw then freeze the cluster
+ // start then stop the cluster
thaw(CLUSTER,
[
@@ -151,7 +154,7 @@
// condition returns false if it is required to be live
exists(EXIT_FALSE, CLUSTER, true)
- // thaw with a restart count set to enable restart
+ // start with a restart count set to enable restart
describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
thaw(CLUSTER,
diff --git a/slider-providers/hbase/slider-hbase-provider/pom.xml b/slider-providers/hbase/slider-hbase-provider/pom.xml
index b9986f3..71c4d27 100644
--- a/slider-providers/hbase/slider-hbase-provider/pom.xml
+++ b/slider-providers/hbase/slider-hbase-provider/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.50.2-incubating</version>
+ <version>0.60.0-incubating</version>
<relativePath>../../../</relativePath>
</parent>
@@ -106,25 +106,7 @@
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>src/main/resources/org/apache/slider/providers/hbase/conf/regionservers</exclude>
- </excludes>
- </configuration>
- </plugin>
+
</plugins>
</build>
@@ -243,5 +225,32 @@
</dependencies>
-
+ <profiles>
+ <profile>
+ <id>rat</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>src/main/resources/org/apache/slider/providers/hbase/conf/regionservers</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index 9ad872f..e0acd06 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -224,9 +224,9 @@
* @param instanceDefinition instance definition
*/
@Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
SliderException {
- super.validateInstanceDefinition(instanceDefinition);
+ super.validateInstanceDefinition(instanceDefinition, fs);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
Set<String> unknownRoles = resources.getComponentNames();
@@ -314,7 +314,7 @@
//now, if there is an extra client conf, merge it in too
if (clientConfExtras != null) {
ConfigHelper.mergeConfigurations(siteConf, clientConfExtras,
- "Slider Client");
+ "Slider Client", true);
}
if (log.isDebugEnabled()) {
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
index 2a20438..064414e 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
@@ -88,7 +88,7 @@
/**
* Service type used in registry
*/
- String HBASE_SERVICE_TYPE = "org.apache.hbase";
+ String HBASE_SERVICE_TYPE = "org-apache-hbase";
String HBASE_SITE_PUBLISHED_CONFIG = "hbase-site";
}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 82e535f..3b6cf8b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -18,10 +18,13 @@
package org.apache.slider.providers.hbase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.api.ClusterDescription;
@@ -36,9 +39,8 @@
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
@@ -56,6 +58,7 @@
import java.io.File;
import java.io.IOException;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.List;
@@ -106,9 +109,9 @@
* @param instanceDefinition the instance definition to validate
*/
@Override // Client and Server
- public void validateInstanceDefinition(AggregateConf instanceDefinition)
+ public void validateInstanceDefinition(AggregateConf instanceDefinition)
throws SliderException {
- clientProvider.validateInstanceDefinition(instanceDefinition);
+ clientProvider.validateInstanceDefinition(instanceDefinition, null);
}
@Override
@@ -245,14 +248,6 @@
}
@Override
- public void applyInitialRegistryDefinitions(URL web,
- URL secureWebAPI,
- ServiceInstanceData instanceData) throws
- IOException {
- super.applyInitialRegistryDefinitions(web, secureWebAPI, instanceData);
- }
-
- @Override
protected void serviceStart() throws Exception {
registerHBaseServiceEntry();
super.serviceStart();
@@ -260,46 +255,37 @@
private void registerHBaseServiceEntry() throws IOException {
- String name = amState.getApplicationName() ;
-// name += ".hbase";
- ServiceInstanceData instanceData = new ServiceInstanceData(name,
- HBASE_SERVICE_TYPE);
- log.info("registering {}/{}", name, HBASE_SERVICE_TYPE );
+ String name = amState.getApplicationName() ;
+ ServiceRecord serviceRecord = new ServiceRecord();
+ // bond lifespan to the application
+ serviceRecord.set(YarnRegistryAttributes.YARN_ID,
+ yarnRegistry.getApplicationAttemptId()
+ .getApplicationId().toString());
+ serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.APPLICATION);
+ try {
+ URL configURL = new URL(amWebAPI,
+ SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE);
+
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API,
+ configURL.toURI()));
+ } catch (URISyntaxException e) {
+ log.warn("failed to create config URL: {}", e, e);
+ }
+ log.info("registering {}/{}", name, HBASE_SERVICE_TYPE);
+ yarnRegistry.putService(HBASE_SERVICE_TYPE, name, serviceRecord, true);
+
PublishedConfiguration publishedSite =
new PublishedConfiguration("HBase site", siteConf);
PublishedConfigSet configSet =
amState.getOrCreatePublishedConfigSet(HBASE_SERVICE_TYPE);
- instanceData.externalView.configurationsURL = SliderUtils.appendToURL(
- amWebAPI.toExternalForm(), SLIDER_PATH_PUBLISHER, HBASE_SERVICE_TYPE);
- configSet.put(HBASE_SITE_PUBLISHED_CONFIG, publishedSite);
- registry.registerServiceInstance(instanceData, null);
+ configSet.put(HBASE_SITE_PUBLISHED_CONFIG, publishedSite);
}
/**
- * Run this service
- *
- *
- * @param instanceDefinition component description
- * @param confDir local dir with the config
- * @param env environment variables above those generated by
- * @param execInProgress callback for the event notification
- * @throws IOException IO problems
- * @throws SliderException anything internal
- */
- @Override
- public boolean exec(AggregateConf instanceDefinition,
- File confDir,
- Map<String, String> env,
- ProviderCompleted execInProgress) throws
- IOException,
- SliderException {
-
- return false;
- }
-
-
- /**
* This is a validation of the application configuration on the AM.
* Here is where things like the existence of keytabs and other
* not-seen-client-side properties can be tested, before
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 6a69e17..7712a83 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -30,6 +30,7 @@
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseTestUtils
import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.junit.internal.AssumptionViolatedException
import static org.apache.slider.common.params.Arguments.*
import static org.apache.slider.test.SliderTestUtils.*
@@ -76,8 +77,12 @@
void teardown() {
super.teardown();
if (teardownKillall) {
- killAllRegionServers();
- killAllMasterServers();
+ try {
+ killAllRegionServers();
+ killAllMasterServers();
+ } catch (AssumptionViolatedException e) {
+ log.info e.toString();
+ }
}
}
@@ -348,7 +353,6 @@
int masterFlexTarget,
int workerFlexTarget,
boolean testHBaseAfter) {
- int flexTarget
describe(
"Flexing masters -> $masterFlexTarget ; workers -> ${workerFlexTarget}");
boolean flexed;
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy
index 85726a7..09a87a9 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy
@@ -51,7 +51,7 @@
hbaseClusterStartupToLiveTime)
- clusterActionFreeze(sliderClient, clustername, "test freeze")
+ clusterActionFreeze(sliderClient, clustername, "test stop")
describe("Restarting cluster")
killAllRegionServers();
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
index c305b5b..0f3e06b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
@@ -28,7 +28,6 @@
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
-import static HBaseKeys.PROVIDER_HBASE
import static org.apache.slider.common.params.Arguments.ARG_PROVIDER
@CompileStatic
@@ -50,18 +49,18 @@
(HBaseKeys.ROLE_WORKER): 1,
],
[
- ARG_PROVIDER, PROVIDER_HBASE
+ ARG_PROVIDER, HBaseKeys.PROVIDER_HBASE
],
true,
false,
[:])
SliderClient sliderClient = launcher.service
addToTeardown(sliderClient);
- def serviceRegistryClient = sliderClient.YARNRegistryClient
+ def serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport report = serviceRegistryClient.findInstance(clustername)
assert report == null;
- //thaw time
+ //start time
ServiceLauncher<SliderClient> l2 = thawCluster(clustername, [], true)
SliderClient thawed = l2.service
addToTeardown(thawed);
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseAM.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseAM.groovy
index d219636..6d3a12b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseAM.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseAM.groovy
@@ -47,7 +47,6 @@
@Test
public void testKilledHBaseAM() throws Throwable {
- skip("SLIDER-66: AM Restart Failing -YARN issues")
int regionServerCount = 1
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
index fdbbce8..c1265de 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
@@ -33,7 +33,7 @@
@Test
public void testClusterFlex1To1() throws Throwable {
- assert !flexHBaseClusterTestRun(
+ assert flexHBaseClusterTestRun(
"",
1,
1,
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
index f6748e0..d01716b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
@@ -54,8 +54,8 @@
conf.setInt("yarn.nodemanager.resource.cpu-vcores", 1)
String clustername = createMiniCluster("", conf, nodemanagers, true)
describe(
- "Create a $regionServerCount node cluster, freeze it, patch the configuration files," +
- " thaw it and verify that it came back with the new settings")
+ "Create a $regionServerCount node cluster, stop it, patch the configuration files," +
+ " start it and verify that it came back with the new settings")
ServiceLauncher<SliderClient> launcher = createHBaseCluster(
clustername,
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy
index 66dd4f0..9a0e1db 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy
@@ -43,7 +43,7 @@
public void testFreezeThawLiveRegionService() throws Throwable {
int regionServerCount = 2
String clustername = createMiniCluster("", configuration, 1, true)
- describe("Create a cluster, freeze it, thaw it and verify that it came back ")
+ describe("Create a cluster, stop it, start it and verify that it came back ")
//use a smaller AM HEAP to include it in the test cycle
ServiceLauncher launcher = createHBaseCluster(clustername, regionServerCount,
[
@@ -84,7 +84,7 @@
waitForHBaseRegionServerCount(newCluster, clustername, regionServerCount,
hbaseClusterStartupToLiveTime)
- // finally, attempt to thaw it while it is running
+ //finally, attempt to start it while it is running
//now let's start the cluster up again
try {
ServiceLauncher launcher3 = thawCluster(clustername, [], true);
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
index 47f5e1c..10a502b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
@@ -22,11 +22,11 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.ClusterStatus
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.providers.hbase.HBaseTestUtils
-import org.apache.slider.common.tools.Duration
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.tools.Duration
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.HBaseTestUtils
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
index 634ebe2..db102cb 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
@@ -20,20 +20,19 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.registry.docstore.PublishedConfigSet
-import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.core.registry.retrieve.RegistryRetriever
import org.apache.slider.providers.hbase.HBaseKeys
-import org.apache.slider.core.zk.ZKIntegration
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.apache.slider.server.services.registry.SliderRegistryService
import org.junit.Test
/**
@@ -47,8 +46,6 @@
@Test
public void testHBaseMaster() throws Throwable {
String clustername = createMiniCluster("", configuration, 1, true)
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
//now launch the cluster with 1 region server
int regionServerCount = 1
ServiceLauncher<SliderClient> launcher = createHBaseCluster(clustername, regionServerCount,
@@ -77,28 +74,24 @@
// look up the registry entries for HBase
describe "service registry names"
- SliderRegistryService registryService = client.registry
- def names = registryService.getServiceTypes();
+ Map<String, ServiceRecord> records = RegistryUtils.listServiceRecords(
+ client.registryOperations,
+ RegistryUtils.serviceclassPath(
+ RegistryUtils.homePathForCurrentUser(),
+ HBaseKeys.HBASE_SERVICE_TYPE
+ )
+ )
+
+ def names = records.keySet()
dumpRegistryServiceTypes(names)
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
- client.listRegistryInstances();
- def hbaseInstances = registryService.findInstances( HBaseKeys.HBASE_SERVICE_TYPE, null)
+
+ def hbaseInstances = records.values()
assert hbaseInstances.size() == 1
- def hbaseService = hbaseInstances[0]
- assert hbaseService
- def hbaseServiceData = hbaseService.payload
+ ServiceRecord hbaseServiceData = hbaseInstances[0]
log.info "HBase service 0 == $hbaseServiceData"
- assert hbaseServiceData.id
- assert hbaseServiceData.serviceType == HBaseKeys.HBASE_SERVICE_TYPE
-
- hbaseInstances = registryService.findInstances(
- HBaseKeys.HBASE_SERVICE_TYPE,
- clustername)
- assert hbaseInstances.size() == 1
- def hbaseServiceData2 = hbaseInstances[0].payload
- assert hbaseServiceData == hbaseServiceData2
+ assert hbaseServiceData[YarnRegistryAttributes.YARN_ID]
RegistryRetriever retriever = new RegistryRetriever(hbaseServiceData)
log.info retriever.toString()
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
index 7f51f95..5e8adca 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
@@ -22,8 +22,8 @@
import groovy.util.logging.Slf4j
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
/**
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
index 957d167..1392535 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
@@ -22,10 +22,9 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.hbase.ClusterStatus
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.core.zk.ZKIntegration
import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
/**
@@ -42,8 +41,6 @@
"", configuration, 1, 1, 1, true, true)
describe(" Create a single region service cluster");
- //make sure that ZK is up and running at the binding string
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
//now launch the cluster
ServiceLauncher<SliderClient> launcher = createHBaseCluster(clustername, regionServerCount, [], true, true)
SliderClient sliderClient = launcher.service
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
index 3561d2f..f10e296 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
@@ -23,11 +23,11 @@
import org.apache.hadoop.hbase.ClusterStatus
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
-import org.apache.slider.providers.hbase.HBaseKeys
-import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.common.params.Arguments
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
/**
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
index 7e4c5ed..029f424 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
@@ -20,14 +20,16 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.common.params.Arguments
+import org.apache.hadoop.registry.client.api.RegistryOperations
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.apache.slider.server.services.registry.SliderRegistryService
import org.junit.Test
@CompileStatic
@@ -83,34 +85,47 @@
// registry instances def names = client.listRegistryNames(clustername)
describe "service registry names"
- SliderRegistryService registry = cluster2Client.registry
- def names = registry.getServiceTypes();
- dumpRegistryServiceTypes(names)
+ RegistryOperations registry = cluster2Client.registryOperations
+ def home = RegistryUtils.homePathForCurrentUser()
+ def userSliderInstancesPath = RegistryUtils.serviceclassPath(
+ RegistryUtils.currentUser(), SliderKeys.APP_TYPE)
+
+
+ def names = RegistryUtils.listServiceRecords(registry,
+ userSliderInstancesPath)
+ dumpMap(names)
+ def statMap = RegistryUtils.statChildren(registry, userSliderInstancesPath)
+ assert statMap.size() == 2
List<String> instanceIds = sliderClient.listRegisteredSliderInstances()
-
dumpRegistryInstanceIDs(instanceIds)
assert names.size() == 2
assert instanceIds.size() == 2
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
+ Map<String, ServiceRecord> instances =
sliderClient.listRegistryInstances()
dumpRegistryInstances(instances)
assert instances.size() == 2
- def hbaseInstances = registry.findInstances(
- HBaseKeys.HBASE_SERVICE_TYPE, null)
- assert hbaseInstances.size() == 2
- def hbase1ServiceData = registry.findInstance(
- HBaseKeys.HBASE_SERVICE_TYPE, clustername1).payload
- def hbase2ServiceData = registry.findInstance(
- HBaseKeys.HBASE_SERVICE_TYPE, clustername2).payload
- assert !(hbase1ServiceData == hbase2ServiceData)
- clusterActionFreeze(cluster2Client, clustername2,"freeze cluster 2")
- clusterActionFreeze(sliderClient, clustername1,"Freeze cluster 1")
+ def hbaseServicePath = RegistryUtils.serviceclassPath(
+ RegistryUtils.currentUser(),
+ HBaseKeys.HBASE_SERVICE_TYPE)
+ Map<String, ServiceRecord> hbaseInstances =
+ RegistryUtils.listServiceRecords(registry,
+ hbaseServicePath);
+
+ assert hbaseInstances.size() == 2
+ String clusterPath1 = RegistryPathUtils.join(hbaseServicePath, clustername1)
+ String clusterPath2 = RegistryPathUtils.join(hbaseServicePath, clustername2)
+ assert hbaseInstances[clusterPath1] != null
+ assert hbaseInstances[clusterPath2] != null
+ assert hbaseInstances[clusterPath1] != hbaseInstances[clusterPath2]
+
+ clusterActionFreeze(cluster2Client, clustername2, "stop cluster 2")
+ clusterActionFreeze(sliderClient, clustername1, "Stop cluster 1")
}
diff --git a/src/test/clusters/morzine/slider/slider-client.xml b/src/test/clusters/morzine/slider/slider-client.xml
index 4d7ab41..dbf25bd 100644
--- a/src/test/clusters/morzine/slider/slider-client.xml
+++ b/src/test/clusters/morzine/slider/slider-client.xml
@@ -73,4 +73,21 @@
</property>
+ <property>
+ <name>slider.am.login.keytab.name</name>
+ <value>Location of keytab in HDFS</value>
+ </property>
+
+ <property>
+ <name>slider.am.keytab.local.path</name>
+ <description>absolute path to keytab</description>
+ <value></value>
+ </property>
+
+ <property>
+ <name>slider.keytab.principal.name</name>
+ <value>Optional principal name in keytab</value>
+ </property>
+
+
</configuration>
diff --git a/src/test/clusters/offline/slider/slider-client.xml b/src/test/clusters/offline/slider/slider-client.xml
index 25c01cf..0f780f0 100644
--- a/src/test/clusters/offline/slider/slider-client.xml
+++ b/src/test/clusters/offline/slider/slider-client.xml
@@ -90,6 +90,22 @@
<value>file://${user.dir}/src/test/configs/sandbox/accumulo</value>
</property>
+ <property>
+ <name>slider.am.login.keytab.name</name>
+ <value>Location of keytab in HDFS</value>
+ </property>
+
+ <property>
+ <name>slider.am.keytab.local.path</name>
+ <description>absolute path to keytab</description>
+ <value></value>
+ </property>
+
+ <property>
+ <name>slider.keytab.principal.name</name>
+ <value>Optional principal name in keytab</value>
+ </property>
+
<property>
<name>zk.home</name>
diff --git a/src/test/clusters/remote/slider/slider-client.xml b/src/test/clusters/remote/slider/slider-client.xml
index 5ed4d10..f8d88eb 100644
--- a/src/test/clusters/remote/slider/slider-client.xml
+++ b/src/test/clusters/remote/slider/slider-client.xml
@@ -53,7 +53,25 @@
<name>slider.test.agent.enabled</name>
<value>true</value>
</property>
-
+
+ <property>
+ <name>slider.test.agent.labeled.queue.enabled</name>
+ <value>false</value>
+ <description>Add a queue named labeled</description>
+ </property>
+
+ <property>
+ <name>slider.test.agent.labels.defined</name>
+ <value>false</value>
+ <description>Add node labels red and blue and ensure enough capacities are allocated</description>
+ </property>
+
+ <property>
+ <name>slider.test.agent.am.failures.enabled</name>
+ <value>false</value>
+ <description>AM failure test scenarios - if enabled vagrant.current.working.dir needs to be set as well</description>
+ </property>
+
<property>
<name>slider.test.agent.tar</name>
<value>hdfs://c6403.ambari.apache.org:8020/slider/agent/slider-agent.tar.gz</value>
diff --git a/src/test/clusters/sandbox/operations.md b/src/test/clusters/sandbox/operations.md
index 0ff5a3a..932f6d5 100644
--- a/src/test/clusters/sandbox/operations.md
+++ b/src/test/clusters/sandbox/operations.md
@@ -118,10 +118,10 @@
- bin/slider thaw cl1
+ bin/slider start cl1
- bin/slider freeze cl1
- bin/slider freeze cluster3
+ bin/slider stop cl1
+ bin/slider stop cluster3
bin/slider destroy cl1
@@ -149,20 +149,20 @@
--component master 1 \
--component worker 2
-### freeze
+### stop
- bin/slider freeze cl1
+ bin/slider stop cl1
- bin/slider freeze cl1 --force
+ bin/slider stop cl1 --force
-### thaw
+### start
- bin/slider thaw cl1 -D slider.yarn.queue.priority=5 -D slider.yarn.queue=default
+ bin/slider start cl1 -D slider.yarn.queue.priority=5 -D slider.yarn.queue=default
-### thaw with bad queue: _MUST_ fail
+### start with bad queue: _MUST_ fail
- bin/slider thaw cl1 -D slider.yarn.queue=unknown
+ bin/slider start cl1 -D slider.yarn.queue=unknown
### monitor
diff --git a/src/test/clusters/sandbox/slider/slider-client.xml b/src/test/clusters/sandbox/slider/slider-client.xml
index 5ac5d59..41629ce 100644
--- a/src/test/clusters/sandbox/slider/slider-client.xml
+++ b/src/test/clusters/sandbox/slider/slider-client.xml
@@ -107,6 +107,23 @@
<property>
+ <name>slider.am.login.keytab.name</name>
+ <value>Location of keytab in HDFS</value>
+ </property>
+
+ <property>
+ <name>slider.am.keytab.local.path</name>
+ <description>absolute path to keytab</description>
+ <value></value>
+ </property>
+
+ <property>
+ <name>slider.keytab.principal.name</name>
+ <value>Optional principal name in keytab</value>
+ </property>
+
+
+ <property>
<name>zk.home</name>
<value>/usr/lib/zookeeper</value>
<description>Zookeeper home dir on target systems</description>
diff --git a/src/test/clusters/script.md b/src/test/clusters/script.md
index 634658a..1e9f567 100644
--- a/src/test/clusters/script.md
+++ b/src/test/clusters/script.md
@@ -53,5 +53,5 @@
./zookeeper-3.4.5/bin/zkServer.sh stop
-
+## Windows
diff --git a/src/test/clusters/ubuntu-secure/operations.md b/src/test/clusters/ubuntu-secure/operations.md
index d894038..1f92290 100644
--- a/src/test/clusters/ubuntu-secure/operations.md
+++ b/src/test/clusters/ubuntu-secure/operations.md
@@ -198,21 +198,21 @@
- bin/slider thaw cl1 \
+ bin/slider start cl1 \
--manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
\
-S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
-D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
-D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
- bin/slider freeze cl1 \
+ bin/slider stop cl1 \
--manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
\
-S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
-D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
-D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
- bin/slider freeze cluster3 \
+ bin/slider stop cluster3 \
--manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
\
-S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
@@ -259,13 +259,13 @@
--role master 1 \
--role worker 2
-# freeze
+# stop
- bin/slider freeze cl1
+ bin/slider stop cl1
-# thaw
+# start
- bin/slider thaw cl1
+ bin/slider start cl1
# monitor
