jaas fix for StormServer
diff --git a/README.md b/README.md
index b91e477..a25b83a 100644
--- a/README.md
+++ b/README.md
@@ -22,9 +22,9 @@
monitor them and make them larger or smaller as desired -even while
the cluster is running.
-Clusters can be stopped, "frozen" and restarted, "thawed" later; the distribution
+Clusters can be stopped and restarted later; the distribution
of the deployed application across the YARN cluster is persisted -enabling
-a best-effort placement close to the previous locations on a cluster thaw.
+a best-effort placement close to the previous locations on a cluster start.
Applications which remember the previous placement of data (such as HBase)
can exhibit fast start-up times from this feature.
diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json
index a44dbad..8828a64 100644
--- a/app-packages/accumulo/appConfig.json
+++ b/app-packages/accumulo/appConfig.json
@@ -9,7 +9,7 @@
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
- "site.global.tserver_heapsize": "128m",
+ "site.global.tserver_heapsize": "256m",
"site.global.master_heapsize": "128m",
"site.global.monitor_heapsize": "64m",
"site.global.gc_heapsize": "64m",
@@ -27,18 +27,21 @@
"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
"site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
"site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
"site.accumulo-site.tserver.memory.maps.max": "80M",
"site.accumulo-site.tserver.cache.data.size": "7M",
"site.accumulo-site.tserver.cache.index.size": "20M",
"site.accumulo-site.trace.user": "root",
"site.accumulo-site.tserver.sort.buffer.size": "50M",
- "site.accumulo-site.tserver.walog.max.size": "100M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
"site.accumulo-site.master.port.client": "0",
"site.accumulo-site.trace.port.client": "0",
"site.accumulo-site.tserver.port.client": "0",
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
diff --git a/app-packages/accumulo/src/test/resources/appConfig_kerberos.json b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
new file mode 100644
index 0000000..b1bff37
--- /dev/null
+++ b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
@@ -0,0 +1,57 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": "${app.package.name}.zip",
+ "java_home": "/usr/lib/jvm/java",
+ "site.global.app_user": "accumulo",
+ "site.global.app_log_dir": "${AGENT_LOG_ROOT}",
+ "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
+ "site.global.tserver_heapsize": "256m",
+ "site.global.master_heapsize": "128m",
+ "site.global.monitor_heapsize": "64m",
+ "site.global.gc_heapsize": "64m",
+ "site.global.other_heapsize": "128m",
+ "site.global.hadoop_prefix": "/usr/lib/hadoop",
+ "site.global.hadoop_conf_dir": "/etc/hadoop/conf",
+ "site.global.zookeeper_home": "/usr/lib/zookeeper",
+ "site.global.accumulo_instance_name": "${USER}-${CLUSTER_NAME}",
+ "site.global.accumulo_root_password": "NOT_USED",
+ "site.global.user_group": "accumulo",
+ "site.global.security_enabled": "false",
+ "site.global.ssl_cert_dir": "ssl",
+ "site.global.monitor_protocol": "http",
+ "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
+ "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
+ "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
+ "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.general.kerberos.keytab": "/etc/security/keytabs/accumulo.service.keytab",
+ "site.accumulo-site.general.kerberos.principal": "accumulo/_HOST@EXAMPLE.COM",
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
+ "site.accumulo-site.tserver.memory.maps.max": "80M",
+ "site.accumulo-site.tserver.cache.data.size": "7M",
+ "site.accumulo-site.tserver.cache.index.size": "20M",
+ "site.accumulo-site.trace.user": "root",
+ "site.accumulo-site.tserver.sort.buffer.size": "50M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
+ "site.accumulo-site.master.port.client": "0",
+ "site.accumulo-site.trace.port.client": "0",
+ "site.accumulo-site.tserver.port.client": "0",
+ "site.accumulo-site.gc.port.client": "0",
+ "site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
+ "site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
+ "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
+ },
+ "credentials": {
+ "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root.initial.password", "instance.secret", "trace.token.property.password"]
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
diff --git a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
index c6a2639..61a9bb1 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
@@ -9,7 +9,7 @@
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
- "site.global.tserver_heapsize": "128m",
+ "site.global.tserver_heapsize": "256m",
"site.global.master_heapsize": "128m",
"site.global.monitor_heapsize": "64m",
"site.global.gc_heapsize": "64m",
@@ -27,18 +27,21 @@
"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
"site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
"site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
"site.accumulo-site.tserver.memory.maps.max": "80M",
"site.accumulo-site.tserver.cache.data.size": "7M",
"site.accumulo-site.tserver.cache.index.size": "20M",
"site.accumulo-site.trace.user": "root",
"site.accumulo-site.tserver.sort.buffer.size": "50M",
- "site.accumulo-site.tserver.walog.max.size": "100M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
"site.accumulo-site.master.port.client": "0",
"site.accumulo-site.trace.port.client": "0",
"site.accumulo-site.tserver.port.client": "0",
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
diff --git a/app-packages/accumulo/src/test/resources/appConfig_ssl.json b/app-packages/accumulo/src/test/resources/appConfig_ssl.json
index f48216d..e46d40e 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_ssl.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_ssl.json
@@ -9,7 +9,7 @@
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
- "site.global.tserver_heapsize": "128m",
+ "site.global.tserver_heapsize": "256m",
"site.global.master_heapsize": "128m",
"site.global.monitor_heapsize": "64m",
"site.global.gc_heapsize": "64m",
@@ -29,18 +29,21 @@
"site.accumulo-site.instance.rpc.ssl.enabled": "true",
"site.accumulo-site.instance.rpc.ssl.clientAuth": "true",
"site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
"site.accumulo-site.tserver.memory.maps.max": "80M",
"site.accumulo-site.tserver.cache.data.size": "7M",
"site.accumulo-site.tserver.cache.index.size": "20M",
"site.accumulo-site.trace.user": "root",
"site.accumulo-site.tserver.sort.buffer.size": "50M",
- "site.accumulo-site.tserver.walog.max.size": "100M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
"site.accumulo-site.master.port.client": "0",
"site.accumulo-site.trace.port.client": "0",
"site.accumulo-site.tserver.port.client": "0",
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 9c37ed9..07d1b4e 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -1,43 +1,47 @@
{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": "${app.package.name}.zip",
- "create.default.zookeeper.node": "true",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": "${app.package.name}.zip",
+ "create.default.zookeeper.node": "true",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "system_configs": "core-site",
- "site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
+ "site.global.app_user": "yarn",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
- "site.global.hbase_instance_name": "instancename",
- "site.global.hbase_root_password": "secret",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.global.monitor_protocol": "http",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
- "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
- "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_port": "8667",
+ "site.global.ganglia_server_id": "Application1",
+ "site.global.ganglia_enabled":"true",
- "site.hbase-env.hbase_master_heapsize": "1024m",
- "site.hbase-env.hbase_regionserver_heapsize": "1024m",
+ "site.global.hbase_instance_name": "instancename",
+ "site.global.hbase_root_password": "secret",
+ "site.global.user_group": "hadoop",
+ "site.global.security_enabled": "false",
+ "site.global.monitor_protocol": "http",
+ "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
+ "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
+ "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
- "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
- "site.hbase-site.hbase.superuser": "yarn",
- "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
- "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
- "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
- "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
- "site.hbase-site.hbase.regionserver.info.port": "0",
- "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
- "site.hbase-site.hbase.regionserver.port": "0"
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
+ "site.hbase-env.hbase_master_heapsize": "1024m",
+ "site.hbase-env.hbase_regionserver_heapsize": "1024m",
+
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
+ "site.hbase-site.hbase.superuser": "yarn",
+ "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+ "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+ "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
+ "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
+ "site.hbase-site.hbase.regionserver.info.port": "0",
+ "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
+ "site.hbase-site.hbase.regionserver.port": "0",
+ "site.hbase-site.hbase.master.port": "0"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
}
- }
}
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 442b0f2..6f1da9c 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.41.0-incubating-SNAPSHOT</version>
+ <version>0.51.0-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index 2bae3d2..288604a 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -23,7 +23,7 @@
Replace the placeholder tarball for Storm.
cp ~/Downloads/apache-storm-0.9.3.0.2.5.0-100.tar.gz package/files/
- rm package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE
+ rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/)
zip -r Apache_Storm_v_0_9_3.zip .
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index d6dee84..33e38ac 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,20 +3,19 @@
"metadata": {
},
"global": {
- "application.def": "Apache_Storm_v_0_9_3.zip",
+ "application.def": "slider-storm-app-package-${pkg.version}.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
+ "system_configs": "core-site",
"site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
"site.global.ganglia_enabled":"true",
"site.global.ganglia_server_port": "8668",
- "site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
- "site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
"site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
"site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
@@ -24,8 +23,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -33,7 +32,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 89ee01e..2c314b4 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -21,7 +21,7 @@
<application>
<name>STORM</name>
<comment>Apache Hadoop Stream processing framework</comment>
- <version>0.9.1.2.1</version>
+ <version>0.9.3</version>
<exportedConfigs>storm-site</exportedConfigs>
<exportGroups>
@@ -29,10 +29,6 @@
<name>QuickLinks</name>
<exports>
<export>
- <name>app.jmx</name>
- <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
- </export>
- <export>
<name>app.monitor</name>
<value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
</export>
@@ -55,7 +51,7 @@
<commandOrders>
<commandOrder>
<command>NIMBUS-START</command>
- <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED
+ <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED
</requires>
</commandOrder>
<commandOrder>
@@ -67,10 +63,6 @@
<requires>NIMBUS-STARTED</requires>
</commandOrder>
<commandOrder>
- <command>STORM_REST_API-START</command>
- <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires>
- </commandOrder>
- <commandOrder>
<command>STORM_UI_SERVER-START</command>
<requires>NIMBUS-STARTED</requires>
</commandOrder>
@@ -91,18 +83,6 @@
</component>
<component>
- <name>STORM_REST_API</name>
- <category>MASTER</category>
- <autoStartOnFailure>true</autoStartOnFailure>
- <appExports>QuickLinks-app.jmx</appExports>
- <commandScript>
- <script>scripts/rest_api.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
<name>SUPERVISOR</name>
<category>SLAVE</category>
<autoStartOnFailure>true</autoStartOnFailure>
@@ -150,7 +130,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.3.0.2.5.0-100.tar.gz</name>
+ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
</package>
</packages>
</osSpecific>
diff --git a/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE
deleted file mode 100644
index dd934d5..0000000
--- a/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Replace with the actual storm package.
diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py
index e0bf9e4..ff85765 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -34,11 +34,7 @@
java64_home = config['hostLevelParams']['java_home']
nimbus_host = config['configurations']['storm-site']['nimbus.host']
nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
-rest_api_port = config['configurations']['global']['rest_api_port']
-rest_api_admin_port = config['configurations']['global']['rest_api_admin_port']
-rest_api_conf_file = format("{conf_dir}/config.yaml")
-rest_lib_dir = format("{app_root}/contrib/storm-rest")
-storm_bin = format("{app_root}/bin/storm")
+storm_bin = format("{app_root}/bin/storm.py")
storm_env_sh_template = config['configurations']['storm-env']['content']
ganglia_installed = config['configurations']['global']['ganglia_enabled']
@@ -57,7 +53,7 @@
storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase)
storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab']
- storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase)
+ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST',nimbus_host.lower())
storm_jaas_stormclient_servicename = storm_jaas_server_principal.split("/")[0]
storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab']
kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index 0ec6413..2150cd4 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -21,6 +21,8 @@
from resource_management import *
import time
+import os
+import sys
"""
Slider package uses jps as pgrep does not list the whole process start command
@@ -31,6 +33,7 @@
import params
import status_params
+ python_binary = os.environ['PYTHON_EXE'] if 'PYTHON_EXE' in os.environ else sys.executable
pid_file = status_params.pid_files[name]
container_id = status_params.container_id
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
@@ -40,17 +43,11 @@
if name == 'ui':
crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.ui.core\" {grep_and_awk} > {pid_file}")
- elif name == "rest_api":
- rest_process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server")
- crt_pid_cmd = format("pgrep -f \"{rest_process_cmd}\" > {pid_file}")
else:
crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk} > {pid_file}")
if action == "start":
- if name == "rest_api":
- cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
- else:
- cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
+ cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
if params.security_enabled:
if name == "nimbus":
@@ -67,30 +64,22 @@
wait_for_finish=False
)
- if name == "rest_api":
+ content = None
+ for i in xrange(12):
Execute(crt_pid_cmd,
user=params.storm_user,
- logoutput=True,
- tries=6,
- try_sleep=10
+ logoutput=True
)
- else:
- content = None
- for i in xrange(12):
- Execute(crt_pid_cmd,
- user=params.storm_user,
- logoutput=True
- )
- with open(pid_file) as f:
- content = f.readline().strip()
- if content.isdigit():
- break;
- File(pid_file, action = "delete")
- time.sleep(10)
- pass
+ with open(pid_file) as f:
+ content = f.readline().strip()
+ if content.isdigit():
+ break;
+ File(pid_file, action = "delete")
+ time.sleep(10)
+ pass
- if not content.isdigit():
- raise Fail(format("Unable to start {name}"))
+ if not content.isdigit():
+ raise Fail(format("Unable to start {name}"))
elif action == "stop":
process_dont_exist = format("! ({no_op_test})")
diff --git a/app-packages/storm/package/scripts/status_params.py b/app-packages/storm/package/scripts/status_params.py
index 5907446..2bf6870 100644
--- a/app-packages/storm/package/scripts/status_params.py
+++ b/app-packages/storm/package/scripts/status_params.py
@@ -28,10 +28,8 @@
pid_drpc = format("{pid_dir}/drpc.pid")
pid_ui = format("{pid_dir}/ui.pid")
pid_logviewer = format("{pid_dir}/logviewer.pid")
-pid_rest_api = format("{pid_dir}/restapi.pid")
pid_files = {"logviewer":pid_logviewer,
"ui": pid_ui,
"nimbus": pid_nimbus,
"supervisor": pid_supervisor,
- "drpc": pid_drpc,
- "rest_api": pid_rest_api}
\ No newline at end of file
+ "drpc": pid_drpc}
\ No newline at end of file
diff --git a/app-packages/storm/package/templates/config.yaml.j2 b/app-packages/storm/package/templates/config.yaml.j2
index 32d2c99..aa4ec46 100644
--- a/app-packages/storm/package/templates/config.yaml.j2
+++ b/app-packages/storm/package/templates/config.yaml.j2
@@ -16,15 +16,6 @@
nimbusHost: {{nimbus_host}}
nimbusPort: {{nimbus_port}}
-# HTTP-specific options.
-http:
-
- # The port on which the HTTP server listens for service requests.
- port: {{rest_api_port}}
-
- # The port on which the HTTP server listens for administrative requests.
- adminPort: {{rest_api_admin_port}}
-
{% if ganglia_installed %}
enableGanglia: {{ganglia_installed}}
diff --git a/app-packages/storm/pom.xml b/app-packages/storm/pom.xml
new file mode 100644
index 0000000..c08ed3d
--- /dev/null
+++ b/app-packages/storm/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.51.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-storm-app-package</artifactId>
+ <packaging>pom</packaging>
+ <name>Slider Storm App Package</name>
+ <description>Slider Storm App Package</description>
+ <version>${pkg.version}</version>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>storm-app-package</id>
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/storm.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ </build>
+
+ <dependencies>
+ </dependencies>
+
+</project>
diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json
index 606dc39..f14b2c8 100644
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm/resources.json
@@ -11,21 +11,17 @@
"yarn.role.priority": "1",
"yarn.component.instances": "1"
},
- "STORM_REST_API": {
+ "STORM_UI_SERVER": {
"yarn.role.priority": "2",
"yarn.component.instances": "0"
},
- "STORM_UI_SERVER": {
+ "DRPC_SERVER": {
"yarn.role.priority": "3",
"yarn.component.instances": "1"
},
- "DRPC_SERVER": {
- "yarn.role.priority": "4",
- "yarn.component.instances": "1"
- },
"SUPERVISOR": {
- "yarn.role.priority": "5",
+ "yarn.role.priority": "4",
"yarn.component.instances": "1"
}
}
-}
\ No newline at end of file
+}
diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml
new file mode 100644
index 0000000..47d223a
--- /dev/null
+++ b/app-packages/storm/src/assembly/storm.xml
@@ -0,0 +1,68 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>slider-storm-v${storm.version}</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>appConfig.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>metainfo.xml</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>pom.xml</exclude>
+ <exclude>src/**</exclude>
+ <exclude>target/**</exclude>
+ <exclude>appConfig.json</exclude>
+ <exclude>metainfo.xml</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ </fileSets>
+</assembly>
diff --git a/pom.xml b/pom.xml
index 8292073..d9b3813 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,6 +72,19 @@
<url>http://slider.incubator.apache.org/</url>
</site>
<downloadUrl>http://git-wip-us.apache.org/repos/asf/incubator-slider.git</downloadUrl>
+ <repository>
+ <id>${distMgmtStagingId}</id>
+ <name>${distMgmtStagingName}</name>
+ <url>${distMgmtStagingUrl}</url>
+ </repository>
+ <snapshotRepository>
+ <id>${distMgmtSnapshotsId}</id>
+ <name>${distMgmtSnapshotsName}</name>
+ <url>${distMgmtSnapshotsUrl}</url>
+ </snapshotRepository>
+
+
+
</distributionManagement>
<mailingLists>
@@ -92,7 +105,12 @@
</mailingLists>
<properties>
-
+ <distMgmtSnapshotsId>apache.snapshots.https</distMgmtSnapshotsId>
+ <distMgmtSnapshotsName>Apache Development Snapshot Repository</distMgmtSnapshotsName>
+ <distMgmtSnapshotsUrl>https://repository.apache.org/content/repositories/snapshots</distMgmtSnapshotsUrl>
+ <distMgmtStagingId>apache.staging.https</distMgmtStagingId>
+ <distMgmtStagingName>Apache Release Distribution Repository</distMgmtStagingName>
+ <distMgmtStagingUrl>https://repository.apache.org/service/local/staging/deploy/maven2</distMgmtStagingUrl>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -182,6 +200,7 @@
<maven-deploy-plugin.version>2.7</maven-deploy-plugin.version>
<maven-doxia-module-markdown.version>1.4</maven-doxia-module-markdown.version>
<maven-enforcer-plugin.version>1.0</maven-enforcer-plugin.version>
+ <maven-exec-plugin.version>1.2.1</maven-exec-plugin.version>
<maven-jar-plugin.version>2.3.1</maven-jar-plugin.version>
<maven.javadoc.version>2.8</maven.javadoc.version>
<maven.project.version>2.4</maven.project.version>
@@ -1397,7 +1416,26 @@
</build>
</profile>
-
+ <profile>
+ <id>sign</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>sign-artifacts</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>sign</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
index f26cb11..48dd301 100644
--- a/slider-agent/pom.xml
+++ b/slider-agent/pom.xml
@@ -59,11 +59,11 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2</version>
+ <version>${maven-exec-plugin.version}</version>
<executions>
<execution>
<configuration>
- <executable>${project.basedir}/src/test/python/python-wrap</executable>
+ <executable>python</executable>
<workingDirectory>src/test/python</workingDirectory>
<arguments>
<argument>unitTests.py</argument>
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index a3fb90d..11db21c 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -122,6 +122,7 @@
self.componentActualState,
self.componentExpectedState,
self.actionQueue.customServiceOrchestrator.allocated_ports,
+ self.actionQueue.customServiceOrchestrator.log_folders,
id))
logger.info("Registering with the server at " + self.registerUrl +
" with data " + pprint.pformat(data))
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 15f1664..dd8e9b9 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -26,6 +26,7 @@
import socket
import posixpath
import platform
+import copy
from AgentConfig import AgentConfig
from AgentException import AgentException
from PythonExecutor import PythonExecutor
@@ -58,6 +59,7 @@
self.public_fqdn = hostname.public_hostname()
self.stored_command = {}
self.allocated_ports = {}
+ self.log_folders = {}
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
@@ -133,15 +135,17 @@
}
if Constants.EXIT_CODE in ret and ret[Constants.EXIT_CODE] == 0:
- ret[Constants.ALLOCATED_PORTS] = allocated_ports
- self.allocated_ports = allocated_ports
+ ret[Constants.ALLOCATED_PORTS] = copy.deepcopy(allocated_ports)
+ ## Generally all ports are allocated at once but just in case
+ self.allocated_ports.update(allocated_ports)
# Irrespective of the outcome report the folder paths
if command_name == 'INSTALL':
- ret[Constants.FOLDERS] = {
+ self.log_folders = {
Constants.AGENT_LOG_ROOT: self.config.getLogPath(),
Constants.AGENT_WORK_ROOT: self.config.getWorkRootPath()
}
+ ret[Constants.FOLDERS] = copy.deepcopy(self.log_folders)
return ret
diff --git a/slider-agent/src/main/python/agent/Register.py b/slider-agent/src/main/python/agent/Register.py
index b59154f..c8246c7 100644
--- a/slider-agent/src/main/python/agent/Register.py
+++ b/slider-agent/src/main/python/agent/Register.py
@@ -29,7 +29,7 @@
def __init__(self, config):
self.config = config
- def build(self, actualState, expectedState, allocated_ports, id='-1'):
+ def build(self, actualState, expectedState, allocated_ports, log_folders, id='-1'):
timestamp = int(time.time() * 1000)
version = self.read_agent_version()
@@ -41,7 +41,8 @@
'agentVersion': version,
'actualState': actualState,
'expectedState': expectedState,
- 'allocatedPorts': allocated_ports
+ 'allocatedPorts': allocated_ports,
+ 'logFolders': log_folders
}
return register
diff --git a/slider-agent/src/main/python/setup.py b/slider-agent/src/main/python/setup.py
index 421b5f9..56969b6 100644
--- a/slider-agent/src/main/python/setup.py
+++ b/slider-agent/src/main/python/setup.py
@@ -17,7 +17,7 @@
setup(
name = "slider-agent",
- version = "0.31.0-incubating-SNAPSHOT",
+ version = "0.51.0-incubating-SNAPSHOT",
packages = ['agent'],
# metadata for upload to PyPI
author = "Apache Software Foundation",
diff --git a/slider-agent/src/test/python/agent/TestRegistration.py b/slider-agent/src/test/python/agent/TestRegistration.py
index 7b3c875..2c98978 100644
--- a/slider-agent/src/test/python/agent/TestRegistration.py
+++ b/slider-agent/src/test/python/agent/TestRegistration.py
@@ -39,7 +39,7 @@
config.set('agent', 'current_ping_port', '33777')
register = Register(config)
- data = register.build(State.INIT, State.INIT, {}, 1)
+ data = register.build(State.INIT, State.INIT, {}, {}, 1)
#print ("Register: " + pprint.pformat(data))
self.assertEquals(data['hostname'] != "", True, "hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty")
@@ -49,7 +49,8 @@
self.assertEquals(data['actualState'], State.INIT, "actualState should not be empty")
self.assertEquals(data['expectedState'], State.INIT, "expectedState should not be empty")
self.assertEquals(data['allocatedPorts'], {}, "allocatedPorts should be empty")
- self.assertEquals(len(data), 8)
+ self.assertEquals(data['logFolders'], {}, "allocated log should be empty")
+ self.assertEquals(len(data), 9)
self.assertEquals(os.path.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir"))
self.assertEquals(os.path.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir"))
diff --git a/slider-agent/src/test/python/python-wrap b/slider-agent/src/test/python/python-wrap.sh
similarity index 84%
rename from slider-agent/src/test/python/python-wrap
rename to slider-agent/src/test/python/python-wrap.sh
index 40dc785..8f30d67 100755
--- a/slider-agent/src/test/python/python-wrap
+++ b/slider-agent/src/test/python/python-wrap.sh
@@ -17,7 +17,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-export PYTHONPATH=/usr/lib/python2.6/site-packages/common_functions:$PYTHONPATH
+export PYTHONPATH=/usr/lib/python2.7/site-packages/common_functions:$PYTHONPATH
# reset settings
unset PYTHON
@@ -25,10 +25,12 @@
# checking for preferable python versions
if [ -a /usr/bin/python2.7 ] && [ -z "$PYTHON" ]; then
PYTHON=/usr/bin/python2.7
+ export PYTHONPATH=/usr/lib/python2.7/site-packages/common_functions:$PYTHONPATH
fi
if [ -a /usr/bin/python2.6 ] && [ -z "$PYTHON" ]; then
PYTHON=/usr/bin/python2.6
+ export PYTHONPATH=/usr/lib/python2.6/site-packages/common_functions:$PYTHONPATH
fi
# if no preferable python versions found, try to use system one
diff --git a/slider-agent/src/test/python/unitTests.py b/slider-agent/src/test/python/unitTests.py
index b01438e..6aa0167 100644
--- a/slider-agent/src/test/python/unitTests.py
+++ b/slider-agent/src/test/python/unitTests.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-'''
+"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -16,14 +16,13 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-'''
+"""
import unittest
-import doctest
-from os.path import dirname, split, isdir
-import logging.handlers
+from os.path import isdir
import logging
-from random import shuffle
+import os
+import sys
LOG_FILE_NAME='tests.log'
SELECTED_PREFIX = "_"
@@ -31,9 +30,9 @@
ignoredDirs = ["mock"]
class TestAgent(unittest.TestSuite):
- def run(self, result):
+ def run(self, result, debug=False):
run = unittest.TestSuite.run
- run(self, result)
+ run(self, result, debug)
return result
@@ -41,19 +40,19 @@
if isdir(path):
if path.endswith(os.sep):
path = os.path.dirname(path)
- parent_dir = os.path.dirname(path)
+ parent = os.path.dirname(path)
else:
- parent_dir = os.path.dirname(os.path.dirname(path))
+ parent = os.path.dirname(os.path.dirname(path))
- return parent_dir
+ return parent
def all_tests_suite():
- src_dir = os.getcwd()
+ root_dir = os.getcwd()
files_list = []
- for directory in os.listdir(src_dir):
+ for directory in os.listdir(root_dir):
if os.path.isdir(directory) and not directory in ignoredDirs:
- files_list += os.listdir(src_dir + os.sep + directory)
+ files_list += os.listdir(root_dir + os.sep + directory)
## temporarily deleting to add more predictability
## shuffle(files_list)
files_list.sort()
@@ -71,8 +70,9 @@
else:
for file_name in files_list:
if file_name.endswith(PY_EXT) and not file_name == __file__:
- logger.info(file_name)
- tests_list.append(file_name.replace(PY_EXT, ''))
+ replaced = file_name.replace(PY_EXT, '')
+ logger.info(replaced)
+ tests_list.append(replaced)
logger.info('------------------------------------------------------------------------')
suite = unittest.TestLoader().loadTestsFromNames(tests_list)
@@ -99,9 +99,7 @@
logger.info('------------------------------------------------------------------------')
if __name__ == '__main__':
- import os
- import sys
- import io
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python')
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python' + os.sep + 'agent')
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index dbbd722..c2d0ecb 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -33,7 +33,7 @@
<rpm.bindir>${rpm.basedir}/bin</rpm.bindir>
<rpm.libdir>${rpm.basedir}/lib</rpm.libdir>
<rpm.agentdir>${rpm.basedir}/agent</rpm.agentdir>
- <rpm.username>mapred</rpm.username>
+ <rpm.username>root</rpm.username>
<rpm.groupname>hadoop</rpm.groupname>
<src.confdir>src/conf-hdp</src.confdir>
<src.libdir>${project.build.directory}/lib</src.libdir>
@@ -287,10 +287,10 @@
<!-- agent -->
<mapping>
<directory>${rpm.agentdir}</directory>
- <configuration>true</configuration>
<filemode>0755</filemode>
<username>${rpm.username}</username>
<groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>false</directoryIncluded>
<sources>
<source>
<location>${project.build.directory}/agent</location>
@@ -308,6 +308,7 @@
<filemode>0755</filemode>
<username>${rpm.username}</username>
<groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>false</directoryIncluded>
<sources>
<source>
<location>${src.agent.ini.dir}</location>
@@ -318,6 +319,24 @@
</source>
</sources>
</mapping>
+
+ <!-- needed to apply attribute to directory -->
+ <mapping>
+ <directory>${rpm.agentdir}</directory>
+ <filemode>0755</filemode>
+ <username>${rpm.username}</username>
+ <groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>true</directoryIncluded>
+ </mapping>
+
+ <mapping>
+ <directory>${rpm.agentdir}/conf</directory>
+ <filemode>0755</filemode>
+ <username>${rpm.username}</username>
+ <groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>true</directoryIncluded>
+ </mapping>
+
</mappings>
<!--
Scripts. Very dangerous in RPMs unless you know exactly what you are doing.
diff --git a/slider-assembly/src/main/bash/README.md b/slider-assembly/src/main/bash/README.md
index a4b7b08..d818aeb 100644
--- a/slider-assembly/src/main/bash/README.md
+++ b/slider-assembly/src/main/bash/README.md
@@ -79,7 +79,7 @@
---------
* slider_destroy will do the following
- 1. Freeze the slider application based on provided name
+ 1. Stop the slider application based on provided name
2. Destory the slider application based on provided name
* The following args are required
diff --git a/slider-assembly/src/main/bash/slider_destroy b/slider-assembly/src/main/bash/slider_destroy
index 9039751..7ca30b7 100755
--- a/slider-assembly/src/main/bash/slider_destroy
+++ b/slider-assembly/src/main/bash/slider_destroy
@@ -53,8 +53,8 @@
#
# Main
#
-echo -e "\n## Freezing app $app_name"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider freeze $app_name --manager $RM_ADDRESS || exit 1
+echo -e "\n## Stopping app $app_name"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider stop $app_name --manager $RM_ADDRESS || exit 1
echo "SUCCESS"
echo -e "\n## Destroying app $app_name"
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
index 6cd55ad..477e237 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -22,6 +22,8 @@
# The env variable SLIDER_JVM_OPTS can be used to override
# the default JVM opts
+export JAVA_HOME=${JAVA_HOME}
+
function usage
{
echo "Usage: slider <action> <arguments>"
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index db4b881..cbc0faf 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -23,6 +23,7 @@
LIB = "lib"
+JAVA_HOME = None
SLIDER_CONF_DIR = "SLIDER_CONF_DIR"
SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS"
SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
@@ -147,9 +148,12 @@
"""
# split the JVM opts by space
# java = "/usr/bin/java"
- prg="java"
- if which("java")==None:
- prg=os.environ["JAVA_HOME"]+"/bin/java"
+ prg = "java"
+ if JAVA_HOME:
+ prg = os.path.join(JAVA_HOME, "bin", "java")
+ else:
+ if which("java") is None:
+ prg = os.path.join(os.environ["JAVA_HOME"], "bin", "java")
commandline = [prg]
commandline.extend(jvm_opts_list)
commandline.append("-classpath")
@@ -203,7 +207,7 @@
try:
returncode = main()
except Exception as e:
- print "Exception: %s " % e.message
+ print "Exception: %s " % str(e)
returncode = -1
sys.exit(returncode)
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index cdcea38..326af88 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -538,7 +538,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2.1</version>
+ <version>${maven-exec-plugin.version}</version>
<configuration>
<executable>java</executable>
<arguments>
@@ -562,7 +562,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2.1</version>
+ <version>${maven-exec-plugin.version}</version>
<executions>
<execution>
<goals>
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index ad384e2..b58f1c6 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -75,6 +75,17 @@
*/
int DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY = 5000;
/**
+ * Time in seconds before a container is considered long-lived.
+ * Shortlived containers are interpreted as a problem with the role
+ * and/or the host: {@value}
+ */
+ String INTERNAL_CONTAINER_FAILURE_SHORTLIFE =
+ "internal.container.failure.shortlife";
+ /**
+ * Default short life threshold: {@value}
+ */
+ int DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE = 60;
+ /**
* Version of the app: {@value}
*/
String KEYTAB_LOCATION = "internal.keytab.location";
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index 3d54140..b542f1a 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -92,19 +92,6 @@
/**
- * Time in seconds before a container is considered long-lived.
- * Shortlived containers are interpreted as a problem with the role
- * and/or the host: {@value}
- */
- String CONTAINER_FAILURE_SHORTLIFE =
- "container.failure.shortlife";
-
- /**
- * Default short life threshold: {@value}
- */
- int DEFAULT_CONTAINER_FAILURE_SHORTLIFE = 60;
-
- /**
* maximum number of failed containers (in a single role)
* before the cluster is deemed to have failed {@value}
*/
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 77cd244..ef6448d 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -46,6 +46,7 @@
import org.apache.slider.common.Constants;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionCreateArgs;
@@ -291,6 +292,12 @@
// choose the action
String action = serviceArgs.getAction();
+
+ AbstractActionArgs coreAction = serviceArgs.getCoreAction();
+ if (coreAction.getHadoopServicesRequired()) {
+ // validate the client
+ SliderUtils.validateSliderClientEnvironment(null);
+ }
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
// actions
@@ -348,7 +355,8 @@
/**
* Delete the zookeeper node associated with the calling user and the cluster
**/
- protected boolean deleteZookeeperNode(String clusterName) throws YarnException, IOException {
+ @VisibleForTesting
+ public boolean deleteZookeeperNode(String clusterName) throws YarnException, IOException {
String user = getUsername();
String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
Exception e = null;
@@ -384,7 +392,8 @@
/**
* Create the zookeeper node associated with the calling user and the cluster
*/
- protected String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException {
+ @VisibleForTesting
+ public String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException {
String user = getUsername();
String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
if(nameOnly) {
@@ -613,7 +622,9 @@
// verify that a live cluster isn't there
SliderUtils.validateClusterName(clustername);
verifyBindingsDefined();
- if (!liveClusterAllowed) verifyNoLiveClusters(clustername);
+ if (!liveClusterAllowed) {
+ verifyNoLiveClusters(clustername);
+ }
Configuration conf = getConfig();
String registryQuorum = lookupZKQuorum();
@@ -1169,7 +1180,7 @@
*/
addConfOptionToCLI(commandLine,
config,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
@@ -1263,10 +1274,11 @@
*/
private void propagatePrincipals(Configuration config,
AggregateConf clusterSpec) {
- String dfsPrincipal = config.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ String dfsPrincipal = config.get(
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY;
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset(
siteDfsPrincipal,
dfsPrincipal);
@@ -1746,10 +1758,10 @@
}
/**
- * Freeze the cluster
+ * Stop the cluster
*
* @param clustername cluster name
- * @param freezeArgs arguments to the freeze
+ * @param freezeArgs arguments to the stop
* @return EXIT_SUCCESS if the cluster was not running by the end of the operation
*/
public int actionFreeze(String clustername,
@@ -1772,10 +1784,10 @@
if (app == null) {
// exit early
log.info("Cluster {} not running", clustername);
- // not an error to freeze a frozen cluster
+ // not an error to stop a stopped cluster
return EXIT_SUCCESS;
}
- log.debug("App to freeze was found: {}:\n{}", clustername,
+ log.debug("App to stop was found: {}:\n{}", clustername,
new SliderUtils.OnDemandReportStringifier(app));
if (app.getYarnApplicationState().ordinal() >=
YarnApplicationState.FINISHED.ordinal()) {
@@ -1789,7 +1801,7 @@
if (forcekill) {
//escalating to forced kill
- application.kill("Forced freeze of " + clustername +
+ application.kill("Forced stop of " + clustername +
": " + text);
} else {
try {
@@ -2002,12 +2014,9 @@
log.info("Flexing running cluster");
SliderClusterProtocol appMaster = connect(instance);
SliderClusterOperations clusterOps = new SliderClusterOperations(appMaster);
- if (clusterOps.flex(instanceDefinition.getResources())) {
- log.info("Cluster size updated");
- exitCode = EXIT_SUCCESS;
- } else {
- log.info("Requested size is the same as current size: no change");
- }
+ clusterOps.flex(instanceDefinition.getResources());
+ log.info("application instance size updated");
+ exitCode = EXIT_SUCCESS;
} else {
log.info("No running instance to update");
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index f4a4569..d18c85b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -148,4 +148,15 @@
public String toString() {
return super.toString() + ": " + getActionName();
}
+
+ /**
+ * Override point:
+ * Flag to indicate that core hadoop API services are needed (HDFS, YARN, etc)
+ * —and that validation of the client state should take place.
+ *
+ * @return a flag to indicate that the core hadoop services will be needed.
+ */
+ public boolean getHadoopServicesRequired() {
+ return true;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
index 04f6305..e3085d9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
@@ -32,7 +32,7 @@
return SliderActions.ACTION_FREEZE;
}
- public static final String FREEZE_COMMAND_ISSUED = "freeze command issued";
+ public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
@ParametersDelegate
public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
index 4d4098f..f59396a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
@@ -20,9 +20,11 @@
import com.beust.jcommander.Parameters;
+/**
+ * The Help command
+ */
@Parameters(commandNames = {SliderActions.ACTION_HELP, SliderActions.ACTION_USAGE},
- commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
+ commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
public class ActionHelpArgs extends AbstractActionArgs {
@Override
public String getActionName() {
@@ -38,4 +40,12 @@
return 0;
}
+ /**
+ * This action does not need hadoop services
+ * @return false
+ */
+ @Override
+ public boolean getHadoopServicesRequired() {
+ return false;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
index c36ef62..b9d212b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
@@ -20,9 +20,11 @@
import com.beust.jcommander.Parameters;
+/**
+ * The version command
+ */
@Parameters(commandNames = {SliderActions.ACTION_VERSION},
commandDescription = SliderActions.DESCRIBE_ACTION_VERSION)
-
public class ActionVersionArgs extends AbstractActionArgs {
@Override
public String getActionName() {
@@ -33,4 +35,12 @@
return 0;
}
+ /**
+ * This action does not need hadoop services
+ * @return false
+ */
+ @Override
+ public boolean getHadoopServicesRequired() {
+ return false;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
index f4ff4ce..c36a968 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
@@ -19,7 +19,7 @@
package org.apache.slider.common.params;
/**
- * Launch args for create and thaw and anything else that can start something
+ * Launch args for create and start and anything else that can start something
*/
public interface LaunchArgsAccessor extends WaitTimeAccessor {
String getRmAddress();
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index 8e50a83..ba2bca9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -32,7 +32,7 @@
String ACTION_ECHO = "echo";
String ACTION_EXISTS = "exists";
String ACTION_FLEX = "flex";
- String ACTION_FREEZE = "freeze";
+ String ACTION_FREEZE = "stop";
String ACTION_GETCONF = "getconf";
String ACTION_HELP = "help";
String ACTION_KILL_CONTAINER = "kill-container";
@@ -41,7 +41,7 @@
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
String ACTION_STATUS = "status";
- String ACTION_THAW = "thaw";
+ String ACTION_THAW = "start";
String ACTION_USAGE = "usage";
String ACTION_VERSION = "version";
String DESCRIBE_ACTION_AM_SUICIDE =
@@ -58,7 +58,7 @@
"Probe for an application running";
String DESCRIBE_ACTION_FLEX = "Flex a Slider application";
String DESCRIBE_ACTION_FREEZE =
- "Freeze/suspend a running application";
+ "Stop a running application";
String DESCRIBE_ACTION_GETCONF =
"Get the configuration of an application";
String DESCRIBE_ACTION_KILL_CONTAINER =
@@ -73,7 +73,7 @@
String DESCRIBE_ACTION_STATUS =
"Get the status of an application";
String DESCRIBE_ACTION_THAW =
- "Thaw a frozen application";
+ "Start a stopped application";
String DESCRIBE_ACTION_VERSION =
"Print the Slider version information";
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index a864878..4cdc580 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -37,6 +37,7 @@
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
@@ -55,7 +56,10 @@
import org.apache.slider.core.exceptions.MissingArgException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ClasspathConstructor;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.server.services.utility.EndOfServiceWaiter;
import org.apache.slider.server.services.utility.PatternValidator;
+import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.zookeeper.server.util.KerberosUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,8 +67,10 @@
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InterruptedIOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.InetSocketAddress;
@@ -86,6 +92,7 @@
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeSet;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -100,12 +107,25 @@
* turned on (prevents re-entrancy)
*/
private static final AtomicBoolean processSecurityAlreadyInitialized =
- new AtomicBoolean(false);
+ new AtomicBoolean(false);
public static final String JAVA_SECURITY_KRB5_REALM =
"java.security.krb5.realm";
public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
-
+ /**
+ * Winutils
+ */
+ public static final String WINUTILS = "WINUTILS.EXE";
+ /**
+ * name of openssl program
+ */
+ public static final String OPENSSL = "openssl";
+
+ /**
+ * name of python program
+ */
+ public static final String PYTHON = "python";
+
private SliderUtils() {
}
@@ -128,7 +148,8 @@
* @param msg the message to be shown in exception
*/
@SuppressWarnings("ResultOfMethodCallIgnored")
- private static void validateNumber(String num, String msg) throws BadConfigException {
+ private static void validateNumber(String num, String msg) throws
+ BadConfigException {
try {
Integer.parseInt(num);
} catch (NumberFormatException nfe) {
@@ -142,15 +163,16 @@
* @param heapsize
* @return heapsize in MB
*/
- public static String translateTrailingHeapUnit(String heapsize) throws BadConfigException {
+ public static String translateTrailingHeapUnit(String heapsize) throws
+ BadConfigException {
String errMsg = "Bad heapsize: ";
if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
- String num = heapsize.substring(0, heapsize.length()-1);
+ String num = heapsize.substring(0, heapsize.length() - 1);
validateNumber(num, errMsg);
return num;
}
if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
- String num = heapsize.substring(0, heapsize.length()-1)+"000";
+ String num = heapsize.substring(0, heapsize.length() - 1) + "000";
validateNumber(num, errMsg);
return num;
}
@@ -219,7 +241,7 @@
ClassLoader loader = my_class.getClassLoader();
if (loader == null) {
throw new IOException(
- "Class " + my_class + " does not have a classloader!");
+ "Class " + my_class + " does not have a classloader!");
}
String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
Enumeration<URL> urlEnumeration = loader.getResources(class_file);
@@ -252,16 +274,16 @@
}
public static void checkPort(String hostname, int port, int connectTimeout)
- throws IOException {
+ throws IOException {
InetSocketAddress addr = new InetSocketAddress(hostname, port);
checkPort(hostname, addr, connectTimeout);
}
@SuppressWarnings("SocketOpenedButNotSafelyClosed")
public static void checkPort(String name,
- InetSocketAddress address,
- int connectTimeout)
- throws IOException {
+ InetSocketAddress address,
+ int connectTimeout)
+ throws IOException {
Socket socket = null;
try {
socket = new Socket();
@@ -271,14 +293,14 @@
+ " at " + address
+ " after " + connectTimeout + "millisconds"
+ ": " + e,
- e);
+ e);
} finally {
IOUtils.closeSocket(socket);
}
}
public static void checkURL(String name, String url, int timeout) throws
- IOException {
+ IOException {
InetSocketAddress address = NetUtils.createSocketAddr(url);
checkPort(name, address, timeout);
}
@@ -291,22 +313,22 @@
* @return the file
*/
public static File requiredFile(String filename, String role) throws
- IOException {
+ IOException {
if (filename.isEmpty()) {
throw new ExitUtil.ExitException(-1, role + " file not defined");
}
File file = new File(filename);
if (!file.exists()) {
throw new ExitUtil.ExitException(-1,
- role + " file not found: " +
- file.getCanonicalPath());
+ role + " file not found: " +
+ file.getCanonicalPath());
}
return file;
}
private static final PatternValidator clusternamePattern
= new PatternValidator("[a-z][a-z0-9_-]*");
-
+
/**
* Normalize a cluster name then verify that it is valid
* @param name proposed cluster name
@@ -315,12 +337,13 @@
public static boolean isClusternameValid(String name) {
return name != null && clusternamePattern.matches(name);
}
+
public static boolean oldIsClusternameValid(String name) {
if (name == null || name.isEmpty()) {
return false;
}
int first = name.charAt(0);
- if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
+ if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
return false;
}
@@ -351,11 +374,11 @@
* @return # of files copies
*/
public static int copyDirectory(Configuration conf,
- Path srcDirPath,
- Path destDirPath,
- FsPermission permission) throws
- IOException,
- BadClusterStateException {
+ Path srcDirPath,
+ Path destDirPath,
+ FsPermission permission) throws
+ IOException,
+ BadClusterStateException {
FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
//list all paths in the src.
@@ -363,7 +386,8 @@
throw new FileNotFoundException("Source dir not found " + srcDirPath);
}
if (!srcFS.isDirectory(srcDirPath)) {
- throw new FileNotFoundException("Source dir not a directory " + srcDirPath);
+ throw new FileNotFoundException(
+ "Source dir not a directory " + srcDirPath);
}
GlobFilter dotFilter = new GlobFilter("[!.]*");
FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
@@ -375,7 +399,8 @@
permission = FsPermission.getDirDefault();
}
if (!destFS.exists(destDirPath)) {
- new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath, permission);
+ new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
+ permission);
}
Path[] sourcePaths = new Path[srcFileCount];
for (int i = 0; i < srcFileCount; i++) {
@@ -391,8 +416,8 @@
sourcePaths[i] = srcFile;
}
log.debug("Copying {} files from {} to dest {}", srcFileCount,
- srcDirPath,
- destDirPath);
+ srcDirPath,
+ destDirPath);
FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
return srcFileCount;
}
@@ -427,7 +452,8 @@
//if the fallback option is NOT set, enable it.
//if it is explicitly set to anything -leave alone
- if (conf.get(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH) == null) {
+ if (conf.get(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH) ==
+ null) {
conf.set(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH, "true");
}
return conf;
@@ -466,7 +492,9 @@
* @param trailing add a trailing entry or not
* @return the joined entries
*/
- public static String join(Collection collection, String separator, boolean trailing) {
+ public static String join(Collection collection,
+ String separator,
+ boolean trailing) {
StringBuilder b = new StringBuilder();
// fast return on empty collection
if (collection.isEmpty()) {
@@ -478,9 +506,9 @@
}
int length = separator.length();
String s = b.toString();
- return (trailing || s.isEmpty())?
+ return (trailing || s.isEmpty()) ?
s
- : (b.substring(0, b.length() - length));
+ : (b.substring(0, b.length() - length));
}
/**
@@ -492,9 +520,10 @@
*/
public static String join(String[] collection, String separator) {
return join(collection, separator, true);
-
-
+
+
}
+
/**
* Join an array of strings with a separator that appears after every
* instance in the list -optionally at the end
@@ -504,7 +533,7 @@
* @return the joined entries
*/
public static String join(String[] collection, String separator,
- boolean trailing) {
+ boolean trailing) {
return join(Arrays.asList(collection), separator, trailing);
}
@@ -516,7 +545,7 @@
* @return the list
*/
public static String joinWithInnerSeparator(String separator,
- Object... collection) {
+ Object... collection) {
StringBuilder b = new StringBuilder();
boolean first = true;
@@ -540,10 +569,15 @@
return v;
}
- public static String appReportToString(ApplicationReport r, String separator) {
+ public static String appReportToString(ApplicationReport r,
+ String separator) {
StringBuilder builder = new StringBuilder(512);
- builder.append("application ").append(
- r.getName()).append("/").append(r.getApplicationType()).append(separator);
+ builder.append("application ")
+ .append(
+ r.getName())
+ .append("/")
+ .append(r.getApplicationType())
+ .append(separator);
Set<String> tags = r.getApplicationTags();
if (!tags.isEmpty()) {
for (String tag : tags) {
@@ -552,12 +586,20 @@
}
builder.append("state: ").append(r.getYarnApplicationState());
builder.append(separator).append("URL: ").append(r.getTrackingUrl());
- builder.append(separator).append("Started ").append(new Date(r.getStartTime()).toGMTString());
+ builder.append(separator)
+ .append("Started ")
+ .append(new Date(r.getStartTime()).toGMTString());
long finishTime = r.getFinishTime();
- if (finishTime>0) {
- builder.append(separator).append("Finished ").append(new Date(finishTime).toGMTString());
+ if (finishTime > 0) {
+ builder.append(separator)
+ .append("Finished ")
+ .append(new Date(finishTime).toGMTString());
}
- builder.append(separator).append("RPC :").append(r.getHost()).append(':').append(r.getRpcPort());
+ builder.append(separator)
+ .append("RPC :")
+ .append(r.getHost())
+ .append(':')
+ .append(r.getRpcPort());
String diagnostics = r.getDiagnostics();
if (!diagnostics.isEmpty()) {
builder.append(separator).append("Diagnostics :").append(diagnostics);
@@ -572,8 +614,8 @@
* @param second the map that is merged in
* @return the first map
*/
- public static Map<String, String> mergeMap(Map<String, String> first,
- Map<String, String> second) {
+ public static Map<String, String> mergeMap(Map<String, String> first,
+ Map<String, String> second) {
first.putAll(second);
return first;
}
@@ -586,8 +628,8 @@
* @return dest -with the entries merged in
*/
public static Map<String, String> mergeEntries(Map<String, String> dest,
- Iterable<Map.Entry<String, String>> entries) {
- for (Map.Entry<String, String> entry: entries) {
+ Iterable<Map.Entry<String, String>> entries) {
+ for (Map.Entry<String, String> entry : entries) {
dest.put(entry.getKey(), entry.getValue());
}
return dest;
@@ -601,8 +643,8 @@
* @param <T2> value type
* @return 'first' merged with the second
*/
- public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
- Map<T1, T2> second) {
+ public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
+ Map<T1, T2> second) {
first.putAll(second);
return first;
}
@@ -616,7 +658,7 @@
* @return 'first' merged with the second
*/
public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
- Map<T1, T2> second) {
+ Map<T1, T2> second) {
Preconditions.checkArgument(first != null, "Null 'first' value");
Preconditions.checkArgument(second != null, "Null 'second' value");
for (Map.Entry<T1, T2> entry : second.entrySet()) {
@@ -634,8 +676,8 @@
* @return a string representation of the map
*/
public static String stringifyMap(Map<String, String> map) {
- StringBuilder builder =new StringBuilder();
- for (Map.Entry<String, String> entry: map.entrySet()) {
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
builder.append(entry.getKey())
.append("=\"")
.append(entry.getValue())
@@ -656,11 +698,11 @@
* @throws BadConfigException if the value could not be parsed
*/
public static int getIntValue(Map<String, String> roleMap,
- String key,
- int defVal,
- int min,
- int max
- ) throws BadConfigException {
+ String key,
+ int defVal,
+ int min,
+ int max
+ ) throws BadConfigException {
String valS = roleMap.get(key);
return parseAndValidate(key, valS, defVal, min, max);
@@ -676,10 +718,10 @@
* @throws BadConfigException if the value could not be parsed
*/
public static int parseAndValidate(String errorKey,
- String valS,
- int defVal,
- int min, int max) throws
- BadConfigException {
+ String valS,
+ int defVal,
+ int min, int max) throws
+ BadConfigException {
if (valS == null) {
valS = Integer.toString(defVal);
}
@@ -706,14 +748,14 @@
public static InetSocketAddress getRmAddress(Configuration conf) {
return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
}
public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
}
/**
@@ -756,11 +798,11 @@
return "null container";
}
return String.format(Locale.ENGLISH,
- "ContainerID=%s nodeID=%s http=%s priority=%s",
- container.getId(),
- container.getNodeId(),
- container.getNodeHttpAddress(),
- container.getPriority());
+ "ContainerID=%s nodeID=%s http=%s priority=%s",
+ container.getId(),
+ container.getNodeId(),
+ container.getNodeHttpAddress(),
+ container.getPriority());
}
/**
@@ -841,12 +883,12 @@
public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
Map<String, String> env = new HashMap<String, String>();
if (roleOpts != null) {
- for (Map.Entry<String, String> entry: roleOpts.entrySet()) {
+ for (Map.Entry<String, String> entry : roleOpts.entrySet()) {
String key = entry.getKey();
if (key.startsWith(RoleKeys.ENV_PREFIX)) {
String envName = key.substring(RoleKeys.ENV_PREFIX.length());
if (!envName.isEmpty()) {
- env.put(envName,entry.getValue());
+ env.put(envName, entry.getValue());
}
}
}
@@ -860,8 +902,8 @@
* @param commandOptions command opts
*/
public static void applyCommandLineRoleOptsToRoleMap(Map<String, Map<String, String>> clusterRoleMap,
- Map<String, Map<String, String>> commandOptions) {
- for (Map.Entry<String, Map<String, String>> entry: commandOptions.entrySet()) {
+ Map<String, Map<String, String>> commandOptions) {
+ for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
String key = entry.getKey();
Map<String, String> optionMap = entry.getValue();
Map<String, String> existingMap = clusterRoleMap.get(key);
@@ -869,7 +911,7 @@
existingMap = new HashMap<String, String>();
}
log.debug("Overwriting role options with command line values {}",
- stringifyMap(optionMap));
+ stringifyMap(optionMap));
mergeMap(existingMap, optionMap);
//set or overwrite the role
clusterRoleMap.put(key, existingMap);
@@ -882,10 +924,10 @@
* @throws BadCommandArgumentsException if it is invalid
*/
public static void validateClusterName(String clustername) throws
- BadCommandArgumentsException {
+ BadCommandArgumentsException {
if (!isClusternameValid(clustername)) {
throw new BadCommandArgumentsException(
- "Illegal cluster name: " + clustername);
+ "Illegal cluster name: " + clustername);
}
}
@@ -897,12 +939,12 @@
* @throws BadConfigException if the key is not set
*/
public static void verifyPrincipalSet(Configuration conf,
- String principal) throws
- BadConfigException {
+ String principal) throws
+ BadConfigException {
String principalName = conf.get(principal);
if (principalName == null) {
throw new BadConfigException("Unset Kerberos principal : %s",
- principal);
+ principal);
}
log.debug("Kerberos princial {}={}", principal, principalName);
}
@@ -925,8 +967,8 @@
* @throws BadConfigException the configuration/process is invalid
*/
public static boolean maybeInitSecurity(Configuration conf) throws
- IOException,
- BadConfigException {
+ IOException,
+ BadConfigException {
boolean clusterSecure = isHadoopClusterSecure(conf);
if (clusterSecure) {
log.debug("Enabling security");
@@ -943,8 +985,8 @@
* @throws BadConfigException the configuration and system state are inconsistent
*/
public static boolean initProcessSecurity(Configuration conf) throws
- IOException,
- BadConfigException {
+ IOException,
+ BadConfigException {
if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
//security is already inited
@@ -956,9 +998,9 @@
//this gets UGI to reset its previous world view (i.e simple auth)
//security
log.debug("java.security.krb5.realm={}",
- System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
+ System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
log.debug("java.security.krb5.kdc={}",
- System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
+ System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
log.debug("hadoop.security.authentication={}",
conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
log.debug("hadoop.security.authorization={}",
@@ -971,20 +1013,20 @@
log.debug("Login user is {}", UserGroupInformation.getLoginUser());
if (!UserGroupInformation.isSecurityEnabled()) {
throw new BadConfigException("Although secure mode is enabled," +
- "the application has already set up its user as an insecure entity %s",
- authUser);
+ "the application has already set up its user as an insecure entity %s",
+ authUser);
}
if (authUser.getAuthenticationMethod() ==
UserGroupInformation.AuthenticationMethod.SIMPLE) {
throw new BadConfigException("Auth User is not Kerberized %s" +
- " -security has already been set up with the wrong authentication method",
- authUser);
+ " -security has already been set up with the wrong authentication method",
+ authUser);
}
SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
SliderUtils.verifyPrincipalSet(conf,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
return true;
}
@@ -1014,24 +1056,25 @@
* @throws IOException trouble copying to HDFS
*/
public static LocalResource putJar(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Class clazz,
- Path tempPath,
- String libdir,
- String jarName
- )
- throws IOException, SliderException {
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
LocalResource res = sliderFileSystem.submitJarWithClass(
- clazz,
- tempPath,
- libdir,
- jarName);
- providerResources.put(libdir + "/"+ jarName, res);
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/" + jarName, res);
return res;
}
- public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
- Map<String, Map<String, String>> dest = new HashMap<String, Map<String, String>>();
+ public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
+ Map<String, Map<String, String>> dest =
+ new HashMap<String, Map<String, String>>();
for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
dest.put(entry.getKey(), stringMapClone(entry.getValue()));
}
@@ -1039,7 +1082,7 @@
}
public static Map<String, String> stringMapClone(Map<String, String> src) {
- Map<String, String> dest = new HashMap<String, String>();
+ Map<String, String> dest = new HashMap<String, String>();
return mergeEntries(dest, src.entrySet());
}
@@ -1143,12 +1186,12 @@
* @return a classpath
*/
public static ClasspathConstructor buildClasspath(String sliderConfDir,
- String libdir,
- Configuration config,
- boolean usingMiniMRCluster) {
+ String libdir,
+ Configuration config,
+ boolean usingMiniMRCluster) {
ClasspathConstructor classpath = new ClasspathConstructor();
-
+
// add the runtime classpath needed for tests to work
if (usingMiniMRCluster) {
// for mini cluster we pass down the java CP properties
@@ -1172,17 +1215,18 @@
* @param errorlog log for output on an error
* @throws FileNotFoundException if it is not a directory
*/
- public static void verifyIsDir(File dir, Logger errorlog) throws FileNotFoundException {
+ public static void verifyIsDir(File dir, Logger errorlog) throws
+ FileNotFoundException {
if (!dir.exists()) {
errorlog.warn("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
+ listDir(dir.getParentFile()));
throw new FileNotFoundException(dir.toString());
}
if (!dir.isDirectory()) {
errorlog.info("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
+ listDir(dir.getParentFile()));
throw new FileNotFoundException(
- "Not a directory: " + dir);
+ "Not a directory: " + dir);
}
}
@@ -1192,10 +1236,11 @@
* @param errorlog log for output on an error
* @throws FileNotFoundException
*/
- public static void verifyFileExists(File file, Logger errorlog) throws FileNotFoundException {
+ public static void verifyFileExists(File file, Logger errorlog) throws
+ FileNotFoundException {
if (!file.exists()) {
errorlog.warn("contents of {}: {}", file,
- listDir(file.getParentFile()));
+ listDir(file.getParentFile()));
throw new FileNotFoundException(file.toString());
}
if (!file.isFile()) {
@@ -1211,15 +1256,15 @@
* @throws BadConfigException if the key is missing
*/
public static String verifyOptionSet(Configuration configuration, String key,
- boolean allowEmpty) throws BadConfigException {
+ boolean allowEmpty) throws BadConfigException {
String val = configuration.get(key);
if (val == null) {
throw new BadConfigException(
- "Required configuration option \"%s\" not defined ", key);
+ "Required configuration option \"%s\" not defined ", key);
}
if (!allowEmpty && val.isEmpty()) {
throw new BadConfigException(
- "Configuration option \"%s\" must not be empty", key);
+ "Configuration option \"%s\" must not be empty", key);
}
return val;
}
@@ -1232,24 +1277,25 @@
* @return the file referenced
* @throws BadConfigException on a failure
*/
- public static File verifyKeytabExists(Configuration siteConf, String prop) throws
- BadConfigException {
+ public static File verifyKeytabExists(Configuration siteConf,
+ String prop) throws
+ BadConfigException {
String keytab = siteConf.get(prop);
if (keytab == null) {
throw new BadConfigException("Missing keytab property %s",
- prop);
+ prop);
}
File keytabFile = new File(keytab);
if (!keytabFile.exists()) {
throw new BadConfigException("Missing keytab file %s defined in %s",
- keytabFile,
- prop);
+ keytabFile,
+ prop);
}
if (keytabFile.length() == 0 || !keytabFile.isFile()) {
throw new BadConfigException("Invalid keytab file %s defined in %s",
- keytabFile,
- prop);
+ keytabFile,
+ prop);
}
return keytabFile;
}
@@ -1275,12 +1321,12 @@
Properties props = SliderVersionInfo.loadVersionProperties();
info.put(prefix + "." + SliderVersionInfo.APP_BUILD_INFO, props.getProperty(
- SliderVersionInfo.APP_BUILD_INFO));
+ SliderVersionInfo.APP_BUILD_INFO));
info.put(prefix + "." + SliderVersionInfo.HADOOP_BUILD_INFO,
- props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO));
+ props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO));
info.put(prefix + "." + SliderVersionInfo.HADOOP_DEPLOYED_INFO,
- VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
+ VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
}
/**
@@ -1292,30 +1338,33 @@
* @param time timestamp
*/
public static void setInfoTime(Map info,
- String keyHumanTime,
- String keyMachineTime,
- long time) {
+ String keyHumanTime,
+ String keyMachineTime,
+ long time) {
info.put(keyHumanTime, SliderUtils.toGMTString(time));
info.put(keyMachineTime, Long.toString(time));
}
- public static Path extractImagePath(CoreFileSystem fs, MapOperations internalOptions) throws
+ public static Path extractImagePath(CoreFileSystem fs,
+ MapOperations internalOptions) throws
SliderException, IOException {
Path imagePath;
String imagePathOption =
internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- String appHomeOption = internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
+ String appHomeOption =
+ internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
if (!isUnset(imagePathOption)) {
imagePath = fs.createPathThatMustExist(imagePathOption);
} else {
imagePath = null;
if (isUnset(appHomeOption)) {
- throw new BadClusterStateException(ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED);
+ throw new BadClusterStateException(
+ ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED);
}
}
return imagePath;
}
-
+
/**
* trigger a JVM halt with no clean shutdown at all
* @param status status code for exit
@@ -1370,7 +1419,7 @@
* @param paths subpaths
* @return base+"/"+paths[0]+"/"+paths[1]...
*/
- public static String appendToURL(String base, String...paths) {
+ public static String appendToURL(String base, String... paths) {
String result = base;
for (String path : paths) {
result = appendToURL(result, path);
@@ -1387,19 +1436,19 @@
* @return
*/
public static String truncate(String toTruncate, int maxSize) {
- if(toTruncate == null || maxSize < 1
- || toTruncate.length() <= maxSize) {
+ if (toTruncate == null || maxSize < 1
+ || toTruncate.length() <= maxSize) {
return toTruncate;
}
String pad = "...";
- if(maxSize < 10) {
+ if (maxSize < 10) {
pad = "";
}
return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
}
-
-
+
+
/**
* Callable for async/scheduled halt
*/
@@ -1433,11 +1482,15 @@
*/
public static int compareTo(long left, long right) {
long diff = left - right;
- if (diff < 0) return -1;
- if (diff > 0) return 1;
+ if (diff < 0) {
+ return -1;
+ }
+ if (diff > 0) {
+ return 1;
+ }
return 0;
}
-
+
/**
* This wrapps ApplicationReports and generates a string version
* iff the toString() operator is invoked
@@ -1456,8 +1509,8 @@
}
public static InputStream getApplicationResourceInputStream(FileSystem fs,
- Path appPath,
- String entry)
+ Path appPath,
+ String entry)
throws IOException {
InputStream is = null;
FSDataInputStream appStream = null;
@@ -1470,7 +1523,8 @@
if (entry.equals(zipEntry.getName())) {
int size = (int) zipEntry.getSize();
if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(), zipEntry.getSize());
+ log.info("Reading {} of size {}", zipEntry.getName(),
+ zipEntry.getSize());
byte[] content = new byte[size];
int offset = 0;
while (offset < size) {
@@ -1499,4 +1553,225 @@
return is;
}
+
+ /**
+ * Strictly verify that windows utils is present.
+ * Checks go as far as opening the file and looking for
+ * the headers.
+ * @throws IOException on any problem reading the file
+ * @throws FileNotFoundException if the file is not considered valid
+ * @param logger
+ */
+ public static void maybeVerifyWinUtilsValid(Logger logger) throws IOException, SliderException {
+ if (!Shell.WINDOWS) {
+ return;
+ }
+ String exePath = Shell.getWinUtilsPath();
+ String program = WINUTILS;
+ if (exePath == null) {
+ throw new FileNotFoundException(program + " not found on Path : " +
+ System.getenv("Path"));
+ }
+ File exe = new File(exePath);
+
+ verifyWindowsExe(program, exe);
+ execCommand(WINUTILS, 0, 5000, log, null, exePath, "systeminfo");
+
+ }
+
+ protected static void verifyIsFile(String program, File exe) throws
+ FileNotFoundException {
+ if (!exe.isFile()) {
+ throw new FileNotFoundException(program
+ + " at " + exe
+ + " is not a file");
+
+ }
+ }
+
+ protected static void verifyFileSize(String program,
+ File exe,
+ int minFileSize) throws FileNotFoundException {
+ if (exe.length() < minFileSize) {
+ throw new FileNotFoundException(program
+ + " at " + exe
+ + " is too short to be an executable");
+ }
+ }
+
+ /**
+ * Look for the windows executable and check it has the right headers.
+ * <code>File.canRead()</code> doesn't work on windows, so the reading
+ * is mandatory.
+ *
+ * @param program program name for errors
+ * @param exe executable
+ * @throws IOException IOE
+ */
+ public static void verifyWindowsExe(String program, File exe)
+ throws IOException {
+ verifyIsFile(program, exe);
+
+ verifyFileSize(program, exe, 0x100);
+
+ // now read two bytes and verify the header.
+
+ FileReader reader = null;
+ try {
+ int[] header = new int[2];
+ reader = new FileReader(exe);
+ header[0] = reader.read();
+ header[1] = reader.read();
+ if ((header[0] != 'M' || header[1] != 'Z')) {
+ throw new FileNotFoundException(program
+ + " at " + exe
+ + " is not a windows executable file");
+ }
+ } finally {
+ IOUtils.closeStream(reader);
+ }
+ }
+
+ /**
+ * Verify that a Unix exe works
+ * @param program program name for errors
+ * @param exe executable
+ * @throws IOException IOE
+
+ */
+ public static void verifyUnixExe(String program, File exe)
+ throws IOException {
+ verifyIsFile(program, exe);
+
+ // read flag
+ if (!exe.canRead()) {
+ throw new IOException("Cannot read " + program + " at " + exe);
+ }
+ // exe flag
+ if (!exe.canExecute()) {
+ throw new IOException("Cannot execute " + program + " at " + exe);
+ }
+ }
+
+ /**
+ * Validate an executable
+ * @param program
+ * @param exe
+ * @throws IOException
+ */
+ public static void validateExe(String program, File exe) throws IOException {
+ if (!Shell.WINDOWS) {
+ verifyWindowsExe(program, exe);
+ } else {
+ verifyUnixExe(program, exe);
+ }
+ }
+
+
+ /**
+ * Execute a command for a test operation
+ * @param name name in error
+ * @param status status code expected
+ * @param timeoutMillis timeout in millis for process to finish
+ * @param logger
+ *@param outputString optional string to grep for (must not span a line)
+ * @param commands commands @return the process
+ * @throws IOException on any failure.
+ */
+ public static ForkedProcessService execCommand(String name,
+ int status,
+ long timeoutMillis,
+ Logger logger,
+ String outputString,
+ String... commands) throws IOException, SliderException {
+ Preconditions.checkArgument(isSet(name), "no name");
+ Preconditions.checkArgument(commands.length > 0, "no commands");
+ Preconditions.checkArgument(isSet(commands[0]), "empty command");
+
+ ForkedProcessService process;
+
+
+ process = new ForkedProcessService(
+ name,
+ new HashMap<String, String>(),
+ Arrays.asList(commands));
+ process.setProcessLog(logger);
+ process.init(new Configuration());
+ String errorText = null;
+ EndOfServiceWaiter waiter = new EndOfServiceWaiter(process);
+ process.start();
+ try {
+ waiter.waitForServiceToStop(timeoutMillis);
+ int exitCode = process.getExitCode();
+ List<String> recentOutput = process.getRecentOutput();
+ if (status != exitCode) {
+ // error condition
+ errorText = "Expected exit code={" + status + "}, "
+ + "actual exit code={" + exitCode + "}";
+ } else {
+ if (isSet(outputString)) {
+ boolean found = false;
+ for (String line : recentOutput) {
+ if (line.contains(outputString)) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ errorText = "Did not find \"" + outputString + "\""
+ + " in output";
+ }
+ }
+ }
+ if (errorText== null) {
+ return process;
+ }
+
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.toString());
+ } catch (TimeoutException e) {
+ log.debug("");
+ errorText = e.toString();
+ }
+ // error text: non null ==> operation failed
+ log.warn(errorText);
+ List<String> recentOutput = process.getRecentOutput();
+ for (String line : recentOutput) {
+ log.info(line);
+ }
+ throw new SliderException(LauncherExitCodes.EXIT_OTHER_FAILURE,
+ "Process %s failed: %s", name, errorText);
+
+ }
+
+
+ /**
+ * Validate the slider client-side execution environment.
+ * This looks for everything felt to be critical for execution, including
+ * native binaries and other essential dependencies.
+ * @param logger logger to log to on normal execution
+ * @throws IOException on IO failures
+ * @throws SliderException on validation failures
+ */
+ public static void validateSliderClientEnvironment(Logger logger) throws
+ IOException,
+ SliderException {
+ maybeVerifyWinUtilsValid(logger);
+ }
+
+ /**
+ * Validate the slider server-side execution environment.
+ * This looks for everything felt to be critical for execution, including
+ * native binaries and other essential dependencies.
+ * @param logger logger to log to on normal execution
+ * @throws IOException on IO failures
+ * @throws SliderException on validation failures
+ */
+ public static void validateSliderServerEnvironment(Logger logger) throws
+ IOException,
+ SliderException {
+ maybeVerifyWinUtilsValid(logger);
+ execCommand(OPENSSL, 0, 5000, logger, "OpenSSL", OPENSSL, "version");
+ execCommand(PYTHON, 0, 5000, logger, "Python", PYTHON, "--version");
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 937b777..bfa7097 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -181,10 +181,11 @@
* Propagate any critical principals from the current site config down to the HBase one.
*/
public void propagatePrincipals() {
- String dfsPrincipal = conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ String dfsPrincipal = conf.get(
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY;
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
instanceDescription.getAppConfOperations().set(siteDfsPrincipal, dfsPrincipal);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
index 61b1ff0..cc1b2c9 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
@@ -27,6 +27,7 @@
import java.util.List;
public class ZookeeperUtils {
+ public static final int DEFAULT_PORT = 2181;
public static String buildConnectionString(String zkHosts, int port) {
String zkPort = Integer.toString(port);
@@ -73,7 +74,7 @@
List<HostAndPort> list = new ArrayList<HostAndPort>(len);
if (strings != null) {
for (String s : strings) {
- list.add(HostAndPort.fromString(s.trim()));
+ list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
}
}
return list;
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 419fa1a..1da99b0 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -85,11 +85,13 @@
String JAVA_HOME = "java_home";
String PACKAGE_LIST = "package_list";
+ String SYSTEM_CONFIGS = "system_configs";
String WAIT_HEARTBEAT = "wait.heartbeat";
String PYTHON_EXE = "python";
String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
+ String AGENT_OUT_FILE = "agent.out";
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 669a19a..063e61c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -92,7 +92,6 @@
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -109,9 +108,9 @@
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
-/** This class implements the server-side logic for application deployment
- * through Slider application package
- **/
+/**
+ * This class implements the server-side logic for application deployment through Slider application package
+ */
public class AgentProviderService extends AbstractProviderService implements
ProviderCore,
AgentKeys,
@@ -194,7 +193,7 @@
// Reads the metainfo.xml in the application package and loads it
private void buildMetainfo(AggregateConf instanceDefinition,
- SliderFileSystem fileSystem) throws IOException, SliderException {
+ SliderFileSystem fileSystem) throws IOException, SliderException {
String appDef = instanceDefinition.getAppConfOperations()
.getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
@@ -211,7 +210,7 @@
"metainfo.xml is required in app package.");
}
commandOrder = new ComponentCommandOrder(metainfo.getApplication()
- .getCommandOrder());
+ .getCommandOrder());
defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metainfo);
monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval());
monitor.start();
@@ -297,8 +296,8 @@
getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
if (org.apache.commons.lang.StringUtils.isNotEmpty(agentConf)) {
LocalResource agentConfRes = fileSystem.createAmResource(fileSystem
- .getFileSystem().resolvePath(new Path(agentConf)),
- LocalResourceType.FILE);
+ .getFileSystem().resolvePath(new Path(agentConf)),
+ LocalResourceType.FILE);
launcher.addLocalResource(AgentKeys.AGENT_CONFIG_FILE, agentConfRes);
}
@@ -334,8 +333,7 @@
operation.add(debugCmd);
}
- String outfile = new File(logDir, "agent.out").toString();
- operation.add("> " + outfile + " 2>&1");
+ operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + AgentKeys.AGENT_OUT_FILE + " 2>&1");
launcher.addCommand(operation.build());
@@ -361,23 +359,23 @@
@Override
public void rebuildContainerDetails(List<Container> liveContainers,
- String applicationId, Map<Integer, ProviderRole> providerRoleMap) {
+ String applicationId, Map<Integer, ProviderRole> providerRoleMap) {
for (Container container : liveContainers) {
// get the role name and label
ProviderRole role = providerRoleMap.get(ContainerPriority
- .extractRole(container));
+ .extractRole(container));
if (role != null) {
String roleName = role.name;
String label = getContainerLabel(container, roleName);
log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
- container.getId(), roleName, applicationId);
+ container.getId(), roleName, applicationId);
getComponentStatuses().put(
label,
new ComponentInstanceState(roleName, container.getId(),
- applicationId));
+ applicationId));
} else {
log.warn("Role not found for container {} in cluster {}",
- container.getId(), applicationId);
+ container.getId(), applicationId);
}
}
}
@@ -411,7 +409,9 @@
/**
* Handle registration calls from the agents
+ *
* @param registration
+ *
* @return
*/
@Override
@@ -426,11 +426,17 @@
componentStatus.heartbeat(System.currentTimeMillis());
updateComponentStatusWithAgentState(componentStatus, agentState);
+ String roleName = getRoleName(label);
+ String containerId = getContainerId(label);
+ String hostFqdn = registration.getPublicHostname();
Map<String, String> ports = registration.getAllocatedPorts();
if (ports != null && !ports.isEmpty()) {
- String roleName = getRoleName(label);
- String containerId = getContainerId(label);
- processAllocatedPorts(registration.getPublicHostname(), roleName, containerId, ports);
+ processAllocatedPorts(hostFqdn, roleName, containerId, ports);
+ }
+
+ Map<String, String> folders = registration.getLogFolders();
+ if (folders != null && folders.size() > 0) {
+ publishLogFolderPaths(folders, containerId, roleName, hostFqdn);
}
} else {
response.setResponseStatus(RegistrationStatus.FAILED);
@@ -443,7 +449,9 @@
/**
* Handle heartbeat response from agents
+ *
* @param heartBeat
+ *
* @return
*/
@Override
@@ -468,7 +476,7 @@
String scriptPath = cmdScript.getScript();
long timeout = cmdScript.getTimeout();
- if(timeout == 0L) {
+ if (timeout == 0L) {
timeout = 600L;
}
@@ -495,7 +503,7 @@
log.info("Component operation. Status: {}", result);
if (command == Command.INSTALL && report.getFolders() != null && report.getFolders().size() > 0) {
- publishLogFolderPaths(report.getFolders(), containerId, heartBeat.getFqdn());
+ publishLogFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn());
}
}
@@ -553,9 +561,9 @@
}
protected void processAllocatedPorts(String fqdn,
- String roleName,
- String containerId,
- Map<String, String> ports) {
+ String roleName,
+ String containerId,
+ Map<String, String> ports) {
RoleInstance instance;
try {
instance = getAmState().getOwnedContainer(containerId);
@@ -569,22 +577,22 @@
log.info("Recording allocated port for {} as {}", portname, portNo);
this.getAllocatedPorts().put(portname, portNo);
this.getAllocatedPorts(containerId).put(portname, portNo);
- if (instance!=null) {
- try {
- instance.registerPortEndpoint(Integer.valueOf(portNo), portname, "");
- } catch (NumberFormatException e) {
- log.warn("Failed to parse {}: {}", portNo, e);
- }
+ if (instance != null) {
+ try {
+ instance.registerPortEndpoint(Integer.valueOf(portNo), portname, "");
+ } catch (NumberFormatException e) {
+ log.warn("Failed to parse {}: {}", portNo, e);
}
+ }
}
// component specific publishes
processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
-
+
// and update registration entries
if (instance != null) {
queueAccess.put(new RegisterComponentInstance(instance.getId(), 0,
- TimeUnit.MILLISECONDS));
+ TimeUnit.MILLISECONDS));
}
}
@@ -653,6 +661,7 @@
/**
* Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
+ *
* @param instanceDefinition
*/
private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
@@ -661,7 +670,7 @@
Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
try {
setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
- }catch (NumberFormatException e) {
+ } catch (NumberFormatException e) {
log.warn(
"Bad value {} for {}. Defaulting to ",
hbMonitorInterval,
@@ -672,6 +681,7 @@
/**
* Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
+ *
* @param instanceDefinition
*/
private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
@@ -703,10 +713,13 @@
/**
* Read all default configs
+ *
* @param fileSystem
* @param appDef
* @param metainfo
+ *
* @return
+ *
* @throws IOException
*/
protected Map<String, DefaultConfig> initializeDefaultConfigs(SliderFileSystem fileSystem,
@@ -748,6 +761,7 @@
/**
* Publish a named property bag that may contain name-value pairs for app configurations such as hbase-site
+ *
* @param name
* @param description
* @param entries
@@ -763,6 +777,7 @@
/**
* Get a list of all hosts for all role/container per role
+ *
* @return
*/
protected Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping() {
@@ -792,11 +807,10 @@
}
/**
- * Lost heartbeat from the container - release it and ask for a replacement
- * (async operation)
- * @param label
- * @param containerId
+ * Lost heartbeat from the container - release it and ask for a replacement (async operation)
*
+ * @param label
+ * @param containerId
*/
protected void lostContainer(
String label,
@@ -818,17 +832,20 @@
/**
* Format the folder locations and publish in the registry service
+ *
* @param folders
* @param containerId
* @param hostFqdn
+ * @param roleName
*/
- private void publishLogFolderPaths(Map<String, String> folders, String containerId, String hostFqdn) {
+ protected void publishLogFolderPaths(
+ Map<String, String> folders, String containerId, String roleName, String hostFqdn) {
for (String key : folders.keySet()) {
- workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
+ workFolders.put(String.format("%s->%s->%s->%s", roleName, hostFqdn, key, containerId), folders.get(key));
}
publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG,
- (new HashMap<String, String>(this.workFolders)).entrySet());
+ (new HashMap<String, String>(this.workFolders)).entrySet());
}
@@ -940,16 +957,16 @@
}
private boolean canBeExported(String exportGroupName, String name, Set<String> appExports) {
- return appExports.contains(String.format("%s-%s", exportGroupName, name));
+ return appExports.contains(String.format("%s-%s", exportGroupName, name));
}
protected Map<String, String> getCurrentExports(String groupName) {
- if(!this.exportGroups.containsKey(groupName)) {
- synchronized (this.exportGroups) {
- if(!this.exportGroups.containsKey(groupName)) {
- this.exportGroups.put(groupName, new ConcurrentHashMap<String, String>());
- }
- }
+ if (!this.exportGroups.containsKey(groupName)) {
+ synchronized (this.exportGroups) {
+ if (!this.exportGroups.containsKey(groupName)) {
+ this.exportGroups.put(groupName, new ConcurrentHashMap<String, String>());
+ }
+ }
}
return this.exportGroups.get(groupName);
@@ -957,7 +974,7 @@
private void publishModifiedExportGroups(Set<String> modifiedGroups) {
synchronized (this.exportGroups) {
- for(String groupName : modifiedGroups) {
+ for (String groupName : modifiedGroups) {
publishApplicationInstanceData(groupName, groupName, this.exportGroups.get(groupName).entrySet());
}
}
@@ -1029,7 +1046,9 @@
/**
* Return Component based on name
+ *
* @param roleName
+ *
* @return
*/
protected Component getApplicationComponent(String roleName) {
@@ -1110,6 +1129,7 @@
/**
* Can any master publish config explicitly, if not a random master is used
+ *
* @return
*/
protected boolean canAnyMasterPublishConfig() {
@@ -1143,10 +1163,12 @@
/**
* Add install command to the heartbeat response
+ *
* @param roleName
* @param containerId
* @param response
* @param scriptPath
+ *
* @throws SliderException
*/
@VisibleForTesting
@@ -1173,7 +1195,8 @@
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- setInstallCommandConfigurations(cmd, containerId);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
+ cmd.setConfigurations(configurations);
cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
@@ -1223,12 +1246,6 @@
return cmdParams;
}
- private void setInstallCommandConfigurations(ExecutionCommand cmd, String containerId) throws SliderException {
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
- cmd.setConfigurations(configurations);
- }
-
@VisibleForTesting
protected void addStatusCommand(String roleName,
String containerId,
@@ -1333,7 +1350,7 @@
synchronized (this.allocatedPorts) {
if (!this.allocatedPorts.containsKey(containerId)) {
this.allocatedPorts.put(containerId,
- new ConcurrentHashMap<String, String>());
+ new ConcurrentHashMap<String, String>());
}
}
}
@@ -1348,17 +1365,45 @@
new TreeMap<String, Map<String, String>>();
Map<String, String> tokens = getStandardTokenMap(appConf);
- List<String> configs = getApplicationConfigurationTypes();
+ Set<String> configs = new HashSet<String>();
+ configs.addAll(getApplicationConfigurationTypes());
+ configs.addAll(getSystemConfigurationsRequested(appConf));
- //Add global
for (String configType : configs) {
addNamedConfiguration(configType, appConf.getGlobalOptions().options,
configurations, tokens, containerId);
}
+ //do a final replacement of re-used configs
+ dereferenceAllConfigs(configurations);
+
return configurations;
}
+ protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) {
+ Map<String, String> allConfigs = new HashMap<String, String>();
+ String lookupFormat = "${@//site/%s/%s}";
+ for (String configType : configurations.keySet()) {
+ Map<String, String> configBucket = configurations.get(configType);
+ for (String configName : configBucket.keySet()) {
+ allConfigs.put(String.format(lookupFormat, configType, configName), configBucket.get(configName));
+ }
+ }
+
+ for (String configType : configurations.keySet()) {
+ Map<String, String> configBucket = configurations.get(configType);
+ for (String configName : configBucket.keySet()) {
+ String configValue = configBucket.get(configName);
+ for (String lookUpKey : allConfigs.keySet()) {
+ if (configValue != null && configValue.contains(lookUpKey)) {
+ configValue = configValue.replace(lookUpKey, allConfigs.get(lookUpKey));
+ }
+ }
+ configBucket.put(configName, configValue);
+ }
+ }
+ }
+
private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) throws SliderException {
Map<String, String> tokens = new HashMap<String, String>();
String nnuri = appConf.get("site.fs.defaultFS");
@@ -1374,12 +1419,28 @@
}
@VisibleForTesting
+ protected List<String> getSystemConfigurationsRequested(ConfTreeOperations appConf) {
+ List<String> configList = new ArrayList<String>();
+
+ String configTypes = appConf.get(AgentKeys.SYSTEM_CONFIGS);
+ if (configTypes != null && configTypes.length() > 0) {
+ String[] configs = configTypes.split(",");
+ for (String config : configs) {
+ configList.add(config.trim());
+ }
+ }
+
+ return new ArrayList<String>(new HashSet<String>(configList));
+ }
+
+
+ @VisibleForTesting
protected List<String> getApplicationConfigurationTypes() {
List<String> configList = new ArrayList<String>();
configList.add(GLOBAL_CONFIG_TAG);
List<ConfigFile> configFiles = getMetainfo().getApplication().getConfigFiles();
- for(ConfigFile configFile : configFiles) {
+ for (ConfigFile configFile : configFiles) {
log.info("Expecting config type {}.", configFile.getDictionaryName());
configList.add(configFile.getDictionaryName());
}
@@ -1404,7 +1465,7 @@
for (String key : config.keySet()) {
String value = config.get(key);
String lookupKey = configName + "." + key;
- if(!value.contains(DO_NOT_PROPAGATE_TAG)) {
+ if (!value.contains(DO_NOT_PROPAGATE_TAG)) {
// If the config property is shared then pass on the already allocated value
// from any container
if (this.getAllocatedPorts().containsKey(lookupKey)) {
@@ -1419,12 +1480,12 @@
}
//apply defaults only if the key is not present and value is not empty
- if(getDefaultConfigs().containsKey(configName)) {
+ if (getDefaultConfigs().containsKey(configName)) {
log.info("Adding default configs for type {}.", configName);
- for(PropertyInfo defaultConfigProp : getDefaultConfigs().get(configName).getPropertyInfos()) {
- if(!config.containsKey(defaultConfigProp.getName())){
- if(!defaultConfigProp.getName().isEmpty() &&
- defaultConfigProp.getValue() != null &&
+ for (PropertyInfo defaultConfigProp : getDefaultConfigs().get(configName).getPropertyInfos()) {
+ if (!config.containsKey(defaultConfigProp.getName())) {
+ if (!defaultConfigProp.getName().isEmpty() &&
+ defaultConfigProp.getValue() != null &&
!defaultConfigProp.getValue().isEmpty()) {
config.put(defaultConfigProp.getName(), defaultConfigProp.getValue());
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index e8b6802..37824c8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -29,7 +29,6 @@
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.actions.ActionStartContainer;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
@@ -39,9 +38,9 @@
import org.slf4j.LoggerFactory;
import java.util.Map;
-import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
/**
* A service for launching containers
@@ -215,9 +214,8 @@
instance.roleId = role.id;
instance.environment = envDescription;
actionQueue.put(new ActionStartContainer("starting " + containerRole,
- 0, container,
- containerLauncher.completeContainerLaunch(),
- instance));
+ container, containerLauncher.completeContainerLaunch(), instance, 0,
+ TimeUnit.MILLISECONDS));
} catch (Exception e) {
log.error("Exception thrown while trying to start {}: {}",
containerRole, e);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 51c3b93..1c60b69 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -106,6 +106,7 @@
import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
+import org.apache.slider.server.appmaster.actions.ReviewAndFlexApplicationSize;
import org.apache.slider.server.appmaster.actions.UnregisterComponentInstance;
import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
@@ -160,9 +161,6 @@
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.WS_AGENT_CONTEXT_ROOT;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.WS_CONTEXT_ROOT;
-
/**
* This is the AM, which directly implements the callbacks from the AM and NM
*/
@@ -392,7 +390,7 @@
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
log.debug("Authenticating as {}", ugi);
SliderUtils.verifyPrincipalSet(conf,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
// always enforce protocol to be token-based.
conf.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -402,6 +400,9 @@
//look at settings of Hadoop Auth, to pick up a problem seen once
checkAndWarnForAuthTokenProblems();
+
+ // validate server env
+ SliderUtils.validateSliderServerEnvironment(log);
executorService = new WorkflowExecutorService<ExecutorService>("AmExecutor",
Executors.newCachedThreadPool(
@@ -1211,11 +1212,7 @@
queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
}
- try {
- reviewRequestAndReleaseNodes();
- } catch (SliderInternalStateException e) {
- log.warn("Exception while flexing nodes", e);
- }
+ reviewRequestAndReleaseNodes("onContainersCompleted");
}
/**
@@ -1223,10 +1220,9 @@
* It should be the only way that anything -even the AM itself on startup-
* asks for nodes.
* @param resources the resource tree
- * @return true if the any requests were made
* @throws IOException
*/
- private boolean flexCluster(ConfTree resources)
+ private void flexCluster(ConfTree resources)
throws IOException, SliderInternalStateException, BadConfigException {
appState.updateResourceDefinitions(resources);
@@ -1238,7 +1234,7 @@
// ask for more containers if needed
- return reviewRequestAndReleaseNodes();
+ reviewRequestAndReleaseNodes("flexCluster");
}
/**
@@ -1269,13 +1265,47 @@
/**
* Look at where the current node state is -and whether it should be changed
+ * @param reason
*/
- private synchronized boolean reviewRequestAndReleaseNodes()
+ private synchronized void reviewRequestAndReleaseNodes(String reason) {
+ log.debug("reviewRequestAndReleaseNodes({})", reason);
+ queue(new ReviewAndFlexApplicationSize(reason, 0, TimeUnit.SECONDS));
+ }
+
+ /**
+ * Handle the event requesting a review ... look at the queue and decide
+ * whether to act or not
+ * @param action action triggering the event. It may be put
+ * back into the queue
+ * @throws SliderInternalStateException
+ */
+ public void handleReviewAndFlexApplicationSize(ReviewAndFlexApplicationSize action)
throws SliderInternalStateException {
- log.debug("in reviewRequestAndReleaseNodes()");
+
+ if ( actionQueues.hasQueuedActionWithAttribute(
+ AsyncAction.ATTR_REVIEWS_APP_SIZE | AsyncAction.ATTR_HALTS_APP)) {
+ // this operation isn't needed at all -existing duplicate or shutdown due
+ return;
+ }
+ // if there is an action which changes cluster size, wait
+ if (actionQueues.hasQueuedActionWithAttribute(
+ AsyncAction.ATTR_CHANGES_APP_SIZE)) {
+ // place the action at the back of the queue
+ actionQueues.put(action);
+ }
+
+ executeNodeReview(action.name);
+ }
+
+ /**
+ * Look at where the current node state is -and whether it should be changed
+ */
+ public synchronized void executeNodeReview(String reason)
+ throws SliderInternalStateException {
+
+ log.debug("in executeNodeReview({})", reason);
if (amCompletionFlag.get()) {
log.info("Ignoring node review operation: shutdown in progress");
- return false;
}
try {
List<AbstractRMOperation> allOperations = appState.reviewRequestAndReleaseNodes();
@@ -1283,16 +1313,17 @@
providerRMOperationHandler.execute(allOperations);
//now apply the operations
executeRMOperations(allOperations);
- return !allOperations.isEmpty();
} catch (TriggerClusterTeardownException e) {
//App state has decided that it is time to exit
log.error("Cluster teardown triggered %s", e);
signalAMComplete(e.getExitCode(), e.toString());
- return false;
}
}
+
+
+
/**
* Shutdown operation: release all containers
*/
@@ -1383,8 +1414,8 @@
String payload = request.getClusterSpec();
ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
ConfTree updatedResources = confTreeSerDeser.fromJson(payload);
- boolean flexed = flexCluster(updatedResources);
- return Messages.FlexClusterResponseProto.newBuilder().setResponse(flexed).build();
+ flexCluster(updatedResources);
+ return Messages.FlexClusterResponseProto.newBuilder().setResponse(true).build();
}
@Override //SliderClusterProtocol
@@ -1609,7 +1640,7 @@
executeRMOperations(appState.releaseContainer(containerId));
// ask for more containers if needed
log.info("Container released; triggering review");
- reviewRequestAndReleaseNodes();
+ reviewRequestAndReleaseNodes("Loss of container");
} else {
log.info("Container not in active set - ignoring");
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
index c21e249..e2ad559 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -25,7 +25,7 @@
import java.util.concurrent.TimeUnit;
/**
- * Exit a JVM halt.
+ * Exit an emergency JVM halt.
* @see ExitUtil#halt(int, String)
*/
public class ActionHalt extends AsyncAction {
@@ -37,7 +37,9 @@
int status,
String text,
long delay, TimeUnit timeUnit) {
- super("Halt", delay, ActionAttributes.HALTS_CLUSTER);
+
+ // do not declare that this action halts the cluster ... keep it a surprise
+ super("Halt", delay, timeUnit);
this.status = status;
this.text = text;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index c1e7e6e..95bf067 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -30,16 +30,34 @@
import java.util.List;
import java.util.concurrent.TimeUnit;
+/**
+ * Kill a specific container
+ */
public class ActionKillContainer extends AsyncAction {
+ /**
+ * container to kill
+ */
private final ContainerId containerId;
+
+ /**
+ * handler for the operation
+ */
private final RMOperationHandler operationHandler;
+
+ /**
+ * Kill a container
+ * @param containerId container to kill
+ * @param delay
+ * @param timeUnit
+ * @param operationHandler
+ */
public ActionKillContainer(
ContainerId containerId,
long delay,
TimeUnit timeUnit,
RMOperationHandler operationHandler) {
- super("kill container", delay, timeUnit);
+ super("kill container", delay, timeUnit, ATTR_CHANGES_APP_SIZE);
this.operationHandler = operationHandler;
Preconditions.checkArgument(containerId != null);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
index d95dc74..358c844 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
@@ -25,6 +25,7 @@
import org.apache.slider.server.appmaster.state.RoleInstance;
import java.util.Locale;
+import java.util.concurrent.TimeUnit;
/**
* Start a container
@@ -37,15 +38,16 @@
private final RoleInstance instance;
public ActionStartContainer(String name,
- long delay,
Container container,
ContainerLaunchContext ctx,
- RoleInstance instance) {
+ RoleInstance instance,
+ long delay, TimeUnit timeUnit) {
super(
String.format(Locale.ENGLISH,
"%s %s: /",
name , container.getId().toString()),
- delay);
+ delay,
+ timeUnit);
this.container = container;
this.ctx = ctx;
this.instance = instance;
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
index f084383..64b8e9e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
@@ -25,15 +25,11 @@
import java.util.concurrent.TimeUnit;
public class ActionStopSlider extends AsyncAction {
- public ActionStopSlider(String message,
- long delay) {
- super(message, delay, ActionAttributes.HALTS_CLUSTER);
- }
public ActionStopSlider(String name,
long delay,
TimeUnit timeUnit) {
- super(name, delay, timeUnit, ActionAttributes.HALTS_CLUSTER);
+ super(name, delay, timeUnit, ATTR_HALTS_APP);
}
@Override
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
index 996390d..c8db42d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
@@ -23,8 +23,6 @@
import org.apache.slider.server.appmaster.state.AppState;
import java.io.IOException;
-import java.util.Collections;
-import java.util.EnumSet;
import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -35,7 +33,7 @@
public final String name;
private long nanos;
- private final EnumSet<ActionAttributes> attrs;
+ public final int attrs;
private final long sequenceNumber = sequencer.incrementAndGet();
@@ -51,34 +49,18 @@
protected AsyncAction(String name,
long delay,
TimeUnit timeUnit) {
- this.name = name;
- this.setNanos(convertAndOffset(delay, timeUnit));
- attrs = EnumSet.noneOf(ActionAttributes.class);
+ this(name, delay, timeUnit, 0);
}
protected AsyncAction(String name,
long delay,
TimeUnit timeUnit,
- EnumSet<ActionAttributes> attrs) {
+ int attrs) {
this.name = name;
this.setNanos(convertAndOffset(delay, timeUnit));
this.attrs = attrs;
}
- protected AsyncAction(String name,
- long delay,
- TimeUnit timeUnit,
- ActionAttributes... attributes) {
- this(name, delay, timeUnit);
- Collections.addAll(attrs, attributes);
- }
-
- protected AsyncAction(String name,
- long delayMillis,
- ActionAttributes... attributes) {
- this(name, delayMillis, TimeUnit.MILLISECONDS);
- }
-
protected long convertAndOffset(long delay, TimeUnit timeUnit) {
return now() + TimeUnit.NANOSECONDS.convert(delay, timeUnit);
}
@@ -118,17 +100,18 @@
return sb.toString();
}
- protected EnumSet<ActionAttributes> getAttrs() {
+ protected int getAttrs() {
return attrs;
}
/**
- * Ask if an action has a specific attribute
+ * Ask if an action has an of the specified bits set.
+ * This is not an equality test.
* @param attr attribute
- * @return true iff the action has the specific attribute
+ * @return true iff the action has any of the bits in the attr arg set
*/
- public boolean hasAttr(ActionAttributes attr) {
- return attrs.contains(attr);
+ public boolean hasAttr(int attr) {
+ return (attrs & attr) != 0;
}
/**
@@ -148,12 +131,8 @@
public void setNanos(long nanos) {
this.nanos = nanos;
}
-
- public enum ActionAttributes {
- SHRINKS_CLUSTER,
- EXPANDS_CLUSTER,
- HALTS_CLUSTER,
- }
-
-
+
+ public static final int ATTR_CHANGES_APP_SIZE = 1;
+ public static final int ATTR_HALTS_APP = 2;
+ public static final int ATTR_REVIEWS_APP_SIZE = 4;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
index 2aa67bb..41fe494 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
@@ -22,6 +22,8 @@
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.state.AppState;
+import java.util.concurrent.TimeUnit;
+
/**
* Report container loss to the AM
* {@link SliderAppMaster#providerLostContainer(ContainerId)}
@@ -31,13 +33,14 @@
private final ContainerId containerId;
public ProviderReportedContainerLoss(ContainerId containerId) {
- super("lost container " + containerId);
- this.containerId = containerId;
+ this("lost container", 0, TimeUnit.MILLISECONDS, containerId);
}
- public ProviderReportedContainerLoss(
- ContainerId containerId, long delayMillis) {
- super("lost container " + containerId, delayMillis);
+ public ProviderReportedContainerLoss(String name,
+ long delay,
+ TimeUnit timeUnit,
+ ContainerId containerId) {
+ super(name, delay, timeUnit);
this.containerId = containerId;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
index 4577025..957a35f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
@@ -27,10 +27,6 @@
super("ProviderStartupCompleted");
}
- public ProviderStartupCompleted(long delayMillis) {
- super("ProviderStartupCompleted", delayMillis);
- }
-
@Override
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
index cffaf5e..0396891 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
@@ -64,4 +64,9 @@
* @return true if the action was found and removed.
*/
boolean removeRenewingAction(String name);
+
+ /**
+ * Look in the immediate queue for any actions of a specific attribute
+ */
+ boolean hasQueuedActionWithAttribute(int attr);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 6ad579d..5b24a35 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -19,7 +19,6 @@
package org.apache.slider.server.appmaster.actions;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.slf4j.Logger;
@@ -149,6 +148,16 @@
}
}
}
+
+ @Override
+ public boolean hasQueuedActionWithAttribute(int attr) {
+ for (AsyncAction action : actionQueue) {
+ if (action.hasAttr(attr)) {
+ return true;
+ }
+ }
+ return false;
+ }
/**
* Run until the queue has been told to stop
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
index a8a6fe2..543c1a8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
@@ -25,6 +25,11 @@
import java.util.concurrent.TimeUnit;
+/**
+ * Notify the app master that it should register a component instance
+ * in the registry
+ * {@link SliderAppMaster#registerComponent(ContainerId)}
+ */
public class RegisterComponentInstance extends AsyncAction {
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java
new file mode 100644
index 0000000..273f599
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tell the AM to execute the full flex review operation
+ */
+public class ReviewAndFlexApplicationSize extends AsyncAction{
+
+ public ReviewAndFlexApplicationSize(String name,
+ long delay,
+ TimeUnit timeUnit) {
+ super(name, delay, timeUnit, ATTR_REVIEWS_APP_SIZE);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster,
+ QueueAccess queueService,
+ AppState appState) throws Exception {
+ appMaster.handleReviewAndFlexApplicationSize(this);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
index 78d9c1c..575fe8f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
@@ -24,12 +24,17 @@
import java.util.concurrent.TimeUnit;
+/**
+ * Tell AM to unregister this component instance
+ * {@link SliderAppMaster#unregisterComponent(ContainerId)}
+ */
public class UnregisterComponentInstance extends AsyncAction {
public final ContainerId containerId;
- public UnregisterComponentInstance(ContainerId containerId, long delay,
+ public UnregisterComponentInstance(ContainerId containerId,
+ long delay,
TimeUnit timeUnit) {
super("UnregisterComponentInstance :" + containerId.toString(),
delay, timeUnit);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
index 5905d2f..2869fe9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -52,7 +52,7 @@
Preconditions.checkArgument(target != null, "null target");
Preconditions.checkArgument(probability > 0, "negative probability");
Preconditions.checkArgument(probability <= ChaosMonkeyService.PERCENT_100,
- "probability over 100%");
+ "probability over 100%: "+ probability);
this.name = name;
this.target = target;
this.probability = probability;
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 07976ef..3e6bb74 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -36,6 +36,7 @@
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterDescriptionOperations;
import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
@@ -520,8 +521,8 @@
instanceDefinition.getResourceOperations().getGlobalOptions();
startTimeThreshold = globalResOpts.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_SHORTLIFE,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE);
+ InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
+ InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
failureThreshold = globalResOpts.getOptionInt(
ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
@@ -1227,7 +1228,7 @@
/**
* Is a role short lived by the threshold set for this application
* @param instance instance
- * @return true if the instance is considered short live
+ * @return true if the instance is considered short lived
*/
@VisibleForTesting
public boolean isShortLived(RoleInstance instance) {
@@ -1236,7 +1237,7 @@
boolean shortlived;
if (started > 0) {
long duration = time - started;
- shortlived = duration < startTimeThreshold;
+ shortlived = duration < (startTimeThreshold * 1000);
} else {
// never even saw a start event
shortlived = true;
@@ -1559,7 +1560,7 @@
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
return resources.getComponentOptInt(roleStatus.getName(),
- ResourceKeys.CONTAINER_FAILURE_SHORTLIFE,
+ ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
failureThreshold);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
index c8ab2a7..83c590b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
@@ -23,7 +23,7 @@
* No fields are synchronized; sync on the instance to work with it
*
The two fields `releasing` and `requested` are used to track the ongoing
- state of YARN requests; they do not need to be persisted across freeze/thaw
+ state of YARN requests; they do not need to be persisted across stop/start
cycles. They may be relevant across AM restart, but without other data
structures in the AM, not enough to track what the AM was up to before
it was restarted. The strategy will be to ignore unexpected allocation
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index edcf7ea..e82162f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -354,7 +354,7 @@
}
/**
- * Handle the thaw process <i>after the history has been rebuilt</i>,
+ * Handle the start process <i>after the history has been rebuilt</i>,
* and after any gc/purge
*/
@VisibleForTesting
@@ -381,7 +381,7 @@
e);
}
- //thaw is then completed
+ //start is then completed
buildAvailableNodeLists();
} else {
//fallback to bootstrap procedure
@@ -392,7 +392,7 @@
/**
- * (After the thaw), rebuild the availability datastructures
+ * (After the start), rebuild the availability data structures
*/
@VisibleForTesting
public synchronized void buildAvailableNodeLists() {
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
index a44c3a4..70d639f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
@@ -38,6 +38,7 @@
private State actualState;
private State expectedState;
private Map<String, String> allocatedPorts;
+ private Map<String, String> logFolders;
@JsonProperty("responseId")
public int getResponseId() {
@@ -133,6 +134,18 @@
this.allocatedPorts = ports;
}
+ /** @return the log folders, or <code>null</code> if none are present */
+ @JsonProperty("logFolders")
+ public Map<String, String> getLogFolders() {
+ return logFolders;
+ }
+
+ /** @param logFolders assigned log folders */
+ @JsonProperty("logFolders")
+ public void setLogFolders(Map<String, String> logFolders) {
+ this.logFolders = logFolders;
+ }
+
@Override
public String toString() {
String ret = "responseId=" + responseId + "\n" +
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
new file mode 100644
index 0000000..6e8add1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceStateChangeListener;
+
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Wait for a service to stop.
+ *
+ * WARNING: the notification may come in as soon as the service enters
+ * the stopped state: it may take some time for the actual stop operation
+ * to complete.
+ */
+public class EndOfServiceWaiter implements ServiceStateChangeListener {
+
+ private final AtomicBoolean finished = new AtomicBoolean(false);
+ private final String name;
+
+ /**
+ * Wait for a service; use the service name as this instance's name
+ * @param service service
+ */
+ public EndOfServiceWaiter(Service service) {
+ this(service.getName(), service);
+ }
+
+
+ /**
+ * Wait for a service
+ * @param name name for messages
+ * @param service service
+ */
+ public EndOfServiceWaiter(String name, Service service) {
+ this.name = name;
+ service.registerServiceListener(this);
+ }
+
+ public synchronized void waitForServiceToStop(long timeout) throws
+ InterruptedException, TimeoutException {
+ if (!finished.get()) {
+ wait(timeout);
+ }
+ if (!finished.get()) {
+ throw new TimeoutException(name
+ + " did not finish after " + timeout +
+ " milliseconds");
+ }
+ }
+
+ /**
+ * Wait for service state change callbacks; notify self if the service has
+ * now stopped
+ * @param service service
+ */
+ @Override
+ public synchronized void stateChanged(Service service) {
+ if (service.isInState(Service.STATE.STOPPED)) {
+ finished.set(true);
+ notify();
+ }
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
index 46c724c..b801993 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
@@ -81,7 +81,11 @@
private LongLivedProcess process;
private int executionTimeout = -1;
private int timeoutCode = 1;
-
+ /**
+ log to log to; defaults to this service log
+ */
+ private Logger processLog = LOG;
+
/**
* Exit code set when the spawned process exits
*/
@@ -102,7 +106,8 @@
* @param env environment variables above those generated by
* @throws IOException IO problems
*/
- public ForkedProcessService(String name, Map<String, String> env,
+ public ForkedProcessService(String name,
+ Map<String, String> env,
List<String> commandList) throws IOException {
super(name);
build(env, commandList);
@@ -130,6 +135,15 @@
}
/**
+ * Set the process log. This may be null for "do not log"
+ * @param processLog process log
+ */
+ public void setProcessLog(Logger processLog) {
+ this.processLog = processLog;
+ process.setProcessLog(processLog);
+ }
+
+ /**
* Set the timeout by which time a process must have finished -or -1 for forever
* @param timeout timeout in milliseconds
*/
@@ -148,7 +162,8 @@
List<String> commandList)
throws IOException {
assert process == null;
- process = new LongLivedProcess(getName(), LOG, commandList);
+
+ process = new LongLivedProcess(getName(), processLog, commandList);
process.setLifecycleCallback(this);
//set the env variable mapping
process.putEnvMap(env);
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
index c8ff758..57b989c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -24,6 +24,7 @@
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
@@ -90,7 +91,7 @@
* Log supplied in the constructor for the spawned process -accessible
* to inner classes
*/
- private final Logger processLog;
+ private Logger processLog;
/**
* Class log -accessible to inner classes
@@ -102,10 +103,15 @@
*/
private final AtomicBoolean finished = new AtomicBoolean(false);
+ /**
+ * Create an instance
+ * @param name process name
+ * @param processLog log for output (or null)
+ * @param commands command list
+ */
public LongLivedProcess(String name,
Logger processLog,
List<String> commands) {
- Preconditions.checkArgument(processLog != null, "processLog");
Preconditions.checkArgument(commands != null, "commands");
this.name = name;
@@ -168,6 +174,14 @@
}
/**
+ * Set the process log. Ignored once the process starts
+ * @param processLog new log ... may be null
+ */
+ public void setProcessLog(Logger processLog) {
+ this.processLog = processLog;
+ }
+
+ /**
* Get the process reference
* @return the process -null if the process is not started
*/
@@ -270,7 +284,8 @@
/**
* Exec the process
* @return the process
- * @throws IOException
+ * @throws IOException on aany failure to start the process
+ * @throws FileNotFoundException if the process could not be found
*/
private Process spawnChildProcess() throws IOException {
if (process != null) {
@@ -279,7 +294,20 @@
if (LOG.isDebugEnabled()) {
LOG.debug("Spawning process:\n " + describeBuilder());
}
- process = processBuilder.start();
+ try {
+ process = processBuilder.start();
+ } catch (IOException e) {
+ // on windows, upconvert DOS error 2 from ::CreateProcess()
+ // to its real meaning: FileNotFound
+ if (e.toString().contains("CreateProcess error=2")) {
+ FileNotFoundException fnfe =
+ new FileNotFoundException(e.toString());
+ fnfe.initCause(e);
+ throw fnfe;
+ } else {
+ throw e;
+ }
+ }
return process;
}
@@ -398,10 +426,11 @@
* something that is only called once per line of IO?
* @param line line to record
* @param isErrorStream is the line from the error stream
- * @param logger logger to log to
+ * @param logger logger to log to - null for no logging
*/
private synchronized void recordRecentLine(String line,
- boolean isErrorStream, Logger logger) {
+ boolean isErrorStream,
+ Logger logger) {
if (line == null) {
return;
}
@@ -410,10 +439,12 @@
if (recentLines.size() > recentLineLimit) {
recentLines.remove(0);
}
- if (isErrorStream) {
- logger.warn(line);
- } else {
- logger.info(line);
+ if (logger != null) {
+ if (isErrorStream) {
+ logger.warn(line);
+ } else {
+ logger.info(line);
+ }
}
}
@@ -428,6 +459,12 @@
private final Logger streamLog;
private final int sleepTime;
+ /**
+ * Create an instance
+ * @param streamLog log -or null to disable logging (recent entries
+ * will still be retained)
+ * @param sleepTime time to sleep when stopping
+ */
private ProcessStreamReader(Logger streamLog, int sleepTime) {
this.streamLog = streamLog;
this.sleepTime = sleepTime;
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index 4566d9d..7786c41 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -27,6 +27,7 @@
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXMLConfKeysForTesting
import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.agent.AgentKeys
import org.apache.slider.test.YarnZKMiniClusterTestBase
@@ -47,6 +48,14 @@
protected static Map<String, String> agentDefOptions
private static TemporaryFolder tempFolder = new TemporaryFolder();
+ /**
+ * Server side test: validate system env before launch
+ */
+ @BeforeClass
+ public static void checkSystem() {
+ SliderUtils.validateSliderServerEnvironment(null)
+ }
+
@BeforeClass
public static void createSubConfFiles() {
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
index 9bfeb8c..a190b7d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
@@ -110,7 +110,7 @@
assert 0 == sliderClient.actionExists(clustername, true)
- // freeze the cluster
+ // stop the cluster
clusterActionFreeze(sliderClient, clustername)
//verify that exists(live) is now false
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy
index 1d50b71..7a2820f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy
@@ -20,10 +20,9 @@
import groovy.util.logging.Slf4j
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.test.YarnMiniClusterTestBase
+import org.apache.slider.test.SliderTestBase
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Before
import org.junit.Test
/**
@@ -31,7 +30,7 @@
*/
@Slf4j
-class TestActionVersion extends YarnMiniClusterTestBase {
+class TestActionVersion extends SliderTestBase {
@Test
public void testVersion() throws Throwable {
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
index f5eff25..60ffa3e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
@@ -45,7 +45,7 @@
YarnConfiguration conf = configuration
String clustername = createMiniCluster("", conf, 1, 1, 1, true, false)
- describe "create a masterless AM, freeze it, try to freeze again"
+ describe "create a masterless AM, stop it, try to stop again"
ServiceLauncher<SliderClient> launcher = createStandaloneAM(
clustername
@@ -59,7 +59,7 @@
assertSucceeded(execSliderCommand(conf,
[SliderActions.ACTION_LIST, clustername]))
- log.info("First Freeze command");
+ log.info("First stop command");
ServiceLauncher freezeCommand = execSliderCommand(conf,
[
SliderActions.ACTION_FREEZE, clustername,
@@ -67,7 +67,7 @@
]);
assertSucceeded(freezeCommand)
- log.info("Second Freeze command");
+ log.info("Second stop command");
ServiceLauncher<SliderClient> freeze2 = execSliderCommand(conf,
[
@@ -94,7 +94,7 @@
assert e.exitCode == LauncherExitCodes.EXIT_FALSE;
}
- log.info("First Thaw");
+ log.info("First Start");
def commands = [
@@ -111,7 +111,7 @@
assertSucceeded(execSliderCommand(conf,
[SliderActions.ACTION_EXISTS, clustername]))
- log.info("Freeze 3");
+ log.info("stop 3");
ServiceLauncher<SliderClient> freeze3 = execSliderCommand(conf,
[
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
index 04be7c0..5a1acef 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
@@ -31,7 +31,7 @@
import org.junit.Test
/**
- * freeze and thaw an AM
+ * stop and start an AM
*/
@CompileStatic
@Slf4j
@@ -52,7 +52,7 @@
YarnConfiguration conf = configuration
String clustername = createMiniCluster("", conf, 1, 1, 1, true, false)
- describe "create a masterless AM, freeze it, thaw it"
+ describe "create a masterless AM, stop it, start it"
//copy the confdir somewhere
Path resConfPath = new Path(resourceConfDirURI)
Path tempConfPath = new Path(confDir)
@@ -81,10 +81,10 @@
// ApplicationReport report = waitForClusterLive(newCluster)
newCluster.getClusterDescription(clustername);
- //freeze
+ //stop
assert 0 == clusterActionFreeze(sliderClient, clustername)
- //freeze again
+ //stop again
assert 0 == clusterActionFreeze(sliderClient, clustername)
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeUnknownCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeUnknownCluster.groovy
index b81bc77..a7bab01 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeUnknownCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeUnknownCluster.groovy
@@ -42,7 +42,7 @@
YarnConfiguration conf = configuration
String clustername = createMiniCluster("", conf, 1, true)
- describe "try to freeze a cluster that isn't defined"
+ describe "try to stop a cluster that isn't defined"
try {
ServiceLauncher<SliderClient> command = execSliderCommand(conf,
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
index 0665a9a..2058caf 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
@@ -96,7 +96,7 @@
- //thaw time
+ //start time
ServiceLauncher<SliderClient> l2 = thawCluster(clustername, [], true)
SliderClient thawed = l2.service
addToTeardown(thawed);
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
index 17e4ff2..aec4930 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -102,8 +102,8 @@
describe "post destroy checks"
sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
- describe "thaw expected to fail"
- //expect thaw to now fail
+ describe "start expected to fail"
+ //expect start to now fail
def ex = launchExpectingException(SliderClient,
configuration,
"",
@@ -115,7 +115,7 @@
])
assert ex instanceof UnknownApplicationInstanceException
- describe "thaw completed, checking dir is still absent"
+ describe "start completed, checking dir is still absent"
sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
index 75f9a2c..7552394 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
@@ -58,7 +58,7 @@
describe("listing Java processes")
lsJavaProcesses();
describe("killing AM")
- killAM(SIGTERM);
+ assert 0 == killAM(SIGTERM);
waitWhileClusterLive(sliderClient);
//give yarn some time to notice
sleep(10000)
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
index 5b17cfa..b37b9a6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
@@ -20,28 +20,22 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.SleepJob
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.InternalKeys
-import org.apache.slider.api.ResourceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.ActionAMSuicideArgs
import org.apache.slider.common.params.Arguments
-import org.apache.slider.core.exceptions.ErrorStrings
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
@CompileStatic
@Slf4j
-
class TestStandaloneAMMonkeyRestart extends AgentMiniClusterTestBase {
-
@Test
public void testStandaloneAMMonkeyRestart() throws Throwable {
describe "Run a Standalone AM with the Chaos monkey set to kill it"
@@ -55,7 +49,7 @@
[
Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_ENABLED, "true",
Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "8",
- Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "75000",
+ Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "7500",
],
true,
false)
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
index b8a590e..cbbd030 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
@@ -342,7 +342,7 @@
registryArgs.getConf = unknownFilename
assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
- describe "freeze cluster"
+ describe "stop cluster"
//now kill that cluster
assert 0 == clusterActionFreeze(client, clustername)
//list it & See if it is still there
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
index d94cd54..ea44489 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
@@ -154,7 +154,7 @@
}
/**
- * Test a thaw command
+ * Test a start command
* @throws Throwable
*/
@Test
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
index 7150b3c..e3bfa1b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
@@ -18,12 +18,15 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.junit.Test
@Slf4j
+@CompileStatic
+
class TestClientResourceRegistration {
/**
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
index 0d21d6c..d78aa1b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
@@ -27,6 +28,8 @@
import org.junit.Test
@Slf4j
+@CompileStatic
+
class TestConfigHelperHDFS extends YarnMiniClusterTestBase {
//diabled for now;
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
new file mode 100644
index 0000000..5489366
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestExecutionEnvironment extends SliderTestBase {
+
+ @Test
+ public void testClientEnv() throws Throwable {
+ SliderUtils.validateSliderClientEnvironment(log)
+ }
+
+
+ @Test
+ public void testServerEnv() throws Throwable {
+ SliderUtils.validateSliderServerEnvironment(log)
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
index 24367a3..9042af5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
@@ -18,12 +18,15 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.slider.test.SliderTestBase
import org.junit.Test
+@CompileStatic
+
class TestMiscSliderUtils extends SliderTestBase {
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 7f5edf3..98fa183 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -27,14 +27,16 @@
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.util.Shell
import org.apache.slider.providers.agent.AgentUtils
-import org.apache.slider.test.SliderTestUtils
+import org.apache.slider.server.services.utility.EndOfServiceWaiter
+import org.apache.slider.server.services.workflow.ForkedProcessService
+import org.apache.slider.test.SliderTestBase
import org.junit.Test
import java.util.regex.Pattern
@CompileStatic
@Slf4j
-class TestWindowsSupport extends SliderTestUtils {
+class TestWindowsSupport extends SliderTestBase {
private static final Pattern hasDriveLetterSpecifier =
Pattern.compile("^/?[a-zA-Z]:");
@@ -94,7 +96,6 @@
@Test
public void testSliderFS() throws Throwable {
assume(Shell.WINDOWS, "not windows")
-
SliderFileSystem sfs = new SliderFileSystem(new Configuration())
try {
def metainfo = AgentUtils.getApplicationMetainfo(sfs, windowsFile)
@@ -104,9 +105,78 @@
}
+
+ @Test
+ public void testHasGawkInstalled() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ exec(0, ["gawk", "--version"])
+ }
+
+ @Test
+ public void testHasXargsInstalled() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ exec(0, ["xargs", "--version"])
+ }
+
+
@Test
public void testEmitKillCommand() throws Throwable {
killJavaProcesses("regionserver", 9)
+ }
+ @Test
+ public void testHadoopHomeDefined() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ def hadoopHome = Shell.hadoopHome
+ log.info("HADOOP_HOME=$hadoopHome")
+ }
+
+ @Test
+ public void testHasWinutils() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ SliderUtils.maybeVerifyWinUtilsValid(log)
+ }
+
+ @Test
+ public void testExecWinutils() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ def winUtilsPath = Shell.winUtilsPath
+ assert winUtilsPath
+ File winUtils = new File(winUtilsPath)
+ log.debug("Winutils is at $winUtils)")
+
+ exec(0, [winUtilsPath, "systeminfo"])
+ }
+
+
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param status code
+ * @param commands
+ * @return the process
+ */
+ public ForkedProcessService exec(int status, List<String> commands) {
+ ForkedProcessService process = exec(commands)
+ assert status == process.exitCode
+ return process
+ }
+
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param commands
+ * @return
+ */
+
+ public ForkedProcessService exec(List<String> commands) {
+ ForkedProcessService process;
+ process = new ForkedProcessService(
+ methodName.methodName,
+ [:],
+ commands);
+ process.init(new Configuration());
+ EndOfServiceWaiter waiter = new EndOfServiceWaiter(process);
+ process.start();
+ waiter.waitForServiceToStop(5000);
+ process
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index fe3bef7..836edbd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.slider.client.SliderClient
@@ -31,20 +32,25 @@
import org.junit.Test
@Slf4j
-
+@CompileStatic
class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTests {
+
+ // as the static compiler doesn't resolve consistently
+ public static final String USER = KeysForTests.USERNAME
+
@Before
void createCluster() {
- Configuration conf = getConfiguration()
+ Configuration conf = configuration
createMicroZKCluster(conf)
}
@Test
public void testIntegrationCreate() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "cluster1", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ ZKIntegration zki = createZKIntegrationInstance(
+ getZKBinding(), "cluster1", true, false, 5000)
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
Stat stat = zki.stat(userPath)
assert stat != null
log.info("User path $userPath has stat $stat")
@@ -54,8 +60,9 @@
public void testListUserClustersWithoutAnyClusters() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ ZKIntegration zki = createZKIntegrationInstance(
+ getZKBinding(), "", true, false, 5000)
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
List<String> clusters = zki.clusters
assert clusters.empty
}
@@ -64,8 +71,9 @@
public void testListUserClustersWithOneCluster() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ ZKIntegration zki = createZKIntegrationInstance(
+ getZKBinding(), "", true, false, 5000)
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
String fullPath = zki.createPath(userPath, "/cluster-",
ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL_SEQUENTIAL)
@@ -77,8 +85,9 @@
@Test
public void testListUserClustersWithTwoCluster() throws Throwable {
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ ZKIntegration zki = createZKIntegrationInstance(
+ getZKBinding(), "", true, false, 5000)
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
String c1 = createEphemeralChild(zki, userPath)
log.info("Ephemeral path $c1")
String c2 = createEphemeralChild(zki, userPath)
@@ -94,25 +103,25 @@
MockSliderClient client = new MockSliderClient()
String path = client.createZookeeperNode("cl1", true)
- ZKIntegration zki = client.getLastZKIntegration()
+ ZKIntegration zki = client.lastZKIntegration
- String zkPath = ZKIntegration.mkClusterPath(USERNAME, "cl1")
- assert zkPath == "/services/slider/users/" + USERNAME + "/cl1", "zkPath must be as expected"
+ String zkPath = ZKIntegration.mkClusterPath(USER, "cl1")
+ assert zkPath == "/services/slider/users/" + USER + "/cl1", "zkPath must be as expected"
assert path == zkPath
assert zki == null, "ZKIntegration should be null."
zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, 5000);
- assert false == zki.exists(zkPath), "zkPath should not exist"
+ assert !zki.exists(zkPath)
path = client.createZookeeperNode("cl1", false)
- zki = client.getLastZKIntegration()
- assert zkPath == "/services/slider/users/" + USERNAME + "/cl1", "zkPath must be as expected"
+ zki = client.lastZKIntegration
+ assert zki
+ assert zkPath == "/services/slider/users/" + USER + "/cl1", "zkPath must be as expected"
assert path == zkPath
- assert true == zki.exists(zkPath), "zkPath must exist"
+ assert zki.exists(zkPath)
zki.createPath(zkPath, "/cn", ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT)
- assert true == zki.exists(zkPath + "/cn"), "zkPath with child node must exist"
+ assert zki.exists(zkPath + "/cn")
client.deleteZookeeperNode("cl1")
- assert false == zki.exists(zkPath), "zkPath must not exist"
-
+ assert !zki.exists(zkPath)
}
public String createEphemeralChild(ZKIntegration zki, String userPath) {
@@ -126,7 +135,7 @@
@Override
public String getUsername() {
- return USERNAME
+ return USER
}
@Override
@@ -137,8 +146,7 @@
@Override
public synchronized Configuration getConfig() {
- Configuration conf = new Configuration();
- return conf;
+ new Configuration();
}
public ZKIntegration getLastZKIntegration() {
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index 6dee64f..ff67a71 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -26,9 +26,11 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.client.SliderClient
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.Before
+import org.junit.BeforeClass
import org.junit.Rule
import org.junit.rules.TemporaryFolder
@@ -47,6 +49,14 @@
@Rule
public TemporaryFolder folder = new TemporaryFolder();
+ /**
+ * Server side test: validate system env before launch
+ */
+ @BeforeClass
+ public static void checkServerEnv() {
+ SliderUtils.validateSliderServerEnvironment(null)
+ }
+
public String app_def_pkg_path;
@Before
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 1cd8de6..6720bda 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -117,9 +117,9 @@
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
- ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
],
true, true,
true)
@@ -163,4 +163,6 @@
assert RegistrationStatus.FAILED == response.getResponseStatus();
}
+
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index a29c8cb..a7b5fe3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -72,9 +72,9 @@
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
- ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
index 1a1e5aa..0b77adc 100644
--- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
@@ -88,9 +88,9 @@
roles,
[
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
- ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
],
true, true,
true)
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index 4242ba1..5575076 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -117,9 +117,9 @@
assert loadedNE2.lastUsed == savetime
assert rh2.thawedDataTime == savetime
- // now thaw it
+ // now start it
rh2.buildAvailableNodeLists();
- describe("thawing")
+ describe("starting")
rh2.dump();
List<NodeInstance> available0 = rh2.cloneAvailableList(0)
assert available0.size() == 1
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index 21881be..44f1214 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -23,6 +23,7 @@
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.util.Shell
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
@@ -71,9 +72,9 @@
[
ARG_PROVIDER, "org.apache.slider.server.appmaster.web.rest.publisher.TestSliderProviderFactory",
ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, toFileURI(app_def_path),
- ARG_OPTION, AGENT_CONF, toFileURI(agt_conf_path),
- ARG_OPTION, AGENT_VERSION, toFileURI(agt_ver_path)
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path)
],
true, true,
true)
@@ -134,17 +135,14 @@
Set uris = webResource.type(MediaType.APPLICATION_JSON)
.get(Set.class)
assert uris.size() > 0
- log.info("Classpath URIs: {}", uris)
- // check for some expected classpath elements
- assert uris.any {it =~ /curator-x-discovery/}
- assert uris.any {it =~ /hadoop-yarn-api/}
- assert uris.any {it =~ /hadoop-hdfs/}
- // and a negative test...
- assert !uris.any {it =~ /foo-bar/}
- }
-
- public String toFileURI(File filename) {
- filename.toURI().toString()
+ if (!Shell.WINDOWS) {
+ log.info("Classpath URIs: {}", uris)
+ // check for some expected classpath elements
+ assert uris.any {it =~ /hadoop-yarn-api/}
+ assert uris.any {it =~ /hadoop-hdfs/}
+ // and a negative test...
+ assert !uris.any {it =~ /foo-bar/}
+ }
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 0036db7..cbf4749 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -755,7 +755,7 @@
/*
"jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
*/
- GString killCommand = "\"jps -l | grep \"${grepString}\" | awk \"{print \$1}\" | xargs -n 1 taskkill /PID\""
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID\""
commandString = ["CMD", "/C", killCommand]
}
Process command = commandString.execute()
@@ -776,4 +776,13 @@
killJavaProcesses(grep,signal)
}
}
+
+ /**
+ * Convert a file to a URI suitable for use in an argument
+ * @param file file
+ * @return a URI string valid on all platforms
+ */
+ public String toURIArg(File file) {
+ file.absoluteFile.toURI().toString()
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 5a091bf..97cc853 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -49,11 +49,8 @@
import org.apache.slider.core.main.ServiceLauncherBaseTest
import org.apache.slider.server.appmaster.SliderAppMaster
import org.junit.After
-import org.junit.Assert
-import org.junit.Before
import org.junit.BeforeClass
import org.junit.Rule
-import org.junit.rules.TestName
import org.junit.rules.Timeout
import static org.apache.slider.test.KeysForTests.*
@@ -129,12 +126,14 @@
KEY_TEST_TIMEOUT,
DEFAULT_TEST_TIMEOUT_SECONDS * 1000)
)
+
+ /**
+ * Clent side test: validate system env before launch
+ */
@BeforeClass
- public static void checkWindowsSupport() {
- if (Shell.WINDOWS) {
- assertNotNull("winutils.exe not found", Shell.WINUTILS)
- }
- }
+ public static void checkClientEnv() {
+ SliderUtils.validateSliderClientEnvironment(null)
+ }
protected String buildClustername(String clustername) {
return clustername ?: createClusterName()
@@ -240,11 +239,16 @@
int numLocalDirs,
int numLogDirs,
boolean startHDFS) {
+
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
SliderUtils.patchConfiguration(conf)
name = buildClustername(name)
- miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs)
+ miniCluster = new MiniYARNCluster(
+ name,
+ noOfNodeManagers,
+ numLocalDirs,
+ numLogDirs)
miniCluster.init(conf)
miniCluster.start();
if (startHDFS) {
@@ -314,10 +318,10 @@
/**
- * Kill all Slider Services. That i
+ * Kill all Slider Services.
* @param signal
*/
- public void killAM(int signal) {
+ public int killAM(int signal) {
killJavaProcesses(SliderAppMaster.SERVICE_CLASSNAME_SHORT, signal)
}
@@ -705,15 +709,15 @@
* @return the exit code
*/
public int clusterActionFreeze(SliderClient sliderClient, String clustername,
- String message = "action freeze") {
- log.info("Freezing cluster $clustername: $message")
+ String message = "action stop") {
+ log.info("Stopping cluster $clustername: $message")
ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
freezeArgs.waittime = CLUSTER_STOP_TIME
freezeArgs.message = message
int exitCode = sliderClient.actionFreeze(clustername,
freezeArgs);
if (exitCode != 0) {
- log.warn("Cluster freeze failed with error code $exitCode")
+ log.warn("Cluster stop failed with error code $exitCode")
}
return exitCode
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
index 0259fb7..9b5f082 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
@@ -147,7 +147,7 @@
assert microZKCluster != null
}
- protected String getZKBinding() {
+ public String getZKBinding() {
if (!microZKCluster) {
return "localhost:1"
} else {
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index b0b059d..3711e12 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -18,6 +18,7 @@
package org.apache.slider.providers.agent;
+import com.sun.jersey.spi.container.servlet.WebConfig;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
@@ -309,6 +310,12 @@
anyString(),
anyMap()
);
+
+ doNothing().when(mockAps).publishLogFolderPaths(anyMap(),
+ anyString(),
+ anyString(),
+ anyString()
+ );
expect(access.isApplicationLive()).andReturn(true).anyTimes();
ClusterDescription desc = new ClusterDescription();
desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
@@ -343,9 +350,12 @@
Register reg = new Register();
reg.setResponseId(0);
reg.setHostname("mockcontainer_1___HBASE_MASTER");
- Map<String,String> ports = new HashMap();
+ Map<String,String> ports = new HashMap<String, String>();
ports.put("a","100");
reg.setAllocatedPorts(ports);
+ Map<String, String> folders = new HashMap<String, String>();
+ folders.put("F1", "F2");
+ reg.setLogFolders(folders);
RegistrationResponse resp = mockAps.handleRegistration(reg);
Assert.assertEquals(0, resp.getResponseId());
Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
@@ -357,6 +367,13 @@
anyMap()
);
+ Mockito.verify(mockAps, Mockito.times(1)).publishLogFolderPaths(
+ anyMap(),
+ anyString(),
+ anyString(),
+ anyString()
+ );
+
HeartBeat hb = new HeartBeat();
hb.setResponseId(1);
hb.setHostname("mockcontainer_1___HBASE_MASTER");
@@ -1057,7 +1074,10 @@
List<String> configurations = new ArrayList<String>();
configurations.add("hbase-site");
configurations.add("global");
+ List<String> sysConfigurations = new ArrayList<String>();
+ configurations.add("core-site");
doReturn(configurations).when(mockAps).getApplicationConfigurationTypes();
+ doReturn(sysConfigurations).when(mockAps).getSystemConfigurationsRequested(any(ConfTreeOperations.class));
Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
Map<String, ClusterNode> container = new HashMap<String, ClusterNode>();
@@ -1078,6 +1098,7 @@
mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", 0, Boolean.FALSE);
Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("hbase-site"));
+ Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("core-site"));
Map<String, String> hbaseSiteConf = hbr.getExecutionCommands().get(0).getConfigurations().get("hbase-site");
Assert.assertTrue(hbaseSiteConf.containsKey("a.port"));
Assert.assertEquals("10023", hbaseSiteConf.get("a.port"));
@@ -1095,4 +1116,45 @@
Assert.assertFalse(cmd.getConfigurations().get("hbase-site").containsKey("defaultB"));
}
+ @Test
+ public void testParameterParsing() {
+ AgentProviderService aps = new AgentProviderService();
+ AggregateConf aggConf = new AggregateConf();
+ ConfTreeOperations treeOps = aggConf.getAppConfOperations();
+ treeOps.getGlobalOptions().put(AgentKeys.SYSTEM_CONFIGS, "core-site,yarn-site, core-site ");
+ List<String> configs = aps.getSystemConfigurationsRequested(treeOps);
+ Assert.assertEquals(2, configs.size());
+ Assert.assertTrue(configs.contains("core-site"));
+ Assert.assertFalse(configs.contains("bore-site"));
+ }
+
+ @Test
+ public void testDereferenceAllConfig() {
+ AgentProviderService aps = new AgentProviderService();
+ Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>();
+ Map<String, String> cfg1 = new HashMap<String, String>();
+ cfg1.put("a1", "${@//site/cfg-2/A1}");
+ cfg1.put("b1", "22");
+ cfg1.put("c1", "33");
+ cfg1.put("d1", "${@//site/cfg1/c1}AA");
+ Map<String, String> cfg2 = new HashMap<String, String>();
+ cfg2.put("A1", "11");
+ cfg2.put("B1", "${@//site/cfg-2/A1},${@//site/cfg-2/A1},AA,${@//site/cfg1/c1}");
+ cfg2.put("C1", "DD${@//site/cfg1/c1}");
+ cfg2.put("D1", "${14}");
+
+ allConfigs.put("cfg1", cfg1);
+ allConfigs.put("cfg-2", cfg2);
+ aps.dereferenceAllConfigs(allConfigs);
+ Assert.assertEquals("11", cfg1.get("a1"));
+ Assert.assertEquals("22", cfg1.get("b1"));
+ Assert.assertEquals("33", cfg1.get("c1"));
+ Assert.assertEquals("33AA", cfg1.get("d1"));
+
+ Assert.assertEquals("11", cfg2.get("A1"));
+ Assert.assertEquals("11,11,AA,33", cfg2.get("B1"));
+ Assert.assertEquals("DD33", cfg2.get("C1"));
+ Assert.assertEquals("${14}", cfg2.get("D1"));
+ }
+
}
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/EndOfServiceWaiter.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/EndOfServiceWaiter.java
deleted file mode 100644
index 5e6df3b..0000000
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/EndOfServiceWaiter.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-import org.junit.Assert;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Wait for a service to stop
- */
-public class EndOfServiceWaiter implements ServiceStateChangeListener {
-
- private final AtomicBoolean finished = new AtomicBoolean(false);
-
- public EndOfServiceWaiter(Service svc) {
- svc.registerServiceListener(this);
- }
-
- public synchronized void waitForServiceToStop(long timeout) throws
- InterruptedException {
- if (!finished.get()) {
- wait(timeout);
- }
- Assert.assertTrue("Service did not finish in time period",
- finished.get());
- }
-
- @Override
- public synchronized void stateChanged(Service service) {
- if (service.isInState(Service.STATE.STOPPED)) {
- finished.set(true);
- notify();
- }
- }
-
-
-}
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
index 45fdc86..4a19417 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
@@ -18,8 +18,11 @@
package org.apache.slider.server.services.workflow;
+import org.apache.hadoop.util.Shell;
+
import java.io.File;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
/**
@@ -37,10 +40,12 @@
* @return commands
*/
public List<String> ls(File dir) {
- List<String> commands = new ArrayList<String>(5);
- commands.add("ls");
- commands.add("-1");
- commands.add(dir.getAbsolutePath());
+ List<String> commands;
+ if (!Shell.WINDOWS) {
+ commands = Arrays.asList("ls","-1", dir.getAbsolutePath());
+ } else {
+ commands = Arrays.asList("cmd", "/c", "dir", dir.getAbsolutePath());
+ }
return commands;
}
@@ -61,8 +66,12 @@
* @return commands
*/
public List<String> env() {
- List<String> commands = new ArrayList<String>(1);
- commands.add("env");
+ List<String> commands;
+ if (!Shell.WINDOWS) {
+ commands = Arrays.asList("env");
+ } else {
+ commands = Arrays.asList("cmd", "/c", "set");
+ }
return commands;
}
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
index 668bcca..9019124 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestLongLivedProcess.java
@@ -33,11 +33,9 @@
*/
public class TestLongLivedProcess extends WorkflowServiceTestBase implements
LongLivedProcessLifecycleEvent {
- private static final Logger
- log = LoggerFactory.getLogger(TestLongLivedProcess.class);
+ private static final Logger log = LoggerFactory.getLogger(TestLongLivedProcess.class);
- private static final Logger
- processLog =
+ private static final Logger processLog =
LoggerFactory.getLogger("org.apache.hadoop.services.workflow.Process");
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
index 6d08156..e8f7d88 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.ServiceOperations;
+import org.apache.slider.server.services.utility.EndOfServiceWaiter;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -31,6 +32,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeoutException;
/**
* Test the long lived process by executing a command that works and a command
@@ -131,7 +133,7 @@
return process;
}
- public void exec() throws InterruptedException {
+ public void exec() throws InterruptedException, TimeoutException {
assertNotNull(process);
EndOfServiceWaiter waiter = new EndOfServiceWaiter(process);
process.start();
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
index 3049d8f..f38bd9d 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
@@ -29,6 +29,7 @@
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Locale;
import java.util.concurrent.Callable;
/**
@@ -122,7 +123,7 @@
boolean found = false;
StringBuilder builder = new StringBuilder();
for (String s : output) {
- builder.append(s).append('\n');
+ builder.append(s.toLowerCase(Locale.ENGLISH)).append('\n');
if (s.contains(text)) {
found = true;
break;
diff --git a/slider-core/src/test/resources/example-slider-test.xml b/slider-core/src/test/resources/example-slider-test.xml
index a752cfd..abf42f9 100644
--- a/slider-core/src/test/resources/example-slider-test.xml
+++ b/slider-core/src/test/resources/example-slider-test.xml
@@ -40,13 +40,13 @@
<property>
<name>slider.test.thaw.wait.seconds</name>
- <description>Time to wait for a thaw to work</description>
+ <description>Time to wait for a start to work</description>
<value>60</value>
</property>
<property>
<name>slider.test.freeze.wait.seconds</name>
- <description>Time to wait for a freeze to work</description>
+ <description>Time to wait for a stop to work</description>
<value>60</value>
</property>
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 278bd2b..3b52912 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -220,7 +220,7 @@
}
/**
- * Freeze cluster: no exit code checking
+ * Stop cluster: no exit code checking
* @param name
* @param args
* @return
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index 6b0f2bd..51f1e94 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -88,7 +88,7 @@
//destroy will fail in use
destroy(EXIT_APPLICATION_IN_USE, CLUSTER)
- //thaw will fail as cluster is in use
+ //start will fail as cluster is in use
thaw(EXIT_APPLICATION_IN_USE, CLUSTER)
//it's still there
@@ -125,7 +125,7 @@
log.info("Connected via Client {}", sliderClient.toString())
- //freeze
+ //stop
freeze(0, CLUSTER, [
ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
ARG_MESSAGE, "freeze-in-test-cluster-lifecycle"
@@ -137,7 +137,7 @@
//condition returns false if it is required to be live
exists(EXIT_FALSE, CLUSTER, true)
- //thaw then freeze the cluster
+ //start then stop the cluster
thaw(CLUSTER,
[
ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
@@ -160,7 +160,7 @@
//condition returns false if it is required to be live
exists(EXIT_FALSE, CLUSTER, true)
- //thaw with a restart count set to enable restart
+ //start with a restart count set to enable restart
describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
thaw(CLUSTER,
[
diff --git a/slider-funtest/src/test/manual/python/SliderTester.py b/slider-funtest/src/test/manual/python/SliderTester.py
index 40bdf2b..1dfd0b5 100644
--- a/slider-funtest/src/test/manual/python/SliderTester.py
+++ b/slider-funtest/src/test/manual/python/SliderTester.py
@@ -235,7 +235,7 @@
pass
def clean_up(self):
- (retcode, out, err) = self.run_slider_command(" ".join([self.slider_exec, "freeze", self.cluster_name]),
+ (retcode, out, err) = self.run_slider_command(" ".join([self.slider_exec, "stop", self.cluster_name]),
self.slider_user)
if retcode != 0:
raise Exception("Could not clean cluster. Out: " + out + " Err: " + err)
@@ -352,7 +352,7 @@
# Finalize resources and appConf json files
# Call create
# Validate existence of the app
- # Call freeze
+ # Call start
if __name__ == "__main__":
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
index 93b2798..3c5606b 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
@@ -228,9 +228,7 @@
//now flex
describe(
"Flexing " + roleMapToString(flexTarget));
- boolean flexed = 0 == sliderClient.flex(clustername,
- flexTarget
- );
+ sliderClient.flex(clustername, flexTarget);
cd = waitForRoleCount(sliderClient, flexTarget,
accumulo_cluster_startup_to_live_time);
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
index 6da00fb..108bab2 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
@@ -68,8 +68,8 @@
AccumuloKeys.MONITOR_PAGE_JSON)
log.info(page);
- log.info("Freezing")
- clusterActionFreeze(sliderClient, clustername, "freeze");
+ log.info("Stopping")
+ clusterActionFreeze(sliderClient, clustername, "stop");
waitForAppToFinish(sliderClient)
//make sure the fetch fails
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
index 63b5fb6..c65593f 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
@@ -80,7 +80,7 @@
destroy(EXIT_APPLICATION_IN_USE, CLUSTER)
- //thaw will fail as cluster is in use
+ //start will fail as cluster is in use
thaw(EXIT_APPLICATION_IN_USE, CLUSTER)
//it's still there
@@ -117,7 +117,7 @@
log.info("Connected via Client {}", sliderClient.toString())
- //freeze
+ //stop
def frozen = freeze(0, CLUSTER, [
ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
ARG_MESSAGE, "freeze-in-test-cluster-lifecycle"
@@ -131,7 +131,7 @@
exists(EXIT_FALSE, CLUSTER, true)
- // thaw then freeze the cluster
+ // start then stop the cluster
thaw(CLUSTER,
[
@@ -151,7 +151,7 @@
// condition returns false if it is required to be live
exists(EXIT_FALSE, CLUSTER, true)
- // thaw with a restart count set to enable restart
+ // start with a restart count set to enable restart
describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
thaw(CLUSTER,
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 10de9ac..7712a83 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -353,7 +353,6 @@
int masterFlexTarget,
int workerFlexTarget,
boolean testHBaseAfter) {
- int flexTarget
describe(
"Flexing masters -> $masterFlexTarget ; workers -> ${workerFlexTarget}");
boolean flexed;
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy
index 85726a7..09a87a9 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy
@@ -51,7 +51,7 @@
hbaseClusterStartupToLiveTime)
- clusterActionFreeze(sliderClient, clustername, "test freeze")
+ clusterActionFreeze(sliderClient, clustername, "test stop")
describe("Restarting cluster")
killAllRegionServers();
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
index c305b5b..e4acb6b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
@@ -61,7 +61,7 @@
ApplicationReport report = serviceRegistryClient.findInstance(clustername)
assert report == null;
- //thaw time
+ //start time
ServiceLauncher<SliderClient> l2 = thawCluster(clustername, [], true)
SliderClient thawed = l2.service
addToTeardown(thawed);
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
index fdbbce8..c1265de 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
@@ -33,7 +33,7 @@
@Test
public void testClusterFlex1To1() throws Throwable {
- assert !flexHBaseClusterTestRun(
+ assert flexHBaseClusterTestRun(
"",
1,
1,
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
index f6748e0..d01716b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
@@ -54,8 +54,8 @@
conf.setInt("yarn.nodemanager.resource.cpu-vcores", 1)
String clustername = createMiniCluster("", conf, nodemanagers, true)
describe(
- "Create a $regionServerCount node cluster, freeze it, patch the configuration files," +
- " thaw it and verify that it came back with the new settings")
+ "Create a $regionServerCount node cluster, stop it, patch the configuration files," +
+ " start it and verify that it came back with the new settings")
ServiceLauncher<SliderClient> launcher = createHBaseCluster(
clustername,
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy
index 66dd4f0..9a0e1db 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy
@@ -43,7 +43,7 @@
public void testFreezeThawLiveRegionService() throws Throwable {
int regionServerCount = 2
String clustername = createMiniCluster("", configuration, 1, true)
- describe("Create a cluster, freeze it, thaw it and verify that it came back ")
+ describe("Create a cluster, stop it, start it and verify that it came back ")
//use a smaller AM HEAP to include it in the test cycle
ServiceLauncher launcher = createHBaseCluster(clustername, regionServerCount,
[
@@ -84,7 +84,7 @@
waitForHBaseRegionServerCount(newCluster, clustername, regionServerCount,
hbaseClusterStartupToLiveTime)
- // finally, attempt to thaw it while it is running
+ //finally, attempt to start it while it is running
//now let's start the cluster up again
try {
ServiceLauncher launcher3 = thawCluster(clustername, [], true);
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
index 7e4c5ed..f7bb8d2 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
@@ -109,8 +109,8 @@
HBaseKeys.HBASE_SERVICE_TYPE, clustername2).payload
assert !(hbase1ServiceData == hbase2ServiceData)
- clusterActionFreeze(cluster2Client, clustername2,"freeze cluster 2")
- clusterActionFreeze(sliderClient, clustername1,"Freeze cluster 1")
+ clusterActionFreeze(cluster2Client, clustername2, "stop cluster 2")
+ clusterActionFreeze(sliderClient, clustername1, "Stop cluster 1")
}
diff --git a/src/test/clusters/sandbox/operations.md b/src/test/clusters/sandbox/operations.md
index 0ff5a3a..932f6d5 100644
--- a/src/test/clusters/sandbox/operations.md
+++ b/src/test/clusters/sandbox/operations.md
@@ -118,10 +118,10 @@
- bin/slider thaw cl1
+ bin/slider start cl1
- bin/slider freeze cl1
- bin/slider freeze cluster3
+ bin/slider stop cl1
+ bin/slider stop cluster3
bin/slider destroy cl1
@@ -149,20 +149,20 @@
--component master 1 \
--component worker 2
-### freeze
+### stop
- bin/slider freeze cl1
+ bin/slider stop cl1
- bin/slider freeze cl1 --force
+ bin/slider stop cl1 --force
-### thaw
+### start
- bin/slider thaw cl1 -D slider.yarn.queue.priority=5 -D slider.yarn.queue=default
+ bin/slider start cl1 -D slider.yarn.queue.priority=5 -D slider.yarn.queue=default
-### thaw with bad queue: _MUST_ fail
+### start with bad queue: _MUST_ fail
- bin/slider thaw cl1 -D slider.yarn.queue=unknown
+ bin/slider start cl1 -D slider.yarn.queue=unknown
### monitor
diff --git a/src/test/clusters/ubuntu-secure/operations.md b/src/test/clusters/ubuntu-secure/operations.md
index d894038..1f92290 100644
--- a/src/test/clusters/ubuntu-secure/operations.md
+++ b/src/test/clusters/ubuntu-secure/operations.md
@@ -198,21 +198,21 @@
- bin/slider thaw cl1 \
+ bin/slider start cl1 \
--manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
\
-S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
-D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
-D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
- bin/slider freeze cl1 \
+ bin/slider stop cl1 \
--manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
\
-S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
-D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
-D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
- bin/slider freeze cluster3 \
+ bin/slider stop cluster3 \
--manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
\
-S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
@@ -259,13 +259,13 @@
--role master 1 \
--role worker 2
-# freeze
+# stop
- bin/slider freeze cl1
+ bin/slider stop cl1
-# thaw
+# start
- bin/slider thaw cl1
+ bin/slider start cl1
# monitor
