initial donation of slider code, RAT-validated already
git-svn-id: https://svn.apache.org/repos/asf/incubator/slider/trunk@1591904 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..9dbb730
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,26 @@
+*.iml
+*.ipr
+*.iws
+*.pyc
+.idea
+.svn
+.classpath
+.project
+.settings
+/target
+/conf
+/slider-core/target
+/slider-core/src/test/resources/slider-test.xml
+/slider-agent/target
+/slider-funtest/target
+/slider-assembly/target
+/slider-providers/hbase/slider-hbase-provider/target
+/slider-providers/hbase/hbase-funtests/target
+/slider-providers/accumulo/slider-accumulo-provider/target
+/slider-providers/accumulo/accumulo-funtests/target
+/test-configs
+release.properties
+*.backup
+*.versionsBackup
+slider-test.xml
+build.properties
diff --git a/LICENSE.TXT b/LICENSE.TXT
new file mode 100644
index 0000000..ddf56c8
--- /dev/null
+++ b/LICENSE.TXT
@@ -0,0 +1,12 @@
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..854d70e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,53 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Slider
+
+
+Slider is a YARN application to deploy existing distributed applications on YARN,
+monitor them and make them larger or smaller as desired -even while
+the cluster is running.
+
+Clusters can be stopped, "frozen" and restarted, "thawed" later; the distribution
+of the deployed application across the YARN cluster is persisted -enabling
+a best-effort placement close to the previous locations on a cluster thaw.
+Applications which remember the previous placement of data (such as HBase)
+can exhibit fast start-up times from this feature.
+
+YARN itself monitors the health of 'YARN containers" hosting parts of
+the deployed application -it notifies the Slider manager application of container
+failure. Slider then asks YARN for a new container, into which Slider deploys
+a replacement for the failed component. As a result, Slider can keep the
+size of managed applications consistent with the specified configuration, even
+in the face of failures of servers in the cluster -as well as parts of the
+application itself
+
+
+# License
+
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ (http://www.apache.org/licenses/LICENSE-2.0)
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
diff --git a/app-packages/accumulo-v1_5/appConfig.json b/app-packages/accumulo-v1_5/appConfig.json
new file mode 100644
index 0000000..18a5c8f
--- /dev/null
+++ b/app-packages/accumulo-v1_5/appConfig.json
@@ -0,0 +1,71 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "agent.conf": "/slider/agent/conf/agent.ini",
+ "application.def": "/slider/accumulo_v151.tar",
+ "config_types": "accumulo-site",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "package_list": "files/accumulo-1.5.1-bin.tar.gz",
+ "site.global.app_user": "yarn",
+ "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
+ "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-1.5.1",
+ "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
+ "site.global.tserver_heapsize": "128m",
+ "site.global.master_heapsize": "128m",
+ "site.global.monitor_heapsize": "64m",
+ "site.global.gc_heapsize": "64m",
+ "site.global.other_heapsize": "128m",
+ "site.global.hadoop_prefix": "/usr/lib/hadoop",
+ "site.global.hadoop_conf_dir": "/etc/hadoop/conf",
+ "site.global.zookeeper_home": "/usr/lib/zookeeper",
+ "site.global.accumulo_instance_name": "instancename",
+ "site.global.accumulo_root_password": "secret",
+ "site.global.user_group": "hadoop",
+ "site.global.security_enabled": "false",
+ "site.accumulo-site.instance.dfs.dir": "/apps/accumulo/data",
+ "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
+ "site.accumulo-site.instance.secret": "DEFAULT",
+ "site.accumulo-site.tserver.memory.maps.max": "80M",
+ "site.accumulo-site.tserver.cache.data.size": "7M",
+ "site.accumulo-site.tserver.cache.index.size": "20M",
+ "site.accumulo-site.trace.token.property.password": "secret",
+ "site.accumulo-site.trace.user": "root",
+ "site.accumulo-site.tserver.sort.buffer.size": "50M",
+ "site.accumulo-site.tserver.walog.max.size": "100M",
+ "site.accumulo-site.master.port.client": "0",
+ "site.accumulo-site.trace.port.client": "0",
+ "site.accumulo-site.tserver.port.client": "0",
+ "site.accumulo-site.gc.port.client": "0",
+ "site.accumulo-site.monitor.port.client": "0",
+ "site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
+ },
+ "components": {
+ "ACCUMULO_MASTER": {
+ "wait.heartbeat": "3",
+ "role.script": "scripts/accumulo_master.py"
+ },
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ },
+ "ACCUMULO_TSERVER": {
+ "wait.heartbeat": "6",
+ "role.script": "scripts/accumulo_tserver.py"
+ },
+ "ACCUMULO_MONITOR": {
+ "wait.heartbeat": "8",
+ "role.script": "scripts/accumulo_monitor.py"
+ },
+ "ACCUMULO_GC": {
+ "wait.heartbeat": "6",
+ "role.script": "scripts/accumulo_gc.py"
+ },
+ "ACCUMULO_TRACER": {
+ "wait.heartbeat": "8",
+ "role.script": "scripts/accumulo_tracer.py"
+ }
+ }
+}
diff --git a/app-packages/accumulo-v1_5/configuration/accumulo-site.xml b/app-packages/accumulo-v1_5/configuration/accumulo-site.xml
new file mode 100644
index 0000000..269cc2b
--- /dev/null
+++ b/app-packages/accumulo-v1_5/configuration/accumulo-site.xml
@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- Put your site-specific accumulo configurations here. The available configuration values along with their defaults are documented in docs/config.html Unless
+ you are simply testing at your workstation, you will most definitely need to change the three entries below. -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers
+ must know in order to communicate with one another.
+ Change it before initialization. To
+ change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.token.property.password</name>
+ <!-- change this to the root user's password, and/or change the user below -->
+ <value>secret</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <!--
+ Add the following for hadoop-2.0
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+ -->
+ <value>
+ $ACCUMULO_HOME/lib/accumulo-server.jar,
+ $ACCUMULO_HOME/lib/accumulo-core.jar,
+ $ACCUMULO_HOME/lib/accumulo-start.jar,
+ $ACCUMULO_HOME/lib/accumulo-fate.jar,
+ $ACCUMULO_HOME/lib/accumulo-proxy.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+ /usr/lib/hadoop/.*.jar,
+ /usr/lib/hadoop/lib/.*.jar,
+ /usr/lib/hadoop-hdfs/.*.jar,
+ /usr/lib/hadoop-mapreduce/.*.jar,
+ /usr/lib/hadoop-yarn/.*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.</description>
+ </property>
+</configuration>
diff --git a/app-packages/accumulo-v1_5/configuration/global.xml b/app-packages/accumulo-v1_5/configuration/global.xml
new file mode 100644
index 0000000..5d39dca
--- /dev/null
+++ b/app-packages/accumulo-v1_5/configuration/global.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>app_log_dir</name>
+ <value>/var/log/accumulo</value>
+ <description>Log Directories for Accumulo.</description>
+ </property>
+ <property>
+ <name>app_pid_dir</name>
+ <value>/var/run/accumulo</value>
+ <description>Pid Directories for Accumulo.</description>
+ </property>
+ <property>
+ <name>tserver_heapsize</name>
+ <value>128m</value>
+ <description>TServer heap size.</description>
+ </property>
+ <property>
+ <name>master_heapsize</name>
+ <value>128m</value>
+ <description>Master Heap Size</description>
+ </property>
+ <property>
+ <name>monitor_heapsize</name>
+ <value>64m</value>
+ <description>Monitor Heap Size</description>
+ </property>
+ <property>
+ <name>gc_heapsize</name>
+ <value>64m</value>
+ <description>GC Heap Size</description>
+ </property>
+ <property>
+ <name>other_heapsize</name>
+ <value>128m</value>
+ <description>Other Heap Size</description>
+ </property>
+ <property>
+ <name>accumulo_hdfs_root_dir</name>
+ <value>/apps/accumulo/data</value>
+ <description>Accumulo Relative Path to HDFS.</description>
+ </property>
+ <property>
+ <name>accumulo_conf_dir</name>
+ <value>/etc/accumulo</value>
+ <description>Config Directory for Accumulo.</description>
+ </property>
+ <property>
+ <name>app_user</name>
+ <value>yarn</value>
+ <description>Accumulo User Name.</description>
+ </property>
+ <property>
+ <name>hadoop_prefix</name>
+ <value>/usr/lib/hadoop</value>
+ <description>Hadoop directory.</description>
+ </property>
+ <property>
+ <name>hadoop_conf_dir</name>
+ <value>/etc/hadoop/conf</value>
+ <description>Hadoop configuration directory.</description>
+ </property>
+ <property>
+ <name>zookeeper_home</name>
+ <value>/usr/lib/zookeeper</value>
+ <description>Zookeeper directory.</description>
+ </property>
+ <property>
+ <name>accumulo_instance_name</name>
+ <value>accumulo-instance</value>
+ <description>Accumulo Instance Name.</description>
+ </property>
+</configuration>
diff --git a/app-packages/accumulo-v1_5/metainfo.xml b/app-packages/accumulo-v1_5/metainfo.xml
new file mode 100644
index 0000000..dc59210
--- /dev/null
+++ b/app-packages/accumulo-v1_5/metainfo.xml
@@ -0,0 +1,105 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ACCUMULO</name>
+ <comment>
+ The Apache Accumulo sorted, distributed key/value store is a robust,
+ scalable, high performance data storage system that features cell-based
+ access control and customizable server-side processing. It is based on
+ Google's BigTable design and is built on top of Apache Hadoop,
+ Zookeeper, and Thrift.
+ </comment>
+ <version>1.5.1</version>
+ <components>
+ <component>
+ <name>ACCUMULO_MASTER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/accumulo_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>ACCUMULO_MONITOR</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/accumulo_monitor.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>ACCUMULO_GC</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/accumulo_gc.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>ACCUMULO_TRACER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/accumulo_tracer.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>ACCUMULO_TSERVER</name>
+ <category>SLAVE</category>
+ <commandScript>
+ <script>scripts/accumulo_tserver.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>ACCUMULO_CLIENT</name>
+ <category>CLIENT</category>
+ <commandScript>
+ <script>scripts/accumulo_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/accumulo-1.5.1-bin.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ </service>
+ </services>
+</metainfo>
diff --git a/app-packages/accumulo-v1_5/package/files/accumulo-1.5.1-bin.tar.gz.REPLACE b/app-packages/accumulo-v1_5/package/files/accumulo-1.5.1-bin.tar.gz.REPLACE
new file mode 100644
index 0000000..ae1e83e
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/accumulo-1.5.1-bin.tar.gz.REPLACE
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/app-packages/accumulo-v1_5/package/files/accumulo-metrics.xml b/app-packages/accumulo-v1_5/package/files/accumulo-metrics.xml
new file mode 100644
index 0000000..60f9f8d
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/accumulo-metrics.xml
@@ -0,0 +1,60 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ This file follows the conventions for XMLConfiguration files specified in the Apache Commons Configuration 1.5 Library. Changes to this file will be noticed
+ at runtime (see the FileChangedReloadingStrategy class in Commons Configuration).
+-->
+<config>
+<!--
+ Metrics log directory
+-->
+ <logging>
+ <dir>${ACCUMULO_HOME}/metrics</dir>
+ </logging>
+<!--
+ Enable/Disable metrics accumulation on the different servers and their components
+ NOTE: Turning on logging can be expensive because it will use several more file handles and will create a lot of short lived objects.
+-->
+ <master>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </master>
+ <logger>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </logger>
+ <tserver>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ <update>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </update>
+ <scan>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </scan>
+ <minc>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </minc>
+ </tserver>
+ <thrift>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </thrift>
+</config>
diff --git a/app-packages/accumulo-v1_5/package/files/auditLog.xml b/app-packages/accumulo-v1_5/package/files/auditLog.xml
new file mode 100644
index 0000000..9b7987e
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/auditLog.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+
+
+ <!-- Write out Audit info to an Audit file -->
+ <appender name="Audit" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.ip.localhost.hostname}.audit"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="DatePattern" value="'.'yyyy-MM-dd"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss,SSS/Z} [%c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+ <logger name="Audit" additivity="false">
+ <appender-ref ref="Audit" />
+ <level value="OFF"/>
+ </logger>
+
+
+
+
+
+</log4j:configuration>
diff --git a/app-packages/accumulo-v1_5/package/files/gc b/app-packages/accumulo-v1_5/package/files/gc
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/gc
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/app-packages/accumulo-v1_5/package/files/generic_logger.xml b/app-packages/accumulo-v1_5/package/files/generic_logger.xml
new file mode 100644
index 0000000..db79efe
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/generic_logger.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.debug.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Send all logging data to a centralized logger -->
+ <appender name="N1" class="org.apache.log4j.net.SocketAppender">
+ <param name="remoteHost" value="${org.apache.accumulo.core.host.log}"/>
+ <param name="port" value="${org.apache.accumulo.core.host.log.port}"/>
+ <param name="application" value="${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname}"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- If the centralized logger is down, buffer the log events, but drop them if it stays down -->
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="N1" />
+ </appender>
+
+ <!-- Log accumulo events to the debug, normal and remote logs. -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="ASYNC" />
+ </logger>
+
+ <logger name="org.apache.accumulo.core.file.rfile.bcfile">
+ <level value="INFO"/>
+ </logger>
+
+ <logger name="org.mortbay.log">
+ <level value="WARN"/>
+ </logger>
+
+ <logger name="org.apache.zookeeper">
+ <level value="ERROR"/>
+ </logger>
+
+ <!-- Log non-accumulo events to the debug and normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
diff --git a/app-packages/accumulo-v1_5/package/files/log4j.properties b/app-packages/accumulo-v1_5/package/files/log4j.properties
new file mode 100644
index 0000000..a4bcb2e
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/log4j.properties
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# default logging properties:
+# by default, log everything at INFO or higher to the console
+log4j.rootLogger=INFO,A1
+
+# hide Jetty junk
+log4j.logger.org.mortbay.log=WARN,A1
+
+# hide "Got brand-new compresssor" messages
+log4j.logger.org.apache.hadoop.io.compress=WARN,A1
+
+# hide junk from TestRandomDeletes
+log4j.logger.org.apache.accumulo.test.TestRandomDeletes=WARN,A1
+
+# hide junk from VFS
+log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN,A1
+
+# hide almost everything from zookeeper
+log4j.logger.org.apache.zookeeper=ERROR,A1
+
+# hide AUDIT messages in the shell, alternatively you could send them to a different logger
+log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=WARN,A1
+
+# Send most things to the console
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.appender.A1.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
diff --git a/app-packages/accumulo-v1_5/package/files/masters b/app-packages/accumulo-v1_5/package/files/masters
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/masters
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/app-packages/accumulo-v1_5/package/files/monitor b/app-packages/accumulo-v1_5/package/files/monitor
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/monitor
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/app-packages/accumulo-v1_5/package/files/monitor_logger.xml b/app-packages/accumulo-v1_5/package/files/monitor_logger.xml
new file mode 100644
index 0000000..91a7671
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/monitor_logger.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.debug.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Keep the last few log messages for display to the user -->
+ <appender name="GUI" class="org.apache.accumulo.server.monitor.LogService">
+ <param name="keep" value="40"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- Log accumulo messages to debug, normal and GUI -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="GUI" />
+ </logger>
+
+ <!-- Log non-accumulo messages to debug, normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
diff --git a/app-packages/accumulo-v1_5/package/files/slaves b/app-packages/accumulo-v1_5/package/files/slaves
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/slaves
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/app-packages/accumulo-v1_5/package/files/tracers b/app-packages/accumulo-v1_5/package/files/tracers
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/files/tracers
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/app-packages/accumulo-v1_5/package/scripts/__init__.py b/app-packages/accumulo-v1_5/package/scripts/__init__.py
new file mode 100644
index 0000000..5561e10
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/__init__.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
diff --git a/app-packages/accumulo-v1_5/package/scripts/accumulo_client.py b/app-packages/accumulo-v1_5/package/scripts/accumulo_client.py
new file mode 100644
index 0000000..45d07dd
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/accumulo_client.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from accumulo_configuration import setup_conf_dir
+
+
+class AccumuloClient(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ setup_conf_dir(name='client')
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+
+if __name__ == "__main__":
+ AccumuloClient().execute()
diff --git a/app-packages/accumulo-v1_5/package/scripts/accumulo_configuration.py b/app-packages/accumulo-v1_5/package/scripts/accumulo_configuration.py
new file mode 100644
index 0000000..4e6bb5c
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/accumulo_configuration.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+def setup_conf_dir(name=None # 'master' or 'tserver' or 'monitor' or 'gc' or 'tracer' or 'client'
+ ):
+ import params
+
+ # create the conf directory
+ Directory( params.conf_dir,
+ owner = params.accumulo_user,
+ group = params.user_group,
+ recursive = True
+ )
+
+ if name != "client":
+ # create pid dir
+ Directory( params.pid_dir,
+ owner = params.accumulo_user,
+ group = params.user_group,
+ recursive = True
+ )
+
+ # create log dir
+ Directory (params.log_dir,
+ owner = params.accumulo_user,
+ group = params.user_group,
+ recursive = True
+ )
+
+ # create a site file for server processes
+ XmlConfig( "accumulo-site.xml",
+ conf_dir = params.conf_dir,
+ configurations = params.config['configurations']['accumulo-site'],
+ owner = params.accumulo_user,
+ group = params.user_group,
+ mode=0600
+ )
+ else:
+ # create a minimal site file for client processes
+ client_configurations = {}
+ client_configurations['instance.zookeeper.host'] = params.config['configurations']['accumulo-site']['instance.zookeeper.host']
+ client_configurations['instance.dfs.dir'] = params.config['configurations']['accumulo-site']['instance.dfs.dir']
+ client_configurations['general.classpaths'] = params.config['configurations']['accumulo-site']['general.classpaths']
+ XmlConfig( "accumulo-site.xml",
+ conf_dir = params.conf_dir,
+ configurations = client_configurations,
+ owner = params.accumulo_user,
+ group = params.user_group
+ )
+
+ # create env file
+ accumulo_TemplateConfig( 'accumulo-env.sh')
+
+ # create host files
+ accumulo_StaticFile( 'masters')
+ accumulo_StaticFile( 'slaves')
+ accumulo_StaticFile( 'monitor')
+ accumulo_StaticFile( 'gc')
+ accumulo_StaticFile( 'tracers')
+
+ # create log4j.properties files
+ if (params.log4j_props != None):
+ File(format("{params.conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.accumulo_user,
+ content=params.log4j_props
+ )
+ else:
+ accumulo_StaticFile("log4j.properties")
+
+ # create other logging configuration files
+ accumulo_StaticFile("auditLog.xml")
+ accumulo_StaticFile("generic_logger.xml")
+ accumulo_StaticFile("monitor_logger.xml")
+ accumulo_StaticFile("accumulo-metrics.xml")
+
+ # create the policy file
+ if 'accumulo-policy' in params.config['configurations']:
+ XmlConfig( "accumulo-policy.xml",
+ configurations = params.config['configurations']['accumulo-policy'],
+ owner = params.accumulo_user,
+ group = params.user_group
+ )
+
+# create file 'name' from template
+def accumulo_TemplateConfig(name,
+ tag=None
+ ):
+ import params
+
+ TemplateConfig( format("{params.conf_dir}/{name}"),
+ owner = params.accumulo_user,
+ group = params.user_group,
+ template_tag = tag
+ )
+
+# create static file 'name'
+def accumulo_StaticFile(name):
+ import params
+
+ File(format("{params.conf_dir}/{name}"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.accumulo_user,
+ content=StaticFile(name)
+ )
diff --git a/app-packages/accumulo-v1_5/package/scripts/accumulo_gc.py b/app-packages/accumulo-v1_5/package/scripts/accumulo_gc.py
new file mode 100644
index 0000000..f8fe499
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/accumulo_gc.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from accumulo_script import AccumuloScript
+
+if __name__ == "__main__":
+ AccumuloScript('gc').execute()
diff --git a/app-packages/accumulo-v1_5/package/scripts/accumulo_master.py b/app-packages/accumulo-v1_5/package/scripts/accumulo_master.py
new file mode 100644
index 0000000..ea8935b
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/accumulo_master.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from accumulo_script import AccumuloScript
+
+if __name__ == "__main__":
+ AccumuloScript('master').execute()
diff --git a/app-packages/accumulo-v1_5/package/scripts/accumulo_monitor.py b/app-packages/accumulo-v1_5/package/scripts/accumulo_monitor.py
new file mode 100644
index 0000000..c8e7bed
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/accumulo_monitor.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from accumulo_script import AccumuloScript
+
+if __name__ == "__main__":
+ AccumuloScript('monitor').execute()
diff --git a/app-packages/accumulo-v1_5/package/scripts/accumulo_script.py b/app-packages/accumulo-v1_5/package/scripts/accumulo_script.py
new file mode 100644
index 0000000..2af7a1a
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/accumulo_script.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+from accumulo_configuration import setup_conf_dir
+from accumulo_service import accumulo_service
+
+
+class AccumuloScript(Script):
+ def __init__(self, component):
+ self.component = component
+
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ setup_conf_dir(name=self.component)
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ if self.component == 'master':
+ Execute( format("{daemon_script} init --instance-name {accumulo_instance_name} --password {accumulo_root_password} --clear-instance-name"),
+ not_if=format("hadoop fs -stat {accumulo_hdfs_root_dir}"),
+ user=params.accumulo_user)
+
+ accumulo_service( self.component,
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ accumulo_service( self.component,
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ component = self.component
+ pid_file = format("{pid_dir}/accumulo-{accumulo_user}-{component}.pid")
+ check_process_status(pid_file)
+
+
+if __name__ == "__main__":
+ self.fail_with_error('component unspecified')
diff --git a/app-packages/accumulo-v1_5/package/scripts/accumulo_service.py b/app-packages/accumulo-v1_5/package/scripts/accumulo_service.py
new file mode 100644
index 0000000..562ef5d
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/accumulo_service.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+def accumulo_service( name,
+ action = 'start'): # 'start' or 'stop' or 'status'
+ import params
+
+ role = name
+ pid_file = format("{pid_dir}/accumulo-{accumulo_user}-{role}.pid")
+
+ pid_exists = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
+
+ if action == 'start':
+ daemon_cmd = format("{daemon_script} {role} > {log_dir}/accumulo-{accumulo_user}-{role}.out 2>{log_dir}/accumulo-{accumulo_user}-{role}.err & echo $! > {pid_file}")
+ Execute ( daemon_cmd,
+ not_if=pid_exists,
+ user=params.accumulo_user
+ )
+
+ elif action == 'stop':
+ no_pid_exists = format("! ({pid_exists})")
+ pid = format("`cat {pid_file}` >/dev/null 2>&1")
+ Execute(format("kill {pid}"),
+ not_if=no_pid_exists,
+ user=params.accumulo_user
+ )
+ Execute(format("kill -9 {pid}"),
+ not_if=format("sleep 2; {no_pid_exists} || sleep 20; {no_pid_exists}"),
+ ignore_failures=True,
+ user=params.accumulo_user
+ )
+ Execute(format("rm -f {pid_file}"),
+ user=params.accumulo_user)
diff --git a/app-packages/accumulo-v1_5/package/scripts/accumulo_tracer.py b/app-packages/accumulo-v1_5/package/scripts/accumulo_tracer.py
new file mode 100644
index 0000000..b8bb9a0
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/accumulo_tracer.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from accumulo_script import AccumuloScript
+
+if __name__ == "__main__":
+ AccumuloScript('tracer').execute()
diff --git a/app-packages/accumulo-v1_5/package/scripts/accumulo_tserver.py b/app-packages/accumulo-v1_5/package/scripts/accumulo_tserver.py
new file mode 100644
index 0000000..3117e35
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/accumulo_tserver.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from accumulo_script import AccumuloScript
+
+if __name__ == "__main__":
+ AccumuloScript('tserver').execute()
diff --git a/app-packages/accumulo-v1_5/package/scripts/params.py b/app-packages/accumulo-v1_5/package/scripts/params.py
new file mode 100644
index 0000000..556c8d2
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/params.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+import status_params
+
+# server configurations
+config = Script.get_config()
+
+# user and status
+accumulo_user = status_params.accumulo_user
+user_group = config['configurations']['global']['user_group']
+pid_dir = status_params.pid_dir
+
+# accumulo env
+java64_home = config['hostLevelParams']['java_home']
+hadoop_prefix = config['configurations']['global']['hadoop_prefix']
+hadoop_conf_dir = config['configurations']['global']['hadoop_conf_dir']
+zookeeper_home = config['configurations']['global']['zookeeper_home']
+master_heapsize = config['configurations']['global']['master_heapsize']
+tserver_heapsize = config['configurations']['global']['tserver_heapsize']
+monitor_heapsize = config['configurations']['global']['monitor_heapsize']
+gc_heapsize = config['configurations']['global']['gc_heapsize']
+other_heapsize = config['configurations']['global']['other_heapsize']
+
+# accumulo local directory structure
+accumulo_root = config['configurations']['global']['app_root']
+conf_dir = None
+if ('accumulo_conf_dir' in config['configurations']['global']):
+ conf_dir = config['configurations']['global']['accumulo_conf_dir']
+else:
+ conf_dir = format("{accumulo_root}/conf")
+log_dir = config['configurations']['global']['app_log_dir']
+daemon_script = format("{accumulo_root}/bin/accumulo")
+
+# accumulo initialization parameters
+accumulo_instance_name = config['configurations']['global']['accumulo_instance_name']
+accumulo_root_password = config['configurations']['global']['accumulo_root_password']
+accumulo_hdfs_root_dir = config['configurations']['accumulo-site']['instance.dfs.dir']
+
+#log4j.properties
+if (('accumulo-log4j' in config['configurations']) and ('content' in config['configurations']['accumulo-log4j'])):
+ log4j_props = config['configurations']['accumulo-log4j']['content']
+else:
+ log4j_props = None
diff --git a/app-packages/accumulo-v1_5/package/scripts/status_params.py b/app-packages/accumulo-v1_5/package/scripts/status_params.py
new file mode 100644
index 0000000..6131880
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/scripts/status_params.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+pid_dir = config['configurations']['global']['app_pid_dir']
+accumulo_user = config['configurations']['global']['app_user']
diff --git a/app-packages/accumulo-v1_5/package/templates/accumulo-env.sh.j2 b/app-packages/accumulo-v1_5/package/templates/accumulo-env.sh.j2
new file mode 100755
index 0000000..7ffec53
--- /dev/null
+++ b/app-packages/accumulo-v1_5/package/templates/accumulo-env.sh.j2
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###
+### Configure these environment variables to point to your local installations.
+###
+### The functional tests require conditional values, so keep this style:
+###
+### test -z "$JAVA_HOME" && export JAVA_HOME=/usr/local/lib/jdk-1.6.0
+###
+###
+### Note that the -Xmx -Xms settings below require substantial free memory:
+### you may want to use smaller values, especially when running everything
+### on a single machine.
+###
+
+export HADOOP_PREFIX={{hadoop_prefix}}
+export HADOOP_CONF_DIR={{hadoop_conf_dir}}
+export JAVA_HOME={{java64_home}}
+export ZOOKEEPER_HOME={{zookeeper_home}}
+export ACCUMULO_LOG_DIR={{log_dir}}
+export ACCUMULO_CONF_DIR={{conf_dir}}
+export ACCUMULO_TSERVER_OPTS="-Xmx{{tserver_heapsize}} -Xms{{tserver_heapsize}}"
+export ACCUMULO_MASTER_OPTS="-Xmx{{master_heapsize}} -Xms{{master_heapsize}}"
+export ACCUMULO_MONITOR_OPTS="-Xmx{{monitor_heapsize}} -Xms{{monitor_heapsize}}"
+export ACCUMULO_GC_OPTS="-Xmx{{gc_heapsize}} -Xms{{gc_heapsize}}"
+export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75"
+export ACCUMULO_OTHER_OPTS="-Xmx{{other_heapsize}} -Xms{{other_heapsize}}"
+# what do when the JVM runs out of heap memory
+export ACCUMULO_KILL_CMD='kill -9 %p'
diff --git a/app-packages/accumulo-v1_5/resources.json b/app-packages/accumulo-v1_5/resources.json
new file mode 100644
index 0000000..aac5e2d
--- /dev/null
+++ b/app-packages/accumulo-v1_5/resources.json
@@ -0,0 +1,31 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ },
+ "components": {
+ "ACCUMULO_MASTER": {
+ "role.priority": "1",
+ "component.instances": "1"
+ },
+ "slider-appmaster": {
+ },
+ "ACCUMULO_TSERVER": {
+ "role.priority": "2",
+ "component.instances": "1"
+ },
+ "ACCUMULO_MONITOR": {
+ "role.priority": "3",
+ "component.instances": "1"
+ },
+ "ACCUMULO_GC": {
+ "role.priority": "4",
+ "component.instances": "1"
+ },
+ "ACCUMULO_TRACER": {
+ "role.priority": "5",
+ "component.instances": "1"
+ }
+ }
+}
diff --git a/app-packages/hbase-v0_96/appConfig.json b/app-packages/hbase-v0_96/appConfig.json
new file mode 100644
index 0000000..5a0a9eb
--- /dev/null
+++ b/app-packages/hbase-v0_96/appConfig.json
@@ -0,0 +1,68 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "agent.conf": "/slider/agent/conf/agent.ini",
+ "application.def": "/slider/hbase_v096.tar",
+ "config_types": "core-site,hdfs-site,hbase-site",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "package_list": "files/hbase-0.96.1-hadoop2-bin.tar.gz",
+ "site.global.app_user": "yarn",
+ "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
+ "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-0.96.1-hadoop2",
+ "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
+ "site.global.hbase_master_heapsize": "1024m",
+ "site.global.hbase_regionserver_heapsize": "1024m",
+ "site.global.user_group": "hadoop",
+ "site.global.security_enabled": "false",
+ "site.hbase-site.hbase.hstore.flush.retries.number": "120",
+ "site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
+ "site.hbase-site.hbase.hstore.compactionThreshold": "3",
+ "site.hbase-site.hbase.rootdir": "${NN_URI}/apps/hbase/data",
+ "site.hbase-site.hbase.stagingdir": "${NN_URI}/apps/hbase/staging",
+ "site.hbase-site.hbase.regionserver.handler.count": "60",
+ "site.hbase-site.hbase.regionserver.global.memstore.lowerLimit": "0.38",
+ "site.hbase-site.hbase.hregion.memstore.block.multiplier": "2",
+ "site.hbase-site.hbase.hregion.memstore.flush.size": "134217728",
+ "site.hbase-site.hbase.superuser": "yarn",
+ "site.hbase-site.hbase.zookeeper.property.clientPort": "2181",
+ "site.hbase-site.hbase.regionserver.global.memstore.upperLimit": "0.4",
+ "site.hbase-site.zookeeper.session.timeout": "30000",
+ "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+ "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+ "site.hbase-site.hbase.hregion.max.filesize": "10737418240",
+ "site.hbase-site.hfile.block.cache.size": "0.40",
+ "site.hbase-site.hbase.security.authentication": "simple",
+ "site.hbase-site.hbase.defaults.for.version.skip": "true",
+ "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
+ "site.hbase-site.zookeeper.znode.parent": "/hbase-unsecure",
+ "site.hbase-site.hbase.hstore.blockingStoreFiles": "10",
+ "site.hbase-site.hbase.hregion.majorcompaction": "86400000",
+ "site.hbase-site.hbase.security.authorization": "false",
+ "site.hbase-site.hbase.cluster.distributed": "true",
+ "site.hbase-site.hbase.hregion.memstore.mslab.enabled": "true",
+ "site.hbase-site.hbase.client.scanner.caching": "100",
+ "site.hbase-site.hbase.zookeeper.useMulti": "true",
+ "site.hbase-site.hbase.regionserver.info.port": "0",
+ "site.hbase-site.hbase.master.info.port": "60010",
+ "site.hbase-site.hbase.regionserver.port": "0",
+ "site.core-site.fs.defaultFS": "${NN_URI}",
+ "site.hdfs-site.dfs.namenode.https-address": "${NN_HOST}:50470",
+ "site.hdfs-site.dfs.namenode.http-address": "${NN_HOST}:50070"
+ },
+ "components": {
+ "HBASE_MASTER": {
+ "wait.heartbeat": "5",
+ "role.script": "scripts/hbase_master.py"
+ },
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ },
+ "HBASE_REGIONSERVER": {
+ "wait.heartbeat": "3",
+ "role.script": "scripts/hbase_regionserver.py"
+ }
+ }
+}
diff --git a/app-packages/hbase-v0_96/configuration/global.xml b/app-packages/hbase-v0_96/configuration/global.xml
new file mode 100644
index 0000000..b2c57bd
--- /dev/null
+++ b/app-packages/hbase-v0_96/configuration/global.xml
@@ -0,0 +1,160 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>hbasemaster_host</name>
+ <value></value>
+ <description>HBase Master Host.</description>
+ </property>
+ <property>
+ <name>regionserver_hosts</name>
+ <value></value>
+ <description>Region Server Hosts</description>
+ </property>
+ <property>
+ <name>hbase_log_dir</name>
+ <value>/var/log/hbase</value>
+ <description>Log Directories for HBase.</description>
+ </property>
+ <property>
+ <name>hbase_pid_dir</name>
+ <value>/var/run/hbase</value>
+ <description>Log Directories for HBase.</description>
+ </property>
+ <property>
+ <name>hbase_log_dir</name>
+ <value>/var/log/hbase</value>
+ <description>Log Directories for HBase.</description>
+ </property>
+ <property>
+ <name>hbase_regionserver_heapsize</name>
+ <value>1024</value>
+ <description>Log Directories for HBase.</description>
+ </property>
+ <property>
+ <name>hbase_master_heapsize</name>
+ <value>1024</value>
+ <description>HBase Master Heap Size</description>
+ </property>
+ <property>
+ <name>hstore_compactionthreshold</name>
+ <value>3</value>
+ <description>HBase HStore compaction threshold.</description>
+ </property>
+ <property>
+ <name>hfile_blockcache_size</name>
+ <value>0.40</value>
+ <description>HFile block cache size.</description>
+ </property>
+ <property>
+ <name>hstorefile_maxsize</name>
+ <value>10737418240</value>
+ <description>Maximum HStoreFile Size</description>
+ </property>
+ <property>
+ <name>regionserver_handlers</name>
+ <value>60</value>
+ <description>HBase RegionServer Handler</description>
+ </property>
+ <property>
+ <name>hregion_majorcompaction</name>
+ <value>604800000</value>
+ <description>The time between major compactions of all HStoreFiles in a region. Set to 0 to disable automated major compactions.</description>
+ </property>
+ <property>
+ <name>hregion_blockmultiplier</name>
+ <value>2</value>
+ <description>HBase Region Block Multiplier</description>
+ </property>
+ <property>
+ <name>hregion_memstoreflushsize</name>
+ <value></value>
+ <description>HBase Region MemStore Flush Size.</description>
+ </property>
+ <property>
+ <name>client_scannercaching</name>
+ <value>100</value>
+ <description>Base Client Scanner Caching</description>
+ </property>
+ <property>
+ <name>zookeeper_sessiontimeout</name>
+ <value>30000</value>
+ <description>ZooKeeper Session Timeout</description>
+ </property>
+ <property>
+ <name>hfile_max_keyvalue_size</name>
+ <value>10485760</value>
+ <description>HBase Client Maximum key-value Size</description>
+ </property>
+ <property>
+ <name>hbase_hdfs_root_dir</name>
+ <value>/apps/hbase/data</value>
+ <description>HBase Relative Path to HDFS.</description>
+ </property>
+ <property>
+ <name>hbase_conf_dir</name>
+ <value>/etc/hbase</value>
+ <description>Config Directory for HBase.</description>
+ </property>
+ <property>
+ <name>hdfs_enable_shortcircuit_read</name>
+ <value>true</value>
+ <description>HDFS Short Circuit Read</description>
+ </property>
+ <property>
+ <name>hdfs_support_append</name>
+ <value>true</value>
+ <description>HDFS append support</description>
+ </property>
+ <property>
+ <name>hstore_blockingstorefiles</name>
+ <value>10</value>
+ <description>HStore blocking storefiles.</description>
+ </property>
+ <property>
+ <name>regionserver_memstore_lab</name>
+ <value>true</value>
+ <description>Region Server memstore.</description>
+ </property>
+ <property>
+ <name>regionserver_memstore_lowerlimit</name>
+ <value>0.38</value>
+ <description>Region Server memstore lower limit.</description>
+ </property>
+ <property>
+ <name>regionserver_memstore_upperlimit</name>
+ <value>0.4</value>
+ <description>Region Server memstore upper limit.</description>
+ </property>
+ <property>
+ <name>hbase_conf_dir</name>
+ <value>/etc/hbase</value>
+ <description>HBase conf dir.</description>
+ </property>
+ <property>
+ <name>hbase_user</name>
+ <value>hbase</value>
+ <description>HBase User Name.</description>
+ </property>
+
+</configuration>
diff --git a/app-packages/hbase-v0_96/configuration/hbase-log4j.xml b/app-packages/hbase-v0_96/configuration/hbase-log4j.xml
new file mode 100644
index 0000000..3bbc549
--- /dev/null
+++ b/app-packages/hbase-v0_96/configuration/hbase-log4j.xml
@@ -0,0 +1,142 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+ <property>
+ <name>content</name>
+ <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.security.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+# Rolling File Appender properties
+hbase.log.maxfilesize=256MB
+hbase.log.maxbackupindex=20
+
+# Rolling File Appender
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file}
+
+log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize}
+log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex}
+
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+#
+# Security audit appender
+#
+hbase.security.log.file=SecurityAuth.audit
+hbase.security.log.maxfilesize=256MB
+hbase.security.log.maxbackupindex=20
+log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
+log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file}
+log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize}
+log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex}
+log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+# Enable this to get detailed connection error/retry logging.
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE
+
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of .META. messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
+
+ </value>
+ </property>
+
+</configuration>
diff --git a/app-packages/hbase-v0_96/configuration/hbase-policy.xml b/app-packages/hbase-v0_96/configuration/hbase-policy.xml
new file mode 100644
index 0000000..e45f23c
--- /dev/null
+++ b/app-packages/hbase-v0_96/configuration/hbase-policy.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HRegionInterface protocol implementations (ie.
+ clients talking to HRegionServers)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterInterface protocol implementation (ie.
+ clients talking to HMaster for admin operations).
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterRegionInterface protocol implementations
+ (for HRegionServers communicating with HMaster)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+</configuration>
diff --git a/app-packages/hbase-v0_96/configuration/hbase-site.xml b/app-packages/hbase-v0_96/configuration/hbase-site.xml
new file mode 100644
index 0000000..cf9416e
--- /dev/null
+++ b/app-packages/hbase-v0_96/configuration/hbase-site.xml
@@ -0,0 +1,365 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>hbase.rootdir</name>
+ <value>hdfs://localhost:8020/apps/hbase/data</value>
+ <description>The directory shared by region servers and into
+ which HBase persists. The URL should be 'fully-qualified'
+ to include the filesystem scheme. For example, to specify the
+ HDFS directory '/hbase' where the HDFS instance's namenode is
+ running at namenode.example.org on port 9000, set this value to:
+ hdfs://namenode.example.org:9000/hbase. By default HBase writes
+ into /tmp. Change this configuration else all data will be lost
+ on machine restart.
+ </description>
+ </property>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ <description>The mode the cluster will be in. Possible values are
+ false for standalone mode and true for distributed mode. If
+ false, startup will run all HBase and ZooKeeper daemons together
+ in the one JVM.
+ </description>
+ </property>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>/hadoop/hbase</value>
+ <description>Temporary directory on the local filesystem.
+ Change this setting to point to a location more permanent
+ than '/tmp' (The '/tmp' directory is often cleared on
+ machine restart).
+ </description>
+ </property>
+ <property>
+ <name>hbase.local.dir</name>
+ <value>${hbase.tmp.dir}/local</value>
+ <description>Directory on the local filesystem to be used as a local storage
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.info.bindAddress</name>
+ <value></value>
+ <description>The bind address for the HBase Master web UI
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.info.port</name>
+ <value></value>
+ <description>The port for the HBase Master web UI.</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.info.port</name>
+ <value></value>
+ <description>The port for the HBase RegionServer web UI.</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.global.memstore.upperLimit</name>
+ <value>0.4</value>
+ <description>Maximum size of all memstores in a region server before new
+ updates are blocked and flushes are forced. Defaults to 40% of heap
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>60</value>
+ <description>Count of RPC Listener instances spun up on RegionServers.
+ Same property is used by the Master for count of master handlers.
+ Default is 10.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.majorcompaction</name>
+ <value>86400000</value>
+ <description>The time (in milliseconds) between 'major' compactions of all
+ HStoreFiles in a region. Default: 1 day.
+ Set to 0 to disable automated major compactions.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.global.memstore.lowerLimit</name>
+ <value>0.38</value>
+ <description>When memstores are being forced to flush to make room in
+ memory, keep flushing until we hit this mark. Defaults to 35% of heap.
+ This value equal to hbase.regionserver.global.memstore.upperLimit causes
+ the minimum possible flushing to occur when updates are blocked due to
+ memstore limiting.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.block.multiplier</name>
+ <value>2</value>
+ <description>Block updates if memstore has hbase.hregion.memstore.block.multiplier
+ time hbase.hregion.flush.size bytes. Useful preventing
+ runaway memstore during spikes in update traffic. Without an
+ upper-bound, memstore fills such that when it flushes the
+ resultant flush files take a long time to compact or split, or
+ worse, we OOME
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.flush.size</name>
+ <value>134217728</value>
+ <description>
+ Memstore will be flushed to disk if size of the memstore
+ exceeds this number of bytes. Value is checked by a thread that runs
+ every hbase.server.thread.wakefrequency.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.mslab.enabled</name>
+ <value>true</value>
+ <description>
+ Enables the MemStore-Local Allocation Buffer,
+ a feature which works to prevent heap fragmentation under
+ heavy write loads. This can reduce the frequency of stop-the-world
+ GC pauses on large heaps.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.max.filesize</name>
+ <value>10737418240</value>
+ <description>
+ Maximum HStoreFile size. If any one of a column families' HStoreFiles has
+ grown to exceed this value, the hosting HRegion is split in two.
+ Default: 1G.
+ </description>
+ </property>
+ <property>
+ <name>hbase.client.scanner.caching</name>
+ <value>100</value>
+ <description>Number of rows that will be fetched when calling next
+ on a scanner if it is not served from (local, client) memory. Higher
+ caching values will enable faster scanners but will eat up more memory
+ and some calls of next may take longer and longer times when the cache is empty.
+ Do not set this value such that the time between invocations is greater
+ than the scanner timeout; i.e. hbase.regionserver.lease.period
+ </description>
+ </property>
+ <property>
+ <name>zookeeper.session.timeout</name>
+ <value>30000</value>
+ <description>ZooKeeper session timeout.
+ HBase passes this to the zk quorum as suggested maximum time for a
+ session (This setting becomes zookeeper's 'maxSessionTimeout'). See
+ http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
+ "The client sends a requested timeout, the server responds with the
+ timeout that it can give the client. " In milliseconds.
+ </description>
+ </property>
+ <property>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value>10485760</value>
+ <description>Specifies the combined maximum allowed size of a KeyValue
+ instance. This is to set an upper boundary for a single entry saved in a
+ storage file. Since they cannot be split it helps avoiding that a region
+ cannot be split any further because the data is too large. It seems wise
+ to set this to a fraction of the maximum region size. Setting it to zero
+ or less disables the check.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hstore.compactionThreshold</name>
+ <value>3</value>
+ <description>
+ If more than this number of HStoreFiles in any one HStore
+ (one HStoreFile is written per flush of memstore) then a compaction
+ is run to rewrite all HStoreFiles files as one. Larger numbers
+ put off compaction but when it runs, it takes longer to complete.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hstore.flush.retries.number</name>
+ <value>120</value>
+ <description>
+ The number of times the region flush operation will be retried.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.hstore.blockingStoreFiles</name>
+ <value>10</value>
+ <description>
+ If more than this number of StoreFiles in any one Store
+ (one StoreFile is written per flush of MemStore) then updates are
+ blocked for this HRegion until a compaction is completed, or
+ until hbase.hstore.blockingWaitTime has been exceeded.
+ </description>
+ </property>
+ <property>
+ <name>hfile.block.cache.size</name>
+ <value>0.40</value>
+ <description>
+ Percentage of maximum heap (-Xmx setting) to allocate to block cache
+ used by HFile/StoreFile. Default of 0.25 means allocate 25%.
+ Set to 0 to disable but it's not recommended.
+ </description>
+ </property>
+
+ <!-- The following properties configure authentication information for
+ HBase processes when using Kerberos security. There are no default
+ values, included here for documentation purposes -->
+ <property>
+ <name>hbase.master.keytab.file</name>
+ <value></value>
+ <description>Full path to the kerberos keytab file to use for logging in
+ the configured HMaster server principal.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.kerberos.principal</name>
+ <value></value>
+ <description>Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name
+ that should be used to run the HMaster process. The principal name should
+ be in the form: user/hostname@DOMAIN. If "_HOST" is used as the hostname
+ portion, it will be replaced with the actual hostname of the running
+ instance.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.keytab.file</name>
+ <value></value>
+ <description>Full path to the kerberos keytab file to use for logging in
+ the configured HRegionServer server principal.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.kerberos.principal</name>
+ <value></value>
+ <description>Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name
+ that should be used to run the HRegionServer process. The principal name
+ should be in the form: user/hostname@DOMAIN. If "_HOST" is used as the
+ hostname portion, it will be replaced with the actual hostname of the
+ running instance. An entry for this principal must exist in the file
+ specified in hbase.regionserver.keytab.file
+ </description>
+ </property>
+
+ <!-- Additional configuration specific to HBase security -->
+ <property>
+ <name>hbase.superuser</name>
+ <value>hbase</value>
+ <description>List of users or groups (comma-separated), who are allowed
+ full privileges, regardless of stored ACLs, across the cluster.
+ Only used when HBase security is enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.security.authentication</name>
+ <value>simple</value>
+ <description> Controls whether or not secure authentication is enabled for HBase. Possible values are 'simple'
+ (no authentication), and 'kerberos'.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.security.authorization</name>
+ <value>false</value>
+ <description>Enables HBase authorization. Set the value of this property to false to disable HBase authorization.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.coprocessor.region.classes</name>
+ <value></value>
+ <description>A comma-separated list of Coprocessors that are loaded by
+ default on all tables. For any override coprocessor method, these classes
+ will be called in order. After implementing your own Coprocessor, just put
+ it in HBase's classpath and add the fully qualified class name here.
+ A coprocessor can also be loaded on demand by setting HTableDescriptor.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.coprocessor.master.classes</name>
+ <value></value>
+ <description>A comma-separated list of
+ org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are
+ loaded by default on the active HMaster process. For any implemented
+ coprocessor methods, the listed classes will be called in order. After
+ implementing your own MasterObserver, just put it in HBase's classpath
+ and add the fully qualified class name here.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>2181</value>
+ <description>Property from ZooKeeper's config zoo.cfg.
+ The port at which the clients will connect.
+ </description>
+ </property>
+
+ <!--
+ The following three properties are used together to create the list of
+ host:peer_port:leader_port quorum servers for ZooKeeper.
+ -->
+ <property>
+ <name>hbase.zookeeper.quorum</name>
+ <value>localhost</value>
+ <description>Comma separated list of servers in the ZooKeeper Quorum.
+ For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
+ By default this is set to localhost for local and pseudo-distributed modes
+ of operation. For a fully-distributed setup, this should be set to a full
+ list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
+ this is the list of servers which we will start/stop ZooKeeper on.
+ </description>
+ </property>
+ <!-- End of properties used to generate ZooKeeper host:port quorum list. -->
+
+ <property>
+ <name>hbase.zookeeper.useMulti</name>
+ <value>true</value>
+ <description>Instructs HBase to make use of ZooKeeper's multi-update functionality.
+ This allows certain ZooKeeper operations to complete more quickly and prevents some issues
+ with rare Replication failure scenarios (see the release note of HBASE-2611 for an example).·
+ IMPORTANT: only set this to true if all ZooKeeper servers in the cluster are on version 3.4+
+ and will not be downgraded. ZooKeeper versions before 3.4 do not support multi-update and will
+ not fail gracefully if multi-update is invoked (see ZOOKEEPER-1495).
+ </description>
+ </property>
+ <property>
+ <name>zookeeper.znode.parent</name>
+ <value>/hbase-unsecure</value>
+ <description>Root ZNode for HBase in ZooKeeper. All of HBase's ZooKeeper
+ files that are configured with a relative path will go under this node.
+ By default, all of HBase's ZooKeeper file path are configured with a
+ relative path, so they will all go under this directory unless changed.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.defaults.for.version.skip</name>
+ <value>true</value>
+ <description>Disables version verification.</description>
+ </property>
+
+ <property>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ <description>Path to domain socket.</description>
+ </property>
+
+</configuration>
diff --git a/app-packages/hbase-v0_96/metainfo.xml b/app-packages/hbase-v0_96/metainfo.xml
new file mode 100644
index 0000000..ad440f1
--- /dev/null
+++ b/app-packages/hbase-v0_96/metainfo.xml
@@ -0,0 +1,78 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>HBASE</name>
+ <comment>
+ Apache HBase is the Hadoop database, a distributed, scalable, big data
+ store.
+ </comment>
+ <version>0.96.0.2.1.1</version>
+ <type>YARN-APP</type>
+ <minHadoopVersion>2.1.0</minHadoopVersion>
+ <components>
+ <component>
+ <name>HBASE_MASTER</name>
+ <category>MASTER</category>
+ <minInstanceCount>1</minInstanceCount>
+ <maxInstanceCount>2</maxInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>HBASE_REGIONSERVER</name>
+ <category>SLAVE</category>
+ <minInstanceCount>1</minInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_regionserver.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>HBASE_CLIENT</name>
+ <category>CLIENT</category>
+ <minInstanceCount>0</minInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/hbase-0.96.1-hadoop2-bin.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ </service>
+ </services>
+</metainfo>
diff --git a/app-packages/hbase-v0_96/package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE b/app-packages/hbase-v0_96/package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE
new file mode 100644
index 0000000..5d03caa
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Replace with actual hbase tarball.
diff --git a/app-packages/hbase-v0_96/package/scripts/__init__.py b/app-packages/hbase-v0_96/package/scripts/__init__.py
new file mode 100644
index 0000000..5561e10
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/scripts/__init__.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
diff --git a/app-packages/hbase-v0_96/package/scripts/functions.py b/app-packages/hbase-v0_96/package/scripts/functions.py
new file mode 100644
index 0000000..e6e7fb9
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/scripts/functions.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import math
+import datetime
+
+from resource_management.core.shell import checked_call
+
+def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max):
+ """
+ @param heapsize_str: str (e.g '1000m')
+ @param xmn_percent: float (e.g 0.2)
+ @param xmn_max: integer (e.g 512)
+ """
+ heapsize = int(re.search('\d+',heapsize_str).group(0))
+ heapsize_unit = re.search('\D+',heapsize_str).group(0)
+ xmn_val = int(math.floor(heapsize*xmn_percent))
+ xmn_val -= xmn_val % 8
+
+ result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val
+ return str(result_xmn_val) + heapsize_unit
diff --git a/app-packages/hbase-v0_96/package/scripts/hbase.py b/app-packages/hbase-v0_96/package/scripts/hbase.py
new file mode 100644
index 0000000..3ee0f41
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/scripts/hbase.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+
+from resource_management import *
+import sys
+
+def hbase(name=None # 'master' or 'regionserver' or 'client'
+ ):
+ import params
+
+ if name in ["master","regionserver"]:
+ params.HdfsDirectory(params.hbase_hdfs_root_dir,
+ action="create_delayed"
+ )
+ params.HdfsDirectory(params.hbase_staging_dir,
+ action="create_delayed",
+ mode=0711
+ )
+ params.HdfsDirectory(None, action="create")
+ Directory( params.conf_dir,
+ owner = params.hbase_user,
+ group = params.user_group,
+ recursive = True
+ )
+
+ Directory (params.tmp_dir,
+ owner = params.hbase_user,
+ recursive = True
+ )
+
+ Directory (os.path.join(params.local_dir, "jars"),
+ owner = params.hbase_user,
+ group = params.user_group,
+ mode=0775,
+ recursive = True
+ )
+
+ XmlConfig( "hbase-site.xml",
+ conf_dir = params.conf_dir,
+ configurations = params.config['configurations']['hbase-site'],
+ owner = params.hbase_user,
+ group = params.user_group
+ )
+
+ XmlConfig( "hdfs-site.xml",
+ conf_dir = params.conf_dir,
+ configurations = params.config['configurations']['hdfs-site'],
+ owner = params.hbase_user,
+ group = params.user_group
+ )
+
+ if 'hbase-policy' in params.config['configurations']:
+ XmlConfig( "hbase-policy.xml",
+ configurations = params.config['configurations']['hbase-policy'],
+ owner = params.hbase_user,
+ group = params.user_group
+ )
+ # Manually overriding ownership of file installed by hadoop package
+ else:
+ File( format("{conf_dir}/hbase-policy.xml"),
+ owner = params.hbase_user,
+ group = params.user_group
+ )
+
+ hbase_TemplateConfig( 'hbase-env.sh')
+
+ if params.security_enabled:
+ hbase_TemplateConfig( format("hbase_{name}_jaas.conf"))
+
+ if name != "client":
+ Directory( params.pid_dir,
+ owner = params.hbase_user,
+ recursive = True
+ )
+
+ Directory (params.log_dir,
+ owner = params.hbase_user,
+ recursive = True
+ )
+
+ if (params.log4j_props != None):
+ File(format("{params.conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hbase_user,
+ content=params.log4j_props
+ )
+ elif (os.path.exists(format("{params.conf_dir}/log4j.properties"))):
+ File(format("{params.conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hbase_user
+ )
+
+def hbase_TemplateConfig(name,
+ tag=None
+ ):
+ import params
+
+ TemplateConfig( format("{conf_dir}/{name}"),
+ owner = params.hbase_user,
+ template_tag = tag
+ )
diff --git a/app-packages/hbase-v0_96/package/scripts/hbase_client.py b/app-packages/hbase-v0_96/package/scripts/hbase_client.py
new file mode 100644
index 0000000..043ad11
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/scripts/hbase_client.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+
+
+class HbaseClient(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='client')
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+
+if __name__ == "__main__":
+ HbaseClient().execute()
diff --git a/app-packages/hbase-v0_96/package/scripts/hbase_master.py b/app-packages/hbase-v0_96/package/scripts/hbase_master.py
new file mode 100644
index 0000000..47b2409
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/scripts/hbase_master.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseMaster(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='master')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'master',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'master',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid")
+ check_process_status(pid_file)
+
+
+if __name__ == "__main__":
+ HbaseMaster().execute()
diff --git a/app-packages/hbase-v0_96/package/scripts/hbase_regionserver.py b/app-packages/hbase-v0_96/package/scripts/hbase_regionserver.py
new file mode 100644
index 0000000..8d66dcc
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/scripts/hbase_regionserver.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseRegionServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='regionserver')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'regionserver',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'regionserver',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid")
+ check_process_status(pid_file)
+
+ def decommission(self, env):
+ print "Decommission not yet implemented!"
+
+
+if __name__ == "__main__":
+ HbaseRegionServer().execute()
diff --git a/app-packages/hbase-v0_96/package/scripts/hbase_service.py b/app-packages/hbase-v0_96/package/scripts/hbase_service.py
new file mode 100644
index 0000000..2b30083
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/scripts/hbase_service.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+def hbase_service(
+ name,
+ action = 'start'): # 'start' or 'stop' or 'status'
+
+ import params
+
+ role = name
+ cmd = format("{daemon_script} --config {conf_dir}")
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid")
+
+ daemon_cmd = None
+ no_op_test = None
+
+ if action == 'start':
+ daemon_cmd = format("{cmd} start {role}")
+ no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
+ elif action == 'stop':
+ daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
+
+ if daemon_cmd is not None:
+ Execute ( daemon_cmd,
+ not_if = no_op_test
+ )
diff --git a/app-packages/hbase-v0_96/package/scripts/params.py b/app-packages/hbase-v0_96/package/scripts/params.py
new file mode 100644
index 0000000..6520525
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/scripts/params.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from functions import calc_xmn_from_xms
+from resource_management import *
+import status_params
+
+# server configurations
+config = Script.get_config()
+
+hbase_root = config['configurations']['global']['app_root']
+conf_dir = format("{hbase_root}/conf")
+daemon_script = format("{hbase_root}/bin/hbase-daemon.sh")
+
+hbase_user = status_params.hbase_user
+_authentication = config['configurations']['core-site']['hadoop.security.authentication']
+security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
+user_group = config['configurations']['global']['user_group']
+
+# this is "hadoop-metrics.properties" for 1.x stacks
+metric_prop_file_name = "hadoop-metrics2-hbase.properties"
+
+# not supporting 32 bit jdk.
+java64_home = config['hostLevelParams']['java_home']
+
+log_dir = config['configurations']['global']['app_log_dir']
+master_heapsize = config['configurations']['global']['hbase_master_heapsize']
+
+regionserver_heapsize = config['configurations']['global']['hbase_regionserver_heapsize']
+regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
+
+pid_dir = status_params.pid_dir
+tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
+local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'], config['configurations']['hbase-site'])
+
+client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf_dir}/hbase_client_jaas.conf"))
+master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf"))
+regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', format("{conf_dir}/hbase_regionserver_jaas.conf"))
+
+if security_enabled:
+
+ _use_hostname_in_principal = default('instance_name', True)
+ _master_primary_name = config['configurations']['global']['hbase_master_primary_name']
+ _hostname_lowercase = config['hostname'].lower()
+ _kerberos_domain = config['configurations']['global']['kerberos_domain']
+ _master_principal_name = config['configurations']['global']['hbase_master_principal_name']
+ _regionserver_primary_name = config['configurations']['global']['hbase_regionserver_primary_name']
+
+ if _use_hostname_in_principal:
+ master_jaas_princ = format("{_master_primary_name}/{_hostname_lowercase}@{_kerberos_domain}")
+ regionserver_jaas_princ = format("{_regionserver_primary_name}/{_hostname_lowercase}@{_kerberos_domain}")
+ else:
+ master_jaas_princ = format("{_master_principal_name}@{_kerberos_domain}")
+ regionserver_jaas_princ = format("{_regionserver_primary_name}@{_kerberos_domain}")
+
+master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
+regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+if security_enabled:
+ kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};")
+else:
+ kinit_cmd = ""
+
+#log4j.properties
+if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])):
+ log4j_props = config['configurations']['hbase-log4j']['content']
+else:
+ log4j_props = None
+
+
+hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']
+hbase_staging_dir = config['configurations']['hbase-site']['hbase.stagingdir']
+#for create_hdfs_directory
+hostname = config["hostname"]
+hadoop_conf_dir = "/etc/hadoop/conf"
+hdfs_user_keytab = config['configurations']['global']['hdfs_user_keytab']
+hdfs_user = config['configurations']['global']['hdfs_user']
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+import functools
+#create partial functions with common arguments for every HdfsDirectory call
+#to create hdfs directory we need to call params.HdfsDirectory in code
+HdfsDirectory = functools.partial(
+ HdfsDirectory,
+ conf_dir=hadoop_conf_dir,
+ hdfs_user=hdfs_user,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local
+)
diff --git a/app-packages/hbase-v0_96/package/scripts/status_params.py b/app-packages/hbase-v0_96/package/scripts/status_params.py
new file mode 100644
index 0000000..c18cbb9
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/scripts/status_params.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+pid_dir = config['configurations']['global']['app_pid_dir']
+hbase_user = config['configurations']['global']['app_user']
diff --git a/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
new file mode 100644
index 0000000..2583f44
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See http://wiki.apache.org/hadoop/GangliaMetrics
+#
+# Make sure you know whether you are using ganglia 3.0 or 3.1.
+# If 3.1, you will have to patch your hadoop instance with HADOOP-4675
+# And, yes, this file is named hadoop-metrics.properties rather than
+# hbase-metrics.properties because we're leveraging the hadoop metrics
+# package and hadoop-metrics.properties is an hardcoded-name, at least
+# for the moment.
+#
+# See also http://hadoop.apache.org/hbase/docs/current/metrics.html
+
+# HBase-specific configuration to reset long-running stats (e.g. compactions)
+# If this variable is left out, then the default is no expiration.
+hbase.extendedperiod = 3600
+
+# Configuration of the "hbase" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+hbase.period=10
+hbase.servers={{ganglia_server_host}}:8663
+
+# Configuration of the "jvm" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+jvm.period=10
+jvm.servers={{ganglia_server_host}}:8663
+
+# Configuration of the "rpc" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+rpc.period=10
+rpc.servers={{ganglia_server_host}}:8663
+
+#Ganglia following hadoop example
+hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
+hbase.sink.ganglia.period=10
+
+# default for supportsparse is false
+*.sink.ganglia.supportsparse=true
+
+.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
+.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
+
+hbase.sink.ganglia.servers={{ganglia_server_host}}:8663
diff --git a/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
new file mode 100644
index 0000000..9f2b616
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See http://wiki.apache.org/hadoop/GangliaMetrics
+#
+# Make sure you know whether you are using ganglia 3.0 or 3.1.
+# If 3.1, you will have to patch your hadoop instance with HADOOP-4675
+# And, yes, this file is named hadoop-metrics.properties rather than
+# hbase-metrics.properties because we're leveraging the hadoop metrics
+# package and hadoop-metrics.properties is an hardcoded-name, at least
+# for the moment.
+#
+# See also http://hadoop.apache.org/hbase/docs/current/metrics.html
+
+# HBase-specific configuration to reset long-running stats (e.g. compactions)
+# If this variable is left out, then the default is no expiration.
+hbase.extendedperiod = 3600
+
+# Configuration of the "hbase" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+hbase.period=10
+hbase.servers={{ganglia_server_host}}:8660
+
+# Configuration of the "jvm" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+jvm.period=10
+jvm.servers={{ganglia_server_host}}:8660
+
+# Configuration of the "rpc" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+rpc.period=10
+rpc.servers={{ganglia_server_host}}:8660
+
+#Ganglia following hadoop example
+hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
+hbase.sink.ganglia.period=10
+
+# default for supportsparse is false
+*.sink.ganglia.supportsparse=true
+
+.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
+.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
+
+hbase.sink.ganglia.servers={{ganglia_server_host}}:8660
diff --git a/app-packages/hbase-v0_96/package/templates/hbase-env.sh.j2 b/app-packages/hbase-v0_96/package/templates/hbase-env.sh.j2
new file mode 100644
index 0000000..4aa79ad
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/templates/hbase-env.sh.j2
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set environment variables here.
+
+# The java implementation to use. Java 1.6 required.
+export JAVA_HOME={{java64_home}}
+
+# HBase Configuration directory
+export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{conf_dir}}}
+
+# Extra Java CLASSPATH elements. Optional.
+export HBASE_CLASSPATH=${HBASE_CLASSPATH}
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log"
+export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`"
+# Uncomment below to enable java garbage collection logging.
+# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}"
+export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
+# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored. $HBASE_HOME/logs by default.
+export HBASE_LOG_DIR={{log_dir}}
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+export HBASE_PID_DIR={{pid_dir}}
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+export HBASE_MANAGES_ZK=false
+
+{% if security_enabled %}
+export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}"
+export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}"
+{% endif %}
diff --git a/app-packages/hbase-v0_96/package/templates/hbase_client_jaas.conf.j2 b/app-packages/hbase-v0_96/package/templates/hbase_client_jaas.conf.j2
new file mode 100644
index 0000000..bb4279c
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/templates/hbase_client_jaas.conf.j2
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=false
+useTicketCache=true;
+};
diff --git a/app-packages/hbase-v0_96/package/templates/hbase_master_jaas.conf.j2 b/app-packages/hbase-v0_96/package/templates/hbase_master_jaas.conf.j2
new file mode 100644
index 0000000..91ce3ef
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/templates/hbase_master_jaas.conf.j2
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{master_keytab_path}}"
+principal="{{master_jaas_princ}}";
+};
diff --git a/app-packages/hbase-v0_96/package/templates/hbase_regionserver_jaas.conf.j2 b/app-packages/hbase-v0_96/package/templates/hbase_regionserver_jaas.conf.j2
new file mode 100644
index 0000000..2a9b9f3
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/templates/hbase_regionserver_jaas.conf.j2
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{regionserver_keytab_path}}"
+principal="{{regionserver_jaas_princ}}";
+};
diff --git a/app-packages/hbase-v0_96/package/templates/regionservers.j2 b/app-packages/hbase-v0_96/package/templates/regionservers.j2
new file mode 100644
index 0000000..81d060b
--- /dev/null
+++ b/app-packages/hbase-v0_96/package/templates/regionservers.j2
@@ -0,0 +1,20 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in rs_hosts %}{{host}}
+{% endfor %}
diff --git a/app-packages/hbase-v0_96/resources.json b/app-packages/hbase-v0_96/resources.json
new file mode 100644
index 0000000..4d5617f
--- /dev/null
+++ b/app-packages/hbase-v0_96/resources.json
@@ -0,0 +1,19 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ },
+ "components": {
+ "HBASE_MASTER": {
+ "role.priority": "1",
+ "component.instances": "1"
+ },
+ "slider-appmaster": {
+ },
+ "HBASE_REGIONSERVER": {
+ "role.priority": "2",
+ "component.instances": "1"
+ }
+ }
+}
\ No newline at end of file
diff --git a/app-packages/storm-v0_91/appConfig.json b/app-packages/storm-v0_91/appConfig.json
new file mode 100644
index 0000000..56404d1
--- /dev/null
+++ b/app-packages/storm-v0_91/appConfig.json
@@ -0,0 +1,129 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "agent.conf": "/slider/agent/conf/agent.ini",
+ "application.def": "/slider/storm_v091.tar",
+ "config_types": "storm-site",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "package_list": "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz",
+ "site.global.app_user": "yarn",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237",
+ "site.global.user_group": "hadoop",
+ "site.global.security_enabled": "false",
+ "site.storm-site.topology.tuple.serializer": "backtype.storm.serialization.types.ListDelegateSerializer",
+ "site.storm-site.topology.workers": "1",
+ "site.storm-site.drpc.worker.threads": "64",
+ "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
+ "site.storm-site.supervisor.heartbeat.frequency.secs": "5",
+ "site.storm-site.topology.executor.send.buffer.size": "1024",
+ "site.storm-site.drpc.childopts": "-Xmx768m",
+ "site.storm-site.nimbus.thrift.port": "6627",
+ "site.storm-site.storm.zookeeper.retry.intervalceiling.millis": "30000",
+ "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
+ "site.storm-site.topology.receiver.buffer.size": "8",
+ "site.storm-site.storm.messaging.netty.client_worker_threads": "1",
+ "site.storm-site.transactional.zookeeper.root": "/transactional",
+ "site.storm-site.drpc.request.timeout.secs": "600",
+ "site.storm-site.topology.skip.missing.kryo.registrations": "false",
+ "site.storm-site.worker.heartbeat.frequency.secs": "1",
+ "site.storm-site.zmq.hwm": "0",
+ "site.storm-site.storm.zookeeper.connection.timeout": "15000",
+ "site.storm-site.topology.max.error.report.per.interval": "5",
+ "site.storm-site.storm.messaging.netty.server_worker_threads": "1",
+ "site.storm-site.supervisor.worker.start.timeout.secs": "120",
+ "site.storm-site.zmq.threads": "1",
+ "site.storm-site.topology.acker.executors": "null",
+ "site.storm-site.storm.local.mode.zmq": "false",
+ "site.storm-site.topology.max.task.parallelism": "null",
+ "site.storm-site.storm.zookeeper.port": "2181",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8649,wireformat31x=true,mode=multicast,config=/tmp/container002/work/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8650,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.drpc.queue.size": "128",
+ "site.storm-site.storm.zookeeper.retry.times": "5",
+ "site.storm-site.nimbus.monitor.freq.secs": "10",
+ "site.storm-site.storm.cluster.mode": "distributed",
+ "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
+ "site.storm-site.drpc.invocations.port": "3773",
+ "site.storm-site.storm.zookeeper.root": "/storm",
+ "site.storm-site.logviewer.childopts": "-Xmx128m",
+ "site.storm-site.transactional.zookeeper.port": "null",
+ "site.storm-site.topology.worker.childopts": "null",
+ "site.storm-site.topology.max.spout.pending": "null",
+ "site.storm-site.nimbus.cleanup.inbox.freq.secs": "600",
+ "site.storm-site.storm.messaging.netty.min_wait_ms": "100",
+ "site.storm-site.nimbus.task.timeout.secs": "30",
+ "site.storm-site.nimbus.thrift.max_buffer_size": "1048576",
+ "site.storm-site.topology.sleep.spout.wait.strategy.time.ms": "1",
+ "site.storm-site.topology.optimize": "true",
+ "site.storm-site.nimbus.reassign": "true",
+ "site.storm-site.storm.messaging.transport": "backtype.storm.messaging.netty.Context",
+ "site.storm-site.logviewer.appender.name": "A1",
+ "site.storm-site.nimbus.host": "${NIMBUS_HOST}",
+ "site.storm-site.ui.port": "8744",
+ "site.storm-site.supervisor.slots.ports": "[6700, 6701]",
+ "site.storm-site.nimbus.file.copy.expiration.secs": "600",
+ "site.storm-site.supervisor.monitor.frequency.secs": "3",
+ "site.storm-site.transactional.zookeeper.servers": "null",
+ "site.storm-site.zmq.linger.millis": "5000",
+ "site.storm-site.topology.error.throttle.interval.secs": "10",
+ "site.storm-site.topology.worker.shared.thread.pool.size": "4",
+ "site.storm-site.java.library.path": "/usr/local/lib:/opt/local/lib:/usr/lib",
+ "site.storm-site.topology.spout.wait.strategy": "backtype.storm.spout.SleepSpoutWaitStrategy",
+ "site.storm-site.task.heartbeat.frequency.secs": "3",
+ "site.storm-site.topology.transfer.buffer.size": "1024",
+ "site.storm-site.storm.zookeeper.session.timeout": "20000",
+ "site.storm-site.topology.executor.receive.buffer.size": "1024",
+ "site.storm-site.topology.stats.sample.rate": "0.05",
+ "site.storm-site.topology.fall.back.on.java.serialization": "true",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=56431 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8650,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.topology.enable.message.timeouts": "true",
+ "site.storm-site.storm.messaging.netty.max_wait_ms": "1000",
+ "site.storm-site.nimbus.topology.validator": "backtype.storm.nimbus.DefaultTopologyValidator",
+ "site.storm-site.nimbus.supervisor.timeout.secs": "60",
+ "site.storm-site.topology.disruptor.wait.strategy": "com.lmax.disruptor.BlockingWaitStrategy",
+ "site.storm-site.nimbus.inbox.jar.expiration.secs": "3600",
+ "site.storm-site.drpc.port": "3772",
+ "site.storm-site.topology.kryo.factory": "backtype.storm.serialization.DefaultKryoFactory",
+ "site.storm-site.storm.zookeeper.retry.interval": "1000",
+ "site.storm-site.storm.messaging.netty.max_retries": "30",
+ "site.storm-site.topology.tick.tuple.freq.secs": "null",
+ "site.storm-site.supervisor.enable": "true",
+ "site.storm-site.nimbus.task.launch.secs": "120",
+ "site.storm-site.task.refresh.poll.secs": "10",
+ "site.storm-site.topology.message.timeout.secs": "30",
+ "site.storm-site.storm.messaging.netty.buffer_size": "5242880",
+ "site.storm-site.topology.state.synchronization.timeout.secs": "60",
+ "site.storm-site.supervisor.worker.timeout.secs": "30",
+ "site.storm-site.topology.trident.batch.emit.interval.millis": "500",
+ "site.storm-site.topology.builtin.metrics.bucket.size.secs": "60",
+ "site.storm-site.storm.thrift.transport": "backtype.storm.security.auth.SimpleTransportPlugin",
+ "site.storm-site.logviewer.port": "8000",
+ "site.storm-site.topology.debug": "false"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ },
+ "NIMBUS": {
+ "role.script": "scripts/nimbus.py"
+ },
+ "STORM_REST_API": {
+ "wait.heartbeat": "3",
+ "role.script": "scripts/rest_api.py"
+ },
+ "STORM_UI_SERVER": {
+ "wait.heartbeat": "3",
+ "role.script": "scripts/ui_server.py"
+ },
+ "DRPC_SERVER": {
+ "wait.heartbeat": "7",
+ "role.script": "scripts/drpc_server.py"
+ },
+ "SUPERVISOR": {
+ "wait.heartbeat": "10",
+ "role.script": "scripts/supervisor.py"
+ }
+ }
+}
diff --git a/app-packages/storm-v0_91/configuration/global.xml b/app-packages/storm-v0_91/configuration/global.xml
new file mode 100644
index 0000000..5cc9170
--- /dev/null
+++ b/app-packages/storm-v0_91/configuration/global.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>storm_user</name>
+ <value>storm</value>
+ <description></description>
+ </property>
+ <property>
+ <name>storm_log_dir</name>
+ <value>/var/log/storm</value>
+ <description></description>
+ </property>
+ <property>
+ <name>storm_pid_dir</name>
+ <value>/var/run/storm</value>
+ <description></description>
+ </property>
+</configuration>
diff --git a/app-packages/storm-v0_91/configuration/storm-site.xml b/app-packages/storm-v0_91/configuration/storm-site.xml
new file mode 100644
index 0000000..6eca8f9
--- /dev/null
+++ b/app-packages/storm-v0_91/configuration/storm-site.xml
@@ -0,0 +1,587 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>java.library.path</name>
+ <value>/usr/local/lib:/opt/local/lib:/usr/lib</value>
+ <description>This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers)
+ for the java.library.path value. java.library.path tells the JVM where
+ to look for native libraries. It is necessary to set this config correctly since
+ Storm uses the ZeroMQ and JZMQ native libs. </description>
+ </property>
+ <property>
+ <name>storm.local.dir</name>
+ <value>/hadoop/storm</value>
+ <description>A directory on the local filesystem used by Storm for any local
+ filesystem usage it needs. The directory must exist and the Storm daemons must
+ have permission to read/write from this location.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.servers</name>
+ <value>['localhost']</value>
+ <description>A list of hosts of ZooKeeper servers used to manage the cluster.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.port</name>
+ <value>2181</value>
+ <description>The port Storm will use to connect to each of the ZooKeeper servers.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.root</name>
+ <value>/storm</value>
+ <description>The root location at which Storm stores data in ZooKeeper.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.session.timeout</name>
+ <value>20000</value>
+ <description>The session timeout for clients to ZooKeeper.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.connection.timeout</name>
+ <value>15000</value>
+ <description>The connection timeout for clients to ZooKeeper.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.retry.times</name>
+ <value>5</value>
+ <description>The number of times to retry a Zookeeper operation.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.retry.interval</name>
+ <value>1000</value>
+ <description>The interval between retries of a Zookeeper operation.</description>
+ </property>
+ <property>
+ <name>storm.zookeeper.retry.intervalceiling.millis</name>
+ <value>30000</value>
+ <description>The ceiling of the interval between retries of a Zookeeper operation.</description>
+ </property>
+ <property>
+ <name>storm.cluster.mode</name>
+ <value>distributed</value>
+ <description>The mode this Storm cluster is running in. Either "distributed" or "local".</description>
+ </property>
+ <property>
+ <name>storm.local.mode.zmq</name>
+ <value>false</value>
+ <description>Whether or not to use ZeroMQ for messaging in local mode. If this is set
+ to false, then Storm will use a pure-Java messaging system. The purpose
+ of this flag is to make it easy to run Storm in local mode by eliminating
+ the need for native dependencies, which can be difficult to install.
+ </description>
+ </property>
+ <property>
+ <name>storm.thrift.transport</name>
+ <value>backtype.storm.security.auth.SimpleTransportPlugin</value>
+ <description>The transport plug-in for Thrift client/server communication.</description>
+ </property>
+ <property>
+ <name>storm.messaging.transport</name>
+ <value>backtype.storm.messaging.netty.Context</value>
+ <description>The transporter for communication among Storm tasks.</description>
+ </property>
+ <property>
+ <name>nimbus.host</name>
+ <value>localhost</value>
+ <description>The host that the master server is running on.</description>
+ </property>
+ <property>
+ <name>nimbus.thrift.port</name>
+ <value>6627</value>
+ <description> Which port the Thrift interface of Nimbus should run on. Clients should
+ connect to this port to upload jars and submit topologies.</description>
+ </property>
+ <property>
+ <name>nimbus.thrift.max_buffer_size</name>
+ <value>1048576</value>
+ <description>The maximum buffer size thrift should use when reading messages.</description>
+ </property>
+ <property>
+ <name>nimbus.childopts</name>
+ <value>-Xmx1024m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8649,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM</value>
+ <description>This parameter is used by the storm-deploy project to configure the jvm options for the nimbus daemon.</description>
+ </property>
+ <property>
+ <name>nimbus.task.timeout.secs</name>
+ <value>30</value>
+ <description>How long without heartbeating a task can go before nimbus will consider the task dead and reassign it to another location.</description>
+ </property>
+ <property>
+ <name>nimbus.supervisor.timeout.secs</name>
+ <value>60</value>
+ <description>How long before a supervisor can go without heartbeating before nimbus considers it dead and stops assigning new work to it.</description>
+ </property>
+ <property>
+ <name>nimbus.monitor.freq.secs</name>
+ <value>10</value>
+ <description>
+ How often nimbus should wake up to check heartbeats and do reassignments. Note
+ that if a machine ever goes down Nimbus will immediately wake up and take action.
+ This parameter is for checking for failures when there's no explicit event like that occuring.
+ </description>
+ </property>
+ <property>
+ <name>nimbus.cleanup.inbox.freq.secs</name>
+ <value>600</value>
+ <description>How often nimbus should wake the cleanup thread to clean the inbox.</description>
+ </property>
+ <property>
+ <name>nimbus.inbox.jar.expiration.secs</name>
+ <value>3600</value>
+ <description>
+ The length of time a jar file lives in the inbox before being deleted by the cleanup thread.
+
+ Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS.
+ Note that the time it takes to delete an inbox jar file is going to be somewhat more than
+ NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS (depending on how often NIMBUS_CLEANUP_FREQ_SECS is set to).
+ </description>
+ </property>
+ <property>
+ <name>nimbus.task.launch.secs</name>
+ <value>120</value>
+ <description>A special timeout used when a task is initially launched. During launch, this is the timeout
+ used until the first heartbeat, overriding nimbus.task.timeout.secs.</description>
+ </property>
+ <property>
+ <name>nimbus.reassign</name>
+ <value>true</value>
+ <description>Whether or not nimbus should reassign tasks if it detects that a task goes down.
+ Defaults to true, and it's not recommended to change this value.</description>
+ </property>
+ <property>
+ <name>nimbus.file.copy.expiration.secs</name>
+ <value>600</value>
+ <description>During upload/download with the master, how long an upload or download connection is idle
+ before nimbus considers it dead and drops the connection.</description>
+ </property>
+ <property>
+ <name>nimbus.topology.validator</name>
+ <value>backtype.storm.nimbus.DefaultTopologyValidator</value>
+ <description>A custom class that implements ITopologyValidator that is run whenever a
+ topology is submitted. Can be used to provide business-specific logic for
+ whether topologies are allowed to run or not.</description>
+ </property>
+ <property>
+ <name>ui.port</name>
+ <value>8744</value>
+ <description>Storm UI binds to this port.</description>
+ </property>
+ <property>
+ <name>ui.childopts</name>
+ <value>-Xmx768m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf</value>
+ <description>Childopts for Storm UI Java process.</description>
+ </property>
+ <property>
+ <name>logviewer.port</name>
+ <value>8000</value>
+ <description>HTTP UI port for log viewer.</description>
+ </property>
+ <property>
+ <name>logviewer.childopts</name>
+ <value>-Xmx128m</value>
+ <description>Childopts for log viewer java process.</description>
+ </property>
+ <property>
+ <name>logviewer.appender.name</name>
+ <value>A1</value>
+ <description>Appender name used by log viewer to determine log directory.</description>
+ </property>
+ <property>
+ <name>drpc.port</name>
+ <value>3772</value>
+ <description>This port is used by Storm DRPC for receiving DPRC requests from clients.</description>
+ </property>
+ <property>
+ <name>drpc.worker.threads</name>
+ <value>64</value>
+ <description>DRPC thrift server worker threads.</description>
+ </property>
+ <property>
+ <name>drpc.queue.size</name>
+ <value>128</value>
+ <description>DRPC thrift server queue size.</description>
+ </property>
+ <property>
+ <name>drpc.invocations.port</name>
+ <value>3773</value>
+ <description>This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back.</description>
+ </property>
+ <property>
+ <name>drpc.request.timeout.secs</name>
+ <value>600</value>
+ <description>The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also
+ timeout based on the socket timeout on the DRPC client, and separately based on the topology message
+ timeout for the topology implementing the DRPC function.</description>
+ </property>
+ <property>
+ <name>drpc.childopts</name>
+ <value>-Xmx768m</value>
+ <description>Childopts for Storm DRPC Java process.</description>
+ </property>
+ <property>
+ <name>transactional.zookeeper.root</name>
+ <value>/transactional</value>
+ <description>The root directory in ZooKeeper for metadata about TransactionalSpouts.</description>
+ </property>
+ <property>
+ <name>transactional.zookeeper.servers</name>
+ <value>null</value>
+ <description>The list of zookeeper servers in which to keep the transactional state. If null (which is default),
+ will use storm.zookeeper.servers</description>
+ </property>
+ <property>
+ <name>transactional.zookeeper.port</name>
+ <value>null</value>
+ <description>The port to use to connect to the transactional zookeeper servers. If null (which is default),
+ will use storm.zookeeper.port</description>
+ </property>
+ <property>
+ <name>supervisor.slots.ports</name>
+ <value>[6700, 6701]</value>
+ <description>A list of ports that can run workers on this supervisor. Each worker uses one port, and
+ the supervisor will only run one worker per port. Use this configuration to tune
+ how many workers run on each machine.</description>
+ </property>
+ <property>
+ <name>supervisor.childopts</name>
+ <value>-Xmx256m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=56431 -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM</value>
+ <description>This parameter is used by the storm-deploy project to configure the jvm options for the supervisor daemon.</description>
+ </property>
+ <property>
+ <name>supervisor.worker.start.timeout.secs</name>
+ <value>120</value>
+ <description>How long a worker can go without heartbeating during the initial launch before
+ the supervisor tries to restart the worker process. This value override
+ supervisor.worker.timeout.secs during launch because there is additional
+ overhead to starting and configuring the JVM on launch.</description>
+ </property>
+ <property>
+ <name>supervisor.worker.timeout.secs</name>
+ <value>30</value>
+ <description>How long a worker can go without heartbeating before the supervisor tries to restart the worker process.</description>
+ </property>
+ <property>
+ <name>supervisor.monitor.frequency.secs</name>
+ <value>3</value>
+ <description>How often the supervisor checks the worker heartbeats to see if any of them need to be restarted.</description>
+ </property>
+ <property>
+ <name>supervisor.heartbeat.frequency.secs</name>
+ <value>5</value>
+ <description>How often the supervisor sends a heartbeat to the master.</description>
+ </property>
+ <property>
+ <name>supervisor.enable</name>
+ <value>true</value>
+ <description>Whether or not the supervisor should launch workers assigned to it. Defaults
+ to true -- and you should probably never change this value. This configuration
+ is used in the Storm unit tests.</description>
+ </property>
+ <property>
+ <name>worker.childopts</name>
+ <value>-Xmx768m -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM</value>
+ <description>The jvm opts provided to workers launched by this supervisor. All \"%ID%\" substrings are replaced with an identifier for this worker.</description>
+ </property>
+ <property>
+ <name>worker.heartbeat.frequency.secs</name>
+ <value>1</value>
+ <description>How often this worker should heartbeat to the supervisor.</description>
+ </property>
+ <property>
+ <name>task.heartbeat.frequency.secs</name>
+ <value>3</value>
+ <description>How often a task should heartbeat its status to the master.</description>
+ </property>
+ <property>
+ <name>task.refresh.poll.secs</name>
+ <value>10</value>
+ <description>How often a task should sync its connections with other tasks (if a task is
+ reassigned, the other tasks sending messages to it need to refresh their connections).
+ In general though, when a reassignment happens other tasks will be notified
+ almost immediately. This configuration is here just in case that notification doesn't
+ come through.</description>
+ </property>
+ <property>
+ <name>zmq.threads</name>
+ <value>1</value>
+ <description>The number of threads that should be used by the zeromq context in each worker process.</description>
+ </property>
+ <property>
+ <name>zmq.linger.millis</name>
+ <value>5000</value>
+ <description>How long a connection should retry sending messages to a target host when
+ the connection is closed. This is an advanced configuration and can almost
+ certainly be ignored.</description>
+ </property>
+ <property>
+ <name>zmq.hwm</name>
+ <value>0</value>
+ <description>The high water for the ZeroMQ push sockets used for networking. Use this config to prevent buffer explosion
+ on the networking layer.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.server_worker_threads</name>
+ <value>1</value>
+ <description>Netty based messaging: The # of worker threads for the server.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.client_worker_threads</name>
+ <value>1</value>
+ <description>Netty based messaging: The # of worker threads for the client.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.buffer_size</name>
+ <value>5242880</value>
+ <description>Netty based messaging: The buffer size for send/recv buffer.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.max_retries</name>
+ <value>30</value>
+ <description>Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.max_wait_ms</name>
+ <value>1000</value>
+ <description>Netty based messaging: The max # of milliseconds that a peer will wait.</description>
+ </property>
+ <property>
+ <name>storm.messaging.netty.min_wait_ms</name>
+ <value>100</value>
+ <description>Netty based messaging: The min # of milliseconds that a peer will wait.</description>
+ </property>
+ <property>
+ <name>topology.enable.message.timeouts</name>
+ <value>true</value>
+ <description>True if Storm should timeout messages or not. Defaults to true. This is meant to be used
+ in unit tests to prevent tuples from being accidentally timed out during the test.</description>
+ </property>
+ <property>
+ <name>topology.debug</name>
+ <value>false</value>
+ <description>When set to true, Storm will log every message that's emitted.</description>
+ </property>
+ <property>
+ <name>topology.optimize</name>
+ <value>true</value>
+ <description>Whether or not the master should optimize topologies by running multiple tasks in a single thread where appropriate.</description>
+ </property>
+ <property>
+ <name>topology.workers</name>
+ <value>1</value>
+ <description>How many processes should be spawned around the cluster to execute this
+ topology. Each process will execute some number of tasks as threads within
+ them. This parameter should be used in conjunction with the parallelism hints
+ on each component in the topology to tune the performance of a topology.</description>
+ </property>
+ <property>
+ <name>topology.acker.executors</name>
+ <value>null</value>
+ <description>How many executors to spawn for ackers.
+
+ If this is set to 0, then Storm will immediately ack tuples as soon
+ as they come off the spout, effectively disabling reliability.
+ </description>
+ </property>
+ <property>
+ <name>topology.message.timeout.secs</name>
+ <value>30</value>
+ <description>The maximum amount of time given to the topology to fully process a message
+ emitted by a spout. If the message is not acked within this time frame, Storm
+ will fail the message on the spout. Some spouts implementations will then replay
+ the message at a later time.</description>
+ </property>
+ <property>
+ <name>topology.skip.missing.kryo.registrations</name>
+ <value>false</value>
+ <description> Whether or not Storm should skip the loading of kryo registrations for which it
+ does not know the class or have the serializer implementation. Otherwise, the task will
+ fail to load and will throw an error at runtime. The use case of this is if you want to
+ declare your serializations on the storm.yaml files on the cluster rather than every single
+ time you submit a topology. Different applications may use different serializations and so
+ a single application may not have the code for the other serializers used by other apps.
+ By setting this config to true, Storm will ignore that it doesn't have those other serializations
+ rather than throw an error.</description>
+ </property>
+ <property>
+ <name>topology.max.task.parallelism</name>
+ <value>null</value>
+ <description>The maximum parallelism allowed for a component in this topology. This configuration is
+ typically used in testing to limit the number of threads spawned in local mode.</description>
+ </property>
+ <property>
+ <name>topology.max.spout.pending</name>
+ <value>null</value>
+ <description>The maximum number of tuples that can be pending on a spout task at any given time.
+ This config applies to individual tasks, not to spouts or topologies as a whole.
+
+ A pending tuple is one that has been emitted from a spout but has not been acked or failed yet.
+ Note that this config parameter has no effect for unreliable spouts that don't tag
+ their tuples with a message id.</description>
+ </property>
+ <property>
+ <name>topology.state.synchronization.timeout.secs</name>
+ <value>60</value>
+ <description>The maximum amount of time a component gives a source of state to synchronize before it requests
+ synchronization again.</description>
+ </property>
+ <property>
+ <name>topology.stats.sample.rate</name>
+ <value>0.05</value>
+ <description>The percentage of tuples to sample to produce stats for a task.</description>
+ </property>
+ <property>
+ <name>topology.builtin.metrics.bucket.size.secs</name>
+ <value>60</value>
+ <description>The time period that builtin metrics data in bucketed into.</description>
+ </property>
+ <property>
+ <name>topology.fall.back.on.java.serialization</name>
+ <value>true</value>
+ <description>Whether or not to use Java serialization in a topology.</description>
+ </property>
+ <property>
+ <name>topology.worker.childopts</name>
+ <value>null</value>
+ <description>Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS.</description>
+ </property>
+ <property>
+ <name>topology.executor.receive.buffer.size</name>
+ <value>1024</value>
+ <description>The size of the Disruptor receive queue for each executor. Must be a power of 2.</description>
+ </property>
+ <property>
+ <name>topology.executor.send.buffer.size</name>
+ <value>1024</value>
+ <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description>
+ </property>
+ <property>
+ <name>topology.receiver.buffer.size</name>
+ <value>8</value>
+ <description>The maximum number of messages to batch from the thread receiving off the network to the
+ executor queues. Must be a power of 2.</description>
+ </property>
+ <property>
+ <name>topology.transfer.buffer.size</name>
+ <value>1024</value>
+ <description>The size of the Disruptor transfer queue for each worker.</description>
+ </property>
+ <property>
+ <name>topology.tick.tuple.freq.secs</name>
+ <value>null</value>
+ <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent
+ to tasks. Meant to be used as a component-specific configuration.</description>
+ </property>
+ <property>
+ <name>topology.worker.shared.thread.pool.size</name>
+ <value>4</value>
+ <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed
+ via the TopologyContext.</description>
+ </property>
+ <property>
+ <name>topology.disruptor.wait.strategy</name>
+ <value>com.lmax.disruptor.BlockingWaitStrategy</value>
+ <description>Configure the wait strategy used for internal queuing. Can be used to tradeoff latency
+ vs. throughput.</description>
+ </property>
+ <property>
+ <name>topology.executor.send.buffer.size</name>
+ <value>1024</value>
+ <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description>
+ </property>
+ <property>
+ <name>topology.receiver.buffer.size</name>
+ <value>8</value>
+ <description>The maximum number of messages to batch from the thread receiving off the network to the
+ executor queues. Must be a power of 2.</description>
+ </property>
+ <property>
+ <name>topology.transfer.buffer.size</name>
+ <value>1024</value>
+ <description>The size of the Disruptor transfer queue for each worker.</description>
+ </property>
+ <property>
+ <name>topology.tick.tuple.freq.secs</name>
+ <value>null</value>
+ <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent
+ to tasks. Meant to be used as a component-specific configuration.</description>
+ </property>
+ <property>
+ <name>topology.worker.shared.thread.pool.size</name>
+ <value>4</value>
+ <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed
+ via the TopologyContext.</description>
+ </property>
+ <property>
+ <name>topology.spout.wait.strategy</name>
+ <value>backtype.storm.spout.SleepSpoutWaitStrategy</value>
+ <description>A class that implements a strategy for what to do when a spout needs to wait. Waiting is
+ triggered in one of two conditions:
+
+ 1. nextTuple emits no tuples
+ 2. The spout has hit maxSpoutPending and can't emit any more tuples</description>
+ </property>
+ <property>
+ <name>topology.sleep.spout.wait.strategy.time.ms</name>
+ <value>1</value>
+ <description>The amount of milliseconds the SleepEmptyEmitStrategy should sleep for.</description>
+ </property>
+ <property>
+ <name>topology.error.throttle.interval.secs</name>
+ <value>10</value>
+ <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example,
+ an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be
+ reported to Zookeeper per task for every 10 second interval of time.</description>
+ </property>
+ <property>
+ <name>topology.max.error.report.per.interval</name>
+ <value>5</value>
+ <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example,
+ an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be
+ reported to Zookeeper per task for every 10 second interval of time.</description>
+ </property>
+ <property>
+ <name>topology.kryo.factory</name>
+ <value>backtype.storm.serialization.DefaultKryoFactory</value>
+ <description>Class that specifies how to create a Kryo instance for serialization. Storm will then apply
+ topology.kryo.register and topology.kryo.decorators on top of this. The default implementation
+ implements topology.fall.back.on.java.serialization and turns references off.</description>
+ </property>
+ <property>
+ <name>topology.tuple.serializer</name>
+ <value>backtype.storm.serialization.types.ListDelegateSerializer</value>
+ <description>The serializer class for ListDelegate (tuple payload).
+ The default serializer will be ListDelegateSerializer</description>
+ </property>
+ <property>
+ <name>topology.trident.batch.emit.interval.millis</name>
+ <value>500</value>
+ <description>How often a batch can be emitted in a Trident topology.</description>
+ </property>
+ <property>
+ <name>dev.zookeeper.path</name>
+ <value>/tmp/dev-storm-zookeeper</value>
+ <description>The path to use as the zookeeper dir when running a zookeeper server via
+ "storm dev-zookeeper". This zookeeper instance is only intended for development;
+ it is not a production grade zookeeper setup.</description>
+ </property>
+</configuration>
diff --git a/app-packages/storm-v0_91/metainfo.xml b/app-packages/storm-v0_91/metainfo.xml
new file mode 100644
index 0000000..2fcf4cd
--- /dev/null
+++ b/app-packages/storm-v0_91/metainfo.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>STORM</name>
+ <comment>Apache Hadoop Stream processing framework</comment>
+ <version>0.9.1.2.1</version>
+ <components>
+
+ <component>
+ <name>NIMBUS</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/nimbus.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_REST_API</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/rest_api.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>SUPERVISOR</name>
+ <category>SLAVE</category>
+ <commandScript>
+ <script>scripts/supervisor.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_UI_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/ui_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>DRPC_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/drpc_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <configuration-dependencies>
+ <config-type>storm-site</config-type>
+ <config-type>global</config-type>
+ </configuration-dependencies>
+ </service>
+ </services>
+</metainfo>
diff --git a/app-packages/storm-v0_91/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE b/app-packages/storm-v0_91/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
new file mode 100644
index 0000000..dd934d5
--- /dev/null
+++ b/app-packages/storm-v0_91/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Replace with the actual storm package.
diff --git a/app-packages/storm-v0_91/package/scripts/drpc_server.py b/app-packages/storm-v0_91/package/scripts/drpc_server.py
new file mode 100644
index 0000000..a01d0f0
--- /dev/null
+++ b/app-packages/storm-v0_91/package/scripts/drpc_server.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+
+class DrpcServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("drpc", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("drpc", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.pid_drpc)
+
+if __name__ == "__main__":
+ DrpcServer().execute()
diff --git a/app-packages/storm-v0_91/package/scripts/nimbus.py b/app-packages/storm-v0_91/package/scripts/nimbus.py
new file mode 100644
index 0000000..c7c3120
--- /dev/null
+++ b/app-packages/storm-v0_91/package/scripts/nimbus.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+
+class Nimbus(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("nimbus", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("nimbus", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.pid_nimbus)
+
+if __name__ == "__main__":
+ Nimbus().execute()
diff --git a/app-packages/storm-v0_91/package/scripts/params.py b/app-packages/storm-v0_91/package/scripts/params.py
new file mode 100644
index 0000000..784069d
--- /dev/null
+++ b/app-packages/storm-v0_91/package/scripts/params.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+import status_params
+
+# server configurations
+config = Script.get_config()
+
+app_root = config['configurations']['global']['app_root']
+conf_dir = format("{app_root}/conf")
+storm_user = config['configurations']['global']['app_user']
+log_dir = config['configurations']['global']['app_log_dir']
+pid_dir = status_params.pid_dir
+local_dir = config['configurations']['storm-site']['storm.local.dir']
+user_group = config['configurations']['global']['user_group']
+java64_home = config['hostLevelParams']['java_home']
+nimbus_host = config['configurations']['storm-site']['nimbus.host']
+nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
+nimbus_host = config['configurations']['storm-site']['nimbus.host']
+rest_api_port = "8745"
+rest_api_admin_port = "8746"
+rest_api_conf_file = format("{conf_dir}/config.yaml")
+rest_lib_dir = format("{app_root}/contrib/storm-rest")
+storm_bin = format("{app_root}/bin/storm")
+
+ganglia_installed = False
+
+_authentication = config['configurations']['core-site']['hadoop.security.authentication']
+security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
+
+if security_enabled:
+ _hostname_lowercase = config['hostname'].lower()
+ _kerberos_domain = config['configurations']['global']['kerberos_domain']
+ _storm_principal_name = config['configurations']['global']['storm_principal_name']
+ storm_jaas_principal = _storm_principal_name.replace('_HOST',_hostname_lowercase)
+ storm_keytab_path = config['configurations']['global']['storm_keytab']
diff --git a/app-packages/storm-v0_91/package/scripts/rest_api.py b/app-packages/storm-v0_91/package/scripts/rest_api.py
new file mode 100644
index 0000000..33d8924
--- /dev/null
+++ b/app-packages/storm-v0_91/package/scripts/rest_api.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+
+
+class StormRestApi(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("rest_api", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("rest_api", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.pid_rest_api)
+
+if __name__ == "__main__":
+ StormRestApi().execute()
diff --git a/app-packages/storm-v0_91/package/scripts/service.py b/app-packages/storm-v0_91/package/scripts/service.py
new file mode 100644
index 0000000..10fa5b9
--- /dev/null
+++ b/app-packages/storm-v0_91/package/scripts/service.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+
+from resource_management import *
+import time
+
+
+def service(
+ name,
+ action='start'):
+ import params
+ import status_params
+
+ pid_file = status_params.pid_files[name]
+ no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
+ jps_path = format("{java64_home}/bin/jps")
+ grep_and_awk = "| grep -v grep | awk '{print $1}'"
+
+ if name == 'ui':
+ #process_cmd = "^java.+backtype.storm.ui.core$"
+ pid_chk_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.ui.core\" {grep_and_awk} > {pid_file}")
+ elif name == "rest_api":
+ process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server")
+ crt_pid_cmd = format("pgrep -f \"{process_cmd}\" && pgrep -f \"{process_cmd}\" > {pid_file}")
+ else:
+ #process_cmd = format("^java.+backtype.storm.daemon.{name}$")
+ pid_chk_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk} > {pid_file}")
+
+ if action == "start":
+ if name == "rest_api":
+ cmd = format("{process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
+ else:
+ cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name}")
+
+ Execute(cmd,
+ not_if=no_op_test,
+ user=params.storm_user,
+ logoutput=False,
+ wait_for_finish=False
+ )
+
+ if name == "rest_api":
+ Execute(crt_pid_cmd,
+ user=params.storm_user,
+ logoutput=True,
+ tries=6,
+ try_sleep=10
+ )
+ else:
+ content = None
+ for i in xrange(12):
+ Execute(pid_chk_cmd,
+ user=params.storm_user,
+ logoutput=True
+ )
+ with open(pid_file) as f:
+ content = f.readline().strip()
+ if content.isdigit():
+ break;
+ File(pid_file, action = "delete")
+ time.sleep(10)
+ pass
+
+ if not content.isdigit():
+ raise Fail(format("Unable to start {name}"))
+
+ elif action == "stop":
+ process_dont_exist = format("! ({no_op_test})")
+ pid = format("`cat {pid_file}` >/dev/null 2>&1")
+ Execute(format("kill {pid}"),
+ not_if=process_dont_exist
+ )
+ Execute(format("kill -9 {pid}"),
+ not_if=format("sleep 2; {process_dont_exist} || sleep 20; {process_dont_exist}"),
+ ignore_failures=True
+ )
+ Execute(format("rm -f {pid_file}"))
diff --git a/app-packages/storm-v0_91/package/scripts/status_params.py b/app-packages/storm-v0_91/package/scripts/status_params.py
new file mode 100644
index 0000000..eab83cf
--- /dev/null
+++ b/app-packages/storm-v0_91/package/scripts/status_params.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management import *
+
+config = Script.get_config()
+
+pid_dir = config['configurations']['global']['app_pid_dir']
+pid_nimbus = format("{pid_dir}/nimbus.pid")
+pid_supervisor = format("{pid_dir}/supervisor.pid")
+pid_drpc = format("{pid_dir}/drpc.pid")
+pid_ui = format("{pid_dir}/ui.pid")
+pid_logviewer = format("{pid_dir}/logviewer.pid")
+pid_rest_api = format("{pid_dir}/restapi.pid")
+pid_files = {"logviewer":pid_logviewer,
+ "ui": pid_ui,
+ "nimbus": pid_nimbus,
+ "supervisor": pid_supervisor,
+ "drpc": pid_drpc,
+ "rest_api": pid_rest_api}
\ No newline at end of file
diff --git a/app-packages/storm-v0_91/package/scripts/storm.py b/app-packages/storm-v0_91/package/scripts/storm.py
new file mode 100644
index 0000000..bce272b
--- /dev/null
+++ b/app-packages/storm-v0_91/package/scripts/storm.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from yaml_config import yaml_config
+import sys
+
+def storm():
+ import params
+
+ Directory([params.log_dir, params.pid_dir, params.local_dir, params.conf_dir],
+ owner=params.storm_user,
+ group=params.user_group,
+ recursive=True
+ )
+
+ File(format("{conf_dir}/config.yaml"),
+ content=Template("config.yaml.j2"),
+ owner = params.storm_user,
+ group = params.user_group
+ )
+
+ yaml_config( "storm.yaml",
+ conf_dir = params.conf_dir,
+ configurations = params.config['configurations']['storm-site'],
+ owner = params.storm_user,
+ group = params.user_group
+ )
+
+ if params.security_enabled:
+ TemplateConfig( format("{conf_dir}/storm_jaas.conf"),
+ owner = params.storm_user
+ )
\ No newline at end of file
diff --git a/app-packages/storm-v0_91/package/scripts/supervisor.py b/app-packages/storm-v0_91/package/scripts/supervisor.py
new file mode 100644
index 0000000..47c20c9
--- /dev/null
+++ b/app-packages/storm-v0_91/package/scripts/supervisor.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from yaml_config import yaml_config
+from storm import storm
+from service import service
+
+
+class Supervisor(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("supervisor", action="start")
+ service("logviewer", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("supervisor", action="stop")
+ service("logviewer", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ check_process_status(status_params.pid_supervisor)
+
+
+if __name__ == "__main__":
+ Supervisor().execute()
+
diff --git a/app-packages/storm-v0_91/package/scripts/ui_server.py b/app-packages/storm-v0_91/package/scripts/ui_server.py
new file mode 100644
index 0000000..0fe7cd2
--- /dev/null
+++ b/app-packages/storm-v0_91/package/scripts/ui_server.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+
+class UiServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("ui", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("ui", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.pid_ui)
+
+if __name__ == "__main__":
+ UiServer().execute()
diff --git a/app-packages/storm-v0_91/package/scripts/yaml_config.py b/app-packages/storm-v0_91/package/scripts/yaml_config.py
new file mode 100644
index 0000000..39261be
--- /dev/null
+++ b/app-packages/storm-v0_91/package/scripts/yaml_config.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import re
+from resource_management import *
+
+def escape_yaml_propetry(value):
+ unquouted = False
+ unquouted_values = ["null","Null","NULL","true","True","TRUE","false","False","FALSE","YES","Yes","yes","NO","No","no","ON","On","on","OFF","Off","off"]
+
+ if value in unquouted_values:
+ unquouted = True
+
+ # if is list [a,b,c]
+ if re.match('^\w*\[.+\]\w*$', value):
+ unquouted = True
+
+ try:
+ int(value)
+ unquouted = True
+ except ValueError:
+ pass
+
+ try:
+ float(value)
+ unquouted = True
+ except ValueError:
+ pass
+
+ if not unquouted:
+ value = value.replace("'","''")
+ value = "'"+value+"'"
+
+ return value
+
+def yaml_config(
+ filename,
+ configurations = None,
+ conf_dir = None,
+ mode = None,
+ owner = None,
+ group = None
+):
+ config_content = source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }}
+{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry])
+
+ File (format("{conf_dir}/{filename}"),
+ content = config_content,
+ owner = owner,
+ group = group,
+ mode = mode
+ )
diff --git a/app-packages/storm-v0_91/package/templates/config.yaml.j2 b/app-packages/storm-v0_91/package/templates/config.yaml.j2
new file mode 100644
index 0000000..89c8e67
--- /dev/null
+++ b/app-packages/storm-v0_91/package/templates/config.yaml.j2
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nimbusHost: {{nimbus_host}}
+nimbusPort: {{nimbus_port}}
+
+# HTTP-specific options.
+http:
+
+ # The port on which the HTTP server listens for service requests.
+ port: {{rest_api_port}}
+
+ # The port on which the HTTP server listens for administrative requests.
+ adminPort: {{rest_api_admin_port}}
+
+{% if ganglia_installed %}
+enableGanglia: {{ganglia_installed}}
+
+# ganglia configuration (necessary if ganglia reporting is enabled)
+ganglia:
+
+ # how often to report to ganglia metrics (in seconds)
+ reportInterval: {{ganglia_report_interval}}
+
+ # the hostname of the gmond server where storm cluster metrics will be sent
+ host: "{{ganglia_server}}"
+
+ # address mode
+ # default is MULTICAST
+ addressMode: "UNICAST"
+
+ # an <IP>:<HOSTNAME> pair to spoof
+ # this allows us to simulate storm cluster metrics coming from a specific host
+ #spoof: "192.168.1.1:storm"
+{% endif %}
\ No newline at end of file
diff --git a/app-packages/storm-v0_91/package/templates/storm_jaas.conf.j2 b/app-packages/storm-v0_91/package/templates/storm_jaas.conf.j2
new file mode 100644
index 0000000..4031d22
--- /dev/null
+++ b/app-packages/storm-v0_91/package/templates/storm_jaas.conf.j2
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{storm_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="zookeeper"
+ principal="{{storm_jaas_principal}}";
+};
diff --git a/app-packages/storm-v0_91/resources.json b/app-packages/storm-v0_91/resources.json
new file mode 100644
index 0000000..5b31d93
--- /dev/null
+++ b/app-packages/storm-v0_91/resources.json
@@ -0,0 +1,31 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ },
+ "NIMBUS": {
+ "role.priority": "1",
+ "component.instances": "1"
+ },
+ "STORM_REST_API": {
+ "role.priority": "2",
+ "component.instances": "1"
+ },
+ "STORM_UI_SERVER": {
+ "role.priority": "3",
+ "component.instances": "1"
+ },
+ "DRPC_SERVER": {
+ "role.priority": "4",
+ "component.instances": "1"
+ },
+ "SUPERVISOR": {
+ "role.priority": "5",
+ "component.instances": "1"
+ }
+ }
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..3fb7fbb
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,1340 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <name>Slider</name>
+ <version>0.23.0-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <description>
+ Slider is designed to deploy existing applications onto a YARN cluster without
+ rewriting the application to be YARN-ready.
+ </description>
+ <url>https://github.com/hortonworks/slider/tree/master/</url>
+ <modules>
+ <module>slider-core</module>
+ <module>slider-agent</module>
+ <module>slider-assembly</module>
+ <module>slider-funtest</module>
+ <module>slider-providers/hbase/slider-hbase-provider</module>
+ <module>slider-providers/hbase/hbase-funtests</module>
+ <module>slider-providers/accumulo/slider-accumulo-provider</module>
+ <module>slider-providers/accumulo/accumulo-funtests</module>
+ </modules>
+
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+ </license>
+ </licenses>
+
+
+ <scm>
+ <url>https://github.com/hortonworks/slider</url>
+ <connection>scm:git:git://github.com/hortonworks/slider.git
+ </connection>
+ <developerConnection>
+ scm:git:git@github.com:hortonworks/slider.git
+ </developerConnection>
+ </scm>
+
+ <developers>
+ <developer>
+ <id>steveloughran</id>
+ <name>Steve Loughran</name>
+ <timezone>0</timezone>
+ <organization>Hortonworks</organization>
+ <organizationUrl>http://www.hortonworks.com</organizationUrl>
+ </developer>
+
+ <developer>
+ <id>billierinaldi</id>
+ <name>Billie Rinaldi</name>
+ <timezone>-5</timezone>
+ <organization>Hortonworks</organization>
+ <organizationUrl>http://www.hortonworks.com</organizationUrl>
+ </developer>
+
+ <developer>
+ <id>ddraj</id>
+ <name>Devaraj Das</name>
+ <timezone>-8</timezone>
+ <organization>Hortonworks</organization>
+ <organizationUrl>http://www.hortonworks.com</organizationUrl>
+ </developer>
+
+ <developer>
+ <id>tyu</id>
+ <name>Ted Yu</name>
+ <timezone>-8</timezone>
+ <organization>Hortonworks</organization>
+ <organizationUrl>http://www.hortonworks.com</organizationUrl>
+ </developer>
+
+ <developer>
+ <id>joshelser</id>
+ <name>Josh Elser</name>
+ <timezone>-5</timezone>
+ <organization>Hortonworks</organization>
+ <organizationUrl>http://www.hortonworks.com</organizationUrl>
+ </developer>
+
+ <developer>
+ <id>sumitmohanty</id>
+ <name>Sumit Mohanty</name>
+ <timezone>-8</timezone>
+ <organization>Hortonworks</organization>
+ <organizationUrl>http://www.hortonworks.com</organizationUrl>
+ </developer>
+
+ <developer>
+ <id>jmaron</id>
+ <name>Jon Maron</name>
+ <timezone>-5</timezone>
+ <organization>Hortonworks</organization>
+ <organizationUrl>http://www.hortonworks.com</organizationUrl>
+ </developer>
+
+ </developers>
+
+
+ <distributionManagement>
+ <site>
+ <id>slider.website</id>
+ <name>Slider Website</name>
+ <url>file:///tmp/slider/public_html/</url>
+ </site>
+ <downloadUrl>https://github.com/hortonworks/slider/releases</downloadUrl>
+ </distributionManagement>
+
+ <properties>
+
+
+ <!--
+ options
+ -->
+
+ <test.forkedProcessTimeoutInSeconds>18000</test.forkedProcessTimeoutInSeconds>
+ <test.argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</test.argLine>
+ <test.reuseForks>false</test.reuseForks>
+ <test.failIfNoTests>true</test.failIfNoTests>
+ <test.forkMode>always</test.forkMode>
+
+ <!--
+ core artifacts
+ -->
+ <hadoop.version>2.4.0</hadoop.version>
+
+ <hbase.version>0.98.1-hadoop2</hbase.version>
+ <accumulo.version>1.5.1</accumulo.version>
+
+ <!--
+ artifact versions
+ -->
+ <avro.version>1.7.4</avro.version>
+ <bigtop.version>0.7.0</bigtop.version>
+ <commons-codec.version>1.4</commons-codec.version>
+ <commons-digester.version>1.8</commons-digester.version>
+ <commons-configuration.version>1.6</commons-configuration.version>
+ <commons-lang.version>2.6</commons-lang.version>
+ <curator.version>2.4.1</curator.version>
+ <groovy.version>2.1.9</groovy.version>
+ <guava.version>11.0.2</guava.version>
+ <gson.version>2.2.2</gson.version>
+ <guice.version>3.0</guice.version>
+ <httpclient.version>4.2.5</httpclient.version>
+
+
+<!-- <jackson.version>1.8.8</jackson.version>-->
+ <jackson.version>1.9.13</jackson.version>
+ <jcommander.version>1.30</jcommander.version>
+ <jersey.version>1.9</jersey.version>
+ <junit.version>4.11</junit.version>
+ <log4j.version>1.2.17</log4j.version>
+
+ <!-- ProtocolBuffer version, used to verify the protoc version and -->
+ <!-- define the protobuf JAR version -->
+ <protobuf.version>2.5.0</protobuf.version>
+
+ <slf4j.version>1.7.5</slf4j.version>
+ <stringtemplate.version>2.4.1</stringtemplate.version>
+ <zookeeper.version>3.4.5</zookeeper.version>
+
+
+ <!-- Plugin versions -->
+ <gmavenVersion>1.5</gmavenVersion>
+ <gmavenProviderSelection>2.0</gmavenProviderSelection>
+ <buildnumber-maven-plugin.version>1.2</buildnumber-maven-plugin.version>
+ <groovy-eclipse-compiler.version>2.8.0-01</groovy-eclipse-compiler.version>
+ <groovy-eclipse-batch.version>2.1.3-01</groovy-eclipse-batch.version>
+ <maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
+ <maven.cobertura.version>2.5.2</maven.cobertura.version>
+ <maven-compiler-plugin.version>3.1</maven-compiler-plugin.version>
+ <maven-dependency-plugin.version>2.8</maven-dependency-plugin.version>
+ <maven-enforcer-plugin.version>1.0</maven-enforcer-plugin.version>
+ <maven-jar-plugin.version>2.3.1</maven-jar-plugin.version>
+ <maven.javadoc.version>2.8</maven.javadoc.version>
+ <maven.project.version>2.4</maven.project.version>
+ <maven.properties.version>1.0-alpha-2</maven.properties.version>
+ <maven-project-info-reports-plugin.version>2.6</maven-project-info-reports-plugin.version>
+ <maven-site-plugin.version>3.3</maven-site-plugin.version>
+ <maven-source-plugin.version>2.2.1</maven-source-plugin.version>
+ <maven-surefire-plugin.version>2.16</maven-surefire-plugin.version>
+ <maven-surefire-report-plugin.version>2.16</maven-surefire-report-plugin.version>
+ <maven-failsafe-plugin.version>2.16</maven-failsafe-plugin.version>
+ <apache-rat-plugin.version>0.10</apache-rat-plugin.version>
+
+ <!-- build options-->
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <build.dependency.fail.on.warning>true</build.dependency.fail.on.warning>
+ <project.java.src.version>1.6</project.java.src.version>
+ <build.redirect.test.output.to.file>false</build.redirect.test.output.to.file>
+ <maven-doxia-module-markdown.version>1.4</maven-doxia-module-markdown.version>
+
+ <!-- github options-->
+ <github.global.server>github</github.global.server>
+ <github.site.plugin.version>0.8</github.site.plugin.version>
+ <maven-site-plugin.skipDeploy>true</maven-site-plugin.skipDeploy>
+ <github.downloads.plugin.version>0.6</github.downloads.plugin.version>
+ <maven-deploy-plugin.version>2.7</maven-deploy-plugin.version>
+ </properties>
+
+
+ <repositories>
+ <repository>
+ <id>ASF Staging</id>
+ <url>https://repository.apache.org/content/groups/staging/</url>
+ </repository>
+ </repositories>
+
+
+ <pluginRepositories>
+ <pluginRepository>
+ <id>ASF Staging</id>
+ <url>https://repository.apache.org/content/groups/staging/</url>
+ </pluginRepository>
+ </pluginRepositories>
+
+ <build>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>${maven-source-plugin.version}</version>
+ <inherited>true</inherited>
+ <configuration>
+ <!-- skip test resource configuration-->
+ <excludes>
+ <exclude>**/slider-test.xml</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <id>attach-sources</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>jar</goal>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>${maven-site-plugin.version}</version>
+ <inherited>true</inherited>
+ <configuration>
+ <skipDeploy>${maven-site-plugin.skipDeploy}</skipDeploy>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-module-markdown</artifactId>
+ <version>${maven-doxia-module-markdown.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+ <!--read in a build.properties file if defined-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <version>${maven.properties.version}</version>
+ <executions>
+ <execution>
+ <phase>initialize</phase>
+ <goals>
+ <goal>read-project-properties</goal>
+ </goals>
+ <configuration>
+ <quiet>true</quiet>
+ <files>
+ <file>build.properties</file>
+ </files>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/*.tar</exclude>
+ <exclude>**/build.properties</exclude>
+ <exclude>**/regionservers</exclude>
+ <exclude>**/slaves</exclude>
+ <exclude>**/httpfs-signature.secret</exclude>
+ <exclude>**/dfs.exclude</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-report-plugin</artifactId>
+ <version>${maven-surefire-report-plugin.version}</version>
+ <inherited>true</inherited>
+ <configuration>
+ <showSuccess>false</showSuccess>
+ <outputDirectory>${basedir}/target/surefire-reports</outputDirectory>
+ </configuration>
+ <reportSets>
+ <reportSet>
+ <id>unit-tests</id>
+ <reports>
+ <report>report-only</report>
+ </reports>
+ </reportSet>
+ <reportSet>
+ <id>integration-tests</id>
+ <reports>
+ <report>failsafe-report-only</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <inherited>true</inherited>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>${maven-project-info-reports-plugin.version}</version>
+ <configuration>
+ <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+ </configuration>
+ </plugin>
+
+
+ </plugins>
+ </reporting>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <version>${groovy.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ <version>${jcommander.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <type>pom</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client</artifactId>
+ <version>${hadoop.version}</version>
+ <type>pom</type>
+ <exclusions>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>${avro.version}</version>
+
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>paranamer-ant</artifactId>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>itest-common</artifactId>
+ <version>${bigtop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <version>${commons-configuration.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons-lang.version}</version>
+ <exclusions>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
+ <exclusions>
+ </exclusions>
+ </dependency>
+
+ <!-- ======================================================== -->
+ <!-- HBASE -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.version}</version>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <version>${hbase.version}</version>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.version}</version>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <version>${hbase.version}</version>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.version}</version>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop1-compat</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- hbase uses v2.4, which is better, but ...-->
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop1-compat</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- hbase uses v2.4, which is better, but ...-->
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- ======================================================== -->
+ <!-- ZK -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+
+ <!-- ======================================================== -->
+ <!-- Accumulo -->
+ <!-- ======================================================== -->
+
+ <!--
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo</artifactId>
+ <version>${accumulo.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ -->
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-server</artifactId>
+ <version>${accumulo.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ <version>${accumulo.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.maven.scm</groupId>
+ <artifactId>maven-scm-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.maven.scm</groupId>
+ <artifactId>maven-scm-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.maven.scm</groupId>
+ <artifactId>maven-scm-provider-svnexe</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-fate</artifactId>
+ <version>${accumulo.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-minicluster</artifactId>
+ <version>${accumulo.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ <version>${accumulo.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-trace</artifactId>
+ <version>${accumulo.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-test</artifactId>
+ <version>${accumulo.version}</version>
+ </dependency>
+
+
+ <!-- ======================================================== -->
+ <!-- Others -->
+ <!-- ======================================================== -->
+
+
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>${commons-codec.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-digester</groupId>
+ <artifactId>commons-digester</artifactId>
+ <version>${commons-digester.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-x-discovery</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-x-discovery-server</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>${log4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+<!--
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>stringtemplate</artifactId>
+ <version>${stringtemplate.version}</version>
+ </dependency>
+ -->
+ <!-- Used for unit testing -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <!--
+ <dependency>
+ <groupId>net.sourceforge.htmlunit</groupId>
+ <artifactId>htmlunit</artifactId>
+ <version>2.12</version>
+
+ <exclusions>
+ <exclusion>
+ <groupId>xalan</groupId>
+ <artifactId>xalan</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ <version>2.11.0</version>
+
+ </dependency>
+
+ <dependency>
+ <groupId>xalan</groupId>
+ <artifactId>xalan</artifactId>
+ <version>2.7.1</version>
+
+ </dependency>
+ -->
+
+
+ <!-- ======================================================== -->
+ <!-- Jersey and webapp support -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+ <dependency>
+
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ <version>${jersey.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>stax</groupId>
+ <artifactId>stax-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <version>${guice.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ <version>${guice.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-guice</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-core</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+
+ </dependencies>
+ </dependencyManagement>
+
+ <profiles>
+
+
+ <profile>
+ <!-- local builds of everything -->
+ <id>local</id>
+ <properties>
+ <hadoop.version>2.4.1-SNAPSHOT</hadoop.version>
+ <hbase.version>0.98.0-SNAPSHOT</hbase.version>
+ <accumulo.version>1.6.0-SNAPSHOT</accumulo.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <!-- hadoop 2.4 builds of everything -->
+ <id>branch-2</id>
+ <properties>
+ <hadoop.version>2.4.0</hadoop.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <!-- anything for a jenkins build -->
+ <id>jenkins</id>
+ <properties>
+ <build.redirect.test.output.to.file>true
+ </build.redirect.test.output.to.file>
+ </properties>
+ </profile>
+
+ <profile>
+ <!-- anything for a github release -->
+ <!-- see https://github.com/github/maven-plugins-->
+ <!-- this doesn't seem to work and is not in active us-->
+ <id>github</id>
+ <properties>
+ <maven-site-plugin.skipDeploy>false</maven-site-plugin.skipDeploy>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.github.github</groupId>
+ <artifactId>site-maven-plugin</artifactId>
+ <version>${github.site.plugin.version}</version>
+ <inherited>true</inherited>
+ <configuration>
+ <message>Creating site for ${project.version}</message>
+ <dryRun>false</dryRun>
+ <repositoryName>slider</repositoryName>
+ <repositoryOwner>hortonworks</repositoryOwner>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>site</goal>
+ </goals>
+ <phase>site-deploy</phase>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>${maven-deploy-plugin.version}</version>
+ <configuration>
+ <altDeploymentRepository>
+ internal.repo::default::file://${project.build.directory}/mvn-repo
+ </altDeploymentRepository>
+ </configuration>
+ </plugin>
+
+
+ </plugins>
+ </build>
+
+ </profile>
+
+ <profile>
+ <id>strict</id>
+ <!--enable this if you want to get told off about dependency conflict-->
+ <build>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>${maven-enforcer-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>enforce</id>
+ <configuration>
+ <rules>
+ <DependencyConvergence/>
+ </rules>
+ </configuration>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+
+ </build>
+ </profile>
+
+ </profiles>
+
+
+</project>
diff --git a/slider-agent/conf/agent.ini b/slider-agent/conf/agent.ini
new file mode 100644
index 0000000..87d73a7
--- /dev/null
+++ b/slider-agent/conf/agent.ini
@@ -0,0 +1,49 @@
+; Licensed to the Apache Software Foundation (ASF) under one or more
+; contributor license agreements. See the NOTICE file distributed with
+; this work for additional information regarding copyright ownership.
+; The ASF licenses this file to You under the Apache License, Version 2.0
+; (the "License"); you may not use this file except in compliance with
+; the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+[server]
+hostname=localhost
+port=8440
+secured_port=8441
+check_path=/ws/v1/slider/agents/
+register_path=/ws/v1/slider/agents/{name}/register
+heartbeat_path=/ws/v1/slider/agents/{name}/heartbeat
+
+[agent]
+app_pkg_dir=app/definition
+app_install_dir=app/install
+app_run_dir=app/run
+
+app_task_dir=app/command-log
+app_log_dir=app/log
+app_tmp_dir=app/tmp
+
+log_dir=infra/log
+run_dir=infra/run
+version_file=infra/version
+
+log_level=INFO
+
+[python]
+
+[command]
+max_retries=2
+sleep_between_retries=1
+
+[security]
+
+[heartbeat]
+state_interval=6
+log_lines_count=300
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
new file mode 100644
index 0000000..6cf4082
--- /dev/null
+++ b/slider-agent/pom.xml
@@ -0,0 +1,117 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-agent</artifactId>
+ <packaging>pom</packaging>
+ <version>0.23.0-SNAPSHOT</version>
+ <name>Slider Agent</name>
+ <description>Slider Agent</description>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <final.name>${project.artifactId}-${project.version}</final.name>
+ <package.release>1</package.release>
+ <skipTests>false</skipTests>
+ <python.ver>python >= 2.6</python.ver>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/packages/tarball/all.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.0</version>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2</version>
+ <executions>
+ <execution>
+ <configuration>
+ <executable>python2.6</executable>
+ <workingDirectory>src/test/python</workingDirectory>
+ <arguments>
+ <argument>unitTests.py</argument>
+ </arguments>
+ <environmentVariables>
+ <PYTHONPATH>${project.basedir}/src/main/python/jinja2:${project.basedir}/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/agent:${project.basedir}/src/main/python/resource_management:${project.basedir}/src/test/python/agent:${project.basedir}/src/test/python/resource_management</PYTHONPATH>
+ </environmentVariables>
+ <skip>${skipTests}</skip>
+ </configuration>
+ <id>python-test</id>
+ <phase>test</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>src/test/python/agent/dummy_output_error.txt</exclude>
+ <exclude>src/test/python/agent/dummy_output_good.txt</exclude>
+ <!-- jinja2 files (BSD license) -->
+ <exclude>src/main/python/jinja2/**</exclude>
+ <!-- mock files (BSD license) -->
+ <exclude>src/test/python/mock/**</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ <extensions>
+ <extension>
+ <groupId>org.apache.maven.wagon</groupId>
+ <artifactId>wagon-ssh-external</artifactId>
+ </extension>
+ </extensions>
+ </build>
+</project>
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
new file mode 100644
index 0000000..86a13af
--- /dev/null
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import Queue
+
+import logging
+import traceback
+import threading
+import pprint
+import os
+import time
+
+from shell import shellRunner
+from AgentConfig import AgentConfig
+from CommandStatusDict import CommandStatusDict
+from CustomServiceOrchestrator import CustomServiceOrchestrator
+
+
+logger = logging.getLogger()
+installScriptHash = -1
+
+
+class ActionQueue(threading.Thread):
+ """ Action Queue for the agent. We pick one command at a time from the queue
+ and execute it
+ """
+
+ STATUS_COMMAND = 'STATUS_COMMAND'
+ EXECUTION_COMMAND = 'EXECUTION_COMMAND'
+
+ IN_PROGRESS_STATUS = 'IN_PROGRESS'
+ COMPLETED_STATUS = 'COMPLETED'
+ FAILED_STATUS = 'FAILED'
+
+ STORE_APPLIED_CONFIG = 'record_config'
+
+ def __init__(self, config, controller):
+ super(ActionQueue, self).__init__()
+ self.commandQueue = Queue.Queue()
+ self.commandStatuses = CommandStatusDict(callback_action=
+ self.status_update_callback)
+ self.config = config
+ self.controller = controller
+ self.sh = shellRunner()
+ self._stop = threading.Event()
+ self.tmpdir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
+ self.customServiceOrchestrator = CustomServiceOrchestrator(config,
+ controller)
+
+
+ def stop(self):
+ self._stop.set()
+
+ def stopped(self):
+ return self._stop.isSet()
+
+ def put(self, commands):
+ for command in commands:
+ logger.info("Adding " + command['commandType'] + " for service " + \
+ command['serviceName'] + " of cluster " + \
+ command['clusterName'] + " to the queue.")
+ logger.debug(pprint.pformat(command))
+ self.commandQueue.put(command)
+
+ def empty(self):
+ return self.commandQueue.empty()
+
+
+ def run(self):
+ while not self.stopped():
+ time.sleep(2)
+ command = self.commandQueue.get() # Will block if queue is empty
+ self.process_command(command)
+ logger.info("ActionQueue stopped.")
+
+
+ def process_command(self, command):
+ logger.debug("Took an element of Queue: " + pprint.pformat(command))
+ # make sure we log failures
+ try:
+ if command['commandType'] == self.EXECUTION_COMMAND:
+ self.execute_command(command)
+ elif command['commandType'] == self.STATUS_COMMAND:
+ self.execute_status_command(command)
+ else:
+ logger.error("Unrecognized command " + pprint.pformat(command))
+ except Exception, err:
+ # Should not happen
+ traceback.print_exc()
+ logger.warn(err)
+
+
+ def execute_command(self, command):
+ '''
+ Executes commands of type EXECUTION_COMMAND
+ '''
+ clusterName = command['clusterName']
+ commandId = command['commandId']
+
+ message = "Executing command with id = {commandId} for role = {role} of " \
+ "cluster {cluster}".format(
+ commandId=str(commandId), role=command['role'],
+ cluster=clusterName)
+ logger.info(message)
+ logger.debug(pprint.pformat(command))
+
+ taskId = command['taskId']
+ # Preparing 'IN_PROGRESS' report
+ in_progress_status = self.commandStatuses.generate_report_template(command)
+ in_progress_status.update({
+ 'tmpout': self.tmpdir + os.sep + 'output-' + str(taskId) + '.txt',
+ 'tmperr': self.tmpdir + os.sep + 'errors-' + str(taskId) + '.txt',
+ 'structuredOut': self.tmpdir + os.sep + 'structured-out-' + str(
+ taskId) + '.json',
+ 'status': self.IN_PROGRESS_STATUS
+ })
+ self.commandStatuses.put_command_status(command, in_progress_status)
+ store_config = False
+ if ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']:
+ store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
+
+ # running command
+ commandresult = self.customServiceOrchestrator.runCommand(command,
+ in_progress_status[
+ 'tmpout'],
+ in_progress_status[
+ 'tmperr'],
+ True,
+ store_config)
+ # dumping results
+ status = self.COMPLETED_STATUS
+ if commandresult['exitcode'] != 0:
+ status = self.FAILED_STATUS
+ roleResult = self.commandStatuses.generate_report_template(command)
+ roleResult.update({
+ 'stdout': commandresult['stdout'],
+ 'stderr': commandresult['stderr'],
+ 'exitCode': commandresult['exitcode'],
+ 'status': status,
+ })
+ if roleResult['stdout'] == '':
+ roleResult['stdout'] = 'None'
+ if roleResult['stderr'] == '':
+ roleResult['stderr'] = 'None'
+
+ if 'structuredOut' in commandresult:
+ roleResult['structuredOut'] = str(commandresult['structuredOut'])
+ else:
+ roleResult['structuredOut'] = ''
+ # let server know that configuration tags were applied
+ if status == self.COMPLETED_STATUS:
+ if command.has_key('configurationTags'):
+ roleResult['configurationTags'] = command['configurationTags']
+ self.commandStatuses.put_command_status(command, roleResult)
+
+ # Store action result to agent response queue
+ def result(self):
+ return self.commandStatuses.generate_report()
+
+ def execute_status_command(self, command):
+ '''
+ Executes commands of type STATUS_COMMAND
+ '''
+ try:
+ cluster = command['clusterName']
+ service = command['serviceName']
+ component = command['componentName']
+ reportResult = True
+ if 'auto_generated' in command:
+ reportResult = not command['auto_generated']
+
+ component_status = self.customServiceOrchestrator.requestComponentStatus(command)
+
+ result = {"componentName": component,
+ "msg": "",
+ "clusterName": cluster,
+ "serviceName": service,
+ "reportResult": reportResult,
+ "roleCommand": command['roleCommand']
+ }
+
+ if 'configurations' in component_status:
+ result['configurations'] = component_status['configurations']
+ if 'exitcode' in component_status:
+ result['status'] = component_status['exitcode']
+ logger.debug("Got live status for component " + component + \
+ " of service " + str(service) + \
+ " of cluster " + str(cluster))
+ logger.debug(pprint.pformat(result))
+
+ if result is not None:
+ self.commandStatuses.put_command_status(command, result, reportResult)
+ except Exception, err:
+ traceback.print_exc()
+ logger.warn(err)
+ pass
+
+
+ def status_update_callback(self):
+ """
+ Actions that are executed every time when command status changes
+ """
+ self.controller.heartbeat_wait_event.set()
diff --git a/slider-agent/src/main/python/agent/AgentConfig.py b/slider-agent/src/main/python/agent/AgentConfig.py
new file mode 100644
index 0000000..e0981f6
--- /dev/null
+++ b/slider-agent/src/main/python/agent/AgentConfig.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import ConfigParser
+import StringIO
+import os
+
+config = ConfigParser.RawConfigParser()
+content = """
+
+[server]
+hostname=localhost
+port=8440
+secured_port=8441
+check_path=/ws/v1/slider/agents/
+register_path=/ws/v1/slider/agents/{name}/register
+heartbeat_path=/ws/v1/slider/agents/{name}/heartbeat
+
+[agent]
+app_pkg_dir=app/definition
+app_install_dir=app/install
+app_run_dir=app/run
+
+app_task_dir=app/command-log
+app_log_dir=app/log
+app_tmp_dir=app/tmp
+
+log_dir=infra/log
+run_dir=infra/run
+version_file=infra/version
+
+log_level=INFO
+
+[python]
+
+[command]
+max_retries=2
+sleep_between_retries=1
+
+[security]
+
+[heartbeat]
+state_interval=6
+log_lines_count=300
+
+"""
+s = StringIO.StringIO(content)
+config.readfp(s)
+
+
+class AgentConfig:
+ SERVER_SECTION = "server"
+ AGENT_SECTION = "agent"
+ PYTHON_SECTION = "python"
+ COMMAND_SECTION = "command"
+ SECURITY_SECTION = "security"
+ HEARTBEAT_SECTION = "heartbeat"
+
+ # the location of the app package
+ APP_PACKAGE_DIR = "app_pkg_dir"
+ # the location where the app component is installed
+ APP_INSTALL_DIR = "app_install_dir"
+ # the location to store component instance PID directories
+ APP_RUN_DIR = "app_run_dir"
+
+ # run time dir for command executions
+ APP_TASK_DIR = "app_task_dir"
+ # application log directory
+ APP_LOG_DIR = "app_log_dir"
+ # application tmp directory
+ APP_TMP_DIR = "app_tmp_dir"
+
+ # agent log directory
+ LOG_DIR = "log_dir"
+ # agent run directory
+ RUN_DIR = "run_dir"
+ # agent version file
+ VERSION_FILE = "version_file"
+
+ FOLDER_MAPPING = {
+ APP_PACKAGE_DIR: "WORK",
+ APP_INSTALL_DIR: "WORK",
+ APP_RUN_DIR: "WORK",
+ APP_TMP_DIR: "WORK",
+ RUN_DIR: "WORK",
+ VERSION_FILE: "WORK",
+ APP_TASK_DIR: "LOG",
+ APP_LOG_DIR: "LOG",
+ LOG_DIR: "LOG"
+ }
+
+ def __init__(self, workroot, logroot, label="agent"):
+ self.workroot = workroot
+ self.logroot = logroot
+ self.label = label
+
+ def getWorkRootPath(self):
+ return self.workroot
+
+ def getLogPath(self):
+ return self.logroot
+
+ def getLabel(self):
+ return self.label
+
+ def getResolvedPath(self, name):
+ global config
+
+ relativePath = config.get(AgentConfig.AGENT_SECTION, name)
+ if not os.path.isabs(relativePath):
+ root_folder_to_use = self.workroot
+ if name in AgentConfig.FOLDER_MAPPING and AgentConfig.FOLDER_MAPPING[
+ name] == "LOG":
+ root_folder_to_use = self.logroot
+ return os.path.join(root_folder_to_use, relativePath)
+ else:
+ return relativePath
+
+ def get(self, category, name):
+ global config
+ return config.get(category, name)
+
+ def set(self, category, name, value):
+ global config
+ return config.set(category, name, value)
+
+ def setConfig(self, configFile):
+ global config
+ config.read(configFile)
+
+
+def main():
+ print config
+
+
+if __name__ == "__main__":
+ main()
diff --git a/slider-agent/src/main/python/agent/AgentException.py b/slider-agent/src/main/python/agent/AgentException.py
new file mode 100644
index 0000000..d7455f5
--- /dev/null
+++ b/slider-agent/src/main/python/agent/AgentException.py
@@ -0,0 +1,24 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+class AgentException(Exception):
+ def __init__(self, value):
+ self.parameter = value
+
+ def __str__(self):
+ return repr(self.parameter)
diff --git a/slider-agent/src/main/python/agent/CommandStatusDict.py b/slider-agent/src/main/python/agent/CommandStatusDict.py
new file mode 100644
index 0000000..4f9cf8c
--- /dev/null
+++ b/slider-agent/src/main/python/agent/CommandStatusDict.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import json
+import logging
+import threading
+from Grep import Grep
+
+logger = logging.getLogger()
+
+class CommandStatusDict():
+ """
+ Holds results for all commands that are being executed or have finished
+ execution (but are not yet reported). Implementation is thread-safe.
+ Dict format:
+ task_id -> (command, cmd_report)
+ """
+
+ def __init__(self, callback_action):
+ """
+ callback_action is called every time when status of some command is
+ updated
+ """
+ self.current_state = {} # Contains all statuses
+ self.callback_action = callback_action
+ self.lock = threading.RLock()
+
+
+ def put_command_status(self, command, new_report, wakeupController=True):
+ """
+ Stores new version of report for command (replaces previous)
+ """
+ if 'taskId' in command:
+ key = command['taskId']
+ else: # Status command reports has no task id
+ key = id(command)
+ with self.lock: # Synchronized
+ self.current_state[key] = (command, new_report)
+
+ # Usually, status commands are not reported immediately
+ if wakeupController:
+ self.callback_action()
+
+
+ def generate_report(self):
+ """
+ Generates status reports about commands that are IN_PROGRESS, COMPLETE or
+ FAILED. Statuses for COMPLETE or FAILED commands are forgotten after
+ generation
+ """
+ from ActionQueue import ActionQueue
+ with self.lock: # Synchronized
+ resultReports = []
+ resultComponentStatus = []
+ for key, item in self.current_state.items():
+ command = item[0]
+ report = item[1]
+ if command ['commandType'] == ActionQueue.EXECUTION_COMMAND:
+ if (report['status']) != ActionQueue.IN_PROGRESS_STATUS:
+ resultReports.append(report)
+ # Removing complete/failed command status from dict
+ del self.current_state[key]
+ else:
+ in_progress_report = \
+ self.generate_in_progress_report(command, report)
+ resultReports.append(in_progress_report)
+ elif command ['commandType'] == ActionQueue.STATUS_COMMAND:
+ resultComponentStatus.append(report)
+ # Component status is useful once, removing it
+ del self.current_state[key]
+ result = {
+ 'reports': resultReports,
+ 'componentStatus': resultComponentStatus
+ }
+ return result
+
+
+ def generate_in_progress_report(self, command, report):
+ """
+ Reads stdout/stderr for IN_PROGRESS command from disk file
+ and populates other fields of report.
+ """
+ from ActionQueue import ActionQueue
+ try:
+ tmpout = open(report['tmpout'], 'r').read()
+ tmperr = open(report['tmperr'], 'r').read()
+ except Exception, err:
+ logger.warn(err)
+ tmpout = '...'
+ tmperr = '...'
+ try:
+ with open(report['structuredOut'], 'r') as fp:
+ tmpstructuredout = json.load(fp)
+ except Exception:
+ tmpstructuredout = '{}'
+ grep = Grep()
+ output = grep.tail(tmpout, Grep.OUTPUT_LAST_LINES)
+ inprogress = self.generate_report_template(command)
+ inprogress.update({
+ 'stdout': grep.filterMarkup(output),
+ 'stderr': tmperr,
+ 'structuredOut': tmpstructuredout,
+ 'exitCode': 777,
+ 'status': ActionQueue.IN_PROGRESS_STATUS,
+ })
+ return inprogress
+
+
+ def generate_report_template(self, command):
+ """
+ Generates stub dict for command.
+ Other fields should be populated manually
+ """
+ stub = {
+ 'role': command['role'],
+ 'actionId': command['commandId'],
+ 'taskId': command['taskId'],
+ 'clusterName': command['clusterName'],
+ 'serviceName': command['serviceName'],
+ 'roleCommand': command['roleCommand']
+ }
+ return stub
+
+
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
new file mode 100644
index 0000000..fe5760d
--- /dev/null
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -0,0 +1,404 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import logging
+import signal
+import json
+import sys
+import os
+import time
+import threading
+import urllib2
+import pprint
+from random import randint
+
+from AgentConfig import AgentConfig
+from Heartbeat import Heartbeat
+from Register import Register
+from ActionQueue import ActionQueue
+from NetUtil import NetUtil
+import ssl
+import ProcessHelper
+
+
+logger = logging.getLogger()
+
+AGENT_AUTO_RESTART_EXIT_CODE = 77
+
+
+class State:
+ INIT, INSTALLING, INSTALLED, STARTING, STARTED, FAILED = range(6)
+
+
+class Controller(threading.Thread):
+ def __init__(self, config, range=30):
+ threading.Thread.__init__(self)
+ logger.debug('Initializing Controller RPC thread.')
+ self.lock = threading.Lock()
+ self.safeMode = True
+ self.credential = None
+ self.config = config
+ self.hostname = config.getLabel()
+ server_url = 'http://' + config.get(AgentConfig.SERVER_SECTION,
+ 'hostname') + \
+ ':' + config.get(AgentConfig.SERVER_SECTION,
+ 'port')
+ self.registerUrl = server_url + '/ws/v1/slider/agents/' + self.hostname + '/register'
+ self.heartbeatUrl = server_url + '/ws/v1/slider/agents/' + self.hostname + '/heartbeat'
+ self.netutil = NetUtil()
+ self.responseId = -1
+ self.repeatRegistration = False
+ self.isRegistered = False
+ self.cachedconnect = None
+ self.range = range
+ self.hasMappedComponents = True
+ # Event is used for synchronizing heartbeat iterations (to make possible
+ # manual wait() interruption between heartbeats )
+ self.heartbeat_wait_event = threading.Event()
+ # List of callbacks that are called at agent registration
+ self.registration_listeners = []
+ self.componentExpectedState = State.INIT
+ self.componentActualState = State.INIT
+ self.statusCommand = None
+ self.failureCount = 0
+
+
+ def __del__(self):
+ logger.info("Server connection disconnected.")
+ pass
+
+ def registerWithServer(self):
+ id = -1
+ ret = {}
+
+ while not self.isRegistered:
+ try:
+ data = json.dumps(self.register.build(id))
+ logger.info("Registering with the server " + pprint.pformat(data))
+ response = self.sendRequest(self.registerUrl, data)
+ ret = json.loads(response)
+ exitstatus = 0
+ # exitstatus is a code of error which was rised on server side.
+ # exitstatus = 0 (OK - Default)
+ # exitstatus = 1 (Registration failed because
+ # different version of agent and server)
+ if 'exitstatus' in ret.keys():
+ exitstatus = int(ret['exitstatus'])
+ # log - message, which will be printed to agents log
+ if 'log' in ret.keys():
+ log = ret['log']
+ if exitstatus == 1:
+ logger.error(log)
+ self.isRegistered = False
+ self.repeatRegistration = False
+ return ret
+ logger.info("Registered with the server with " + pprint.pformat(ret))
+ print("Registered with the server")
+ self.responseId = int(ret['responseId'])
+ self.isRegistered = True
+ if 'statusCommands' in ret.keys():
+ logger.info("Got status commands on registration " + pprint.pformat(
+ ret['statusCommands']))
+ self.addToQueue(ret['statusCommands'])
+ pass
+ else:
+ self.hasMappedComponents = False
+ pass
+ except ssl.SSLError:
+ self.repeatRegistration = False
+ self.isRegistered = False
+ return
+ except Exception, err:
+ # try a reconnect only after a certain amount of random time
+ delay = randint(0, self.range)
+ logger.info("Unable to connect to: " + self.registerUrl, exc_info=True)
+ """ Sleeping for {0} seconds and then retrying again """.format(delay)
+ time.sleep(delay)
+ pass
+ pass
+ return ret
+
+
+ def addToQueue(self, commands):
+ """Add to the queue for running the commands """
+ """ Put the required actions into the Queue """
+ if not commands:
+ logger.debug("No commands from the server : " + pprint.pformat(commands))
+ else:
+ """Only add to the queue if not empty list """
+ self.actionQueue.put(commands)
+ pass
+
+ # For testing purposes
+ DEBUG_HEARTBEAT_RETRIES = 0
+ DEBUG_SUCCESSFULL_HEARTBEATS = 0
+ DEBUG_STOP_HEARTBEATING = False
+ MAX_FAILURE_COUNT_TO_STOP = 2
+
+ def shouldStopAgent(self):
+ '''
+ If component has failed after start then stop the agent
+ '''
+ if (self.componentActualState == State.FAILED) \
+ and (self.componentExpectedState == State.STARTED) \
+ and (self.failureCount >= Controller.MAX_FAILURE_COUNT_TO_STOP):
+ return True
+ else:
+ return False
+ pass
+
+ def heartbeatWithServer(self):
+ self.DEBUG_HEARTBEAT_RETRIES = 0
+ self.DEBUG_SUCCESSFULL_HEARTBEATS = 0
+ retry = False
+ certVerifFailed = False
+
+ id = 0
+ while not self.DEBUG_STOP_HEARTBEATING:
+
+ if self.shouldStopAgent():
+ logger.info("Component instance has stopped, stopping the agent ...")
+ ProcessHelper.stopAgent()
+
+ commandResult = {}
+ try:
+ if not retry:
+ data = json.dumps(
+ self.heartbeat.build(commandResult, self.responseId,
+ self.hasMappedComponents))
+ self.updateStateBasedOnResult(commandResult)
+ logger.debug("Sending request: " + data)
+ pass
+ else:
+ self.DEBUG_HEARTBEAT_RETRIES += 1
+ response = self.sendRequest(self.heartbeatUrl, data)
+ response = json.loads(response)
+
+ logger.debug('Got server response: ' + pprint.pformat(response))
+
+ serverId = int(response['responseId'])
+
+ if 'hasMappedComponents' in response.keys():
+ self.hasMappedComponents = response['hasMappedComponents'] != False
+
+ if 'registrationCommand' in response.keys():
+ # check if the registration command is None. If none skip
+ if response['registrationCommand'] is not None:
+ logger.info(
+ "RegistrationCommand received - repeat agent registration")
+ self.isRegistered = False
+ self.repeatRegistration = True
+ return
+
+ if serverId != self.responseId + 1:
+ logger.error("Error in responseId sequence - restarting")
+ self.restartAgent()
+ else:
+ self.responseId = serverId
+
+ if 'executionCommands' in response.keys():
+ self.updateStateBasedOnCommand(response['executionCommands'])
+ self.addToQueue(response['executionCommands'])
+ pass
+ if 'statusCommands' in response.keys() and len(response['statusCommands']) > 0:
+ self.addToQueue(response['statusCommands'])
+ pass
+ if "true" == response['restartAgent']:
+ logger.error("Got restartAgent command")
+ self.restartAgent()
+ else:
+ logger.info("No commands sent from the Server.")
+ pass
+
+ # Add a status command
+ if (self.componentActualState != State.STARTING and \
+ self.componentExpectedState == State.STARTED) and \
+ not self.statusCommand == None:
+ self.addToQueue([self.statusCommand])
+
+ if retry:
+ print("Reconnected to the server")
+ logger.info("Reconnected to the server")
+ retry = False
+ certVerifFailed = False
+ self.DEBUG_SUCCESSFULL_HEARTBEATS += 1
+ self.DEBUG_HEARTBEAT_RETRIES = 0
+ self.heartbeat_wait_event.clear()
+ except ssl.SSLError:
+ self.repeatRegistration = False
+ self.isRegistered = False
+ return
+ except Exception, err:
+ #randomize the heartbeat
+ delay = randint(0, self.range)
+ time.sleep(delay)
+ if "code" in err:
+ logger.error(err.code)
+ else:
+ logger.error(
+ "Unable to connect to: " + self.heartbeatUrl + " due to " + str(
+ err))
+ logger.debug("Details: " + str(err), exc_info=True)
+ if not retry:
+ print("Connection to the server was lost. Reconnecting...")
+ if 'certificate verify failed' in str(err) and not certVerifFailed:
+ print(
+ "Server certificate verify failed. Did you regenerate server certificate?")
+ certVerifFailed = True
+ self.cachedconnect = None # Previous connection is broken now
+ retry = True
+ # Sleep for some time
+ timeout = self.netutil.HEARTBEAT_IDDLE_INTERVAL_SEC \
+ - self.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS
+ self.heartbeat_wait_event.wait(timeout=timeout)
+ # Sleep a bit more to allow STATUS_COMMAND results to be collected
+ # and sent in one heartbeat. Also avoid server overload with heartbeats
+ time.sleep(self.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS)
+ pass
+ logger.info("Controller stopped heart-beating.")
+
+ def updateStateBasedOnCommand(self, commands):
+ for command in commands:
+ if command["roleCommand"] == "START":
+ self.componentExpectedState = State.STARTED
+ self.componentActualState = State.STARTING
+ self.failureCount = 0
+ self.statusCommand = self.createStatusCommand(command)
+
+ if command["roleCommand"] == "INSTALL":
+ self.componentExpectedState = State.INSTALLED
+ self.componentActualState = State.INSTALLING
+ self.failureCount = 0
+ break;
+
+
+ def updateStateBasedOnResult(self, commandResult):
+ if len(commandResult) > 0:
+ if "commandStatus" in commandResult:
+ if commandResult["commandStatus"] == ActionQueue.COMPLETED_STATUS:
+ self.componentActualState = self.componentExpectedState
+ self.logStates()
+ pass
+ pass
+
+ if commandResult["commandStatus"] == ActionQueue.FAILED_STATUS:
+ self.componentActualState = State.FAILED
+ self.failureCount += 1
+ self.logStates()
+ pass
+
+ if "healthStatus" in commandResult:
+ if commandResult["healthStatus"] == "INSTALLED":
+ self.componentActualState = State.FAILED
+ self.failureCount += 1
+ self.logStates()
+ if (commandResult["healthStatus"] == "STARTED") and (self.componentActualState != State.STARTED):
+ self.componentActualState = State.STARTED
+ self.failureCount = 0
+ self.logStates()
+ pass
+ pass
+ pass
+
+ def logStates(self):
+ logger.info("Component states (result): Expected: " + str(self.componentExpectedState) + \
+ " and Actual: " + str(self.componentActualState))
+ pass
+
+ def createStatusCommand(self, command):
+ statusCommand = {}
+ statusCommand["clusterName"] = command["clusterName"]
+ statusCommand["commandParams"] = command["commandParams"]
+ statusCommand["commandType"] = "STATUS_COMMAND"
+ statusCommand["roleCommand"] = "STATUS"
+ statusCommand["componentName"] = command["role"]
+ statusCommand["configurations"] = {}
+ statusCommand["configurations"]["global"] = command["configurations"]["global"]
+ statusCommand["hostLevelParams"] = command["hostLevelParams"]
+ statusCommand["serviceName"] = command["serviceName"]
+ statusCommand["taskId"] = "status"
+ statusCommand['auto_generated'] = True
+ return statusCommand
+ logger.info("Status command: " + pprint.pformat(statusCommand))
+ pass
+
+
+ def run(self):
+ self.actionQueue = ActionQueue(self.config, controller=self)
+ self.actionQueue.start()
+ self.register = Register(self.config)
+ self.heartbeat = Heartbeat(self.actionQueue, self.config)
+
+ opener = urllib2.build_opener()
+ urllib2.install_opener(opener)
+
+ while True:
+ self.repeatRegistration = False
+ self.registerAndHeartbeat()
+ if not self.repeatRegistration:
+ break
+ logger.info("Controller stopped.")
+ pass
+
+ def registerAndHeartbeat(self):
+ registerResponse = self.registerWithServer()
+ message = registerResponse['response']
+ logger.info("Response from server = " + message)
+ if self.isRegistered:
+ # Process callbacks
+ for callback in self.registration_listeners:
+ callback()
+ time.sleep(self.netutil.HEARTBEAT_IDDLE_INTERVAL_SEC)
+ self.heartbeatWithServer()
+ logger.info("Controller stopped heartbeating.")
+
+ def restartAgent(self):
+ os._exit(AGENT_AUTO_RESTART_EXIT_CODE)
+ pass
+
+ def sendRequest(self, url, data):
+ req = urllib2.Request(url, data, {'Content-Type': 'application/json'})
+ f = urllib2.urlopen(req)
+ response = f.read()
+ f.close()
+ return response
+
+
+def main(argv=None):
+ # Allow Ctrl-C
+ signal.signal(signal.SIGINT, signal.SIG_DFL)
+
+ logger.setLevel(logging.INFO)
+ formatter = logging.Formatter("%(asctime)s %(filename)s:%(lineno)d - \
+ %(message)s")
+ stream_handler = logging.StreamHandler()
+ stream_handler.setFormatter(formatter)
+ logger.addHandler(stream_handler)
+
+ logger.info('Starting Server RPC Thread: %s' % ' '.join(sys.argv))
+
+ config = AgentConfig()
+ collector = Controller(config)
+ collector.start()
+ collector.run()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
new file mode 100644
index 0000000..a72cf87
--- /dev/null
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -0,0 +1,226 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import logging
+import os
+import json
+import pprint
+import sys
+from AgentConfig import AgentConfig
+from AgentException import AgentException
+from PythonExecutor import PythonExecutor
+import hostname
+
+
+logger = logging.getLogger()
+
+
+class CustomServiceOrchestrator():
+ """
+ Executes a command for custom service. stdout and stderr are written to
+ tmpoutfile and to tmperrfile respectively.
+ """
+
+ SCRIPT_TYPE_PYTHON = "PYTHON"
+ COMMAND_NAME_STATUS = "STATUS"
+ LIVE_STATUS = "STARTED"
+ DEAD_STATUS = "INSTALLED"
+
+ def __init__(self, config, controller):
+ self.config = config
+ self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
+ self.python_executor = PythonExecutor(self.tmp_dir, config)
+ self.status_commands_stdout = os.path.join(self.tmp_dir,
+ 'status_command_stdout.txt')
+ self.status_commands_stderr = os.path.join(self.tmp_dir,
+ 'status_command_stderr.txt')
+ self.public_fqdn = hostname.public_hostname()
+ self.applied_configs = {}
+ # Clean up old status command files if any
+ try:
+ os.unlink(self.status_commands_stdout)
+ os.unlink(self.status_commands_stderr)
+ except OSError:
+ pass # Ignore fail
+ self.base_dir = os.path.join(
+ config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR), "package")
+
+
+ def runCommand(self, command, tmpoutfile, tmperrfile,
+ override_output_files=True, store_config=False):
+ try:
+ script_type = command['commandParams']['script_type']
+ script = command['commandParams']['script']
+ timeout = int(command['commandParams']['command_timeout'])
+ task_id = command['taskId']
+ command_name = command['roleCommand']
+
+ script_path = self.resolve_script_path(self.base_dir, script, script_type)
+ script_tuple = (script_path, self.base_dir)
+
+ tmpstrucoutfile = os.path.join(self.tmp_dir,
+ "structured-out-{0}.json".format(task_id))
+ if script_type.upper() != self.SCRIPT_TYPE_PYTHON:
+ # We don't support anything else yet
+ message = "Unknown script type {0}".format(script_type)
+ raise AgentException(message)
+ # Execute command using proper interpreter
+ json_path = self.dump_command_to_json(command, store_config)
+ py_file_list = [script_tuple]
+ # filter None values
+ filtered_py_file_list = [i for i in py_file_list if i]
+
+ # Executing hooks and script
+ ret = None
+ for py_file, current_base_dir in filtered_py_file_list:
+ script_params = [command_name, json_path, current_base_dir]
+ python_paths = [os.path.join(self.config.getWorkRootPath(),
+ "infra/agent/slider-agent/jinja2"),
+ os.path.join(self.config.getWorkRootPath(),
+ "infra/agent/slider-agent")]
+ environment_vars = [("PYTHONPATH", ":".join(python_paths))]
+ ret = self.python_executor.run_file(py_file, script_params,
+ tmpoutfile, tmperrfile, timeout,
+ tmpstrucoutfile,
+ override_output_files,
+ environment_vars)
+ # Next run_file() invocations should always append to current output
+ override_output_files = False
+ if ret['exitcode'] != 0:
+ break
+
+ if not ret: # Something went wrong
+ raise AgentException("No script has been executed")
+
+ except Exception: # We do not want to let agent fail completely
+ exc_type, exc_obj, exc_tb = sys.exc_info()
+ message = "Caught an exception while executing " \
+ "command: {0}: {1}".format(exc_type, exc_obj)
+ logger.exception(message)
+ ret = {
+ 'stdout': message,
+ 'stderr': message,
+ 'structuredOut': '{}',
+ 'exitcode': 1,
+ }
+
+ return ret
+
+
+ def resolve_script_path(self, base_dir, script, script_type):
+ """
+ Encapsulates logic of script location determination.
+ """
+ path = os.path.join(base_dir, script)
+ if not os.path.exists(path):
+ message = "Script {0} does not exist".format(path)
+ raise AgentException(message)
+ return path
+
+ def requestComponentStatus(self, command):
+ """
+ Component status is determined by exit code, returned by runCommand().
+ Exit code 0 means that component is running and any other exit code means that
+ component is not running
+ """
+ override_output_files = True # by default, we override status command output
+ if logger.level == logging.DEBUG:
+ override_output_files = False
+
+ if command['roleCommand'] == "GET_CONFIG":
+ logger.info("Requesting applied config ...")
+ return {
+ 'configurations': self.applied_configs
+ }
+
+ else:
+ res = self.runCommand(command, self.status_commands_stdout,
+ self.status_commands_stderr,
+ override_output_files=override_output_files)
+ if res['exitcode'] == 0:
+ res['exitcode'] = CustomServiceOrchestrator.LIVE_STATUS
+ else:
+ res['exitcode'] = CustomServiceOrchestrator.DEAD_STATUS
+
+ return res
+ pass
+
+ def dump_command_to_json(self, command, store_config=False):
+ """
+ Converts command to json file and returns file path
+ """
+ # Perform few modifications to stay compatible with the way in which
+ # site.pp files are generated by manifestGenerator.py
+ public_fqdn = self.public_fqdn
+ command['public_hostname'] = public_fqdn
+ # Now, dump the json file
+ command_type = command['commandType']
+ from ActionQueue import ActionQueue # To avoid cyclic dependency
+
+ if command_type == ActionQueue.STATUS_COMMAND:
+ # These files are frequently created, thats why we don't
+ # store them all, but only the latest one
+ file_path = os.path.join(self.tmp_dir, "status_command.json")
+ else:
+ task_id = command['taskId']
+ file_path = os.path.join(self.tmp_dir, "command-{0}.json".format(task_id))
+ # Json may contain passwords, that's why we need proper permissions
+ if os.path.isfile(file_path):
+ os.unlink(file_path)
+
+ self.finalize_command(command, store_config)
+
+ with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT,
+ 0600), 'w') as f:
+ content = json.dumps(command, sort_keys=False, indent=4)
+ f.write(content)
+ return file_path
+
+ """
+ patch content
+ ${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath()
+ ${AGENT_LOG_ROOT} -> AgentConfig.getLogPath()
+ """
+
+ def finalize_command(self, command, store_config):
+
+ if 'configurations' in command:
+ for key in command['configurations']:
+ if len(command['configurations'][key]) > 0:
+ for k, value in command['configurations'][key].items():
+ if value and len(value) > 0 and isinstance(value, basestring) > 0:
+ value = value.replace("${AGENT_WORK_ROOT}",
+ self.config.getWorkRootPath())
+ value = value.replace("${AGENT_LOG_ROOT}",
+ self.config.getLogPath())
+ command['configurations'][key][k] = value
+ pass
+ pass
+ pass
+ pass
+ pass
+
+ if store_config:
+ logger.info("Storing applied config: " + pprint.pformat(command['configurations']))
+ self.applied_configs = command['configurations']
+
+ pass
+
+
diff --git a/slider-agent/src/main/python/agent/Grep.py b/slider-agent/src/main/python/agent/Grep.py
new file mode 100644
index 0000000..cf79038
--- /dev/null
+++ b/slider-agent/src/main/python/agent/Grep.py
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+class Grep:
+
+ # How many lines from command output send to server
+ OUTPUT_LAST_LINES = 10
+ # How many lines from command error output send to server (before Err phrase)
+ ERROR_LAST_LINES_BEFORE = 30
+ # How many lines from command error output send to server (after Err phrase)
+ ERROR_LAST_LINES_AFTER = 30
+
+ def __init__(self):
+ pass
+
+ def grep(self, string, phrase, before, after):
+ """
+ Tries to find the last occurence of phrase in a given string. String should consist of lines,
+ separated by line separators. String trim is performed before search.
+ If occurence is found, grep () takes not more than 'before' lines above and 'after' lines below.
+ If phrase is not found, returns None. Search is not case sensitive, regexps are not supported.
+ """
+ stripped_string = string.strip()
+ lines = stripped_string.splitlines(True)
+ first_occurence = None
+ for index in range(len(lines)):
+ line = lines[index]
+ if phrase.lower() in line.lower():
+ first_occurence = index
+ break
+ if first_occurence is None:
+ return None
+ bound_a = before
+ if first_occurence < before:
+ bound_a = first_occurence
+ result = None
+ if (len(lines) - first_occurence) < after:
+ result = lines[first_occurence - bound_a :]
+ else:
+ result = lines[first_occurence - bound_a : first_occurence + after + 1]
+ return "".join(result).strip()
+
+ def cleanByTemplate(self, string, template):
+ if string is not None:
+ stripped_string = string.strip()
+ lines = stripped_string.splitlines(True)
+ for line in lines[:]:
+ if template.lower() in line.lower():
+ lines.remove(line)
+ return "".join(lines).strip()
+ else:
+ return string
+ def tail(self, string, n):
+ """
+ Copies last n lines from string to result. Also, string trim is performed.
+ """
+ stripped_string = string.strip()
+ lines = stripped_string.splitlines(True)
+ if len(lines) <= n:
+ return stripped_string
+ else:
+ length = len(lines)
+ tailed = lines[length - n:]
+ return "".join(tailed)
+
+ def filterMarkup(self, string):
+ if string is None:
+ result = None
+ else:
+ regexp = "\x1b" + r"\[[\d;]{1,4}m"
+ result = re.sub(regexp, '', string)
+ return result
diff --git a/slider-agent/src/main/python/agent/Heartbeat.py b/slider-agent/src/main/python/agent/Heartbeat.py
new file mode 100644
index 0000000..dd1e8ab
--- /dev/null
+++ b/slider-agent/src/main/python/agent/Heartbeat.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import json
+import logging
+import time
+from pprint import pformat
+
+from ActionQueue import ActionQueue
+import AgentConfig
+
+
+logger = logging.getLogger()
+
+class Heartbeat:
+ def __init__(self, actionQueue, config=None):
+ self.actionQueue = actionQueue
+ self.config = config
+ self.reports = []
+
+ def build(self, commandResult, id='-1', componentsMapped=False):
+ timestamp = int(time.time() * 1000)
+ queueResult = self.actionQueue.result()
+ logger.info("Queue result: " + pformat(queueResult))
+
+ nodeStatus = {"status": "HEALTHY",
+ "cause": "NONE"}
+
+ heartbeat = {'responseId': int(id),
+ 'timestamp': timestamp,
+ 'hostname': self.config.getLabel(),
+ 'nodeStatus': nodeStatus
+ }
+
+ commandsInProgress = False
+ if not self.actionQueue.commandQueue.empty():
+ commandsInProgress = True
+ if len(queueResult) != 0:
+ heartbeat['reports'] = queueResult['reports']
+ if len(heartbeat['reports']) > 0:
+ # There may be IN_PROGRESS tasks
+ commandsInProgress = True
+ pass
+
+ # For first request/heartbeat assume no components are mapped
+ if int(id) == 0:
+ componentsMapped = False
+
+ # create a report of command results (there is only one component at any point)
+ if len(heartbeat['reports']) > 0:
+ for report in heartbeat['reports']:
+ commandResult["commandStatus"] = report["status"]
+ pass
+
+ # Process component status - it can have internal or external STATUS requests and CONFIG requests
+ if queueResult['componentStatus']:
+ componentStatuses = []
+ for componentStatus in queueResult['componentStatus']:
+ if componentStatus['reportResult']:
+ del componentStatus['reportResult']
+ componentStatuses.append(componentStatus)
+ else:
+ commandResult["healthStatus"] = componentStatus["status"]
+ pass
+ if len(componentStatuses) > 0:
+ heartbeat['componentStatus'] = componentStatuses
+
+ logger.info("Sending heartbeat with response id: " + str(id) + " and "
+ "timestamp: " + str(timestamp) +
+ ". Command(s) in progress: " + repr(commandsInProgress) +
+ ". Components mapped: " + repr(componentsMapped))
+ logger.debug("Heartbeat : " + pformat(heartbeat))
+
+ return heartbeat
+
+
+def main(argv=None):
+ actionQueue = ActionQueue(AgentConfig.getConfig())
+ heartbeat = Heartbeat(actionQueue)
+ print json.dumps(heartbeat.build('3', 3))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/slider-agent/src/main/python/agent/NetUtil.py b/slider-agent/src/main/python/agent/NetUtil.py
new file mode 100644
index 0000000..ed8e687
--- /dev/null
+++ b/slider-agent/src/main/python/agent/NetUtil.py
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from urlparse import urlparse
+import time
+import logging
+import httplib
+
+logger = logging.getLogger()
+
+class NetUtil:
+
+ CONNECT_SERVER_RETRY_INTERVAL_SEC = 10
+ HEARTBEAT_IDDLE_INTERVAL_SEC = 10
+ MINIMUM_INTERVAL_BETWEEN_HEARTBEATS = 0.1
+
+ # For testing purposes
+ DEBUG_STOP_RETRIES_FLAG = False
+
+ def checkURL(self, url):
+ """Try to connect to a given url. Result is True if url returns HTTP code 200, in any other case
+ (like unreachable server or wrong HTTP code) result will be False
+ """
+ logger.info("Connecting to the following url " + url);
+ try:
+ parsedurl = urlparse(url)
+ ca_connection = httplib.HTTPConnection(parsedurl[1])
+ ca_connection.request("GET", parsedurl[2])
+ response = ca_connection.getresponse()
+ status = response.status
+ logger.info("Calling url received " + str(status))
+
+ if status == 200:
+ return True
+ else:
+ return False
+ except Exception, e:
+ logger.info("Failed to connect to " + str(url) + " due to " + str(e))
+ return False
+
+ def try_to_connect(self, server_url, max_retries, logger = None):
+ """Try to connect to a given url, sleeping for CONNECT_SERVER_RETRY_INTERVAL_SEC seconds
+ between retries. No more than max_retries is performed. If max_retries is -1, connection
+ attempts will be repeated forever until server is not reachable
+ Returns count of retries
+ """
+ if logger is not None:
+ logger.info("DEBUG: Trying to connect to the server at " + server_url)
+ retries = 0
+ while (max_retries == -1 or retries < max_retries) and not self.DEBUG_STOP_RETRIES_FLAG:
+ server_is_up = self.checkURL(server_url)
+ if server_is_up:
+ break
+ else:
+ if logger is not None:
+ logger.info('Server at {0} is not reachable, sleeping for {1} seconds...'.format(server_url,
+ self.CONNECT_SERVER_RETRY_INTERVAL_SEC))
+ retries += 1
+ time.sleep(self.CONNECT_SERVER_RETRY_INTERVAL_SEC)
+ return retries
+
diff --git a/slider-agent/src/main/python/agent/ProcessHelper.py b/slider-agent/src/main/python/agent/ProcessHelper.py
new file mode 100644
index 0000000..b6283b0
--- /dev/null
+++ b/slider-agent/src/main/python/agent/ProcessHelper.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import logging
+import traceback
+import sys
+from shell import getTempFiles
+
+logger = logging.getLogger()
+
+if 'AGENT_WORK_ROOT' in os.environ:
+ pidfile = os.path.join(os.environ['AGENT_WORK_ROOT'], "infra/run/agent.pid")
+else:
+ pidfile = None
+
+
+def _clean():
+ if not pidfile == None:
+ logger.info("Removing pid file")
+ try:
+ os.unlink(pidfile)
+ except Exception as ex:
+ traceback.print_exc()
+ logger.warn("Unable to remove pid file: %s", ex)
+
+ logger.info("Removing temp files")
+ for f in getTempFiles():
+ if os.path.exists(f):
+ try:
+ os.unlink(f)
+ except Exception as ex:
+ traceback.print_exc()
+ logger.warn("Unable to remove: %s, %s", f, ex)
+ pass
+ pass
+ pass
+
+
+def stopAgent():
+ _clean()
+ os._exit(0)
+ pass
+
+
+def restartAgent():
+ _clean()
+
+ executable = sys.executable
+ args = sys.argv[:]
+ args.insert(0, executable)
+
+ logger.info("Restarting self: %s %s", executable, args)
+
+ os.execvp(executable, args)
+
+
diff --git a/slider-agent/src/main/python/agent/PythonExecutor.py b/slider-agent/src/main/python/agent/PythonExecutor.py
new file mode 100644
index 0000000..3cf71e5
--- /dev/null
+++ b/slider-agent/src/main/python/agent/PythonExecutor.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import json
+import logging
+import os
+import subprocess
+import pprint
+import threading
+from threading import Thread
+from Grep import Grep
+import shell
+import sys
+
+
+logger = logging.getLogger()
+
+
+class PythonExecutor:
+ """
+ Performs functionality for executing python scripts.
+ Warning: class maintains internal state. As a result, instances should not be
+ used as a singleton for a concurrent execution of python scripts
+ """
+
+ NO_ERROR = "none"
+ grep = Grep()
+ event = threading.Event()
+ python_process_has_been_killed = False
+
+ def __init__(self, tmpDir, config):
+ self.tmpDir = tmpDir
+ self.config = config
+ pass
+
+ def run_file(self, script, script_params, tmpoutfile, tmperrfile, timeout,
+ tmpstructedoutfile, override_output_files=True,
+ environment_vars=None):
+ """
+ Executes the specified python file in a separate subprocess.
+ Method returns only when the subprocess is finished.
+ Params arg is a list of script parameters
+ Timeout meaning: how many seconds should pass before script execution
+ is forcibly terminated
+ override_output_files option defines whether stdout/stderr files will be
+ recreated or appended
+ """
+ if override_output_files: # Recreate files
+ tmpout = open(tmpoutfile, 'w')
+ tmperr = open(tmperrfile, 'w')
+ else: # Append to files
+ tmpout = open(tmpoutfile, 'a')
+ tmperr = open(tmperrfile, 'a')
+ script_params += [tmpstructedoutfile]
+ pythonCommand = self.python_command(script, script_params)
+ logger.info("Running command " + pprint.pformat(pythonCommand))
+ process = self.launch_python_subprocess(pythonCommand, tmpout, tmperr,
+ environment_vars)
+ logger.debug("Launching watchdog thread")
+ self.event.clear()
+ self.python_process_has_been_killed = False
+ thread = Thread(target=self.python_watchdog_func, args=(process, timeout))
+ thread.start()
+ # Waiting for the process to be either finished or killed
+ process.communicate()
+ self.event.set()
+ thread.join()
+ # Building results
+ error = self.NO_ERROR
+ returncode = process.returncode
+ out = open(tmpoutfile, 'r').read()
+ error = open(tmperrfile, 'r').read()
+
+ try:
+ with open(tmpstructedoutfile, 'r') as fp:
+ structured_out = json.load(fp)
+ except Exception:
+ if os.path.exists(tmpstructedoutfile):
+ errMsg = 'Unable to read structured output from ' + tmpstructedoutfile
+ structured_out = {
+ 'msg': errMsg
+ }
+ logger.warn(structured_out)
+ else:
+ structured_out = '{}'
+
+ if self.python_process_has_been_killed:
+ error = str(error) + "\n Python script has been killed due to timeout"
+ returncode = 999
+ result = self.condenseOutput(out, error, returncode, structured_out)
+ logger.info("Result: %s" % result)
+ return result
+
+
+ def launch_python_subprocess(self, command, tmpout, tmperr,
+ environment_vars=None):
+ """
+ Creates subprocess with given parameters. This functionality was moved to separate method
+ to make possible unit testing
+ """
+ env = os.environ.copy()
+ if environment_vars:
+ for k, v in environment_vars:
+ logger.info("Setting env: %s to %s", k, v)
+ env[k] = v
+ return subprocess.Popen(command,
+ stdout=tmpout,
+ stderr=tmperr, close_fds=True, env=env)
+
+ def isSuccessfull(self, returncode):
+ return not self.python_process_has_been_killed and returncode == 0
+
+ def python_command(self, script, script_params):
+ python_binary = sys.executable
+ python_command = [python_binary, "-S", script] + script_params
+ return python_command
+
+ def condenseOutput(self, stdout, stderr, retcode, structured_out):
+ log_lines_count = self.config.get('heartbeat', 'log_lines_count')
+
+ grep = self.grep
+ result = {
+ "exitcode": retcode,
+ "stdout": grep.tail(stdout,
+ log_lines_count) if log_lines_count else stdout,
+ "stderr": grep.tail(stderr,
+ log_lines_count) if log_lines_count else stderr,
+ "structuredOut": structured_out
+ }
+
+ return result
+
+ def python_watchdog_func(self, python, timeout):
+ self.event.wait(timeout)
+ if python.returncode is None:
+ logger.error("Subprocess timed out and will be killed")
+ shell.kill_process_with_children(python.pid)
+ self.python_process_has_been_killed = True
+ pass
diff --git a/slider-agent/src/main/python/agent/Register.py b/slider-agent/src/main/python/agent/Register.py
new file mode 100644
index 0000000..7c7ff06
--- /dev/null
+++ b/slider-agent/src/main/python/agent/Register.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import time
+import subprocess
+import hostname
+from AgentConfig import AgentConfig
+
+
+class Register:
+ def __init__(self, config):
+ self.config = config
+
+ def build(self, id='-1'):
+ timestamp = int(time.time() * 1000)
+
+ version = self.read_agent_version()
+
+ register = {'responseId': int(id),
+ 'timestamp': timestamp,
+ 'hostname': self.config.getLabel(),
+ 'publicHostname': hostname.public_hostname(),
+ 'agentVersion': version
+ }
+ return register
+
+ def read_agent_version(self):
+ ver_file = self.config.getResolvedPath(AgentConfig.VERSION_FILE)
+ if os.path.isfile(ver_file):
+ f = open(ver_file, "r")
+ version = f.read().strip()
+ f.close()
+ return version
+
+ return "1"
diff --git a/slider-agent/src/main/python/agent/__init__.py b/slider-agent/src/main/python/agent/__init__.py
new file mode 100644
index 0000000..7dd2074
--- /dev/null
+++ b/slider-agent/src/main/python/agent/__init__.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Agent
+
+"""
+
+from __future__ import generators
+
+__version__ = "0.1.0"
+__author__ = [
+ "Slider"
+]
+__license__ = "Apache License v2.0"
+__contributors__ = "TBD"
+
diff --git a/slider-agent/src/main/python/agent/hostname.py b/slider-agent/src/main/python/agent/hostname.py
new file mode 100644
index 0000000..6381224
--- /dev/null
+++ b/slider-agent/src/main/python/agent/hostname.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import socket
+import traceback
+import logging
+
+logger = logging.getLogger()
+
+def hostname():
+ return socket.getfqdn()
+
+def public_hostname():
+ return socket.getfqdn()
+
+def main(argv=None):
+ print hostname()
+ print public_hostname()
+
+if __name__ == '__main__':
+ main()
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
new file mode 100644
index 0000000..afe3595
--- /dev/null
+++ b/slider-agent/src/main/python/agent/main.py
@@ -0,0 +1,237 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import logging
+import logging.handlers
+import signal
+from optparse import OptionParser
+import sys
+import traceback
+import os
+import time
+import errno
+import ConfigParser
+import ProcessHelper
+from Controller import Controller
+from AgentConfig import AgentConfig
+from NetUtil import NetUtil
+
+logger = logging.getLogger()
+formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s"
+agentPid = os.getpid()
+
+configFileRelPath = "infra/conf/agent.ini"
+logFileName = "agent.log"
+
+SERVER_STATUS_URL="http://{0}:{1}{2}"
+
+
+def signal_handler(signum, frame):
+ #we want the handler to run only for the agent process and not
+ #for the children (e.g. namenode, etc.)
+ if os.getpid() != agentPid:
+ os._exit(0)
+ logger.info('signal received, exiting.')
+ ProcessHelper.stopAgent()
+
+
+def debug(sig, frame):
+ """Interrupt running process, and provide a python prompt for
+ interactive debugging."""
+ d = {'_frame': frame} # Allow access to frame object.
+ d.update(frame.f_globals) # Unless shadowed by global
+ d.update(frame.f_locals)
+
+ message = "Signal received : entering python shell.\nTraceback:\n"
+ message += ''.join(traceback.format_stack(frame))
+ logger.info(message)
+
+
+def setup_logging(verbose, logfile):
+ formatter = logging.Formatter(formatstr)
+ rotateLog = logging.handlers.RotatingFileHandler(logfile, "a", 10000000, 25)
+ rotateLog.setFormatter(formatter)
+ logger.addHandler(rotateLog)
+
+ if verbose:
+ logging.basicConfig(format=formatstr, level=logging.DEBUG, filename=logfile)
+ logger.setLevel(logging.DEBUG)
+ logger.info("loglevel=logging.DEBUG")
+ else:
+ logging.basicConfig(format=formatstr, level=logging.INFO, filename=logfile)
+ logger.setLevel(logging.INFO)
+ logger.info("loglevel=logging.INFO")
+
+
+def update_log_level(config, logfile):
+ # Setting loglevel based on config file
+ try:
+ loglevel = config.get('agent', 'log_level')
+ if loglevel is not None:
+ if loglevel == 'DEBUG':
+ logging.basicConfig(format=formatstr, level=logging.DEBUG, filename=logfile)
+ logger.setLevel(logging.DEBUG)
+ logger.info("Newloglevel=logging.DEBUG")
+ else:
+ logging.basicConfig(format=formatstr, level=logging.INFO, filename=logfile)
+ logger.setLevel(logging.INFO)
+ logger.debug("Newloglevel=logging.INFO")
+ except Exception, err:
+ logger.info("Default loglevel=DEBUG")
+
+
+def bind_signal_handlers():
+ signal.signal(signal.SIGINT, signal_handler)
+ signal.signal(signal.SIGTERM, signal_handler)
+ signal.signal(signal.SIGUSR1, debug)
+
+
+def update_config_from_file(agentConfig):
+ try:
+ configFile = os.path.join(agentConfig.getWorkRootPath(), configFileRelPath)
+ if os.path.exists(configFile):
+ agentConfig.setConfig(configFile)
+ else:
+ logger.warn("No config found, using default")
+
+ except Exception, err:
+ logger.warn(err)
+
+
+def perform_prestart_checks(config):
+ if os.path.isfile(ProcessHelper.pidfile):
+ print("%s already exists, deleting" % ProcessHelper.pidfile)
+ os.remove(ProcessHelper.pidfile)
+ # check if the key folders exist
+ elif not os.path.isdir(config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR)):
+ msg = "Package dir %s does not exists, can't continue" \
+ % config.getResolvedPath(AgentConfig.APP_PACKAGE_DIR)
+ logger.error(msg)
+ print(msg)
+ sys.exit(1)
+
+def ensure_folder_layout(config):
+ ensure_path_exists(config.getResolvedPath(AgentConfig.APP_INSTALL_DIR))
+ ensure_path_exists(config.getResolvedPath(AgentConfig.APP_LOG_DIR))
+ ensure_path_exists(config.getResolvedPath(AgentConfig.APP_RUN_DIR))
+ ensure_path_exists(config.getResolvedPath(AgentConfig.APP_TASK_DIR))
+ ensure_path_exists(config.getResolvedPath(AgentConfig.LOG_DIR))
+ ensure_path_exists(config.getResolvedPath(AgentConfig.RUN_DIR))
+
+def ensure_path_exists(path):
+ try:
+ os.makedirs(path)
+ except OSError as exception:
+ if exception.errno != errno.EEXIST:
+ raise
+ pass
+ pass
+
+def write_pid():
+ # agent only dumps self pid to file
+ pid = str(os.getpid())
+ file(ProcessHelper.pidfile, 'w').write(pid)
+
+
+def stop_agent():
+# stop existing Slider agent
+ pid = -1
+ try:
+ f = open(ProcessHelper.pidfile, 'r')
+ pid = f.read()
+ pid = int(pid)
+ f.close()
+ os.kill(pid, signal.SIGTERM)
+ time.sleep(5)
+ if os.path.exists(ProcessHelper.pidfile):
+ raise Exception("PID file still exists.")
+ except Exception, err:
+ if pid == -1:
+ print ("Agent process is not running")
+ else:
+ os.kill(pid, signal.SIGKILL)
+ os._exit(1)
+
+
+def main():
+ parser = OptionParser()
+ parser.add_option("-v", "--verbose", dest="verbose", help="verbose log output", default=False)
+ parser.add_option("-l", "--label", dest="label", help="label of the agent", default=None)
+ parser.add_option("--host", dest="host", help="AppMaster host", default=None)
+ parser.add_option("--port", dest="port", help="AppMaster port", default=None)
+ (options, args) = parser.parse_args()
+
+ if not 'AGENT_WORK_ROOT' in os.environ:
+ parser.error("AGENT_WORK_ROOT environment variable must be set.");
+ options.root_folder = os.environ['AGENT_WORK_ROOT']
+ if not 'AGENT_LOG_ROOT' in os.environ:
+ parser.error("AGENT_LOG_ROOT environment variable must be set.");
+ options.log_folder = os.environ['AGENT_LOG_ROOT']
+ if not options.label:
+ parser.error("label is required.");
+
+ bind_signal_handlers()
+
+ # Check for configuration file.
+ agentConfig = AgentConfig(options.root_folder, options.log_folder, options.label)
+ update_config_from_file(agentConfig)
+
+ # update configurations if needed
+ if options.host:
+ agentConfig.set(AgentConfig.SERVER_SECTION, "hostname", options.host)
+
+ if options.port:
+ agentConfig.set(AgentConfig.SERVER_SECTION, "port", options.port)
+
+ logFile = os.path.join(agentConfig.getResolvedPath(AgentConfig.LOG_DIR), logFileName)
+ perform_prestart_checks(agentConfig)
+ ensure_folder_layout(agentConfig)
+
+ setup_logging(options.verbose, logFile)
+ update_log_level(agentConfig, logFile)
+ write_pid()
+
+ logger.info("Using AGENT_WORK_ROOT = " + options.root_folder)
+ logger.info("Using AGENT_LOG_ROOT = " + options.log_folder)
+
+ server_url = SERVER_STATUS_URL.format(
+ agentConfig.get(AgentConfig.SERVER_SECTION, 'hostname'),
+ agentConfig.get(AgentConfig.SERVER_SECTION, 'port'),
+ agentConfig.get(AgentConfig.SERVER_SECTION, 'check_path'))
+ print("Connecting to the server at " + server_url + "...")
+ logger.info('Connecting to the server at: ' + server_url)
+
+ # Wait until server is reachable
+ netutil = NetUtil()
+ netutil.try_to_connect(server_url, -1, logger)
+
+ # Launch Controller communication
+ controller = Controller(agentConfig)
+ controller.start()
+ try:
+ while controller.is_alive():
+ controller.join(timeout=1.0)
+ except (KeyboardInterrupt, SystemExit):
+ logger.info("... agent interrupted")
+ pass
+
+
+if __name__ == "__main__":
+ main()
diff --git a/slider-agent/src/main/python/agent/security.py b/slider-agent/src/main/python/agent/security.py
new file mode 100644
index 0000000..4037733
--- /dev/null
+++ b/slider-agent/src/main/python/agent/security.py
@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import httplib
+import urllib2
+import socket
+import ssl
+import os
+import logging
+import subprocess
+import json
+import pprint
+import traceback
+import hostname
+
+logger = logging.getLogger()
+
+GEN_AGENT_KEY="openssl req -new -newkey rsa:1024 -nodes -keyout %(keysdir)s/%(hostname)s.key\
+ -subj /OU=%(hostname)s/\
+ -out %(keysdir)s/%(hostname)s.csr"
+
+
+class VerifiedHTTPSConnection(httplib.HTTPSConnection):
+ """ Connecting using ssl wrapped sockets """
+ def __init__(self, host, port=None, config=None):
+ httplib.HTTPSConnection.__init__(self, host, port=port)
+ self.config=config
+ self.two_way_ssl_required=False
+
+ def connect(self):
+
+ if not self.two_way_ssl_required:
+ try:
+ sock=self.create_connection()
+ self.sock = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_NONE)
+ logger.info('SSL connection established. Two-way SSL authentication is '
+ 'turned off on the server.')
+ except (ssl.SSLError, AttributeError):
+ self.two_way_ssl_required=True
+ logger.info('Insecure connection to https://' + self.host + ':' + self.port +
+ '/ failed. Reconnecting using two-way SSL authentication..')
+
+ if self.two_way_ssl_required:
+ self.certMan=CertificateManager(self.config)
+ self.certMan.initSecurity()
+ agent_key = self.certMan.getAgentKeyName()
+ agent_crt = self.certMan.getAgentCrtName()
+ server_crt = self.certMan.getSrvrCrtName()
+
+ sock=self.create_connection()
+
+ try:
+ self.sock = ssl.wrap_socket(sock,
+ keyfile=agent_key,
+ certfile=agent_crt,
+ cert_reqs=ssl.CERT_REQUIRED,
+ ca_certs=server_crt)
+ logger.info('SSL connection established. Two-way SSL authentication '
+ 'completed successfully.')
+ except ssl.SSLError as err:
+ logger.error('Two-way SSL authentication failed. Ensure that '
+ 'server and agent certificates were signed by the same CA '
+ 'and restart the agent. '
+ '\nIn order to receive a new agent certificate, remove '
+ 'existing certificate file from keys directory. As a '
+ 'workaround you can turn off two-way SSL authentication in '
+ 'server configuration.'
+ '\nExiting..')
+ raise err
+
+ def create_connection(self):
+ if self.sock:
+ self.sock.close()
+ logger.info("SSL Connect being called.. connecting to the server")
+ sock = socket.create_connection((self.host, self.port), 60)
+ sock.setsockopt( socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+ if self._tunnel_host:
+ self.sock = sock
+ self._tunnel()
+
+ return sock
+
+class CachedHTTPSConnection:
+ """ Caches a ssl socket and uses a single https connection to the server. """
+
+ def __init__(self, config):
+ self.connected = False;
+ self.config = config
+ self.server = config.get('server', 'hostname')
+ self.port = config.get('server', 'secured_port')
+ self.connect()
+
+ def connect(self):
+ if not self.connected:
+ self.httpsconn = VerifiedHTTPSConnection(self.server, self.port, self.config)
+ self.httpsconn.connect()
+ self.connected = True
+ # possible exceptions are caught and processed in Controller
+
+
+
+ def forceClear(self):
+ self.httpsconn = VerifiedHTTPSConnection(self.server, self.port, self.config)
+ self.connect()
+
+ def request(self, req):
+ self.connect()
+ try:
+ self.httpsconn.request(req.get_method(), req.get_full_url(),
+ req.get_data(), req.headers)
+ response = self.httpsconn.getresponse()
+ readResponse = response.read()
+ except Exception as ex:
+ # This exception is caught later in Controller
+ logger.debug("Error in sending/receving data from the server " +
+ traceback.format_exc())
+ logger.info("Encountered communication error. Details: " + repr(ex))
+ self.connected = False
+ raise IOError("Error occured during connecting to the server: " + str(ex))
+ return readResponse
+
+class CertificateManager():
+ def __init__(self, config):
+ self.config = config
+ self.keysdir = self.config.get('security', 'keysdir')
+ self.server_crt=self.config.get('security', 'server_crt')
+ self.server_url = 'https://' + self.config.get('server', 'hostname') + ':' \
+ + self.config.get('server', 'url_port')
+
+ def getAgentKeyName(self):
+ keysdir = self.config.get('security', 'keysdir')
+ return keysdir + os.sep + hostname.hostname() + ".key"
+
+ def getAgentCrtName(self):
+ keysdir = self.config.get('security', 'keysdir')
+ return keysdir + os.sep + hostname.hostname() + ".crt"
+
+ def getAgentCrtReqName(self):
+ keysdir = self.config.get('security', 'keysdir')
+ return keysdir + os.sep + hostname.hostname() + ".csr"
+
+ def getSrvrCrtName(self):
+ keysdir = self.config.get('security', 'keysdir')
+ return keysdir + os.sep + "ca.crt"
+
+ def checkCertExists(self):
+
+ s = self.config.get('security', 'keysdir') + os.sep + "ca.crt"
+
+ server_crt_exists = os.path.exists(s)
+
+ if not server_crt_exists:
+ logger.info("Server certicate not exists, downloading")
+ self.loadSrvrCrt()
+ else:
+ logger.info("Server certicate exists, ok")
+
+ agent_key_exists = os.path.exists(self.getAgentKeyName())
+
+ if not agent_key_exists:
+ logger.info("Agent key not exists, generating request")
+ self.genAgentCrtReq()
+ else:
+ logger.info("Agent key exists, ok")
+
+ agent_crt_exists = os.path.exists(self.getAgentCrtName())
+
+ if not agent_crt_exists:
+ logger.info("Agent certificate not exists, sending sign request")
+ self.reqSignCrt()
+ else:
+ logger.info("Agent certificate exists, ok")
+
+ def loadSrvrCrt(self):
+ get_ca_url = self.server_url + '/cert/ca/'
+ logger.info("Downloading server cert from " + get_ca_url)
+ stream = urllib2.urlopen(get_ca_url)
+ response = stream.read()
+ stream.close()
+ srvr_crt_f = open(self.getSrvrCrtName(), 'w+')
+ srvr_crt_f.write(response)
+
+ def reqSignCrt(self):
+ sign_crt_req_url = self.server_url + '/certs/' + hostname.hostname()
+ agent_crt_req_f = open(self.getAgentCrtReqName())
+ agent_crt_req_content = agent_crt_req_f.read()
+ passphrase_env_var = self.config.get('security', 'passphrase_env_var_name')
+ passphrase = os.environ[passphrase_env_var]
+ register_data = {'csr' : agent_crt_req_content,
+ 'passphrase' : passphrase}
+ data = json.dumps(register_data)
+ req = urllib2.Request(sign_crt_req_url, data, {'Content-Type': 'application/json'})
+ f = urllib2.urlopen(req)
+ response = f.read()
+ f.close()
+ data = json.loads(response)
+ logger.debug("Sign response from Server: \n" + pprint.pformat(data))
+ result=data['result']
+ if result == 'OK':
+ agentCrtContent=data['signedCa']
+ agentCrtF = open(self.getAgentCrtName(), "w")
+ agentCrtF.write(agentCrtContent)
+ else:
+ # Possible exception is catched higher at Controller
+ logger.error('Certificate signing failed.'
+ '\nIn order to receive a new agent'
+ ' certificate, remove existing certificate file from keys '
+ 'directory. As a workaround you can turn off two-way SSL '
+ 'authentication in server configuration.'
+ '\nExiting..')
+ raise ssl.SSLError
+
+ def genAgentCrtReq(self):
+ generate_script = GEN_AGENT_KEY % {'hostname': hostname.hostname(),
+ 'keysdir' : self.config.get('security', 'keysdir')}
+ logger.info(generate_script)
+ p = subprocess.Popen([generate_script], shell=True, stdout=subprocess.PIPE)
+ p.communicate()
+
+ def initSecurity(self):
+ self.checkCertExists()
diff --git a/slider-agent/src/main/python/agent/shell.py b/slider-agent/src/main/python/agent/shell.py
new file mode 100644
index 0000000..341cb0f
--- /dev/null
+++ b/slider-agent/src/main/python/agent/shell.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import logging
+import subprocess
+import os
+import tempfile
+import signal
+import sys
+import threading
+import time
+import traceback
+import pprint
+
+try:
+ import pwd
+except ImportError:
+ import winpwd as pwd
+
+global serverTracker
+serverTracker = {}
+logger = logging.getLogger()
+
+threadLocal = threading.local()
+gracefull_kill_delay = 5 # seconds between SIGTERM and SIGKILL
+tempFiles = []
+def noteTempFile(filename):
+ tempFiles.append(filename)
+
+def getTempFiles():
+ return tempFiles
+
+def kill_process_with_children(parent_pid):
+ def kill_tree_function(pid, signal):
+ '''
+ Kills process tree starting from a given pid.
+ '''
+ # The command below starts 'ps' linux utility and then parses it's
+ # output using 'awk'. AWK recursively extracts PIDs of all children of
+ # a given PID and then passes list of "kill -<SIGNAL> PID" commands to 'sh'
+ # shell.
+ CMD = """ps xf | awk -v PID=""" + str(pid) + \
+ """ ' $1 == PID { P = $1; next } P && /_/ { P = P " " $1;""" + \
+ """K=P } P && !/_/ { P="" } END { print "kill -""" \
+ + str(signal) + """ "K }' | sh """
+ process = subprocess.Popen(CMD, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, shell=True)
+ process.communicate()
+ run_kill_function(kill_tree_function, parent_pid)
+
+def run_kill_function(kill_function, pid):
+ try:
+ kill_function(pid, signal.SIGTERM)
+ except Exception, e:
+ logger.warn("Failed to kill PID %d" % (pid))
+ logger.warn("Reported error: " + repr(e))
+
+ time.sleep(gracefull_kill_delay)
+
+ try:
+ kill_function(pid, signal.SIGKILL)
+ except Exception, e:
+ logger.error("Failed to send SIGKILL to PID %d. Process exited?" % (pid))
+ logger.error("Reported error: " + repr(e))
+
+def changeUid():
+ try:
+ os.setuid(threadLocal.uid)
+ except Exception:
+ logger.warn("can not switch user for running command.")
+
+class shellRunner:
+ # Run any command
+ def run(self, script, user=None):
+ try:
+ if user!=None:
+ user = pwd.getpwnam(user)[2]
+ else:
+ user = os.getuid()
+ threadLocal.uid = user
+ except Exception:
+ logger.warn("can not switch user for RUN_COMMAND.")
+ code = 0
+ cmd = " "
+ cmd = cmd.join(script)
+ p = subprocess.Popen(cmd, preexec_fn=changeUid, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, shell=True, close_fds=True)
+ out, err = p.communicate()
+ code = p.wait()
+ logger.debug("Exitcode for %s is %d" % (cmd,code))
+ return {'exitCode': code, 'output': out, 'error': err}
+
+ def getServerTracker(self):
+ return serverTracker
\ No newline at end of file
diff --git a/slider-agent/src/main/python/jinja2/AUTHORS b/slider-agent/src/main/python/jinja2/AUTHORS
new file mode 100644
index 0000000..c6cd9ba
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/AUTHORS
@@ -0,0 +1,31 @@
+Jinja is written and maintained by the Jinja Team and various
+contributors:
+
+Lead Developer:
+
+- Armin Ronacher <armin.ronacher@active-4.com>
+
+Developers:
+
+- Christoph Hack
+- Georg Brandl
+
+Contributors:
+
+- Bryan McLemore
+- Mickaël Guérin <kael@crocobox.org>
+- Cameron Knight
+- Lawrence Journal-World.
+- David Cramer
+
+Patches and suggestions:
+
+- Ronny Pfannschmidt
+- Axel Böhm
+- Alexey Melchakov
+- Bryan McLemore
+- Clovis Fabricio (nosklo)
+- Cameron Knight
+- Peter van Dijk (Habbie)
+- Stefan Ebner
+- Rene Leonhardt
diff --git a/slider-agent/src/main/python/jinja2/CHANGES b/slider-agent/src/main/python/jinja2/CHANGES
new file mode 100644
index 0000000..2a49c64
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/CHANGES
@@ -0,0 +1,235 @@
+Jinja2 Changelog
+================
+
+Version 2.5.5
+-------------
+(re-release of 2.5.4 with built documentation removed for filesize.
+ Released on October 18th 2010)
+
+- built documentation is no longer part of release.
+
+Version 2.5.4
+-------------
+(bugfix release, released on October 17th 2010)
+
+- Fixed extensions not loading properly with overlays.
+- Work around a bug in cpython for the debugger that causes segfaults
+ on 64bit big-endian architectures.
+
+Version 2.5.3
+-------------
+(bugfix release, released on October 17th 2010)
+
+- fixed an operator precedence error introduced in 2.5.2. Statements
+ like "-foo.bar" had their implicit parentheses applied around the
+ first part of the expression ("(-foo).bar") instead of the more
+ correct "-(foo.bar)".
+
+Version 2.5.2
+-------------
+(bugfix release, released on August 18th 2010)
+
+- improved setup.py script to better work with assumptions people
+ might still have from it (``--with-speedups``).
+- fixed a packaging error that excluded the new debug support.
+
+Version 2.5.1
+-------------
+(bugfix release, released on August 17th 2010)
+
+- StopIteration exceptions raised by functions called from templates
+ are now intercepted and converted to undefineds. This solves a
+ lot of debugging grief. (StopIteration is used internally to
+ abort template execution)
+- improved performance of macro calls slightly.
+- babel extraction can now properly extract newstyle gettext calls.
+- using the variable `num` in newstyle gettext for something else
+ than the pluralize count will no longer raise a :exc:`KeyError`.
+- removed builtin markup class and switched to markupsafe. For backwards
+ compatibility the pure Python implementation still exists but is
+ pulled from markupsafe by the Jinja2 developers. The debug support
+ went into a separate feature called "debugsupport" and is disabled
+ by default because it is only relevant for Python 2.4
+- fixed an issue with unary operators having the wrong precendence.
+
+Version 2.5
+-----------
+(codename Incoherence, relased on May 29th 2010)
+
+- improved the sort filter (should have worked like this for a
+ long time) by adding support for case insensitive searches.
+- fixed a bug for getattribute constant folding.
+- support for newstyle gettext translations which result in a
+ nicer in-template user interface and more consistent
+ catalogs. (:ref:`newstyle-gettext`)
+- it's now possible to register extensions after an environment
+ was created.
+
+Version 2.4.1
+-------------
+(bugfix release, released on April 20th 2010)
+
+- fixed an error reporting bug for undefineds.
+
+Version 2.4
+-----------
+(codename Correlation, released on April 13th 2010)
+
+- the environment template loading functions now transparently
+ pass through a template object if it was passed to it. This
+ makes it possible to import or extend from a template object
+ that was passed to the template.
+- added a :class:`ModuleLoader` that can load templates from
+ precompiled sources. The environment now features a method
+ to compile the templates from a configured loader into a zip
+ file or folder.
+- the _speedups C extension now supports Python 3.
+- added support for autoescaping toggling sections and support
+ for evaluation contexts (:ref:`eval-context`).
+- extensions have a priority now.
+
+Version 2.3.1
+-------------
+(bugfix release, released on February 19th 2010)
+
+- fixed an error reporting bug on all python versions
+- fixed an error reporting bug on Python 2.4
+
+Version 2.3
+-----------
+(3000 Pythons, released on February 10th 2010)
+
+- fixes issue with code generator that causes unbound variables
+ to be generated if set was used in if-blocks and other small
+ identifier problems.
+- include tags are now able to select between multiple templates
+ and take the first that exists, if a list of templates is
+ given.
+- fixed a problem with having call blocks in outer scopes that
+ have an argument that is also used as local variable in an
+ inner frame (#360).
+- greatly improved error message reporting (#339)
+- implicit tuple expressions can no longer be totally empty.
+ This change makes ``{% if %}...{% endif %}`` a syntax error
+ now. (#364)
+- added support for translator comments if extracted via babel.
+- added with-statement extension.
+- experimental Python 3 support.
+
+Version 2.2.1
+-------------
+(bugfix release, released on September 14th 2009)
+
+- fixes some smaller problems for Jinja2 on Jython.
+
+Version 2.2
+-----------
+(codename Kong, released on September 13th 2009)
+
+- Include statements can now be marked with ``ignore missing`` to skip
+ non existing templates.
+- Priority of `not` raised. It's now possible to write `not foo in bar`
+ as an alias to `foo not in bar` like in python. Previously the grammar
+ required parentheses (`not (foo in bar)`) which was odd.
+- Fixed a bug that caused syntax errors when defining macros or using the
+ `{% call %}` tag inside loops.
+- Fixed a bug in the parser that made ``{{ foo[1, 2] }}`` impossible.
+- Made it possible to refer to names from outer scopes in included templates
+ that were unused in the callers frame (#327)
+- Fixed a bug that caused internal errors if names where used as iteration
+ variable and regular variable *after* the loop if that variable was unused
+ *before* the loop. (#331)
+- Added support for optional `scoped` modifier to blocks.
+- Added support for line-comments.
+- Added the `meta` module.
+- Renamed (undocumented) attribute "overlay" to "overlayed" on the
+ environment because it was clashing with a method of the same name.
+- speedup extension is now disabled by default.
+
+Version 2.1.1
+-------------
+(Bugfix release)
+
+- Fixed a translation error caused by looping over empty recursive loops.
+
+Version 2.1
+-----------
+(codename Yasuzō, released on November 23rd 2008)
+
+- fixed a bug with nested loops and the special loop variable. Before the
+ change an inner loop overwrote the loop variable from the outer one after
+ iteration.
+
+- fixed a bug with the i18n extension that caused the explicit pluralization
+ block to look up the wrong variable.
+
+- fixed a limitation in the lexer that made ``{{ foo.0.0 }}`` impossible.
+
+- index based subscribing of variables with a constant value returns an
+ undefined object now instead of raising an index error. This was a bug
+ caused by eager optimizing.
+
+- the i18n extension looks up `foo.ugettext` now followed by `foo.gettext`
+ if an translations object is installed. This makes dealing with custom
+ translations classes easier.
+
+- fixed a confusing behavior with conditional extending. loops were partially
+ executed under some conditions even though they were not part of a visible
+ area.
+
+- added `sort` filter that works like `dictsort` but for arbitrary sequences.
+
+- fixed a bug with empty statements in macros.
+
+- implemented a bytecode cache system. (:ref:`bytecode-cache`)
+
+- the template context is now weakref-able
+
+- inclusions and imports "with context" forward all variables now, not only
+ the initial context.
+
+- added a cycle helper called `cycler`.
+
+- added a joining helper called `joiner`.
+
+- added a `compile_expression` method to the environment that allows compiling
+ of Jinja expressions into callable Python objects.
+
+- fixed an escaping bug in urlize
+
+Version 2.0
+-----------
+(codename jinjavitus, released on July 17th 2008)
+
+- the subscribing of objects (looking up attributes and items) changed from
+ slightly. It's now possible to give attributes or items a higher priority
+ by either using dot-notation lookup or the bracket syntax. This also
+ changed the AST slightly. `Subscript` is gone and was replaced with
+ :class:`~jinja2.nodes.Getitem` and :class:`~jinja2.nodes.Getattr`.
+
+ For more information see :ref:`the implementation details <notes-on-subscriptions>`.
+
+- added support for preprocessing and token stream filtering for extensions.
+ This would allow extensions to allow simplified gettext calls in template
+ data and something similar.
+
+- added :meth:`jinja2.environment.TemplateStream.dump`.
+
+- added missing support for implicit string literal concatenation.
+ ``{{ "foo" "bar" }}`` is equivalent to ``{{ "foobar" }}``
+
+- `else` is optional for conditional expressions. If not given it evaluates
+ to `false`.
+
+- improved error reporting for undefined values by providing a position.
+
+- `filesizeformat` filter uses decimal prefixes now per default and can be
+ set to binary mode with the second parameter.
+
+- fixed bug in finalizer
+
+Version 2.0rc1
+--------------
+(no codename, released on June 9th 2008)
+
+- first release of Jinja2
diff --git a/slider-agent/src/main/python/jinja2/LICENSE b/slider-agent/src/main/python/jinja2/LICENSE
new file mode 100644
index 0000000..31bf900
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/LICENSE
@@ -0,0 +1,31 @@
+Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details.
+
+Some rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ * The names of the contributors may not be used to endorse or
+ promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/slider-agent/src/main/python/jinja2/MANIFEST.in b/slider-agent/src/main/python/jinja2/MANIFEST.in
new file mode 100644
index 0000000..aeb66af
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/MANIFEST.in
@@ -0,0 +1,12 @@
+include MANIFEST.in Makefile CHANGES LICENSE AUTHORS jinja2/_debugsupport.c
+recursive-include docs *
+recursive-include custom_fixers *
+recursive-include ext *
+recursive-include artwork *
+recursive-include examples *
+recursive-include jinja2/testsuite/res *
+recursive-exclude docs/_build *
+recursive-exclude jinja2 *.pyc
+recursive-exclude docs *.pyc
+recursive-exclude jinja2 *.pyo
+recursive-exclude docs *.pyo
diff --git a/slider-agent/src/main/python/jinja2/Makefile b/slider-agent/src/main/python/jinja2/Makefile
new file mode 100644
index 0000000..60ca1d7
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/Makefile
@@ -0,0 +1,4 @@
+test:
+ python setup.py test
+
+.PHONY: test
diff --git a/slider-agent/src/main/python/jinja2/artwork/jinjalogo.svg b/slider-agent/src/main/python/jinja2/artwork/jinjalogo.svg
new file mode 100644
index 0000000..0bc9ea4
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/artwork/jinjalogo.svg
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+<svg
+ xmlns:dc="http://purl.org/dc/elements/1.1/"
+ xmlns:cc="http://web.resource.org/cc/"
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:svg="http://www.w3.org/2000/svg"
+ xmlns="http://www.w3.org/2000/svg"
+ xmlns:xlink="http://www.w3.org/1999/xlink"
+ xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+ xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+ width="300"
+ height="120"
+ id="svg2"
+ sodipodi:version="0.32"
+ inkscape:version="0.45.1"
+ version="1.0"
+ sodipodi:docbase="/Users/mitsuhiko/Development/jinja2/artwork"
+ sodipodi:docname="jinjalogo.svg"
+ inkscape:export-filename="/Users/mitsuhiko/Development/jinja2/docs/_static/jinjabanner.png"
+ inkscape:export-xdpi="60"
+ inkscape:export-ydpi="60"
+ inkscape:output_extension="org.inkscape.output.svg.inkscape">
+ <defs
+ id="defs4">
+ <linearGradient
+ id="linearGradient6558">
+ <stop
+ style="stop-color:#575757;stop-opacity:1;"
+ offset="0"
+ id="stop6560" />
+ <stop
+ style="stop-color:#2f2f2f;stop-opacity:1;"
+ offset="1"
+ id="stop6562" />
+ </linearGradient>
+ <radialGradient
+ inkscape:collect="always"
+ xlink:href="#linearGradient6558"
+ id="radialGradient6564"
+ cx="61.297766"
+ cy="60.910986"
+ fx="61.297766"
+ fy="60.910986"
+ r="44.688254"
+ gradientTransform="matrix(1,0,0,0.945104,0,3.343747)"
+ gradientUnits="userSpaceOnUse" />
+ <radialGradient
+ inkscape:collect="always"
+ xlink:href="#linearGradient6558"
+ id="radialGradient6580"
+ gradientUnits="userSpaceOnUse"
+ gradientTransform="matrix(1,0,0,0.945104,0.355158,3.334402)"
+ cx="61.297766"
+ cy="60.910986"
+ fx="61.297766"
+ fy="60.910986"
+ r="44.688254" />
+ <linearGradient
+ inkscape:collect="always"
+ xlink:href="#linearGradient6558"
+ id="linearGradient4173"
+ x1="255.15521"
+ y1="32.347946"
+ x2="279.8912"
+ y2="32.347946"
+ gradientUnits="userSpaceOnUse"
+ gradientTransform="matrix(0.8073249,0,0,0.8073249,57.960878,7.4036303)" />
+ <linearGradient
+ inkscape:collect="always"
+ xlink:href="#linearGradient6558"
+ id="linearGradient5145"
+ gradientUnits="userSpaceOnUse"
+ gradientTransform="matrix(0.7902775,0,0,0.82474,60.019977,8.0684132)"
+ x1="255.15521"
+ y1="32.347946"
+ x2="279.8912"
+ y2="32.347946" />
+ </defs>
+ <sodipodi:namedview
+ id="base"
+ pagecolor="#ffffff"
+ bordercolor="#666666"
+ borderopacity="1.0"
+ gridtolerance="10000"
+ guidetolerance="10"
+ objecttolerance="10"
+ inkscape:pageopacity="0.0"
+ inkscape:pageshadow="2"
+ inkscape:zoom="2.8"
+ inkscape:cx="137.4752"
+ inkscape:cy="57.574575"
+ inkscape:document-units="px"
+ inkscape:current-layer="layer1"
+ width="300px"
+ height="120px"
+ showguides="true"
+ inkscape:guide-bbox="true"
+ inkscape:window-width="1396"
+ inkscape:window-height="900"
+ inkscape:window-x="0"
+ inkscape:window-y="22" />
+ <metadata
+ id="metadata7">
+ <rdf:RDF>
+ <cc:Work
+ rdf:about="">
+ <dc:format>image/svg+xml</dc:format>
+ <dc:type
+ rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+ </cc:Work>
+ </rdf:RDF>
+ </metadata>
+ <g
+ inkscape:label="Layer 1"
+ inkscape:groupmode="layer"
+ id="layer1">
+ <path
+ style="font-size:12px;font-style:normal;font-weight:normal;fill:#f4f4f4;fill-opacity:1;stroke:#e7e7e7;stroke-width:0.8;stroke-linecap:butt;stroke-linejoin:round;stroke-opacity:1;font-family:Bitstream Vera Sans;stroke-miterlimit:4;stroke-dasharray:none"
+ d="M 165.36463,80.874808 L 165.36463,80.874808 L 153.32556,80.874808 L 153.32556,81.8344 L 147.64994,81.8344 L 147.64994,36.035583 L 165.36463,36.035583 L 165.36463,20.333129 C 170.58154,21.031083 173.07533,22.077914 172.84609,23.473621 C 172.78871,24.055258 172.21545,24.549594 171.12624,24.956624 L 171.12624,36.035583 L 189.09895,36.035583 L 189.09895,82.532286 L 183.33733,82.532286 L 183.33733,80.874808 L 171.12624,80.874808 L 171.12624,102.94548 L 165.36463,102.94548 L 165.36463,80.874808 M 153.32556,55.489173 L 153.32556,55.489173 L 165.36463,55.489173 L 165.36463,41.793146 L 153.32556,41.793146 L 153.32556,55.489173 M 171.12624,55.489173 L 171.12624,55.489173 L 183.33733,55.489173 L 183.33733,41.793146 L 171.12624,41.793146 L 171.12624,55.489173 M 183.33733,61.333977 L 183.33733,61.333977 L 171.12624,61.333977 L 171.12624,75.030006 L 183.33733,75.030006 L 183.33733,61.333977 M 165.36463,61.333977 L 165.36463,61.333977 L 153.32556,61.333977 L 153.32556,75.030006 L 165.36463,75.030006 L 165.36463,61.333977 M 132.85897,59.414792 C 137.33069,63.136883 140.99969,67.934848 143.86618,73.808701 L 139.13654,77.385372 C 137.24467,72.965445 134.6362,69.12707 131.31114,65.87024 L 131.31114,102.94548 L 125.63554,102.94548 L 125.63554,68.57455 C 122.31042,71.947693 118.52671,74.913707 114.28436,77.47261 L 109.64069,73.372526 C 121.50782,67.091566 130.62312,55.489212 136.98668,38.565417 L 116.26221,38.565417 L 116.26221,32.720615 L 125.80754,32.720615 L 125.80754,20.333129 C 130.85245,21.031083 133.31761,22.048838 133.20299,23.386383 C 133.14561,24.026183 132.57235,24.549594 131.48307,24.956624 L 131.48307,32.720615 L 140.77043,32.720615 L 143.60824,36.733469 C 140.68444,45.51526 137.10137,53.075692 132.85897,59.414792 M 254.11016,49.469901 L 254.11016,49.469901 L 254.11016,20.333129 C 259.21243,21.031083 261.67755,22.048838 261.50562,23.386383 C 261.44823,23.909869 261.04699,24.346044 260.30172,24.694917 C 260.30164,24.694986 260.30164,24.694986 260.30172,24.694917 L 260.30172,24.694917 L 259.78578,24.956624 L 259.78578,49.469901 L 277.15652,49.469901 L 277.15652,55.227471 L 259.78578,55.227471 L 259.78578,93.785712 L 281.45616,93.785712 L 281.45616,99.63051 L 232.35378,99.63051 L 232.35378,93.785712 L 254.11016,93.785712 L 254.11016,55.227471 L 236.22346,55.227471 L 236.22346,49.469901 L 254.11016,49.469901 M 225.5603,59.327554 C 231.12111,63.107798 235.62145,67.876693 239.06127,73.634235 L 234.76157,77.647079 C 231.60845,72.180322 227.82475,67.934848 223.41044,64.910648 L 223.41044,102.94548 L 217.73484,102.94548 L 217.73484,67.44049 C 212.91919,71.627831 207.70222,75.030021 202.084,77.647079 L 197.87027,73.198053 C 212.66118,66.917101 224.01239,55.372897 231.92377,38.565417 L 205.35172,38.565417 L 205.35172,32.720615 L 217.99283,32.720615 L 217.99283,20.333129 C 223.03774,21.031083 225.50291,22.048838 225.38829,23.386383 C 225.33089,24.026183 224.75765,24.549594 223.66837,24.956624 L 223.66837,32.720615 L 236.22346,32.720615 L 238.80326,36.733469 C 235.13421,45.51526 230.71987,53.046611 225.5603,59.327554"
+ id="text4761" />
+ <path
+ style="font-size:44.09793472px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;fill:#b41717;fill-opacity:1;stroke:#7f2828;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Candara;stroke-miterlimit:4;stroke-dasharray:none"
+ d="M 149.14708,37.774469 C 148.97807,41.117899 148.84526,44.824225 148.74871,48.893456 C 148.67626,52.962754 148.3818,70.641328 148.38184,75.524422 C 148.3818,79.065795 148.05588,81.991266 147.40406,84.300835 C 146.75219,86.610422 145.72612,88.557071 144.32585,90.140779 C 142.94969,91.724494 141.17522,92.901283 139.00239,93.671139 C 136.82953,94.440996 134.22211,94.825935 131.18014,94.825935 C 128.83828,94.825935 126.73787,94.59498 124.87889,94.133049 L 125.4221,89.31593 C 127.13623,90.0418 128.92278,90.404734 130.78177,90.404733 C 132.85805,90.404734 134.66875,90.140782 136.2139,89.612876 C 137.78315,89.062981 139.02651,88.216133 139.94396,87.072335 C 140.8855,85.928548 141.54942,84.520804 141.93572,82.8491 C 142.34613,81.177412 142.55134,78.988811 142.55136,76.283285 C 142.55134,66.297119 142.62852,44.659257 142.26641,37.774469 L 149.14708,37.774469 M 166.38498,80.732697 L 159.83024,80.732697 C 160.16821,76.333498 160.33723,71.307412 160.33723,65.654424 C 160.33723,59.2976 159.91471,53.963567 159.06973,49.652319 L 166.31257,48.761483 C 166.02284,53.358679 165.87799,58.98965 165.87799,65.654424 C 165.87799,70.933479 166.04699,75.959565 166.38498,80.732697 M 167.90601,39.490159 C 167.90598,40.611994 167.5076,41.590815 166.7109,42.42662 C 165.91418,43.240515 164.79155,43.647442 163.343,43.647399 C 162.11172,43.647442 161.146,43.295504 160.44588,42.591595 C 159.76988,41.865769 159.43188,40.996927 159.43188,39.98507 C 159.43188,38.885304 159.84231,37.928485 160.66315,37.114591 C 161.48399,36.30078 162.61869,35.893853 164.06727,35.893811 C 165.25023,35.893853 166.17975,36.256783 166.85575,36.982609 C 167.55588,37.686526 167.90598,38.522373 167.90601,39.490159 M 206.72748,80.732697 L 200.13651,80.732697 C 200.66763,74.947749 200.93319,68.634899 200.9332,61.794122 C 200.93319,58.406756 200.1727,56.097177 198.65174,54.865371 C 197.15487,53.61163 195.00619,52.984747 192.20564,52.984714 C 188.77731,52.984747 185.61465,54.117535 182.71753,56.383099 C 182.71753,63.883761 182.76583,72.000287 182.86238,80.732697 L 176.27142,80.732697 C 176.68182,73.254058 176.88707,67.843042 176.88707,64.499632 C 176.88707,59.352589 176.3559,54.359493 175.29363,49.520339 L 181.66734,48.695493 L 182.35539,52.720761 L 182.64511,52.720761 C 186.21823,49.773323 190.04483,48.299592 194.12499,48.299567 C 198.13265,48.299592 201.23499,49.113454 203.43201,50.741118 C 205.62895,52.346863 206.72747,55.217334 206.72748,59.352563 C 206.72747,59.770507 206.70331,60.595362 206.65507,61.827118 C 206.60675,63.058915 206.5826,63.883761 206.58262,64.30167 C 206.5826,67.975018 206.63088,73.452022 206.72748,80.732697 M 222.69791,48.695493 C 222.28747,55.514282 222.08225,62.355041 222.08225,69.21778 C 222.08225,71.043461 222.14262,73.463019 222.26332,76.476468 C 222.40822,79.467925 222.4806,81.502559 222.48063,82.580363 C 222.4806,89.685068 219.51105,93.996287 213.57195,95.514024 L 211.76124,93.006484 C 213.90995,91.356766 215.2378,89.597085 215.74478,87.727431 C 216.49321,85.043912 216.86743,79.324953 216.86743,70.570535 C 216.86743,61.178248 216.3846,54.16153 215.41887,49.520339 L 222.69791,48.695493 M 224.2551,39.490159 C 224.2551,40.611994 223.85673,41.590815 223.06006,42.42662 C 222.26332,43.240515 221.14069,43.647442 219.69213,43.647399 C 218.46084,43.647442 217.49515,43.295504 216.795,42.591595 C 216.119,41.865769 215.781,40.996927 215.781,39.98507 C 215.781,38.885304 216.19144,37.928485 217.01231,37.114591 C 217.83316,36.30078 218.96785,35.893853 220.4164,35.893811 C 221.5994,35.893853 222.52889,36.256783 223.20492,36.982609 C 223.90503,37.686526 224.2551,38.522373 224.2551,39.490159 M 259.60008,80.732697 L 253.91446,80.930661 C 253.62473,79.852857 253.47987,78.830045 253.4799,77.862216 L 253.11774,77.862216 C 250.14817,80.325772 246.10427,81.557546 240.98606,81.557547 C 238.20962,81.557546 235.8195,80.820682 233.81563,79.346948 C 231.81178,77.851221 230.80988,75.728607 230.80988,72.979099 C 230.80988,69.591724 232.37914,66.875216 235.51769,64.829574 C 238.65625,62.761967 244.48667,61.67316 253.00913,61.563165 C 253.08155,61.035275 253.11772,60.430386 253.11774,59.748497 C 253.11772,57.043003 252.32104,55.239336 250.72765,54.337474 C 249.15832,53.435661 246.76819,52.984747 243.55721,52.984714 C 239.76681,52.984747 236.03678,53.413668 232.3671,54.271484 L 232.9827,49.718301 C 236.60411,48.77251 240.76873,48.299592 245.47658,48.299567 C 249.77395,48.299592 253.09359,49.113454 255.43545,50.741118 C 257.77728,52.346863 258.94819,55.096363 258.94824,58.989625 C 258.94819,60.023469 258.88785,61.904117 258.76715,64.631608 C 258.67054,67.337133 258.62228,69.140806 258.6223,70.042632 C 258.62228,74.045913 258.94819,77.609265 259.60008,80.732697 M 253.19019,74.331856 C 253.06945,70.988469 253.00909,67.986016 253.00913,65.324484 C 248.47027,65.324498 245.01786,65.632443 242.65187,66.248318 C 238.69248,67.348131 236.71278,69.448748 236.71278,72.550177 C 236.71278,75.541643 239.03044,77.037371 243.66588,77.037366 C 247.64942,77.037371 250.82416,76.135534 253.19019,74.331856"
+ id="text3736"
+ sodipodi:nodetypes="ccsscssccscccsccccsccsccsscsssccccscscccsccccscsssccscscccscccsscsssccccccscsccscsccscscsccccssc" />
+ <path
+ style="fill:url(#radialGradient6564);fill-opacity:1.0;fill-rule:evenodd;stroke:#323232;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:none"
+ d="M 105.45673,18.675923 C 105.45673,18.675923 88.211949,26.918461 74.172834,28.737898 C 60.133727,30.557333 33.360434,32.377571 28.045622,31.093256 C 22.730818,29.808941 18.915645,28.309196 18.915645,28.309196 L 20.021441,32.056583 L 16.609513,35.052471 L 17.2144,36.121726 L 18.61792,36.22764 L 22.92773,36.762252 L 23.532621,38.688909 L 25.937975,38.905784 L 27.143021,42.970927 C 27.143021,42.970927 32.254764,43.399628 33.758953,43.399628 C 35.263142,43.399628 38.271966,43.187802 38.271966,43.187802 L 38.371202,44.791657 L 39.477002,45.003495 L 39.477002,46.824227 L 37.066917,48.967759 L 37.671807,49.073671 L 37.671807,49.820127 C 37.671807,49.820127 32.255457,50.252157 30.049301,49.93109 C 27.843157,49.610006 27.440747,49.608286 27.440747,49.608286 L 27.242258,49.820127 L 27.143021,50.783455 L 27.643946,50.783455 L 27.84242,54.959544 L 38.976091,54.530844 L 38.172728,68.980747 L 38.073481,70.796442 L 28.645781,70.261816 L 28.546544,66.408513 L 30.649462,66.408513 L 30.852673,64.910557 L 32.757107,64.481857 L 33.059555,64.058192 L 25.937975,62.343374 L 20.522364,63.947229 L 21.42496,64.698732 L 22.327572,64.698732 L 22.426809,65.984848 L 24.331254,66.09076 L 24.331254,69.838147 L 22.228335,70.372777 L 22.630009,71.225146 L 23.130934,71.547931 L 23.130934,74.437917 L 24.435218,74.437917 L 24.435218,87.813529 L 22.327572,88.13632 L 22.630009,91.989617 L 23.929569,92.206492 L 23.731093,100.98236 L 29.449141,101.08826 L 28.244105,92.418334 L 36.868446,92.206492 L 36.268285,96.912181 L 35.464925,100.23086 L 44.188501,100.33677 L 44.287739,91.777793 L 50.303506,91.243181 L 50.005786,96.700351 L 49.802585,99.90807 L 54.920484,99.90807 L 54.717274,91.132217 L 55.421397,91.243181 L 55.619882,87.067076 L 54.816521,87.067076 L 54.518798,85.352258 L 54.017874,80.429702 L 54.216359,74.760706 L 55.31743,74.760706 L 55.31743,71.336105 L 53.913913,71.442015 L 54.117112,67.402096 L 55.747469,67.240708 L 55.823083,65.929374 L 56.749319,65.793192 L 57.699176,65.071956 L 51.985842,63.896802 L 46.31977,65.15265 L 46.872668,66.060507 L 47.47283,66.010066 L 48.172228,65.984848 L 48.299828,67.639144 L 49.878196,67.563497 L 49.906548,71.144447 L 43.111042,70.988097 L 43.337879,67.160002 L 43.559978,63.679927 L 43.559978,59.105378 L 43.763188,54.288748 L 57.373101,53.592733 L 73.567955,52.659674 L 73.71917,55.736265 L 73.142647,63.120082 L 72.892183,69.9945 L 66.928387,69.888585 L 66.900039,65.071956 L 69.106918,64.991267 L 69.206169,63.629486 L 70.108765,63.493308 L 70.061506,63.226006 L 70.964116,63.175568 L 71.465028,62.504773 L 64.721507,60.926122 L 58.001612,62.368592 L 58.4789,63.200785 L 59.230285,63.1453 L 59.230285,63.523577 L 60.156518,63.523577 L 60.156518,65.046738 L 62.136575,65.071956 L 62.112937,69.298485 L 60.109259,69.298485 L 60.080907,70.261816 L 60.785031,70.342507 L 60.70942,74.009202 L 62.188552,74.089909 L 62.013701,88.620507 L 60.057282,89.018952 L 60.080907,89.714967 L 60.761406,89.714967 L 60.761406,93.437137 L 61.886113,93.437137 L 61.588391,98.52109 L 61.210343,102.95945 L 68.331912,103.14605 L 68.105084,99.29275 L 67.580538,96.085028 L 67.476575,93.300955 L 73.520696,93.195041 L 73.345845,97.502272 L 73.317494,102.05159 L 76.729426,102.3189 L 81.3653,102.1323 L 82.820807,101.70358 L 82.017437,99.26753 L 81.818959,95.439438 L 81.440912,92.710853 L 87.206218,92.499027 L 86.955759,95.842931 L 86.932133,101.08826 L 89.238253,101.30009 L 91.520751,101.24965 L 92.621828,100.90165 L 91.969693,95.923633 L 91.747577,92.176239 L 92.725793,92.070324 L 92.749427,88.726422 L 93.02352,88.670945 L 92.976244,87.949712 L 91.846823,87.949712 L 91.619996,85.488427 L 91.520751,74.811143 L 92.371377,74.785924 L 92.371377,71.280616 L 92.725793,71.336105 L 92.725793,70.640088 L 91.468773,70.529127 L 91.497126,66.463987 L 93.600043,66.277382 L 93.477182,64.910557 L 94.403419,64.829863 L 94.351424,64.562549 L 95.580099,63.947229 L 89.337489,62.69138 L 82.995657,63.977495 L 83.39733,64.723951 L 84.375543,64.643256 L 84.427528,64.966046 L 85.254515,64.966046 L 85.301775,66.569901 L 87.357445,66.544681 L 87.532293,70.478688 L 80.264217,70.423216 L 79.413593,64.512124 L 78.733106,61.380041 L 78.184923,55.761484 L 78.510996,52.473053 L 92.999878,51.373557 L 93.047136,46.476221 L 93.774891,46.289613 L 93.727651,45.543159 L 93.174743,45.220372 C 93.174629,45.220372 85.252181,46.395266 82.745197,46.66284 C 82.0389,46.738209 82.09239,46.733258 81.516524,46.79397 L 81.440912,45.886118 L 78.444837,44.317564 L 78.482644,42.491786 L 79.512842,42.461518 L 79.588444,39.949808 C 79.588444,39.949808 85.728225,39.546834 88.009582,39.0117 C 90.290937,38.476559 93.524432,37.942456 93.524432,37.942456 L 95.055545,33.79662 L 98.089437,32.913987 L 98.339888,32.217972 L 105.20628,30.316548 L 105.98602,29.676006 L 103.37744,23.976741 L 103.62792,22.690624 L 104.95584,21.994611 L 105.91041,19.079404 L 105.45673,18.675923 z M 72.466874,40.403728 L 72.429067,42.476654 L 73.983813,42.542211 L 73.884576,44.509221 L 70.836515,46.506487 L 70.647496,47.081457 L 71.876167,47.091543 L 71.866712,47.575729 L 62.552432,48.029652 L 62.613863,46.652742 L 63.039175,45.966809 L 63.067524,45.528025 L 63.07698,44.579832 L 63.341609,43.949374 L 63.440849,43.439982 L 63.440849,43.076841 L 63.842533,41.47297 L 72.466874,40.403728 z M 52.987688,42.168984 L 52.760853,43.561027 L 53.488599,44.418431 L 53.441349,45.916386 L 54.117112,46.960408 L 53.942262,48.191039 L 54.443185,48.912273 L 44.939872,49.2855 L 44.916247,48.967759 L 46.017333,48.831579 L 46.069307,48.428097 L 43.66394,47.121797 L 43.536351,45.03375 L 44.689411,44.978276 L 44.788661,42.72883 L 52.987688,42.168984 z M 67.051262,74.276518 L 72.81657,74.649742 L 72.618099,82.411833 L 73.36947,88.776857 L 67.254465,88.565018 L 67.051262,74.276518 z M 28.44258,74.599304 L 37.671807,75.078442 L 36.868446,80.429702 L 36.868446,84.928593 L 37.520583,87.440302 L 28.494569,87.869006 L 28.44258,74.599304 z M 87.508658,74.649742 L 87.508658,87.924488 L 81.644113,88.353194 L 81.440912,81.342592 L 80.788764,74.811143 L 87.508658,74.649742 z M 43.087416,74.947312 L 49.906548,74.972531 L 49.977434,87.278902 L 43.611966,87.389863 L 43.285891,83.400379 L 43.262266,79.441156 L 43.087416,74.947312 z "
+ id="path4735" />
+ </g>
+</svg>
diff --git a/slider-agent/src/main/python/jinja2/custom_fixers/__init__.py b/slider-agent/src/main/python/jinja2/custom_fixers/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/custom_fixers/__init__.py
diff --git a/slider-agent/src/main/python/jinja2/custom_fixers/fix_alt_unicode.py b/slider-agent/src/main/python/jinja2/custom_fixers/fix_alt_unicode.py
new file mode 100644
index 0000000..96a81c1
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/custom_fixers/fix_alt_unicode.py
@@ -0,0 +1,13 @@
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import Name, BlankLine
+
+
+class FixAltUnicode(fixer_base.BaseFix):
+ PATTERN = """
+ func=funcdef< 'def' name='__unicode__'
+ parameters< '(' NAME ')' > any+ >
+ """
+
+ def transform(self, node, results):
+ name = results['name']
+ name.replace(Name('__str__', prefix=name.prefix))
diff --git a/slider-agent/src/main/python/jinja2/custom_fixers/fix_broken_reraising.py b/slider-agent/src/main/python/jinja2/custom_fixers/fix_broken_reraising.py
new file mode 100644
index 0000000..fd0ea68
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/custom_fixers/fix_broken_reraising.py
@@ -0,0 +1,21 @@
+from lib2to3 import fixer_base, pytree
+from lib2to3.fixer_util import Name, BlankLine, Name, Attr, ArgList
+
+
+class FixBrokenReraising(fixer_base.BaseFix):
+ PATTERN = """
+ raise_stmt< 'raise' any ',' val=any ',' tb=any >
+ """
+
+ # run before the broken 2to3 checker with the same goal
+ # tries to rewrite it with a rule that does not work out for jinja
+ run_order = 1
+
+ def transform(self, node, results):
+ tb = results['tb'].clone()
+ tb.prefix = ''
+ with_tb = Attr(results['val'].clone(), Name('with_traceback')) + \
+ [ArgList([tb])]
+ new = pytree.Node(self.syms.simple_stmt, [Name("raise")] + with_tb)
+ new.prefix = node.prefix
+ return new
diff --git a/slider-agent/src/main/python/jinja2/custom_fixers/fix_xrange2.py b/slider-agent/src/main/python/jinja2/custom_fixers/fix_xrange2.py
new file mode 100644
index 0000000..5d35e50
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/custom_fixers/fix_xrange2.py
@@ -0,0 +1,11 @@
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import Name, BlankLine
+
+
+# whyever this is necessary..
+
+class FixXrange2(fixer_base.BaseFix):
+ PATTERN = "'xrange'"
+
+ def transform(self, node, results):
+ node.replace(Name('range', prefix=node.prefix))
diff --git a/slider-agent/src/main/python/jinja2/docs/Makefile b/slider-agent/src/main/python/jinja2/docs/Makefile
new file mode 100644
index 0000000..5e24ec1
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/Makefile
@@ -0,0 +1,75 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d _build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html web pickle htmlhelp latex changes linkcheck
+
+help:
+ @echo "Please use \`make <target>' where <target> is one of"
+ @echo " html to make standalone HTML files"
+ @echo " pickle to make pickle files"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " changes to make an overview over all changed/added/deprecated items"
+ @echo " linkcheck to check all external links for integrity"
+
+clean:
+ -rm -rf _build/*
+
+html:
+ mkdir -p _build/html _build/doctrees
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html
+ @echo
+ @echo "Build finished. The HTML pages are in _build/html."
+
+pickle:
+ mkdir -p _build/pickle _build/doctrees
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) _build/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files"
+
+json:
+ mkdir -p _build/json _build/doctrees
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) _build/json
+ @echo
+ @echo "Build finished; now you can process the json files"
+
+web: pickle
+
+htmlhelp:
+ mkdir -p _build/htmlhelp _build/doctrees
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) _build/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in _build/htmlhelp."
+
+latex:
+ mkdir -p _build/latex _build/doctrees
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) _build/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in _build/latex."
+ @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
+ "run these through (pdf)latex."
+
+changes:
+ mkdir -p _build/changes _build/doctrees
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) _build/changes
+ @echo
+ @echo "The overview file is in _build/changes."
+
+linkcheck:
+ mkdir -p _build/linkcheck _build/doctrees
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) _build/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in _build/linkcheck/output.txt."
diff --git a/slider-agent/src/main/python/jinja2/docs/_build/.ignore b/slider-agent/src/main/python/jinja2/docs/_build/.ignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_build/.ignore
diff --git a/slider-agent/src/main/python/jinja2/docs/_static/.ignore b/slider-agent/src/main/python/jinja2/docs/_static/.ignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_static/.ignore
diff --git a/slider-agent/src/main/python/jinja2/docs/_static/jinja.js b/slider-agent/src/main/python/jinja2/docs/_static/jinja.js
new file mode 100644
index 0000000..1c04218
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_static/jinja.js
@@ -0,0 +1,26 @@
+$(function() {
+
+ var
+ toc = $('#toc').show(),
+ items = $('#toc > ul').hide();
+
+ $('#toc h3')
+ .click(function() {
+ if (items.is(':visible')) {
+ items.animate({
+ height: 'hide',
+ opacity: 'hide'
+ }, 300, function() {
+ toc.removeClass('expandedtoc');
+ });
+ }
+ else {
+ items.animate({
+ height: 'show',
+ opacity: 'show'
+ }, 400);
+ toc.addClass('expandedtoc');
+ }
+ });
+
+});
diff --git a/slider-agent/src/main/python/jinja2/docs/_static/print.css b/slider-agent/src/main/python/jinja2/docs/_static/print.css
new file mode 100644
index 0000000..fb633d8
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_static/print.css
@@ -0,0 +1,5 @@
+div.header, div.relnav, #toc { display: none; }
+#contentwrapper { padding: 0; margin: 0; border: none; }
+body { color: black; background-color: white; }
+div.footer { border-top: 1px solid #888; color: #888; margin-top: 1cm; }
+div.footer a { text-decoration: none; }
diff --git a/slider-agent/src/main/python/jinja2/docs/_static/style.css b/slider-agent/src/main/python/jinja2/docs/_static/style.css
new file mode 100644
index 0000000..a1c4d59
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_static/style.css
@@ -0,0 +1,390 @@
+body {
+ background-color: #222;
+ margin: 0;
+ padding: 0;
+ font-family: 'Georgia', serif;
+ font-size: 15px;
+ color: #eee;
+}
+
+div.footer {
+ border-top: 1px solid #111;
+ padding: 8px;
+ font-size: 11px;
+ text-align: center;
+ letter-spacing: 0.5px;
+}
+
+div.footer a {
+ color: #eee;
+}
+
+div.header {
+ margin: 0 -15px 0 -15px;
+ background: url(headerbg.png) repeat-x;
+ border-top: 6px solid #D20000;
+}
+
+div.relnav {
+ border-bottom: 1px solid #111;
+ background: url(navigation.png);
+ margin: 0 -15px 0 -15px;
+ padding: 2px 20px 0 28px;
+ line-height: 25px;
+ color: #aaa;
+ font-size: 12px;
+ text-align: center;
+}
+
+div.relnav a {
+ color: #eee;
+ font-weight: bold;
+ text-decoration: none;
+}
+
+div.relnav a:hover {
+ text-decoration: underline;
+}
+
+#content {
+ background-color: white;
+ color: #111;
+ border-bottom: 1px solid black;
+ background: url(watermark.png) center 0;
+ padding: 0 15px 0 15px;
+ margin: 0;
+}
+
+h1 {
+ margin: 0;
+ padding: 15px 0 0 0;
+}
+
+h1.heading {
+ margin: 0;
+ padding: 0;
+ height: 80px;
+}
+
+h1.heading:hover {
+ background: #222;
+}
+
+h1.heading a {
+ background: url(jinjabanner.png) no-repeat center 0;
+ display: block;
+ width: 100%;
+ height: 80px;
+}
+
+h1.heading a:focus {
+ -moz-outline: none;
+ outline: none;
+}
+
+h1.heading span {
+ display: none;
+}
+
+#jinjalogo {
+ background-image: url(jinjalogo.png);
+ background-repeat: no-repeat;
+ width: 400px;
+ height: 160px;
+}
+
+#contentwrapper {
+ max-width: 680px;
+ padding: 0 18px 20px 18px;
+ margin: 0 auto 0 auto;
+ border-right: 1px solid #eee;
+ border-left: 1px solid #eee;
+ background: url(watermark_blur.png) center -114px;
+}
+
+#contentwrapper h2,
+#contentwrapper h2 a {
+ color: #222;
+ font-size: 24px;
+ margin: 20px 0 0 0;
+}
+
+#contentwrapper h3,
+#contentwrapper h3 a {
+ color: #b41717;
+ font-size: 20px;
+ margin: 20px 0 0 0;
+}
+
+table.docutils {
+ border-collapse: collapse;
+ border: 2px solid #aaa;
+ margin: 0.5em 1.5em 0.5em 1.5em;
+}
+
+table.docutils td {
+ padding: 2px;
+ border: 1px solid #ddd;
+}
+
+p, li, dd, dt, blockquote {
+ color: #333;
+}
+
+blockquote {
+ margin: 10px 0 10px 20px;
+}
+
+p {
+ line-height: 20px;
+ margin-bottom: 0;
+ margin-top: 10px;
+}
+
+hr {
+ border-top: 1px solid #ccc;
+ border-bottom: 0;
+ border-right: 0;
+ border-left: 0;
+ margin-bottom: 10px;
+ margin-top: 20px;
+}
+
+dl {
+ margin-left: 10px;
+}
+
+li, dt {
+ margin-top: 5px;
+}
+
+dt {
+ font-weight: bold;
+ color: #000;
+}
+
+dd {
+ margin-top: 10px;
+ line-height: 20px;
+}
+
+th {
+ text-align: left;
+ padding: 3px;
+ background-color: #f2f2f2;
+}
+
+a {
+ color: #b41717;
+}
+
+a:hover {
+ color: #444;
+}
+
+pre {
+ background: #ededed url(metal.png);
+ border-top: 1px solid #ccc;
+ border-bottom: 1px solid #ccc;
+ padding: 5px;
+ font-size: 13px;
+ font-family: 'Bitstream Vera Sans Mono', 'Monaco', monospace;
+}
+
+tt {
+ font-size: 13px;
+ font-family: 'Bitstream Vera Sans Mono', 'Monaco', monospace;
+ color: black;
+ padding: 1px 2px 1px 2px;
+ background-color: #fafafa;
+ border-bottom: 1px solid #eee;
+}
+
+a.reference:hover tt {
+ border-bottom-color: #aaa;
+}
+
+cite {
+ /* abusing <cite>, it's generated by ReST for `x` */
+ font-size: 13px;
+ font-family: 'Bitstream Vera Sans Mono', 'Monaco', monospace;
+ font-weight: bold;
+ font-style: normal;
+}
+
+div.admonition {
+ margin: 10px 0 10px 0;
+ padding: 10px 10px 10px 60px;
+ border: 1px solid #ccc;
+}
+
+div.admonition p.admonition-title {
+ background-color: #b41717;
+ color: white;
+ margin: -10px -10px 10px -60px;
+ padding: 4px 10px 4px 10px;
+ font-weight: bold;
+ font-size: 15px;
+}
+
+div.admonition p.admonition-title a {
+ color: white!important;
+}
+
+div.admonition-note {
+ background: url(note.png) no-repeat 10px 40px;
+}
+
+div.admonition-implementation {
+ background: url(implementation.png) no-repeat 10px 40px;
+}
+
+a.headerlink {
+ color: #B4B4B4!important;
+ font-size: 0.8em;
+ padding: 0 4px 0 4px;
+ text-decoration: none!important;
+ visibility: hidden;
+}
+
+h1:hover > a.headerlink,
+h2:hover > a.headerlink,
+h3:hover > a.headerlink,
+h4:hover > a.headerlink,
+h5:hover > a.headerlink,
+h6:hover > a.headerlink,
+dt:hover > a.headerlink,
+dt:hover > a.headerlink {
+ visibility: visible;
+}
+
+a.headerlink:hover {
+ background-color: #B4B4B4;
+ color: #F0F0F0!important;
+}
+
+table.indextable {
+ width: 100%;
+}
+
+table.indextable td {
+ vertical-align: top;
+ width: 50%;
+}
+
+table.indextable dl dd {
+ font-size: 11px;
+}
+
+table.indextable dl dd a {
+ color: #000;
+}
+
+dl.function dt,
+dl.class dt,
+dl.exception dt,
+dl.method dt,
+dl.attribute dt {
+ font-weight: normal;
+}
+
+dt .descname {
+ font-weight: bold;
+ margin-right: 4px;
+}
+
+dt .descname, dt .descclassname {
+ padding: 0;
+ background: transparent;
+ border-bottom: 1px solid #111;
+}
+
+dt .descclassname {
+ margin-left: 2px;
+}
+
+dl dt big {
+ font-size: 100%;
+}
+
+ul.search {
+ margin: 10px 0 0 30px;
+ padding: 0;
+}
+
+ul.search li {
+ margin: 10px 0 0 0;
+ padding: 0;
+}
+
+ul.search div.context {
+ font-size: 12px;
+ padding: 4px 0 0 20px;
+ color: #888;
+}
+
+span.highlight {
+ background-color: #eee;
+ border: 1px solid #ccc;
+}
+
+#toc {
+ margin: 0 -17px 0 -17px;
+ display: none;
+}
+
+#toc h3 {
+ float: right;
+ margin: 5px 5px 0 0;
+ padding: 0;
+ font-size: 12px;
+ color: #777;
+}
+
+#toc h3:hover {
+ color: #333;
+ cursor: pointer;
+}
+
+.expandedtoc {
+ background: #222 url(darkmetal.png);
+ border-bottom: 1px solid #111;
+ outline-bottom: 1px solid #000;
+ padding: 5px;
+}
+
+.expandedtoc h3 {
+ color: #aaa;
+ margin: 0!important;
+}
+
+.expandedtoc h3:hover {
+ color: white!important;
+}
+
+#tod h3:hover {
+ color: white;
+}
+
+#toc a {
+ color: #ddd;
+ text-decoration: none;
+}
+
+#toc a:hover {
+ color: white;
+ text-decoration: underline;
+}
+
+#toc ul {
+ margin: 5px 0 12px 17px;
+ padding: 0 7px 0 7px;
+}
+
+#toc ul ul {
+ margin-bottom: 0;
+}
+
+#toc ul li {
+ margin: 2px 0 0 0;
+}
diff --git a/slider-agent/src/main/python/jinja2/docs/_templates/.ignore b/slider-agent/src/main/python/jinja2/docs/_templates/.ignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_templates/.ignore
diff --git a/slider-agent/src/main/python/jinja2/docs/_templates/genindex.html b/slider-agent/src/main/python/jinja2/docs/_templates/genindex.html
new file mode 100644
index 0000000..9add6e9
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_templates/genindex.html
@@ -0,0 +1,36 @@
+{% extends "layout.html" %}
+{% set title = 'Index' %}
+{% block body %}
+
+ <h1 id="index">Index</h1>
+
+ {% for key, dummy in genindexentries -%}
+ <a href="#{{ key }}"><strong>{{ key }}</strong></a> {% if not loop.last %}| {% endif %}
+ {%- endfor %}
+ <hr>
+
+ {% for key, entries in genindexentries %}
+ <h2 id="{{ key }}">{{ key }}</h2>
+ <table class="indextable"><tr>
+ {%- for column in entries|slice(2) if column %}
+ <td><dl>
+ {%- for entryname, (links, subitems) in column %}
+ <dt>{% if links %}<a href="{{ links[0] }}">{{ entryname|e }}</a>
+ {% for link in links[1:] %}, <a href="{{ link }}">[Link]</a>{% endfor %}
+ {%- else %}{{ entryname|e }}{% endif %}</dt>
+ {%- if subitems %}
+ <dd><dl>
+ {%- for subentryname, subentrylinks in subitems %}
+ <dt><a href="{{ subentrylinks[0] }}">{{ subentryname|e }}</a>
+ {%- for link in subentrylinks[1:] %}, <a href="{{ link }}">[Link]</a>{% endfor -%}
+ </dt>
+ {%- endfor %}
+ </dl></dd>
+ {%- endif -%}
+ {%- endfor %}
+ </dl></td>
+ {%- endfor %}
+ </tr></table>
+ {% endfor %}
+
+{% endblock %}
diff --git a/slider-agent/src/main/python/jinja2/docs/_templates/layout.html b/slider-agent/src/main/python/jinja2/docs/_templates/layout.html
new file mode 100644
index 0000000..f682f90
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_templates/layout.html
@@ -0,0 +1,77 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+ <head>
+ <title>Jinja2 Documentation</title>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+ <link rel="stylesheet" href="{{ pathto('_static/style.css', 1) }}" type="text/css">
+ <link rel="stylesheet" href="{{ pathto('_static/print.css', 1) }}" type="text/css" media="print">
+ <link rel="stylesheet" href="{{ pathto('_static/pygments.css', 1) }}" type="text/css">
+ {%- if builder != 'htmlhelp' %}
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT: '{{ pathto("", 1) }}',
+ VERSION: '{{ release }}'
+ };
+ </script>
+ <script type="text/javascript" src="{{ pathto('_static/jquery.js', 1) }}"></script>
+ <script type="text/javascript" src="{{ pathto('_static/interface.js', 1) }}"></script>
+ <script type="text/javascript" src="{{ pathto('_static/doctools.js', 1) }}"></script>
+ <script type="text/javascript" src="{{ pathto('_static/jinja.js', 1) }}"></script>
+ {%- endif %}
+ {%- if use_opensearch and builder != 'htmlhelp' %}
+ <link rel="search" type="application/opensearchdescription+xml"
+ title="Search within {{ docstitle }}"
+ href="{{ pathto('_static/opensearch.xml', 1) }}">
+ {%- endif %}
+ {%- if hasdoc('about') %}
+ <link rel="author" title="About these documents" href="{{ pathto('about') }}">
+ {%- endif %}
+ <link rel="contents" title="Global table of contents" href="{{ pathto('contents') }}">
+ <link rel="index" title="Global index" href="{{ pathto('genindex') }}">
+ <link rel="search" title="Search" href="{{ pathto('search') }}">
+ {%- if hasdoc('copyright') %}
+ <link rel="copyright" title="Copyright" href="{{ pathto('copyright') }}">
+ {%- endif %}
+ <link rel="top" title="{{ docstitle }}" href="{{ pathto('index') }}">
+ {%- if parents %}
+ <link rel="up" title="{{ parents[-1].title|striptags }}" href="{{ parents[-1].link|e }}">
+ {%- endif %}
+ {%- if next %}
+ <link rel="next" title="{{ next.title|striptags }}" href="{{ next.link|e }}">
+ {%- endif %}
+ {%- if prev %}
+ <link rel="prev" title="{{ prev.title|striptags }}" href="{{ prev.link|e }}">
+ {%- endif %}
+ {% block extrahead %}{% endblock %}
+ </head>
+ <body>
+ <div id="content">
+ <div class="header">
+ <h1 class="heading"><a href="{{ pathto('index') }}"
+ title="back to the documentation overview"><span>Jinja</span></a></h1>
+ </div>
+ <div class="relnav">
+ {%- if prev %}
+ <a href="{{ prev.link|e }}">« {{ prev.title }}</a> |
+ {%- endif %}
+ <a href="{{ pathto(current_page_name) if current_page_name else '#' }}">{{ title }}</a>
+ {%- if next %}
+ | <a href="{{ next.link|e }}">{{ next.title }} »</a>
+ {%- endif %}
+ </div>
+ <div id="contentwrapper">
+ {%- if display_toc %}
+ <div id="toc">
+ <h3>Table Of Contents</h3>
+ {{ toc }}
+ </div>
+ {%- endif %}
+ {% block body %}{% endblock %}
+ </div>
+ </div>
+ <div class="footer">
+ © Copyright 2010 by the <a href="http://pocoo.org/">Pocoo Team</a>,
+ documentation generated by <a href="http://sphinx.pocoo.org/">Sphinx</a>
+ </div>
+ </body>
+</html>
diff --git a/slider-agent/src/main/python/jinja2/docs/_templates/opensearch.xml b/slider-agent/src/main/python/jinja2/docs/_templates/opensearch.xml
new file mode 100644
index 0000000..9f2fa42
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_templates/opensearch.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
+ <ShortName>{{ project }}</ShortName>
+ <Description>Search {{ docstitle }}</Description>
+ <InputEncoding>utf-8</InputEncoding>
+ <Url type="text/html" method="get"
+ template="{{ use_opensearch }}/{{ pathto('search') }}?q={searchTerms}&check_keywords=yes&area=default"/>
+ <LongName>{{ docstitle }}</LongName>
+</OpenSearchDescription>
diff --git a/slider-agent/src/main/python/jinja2/docs/_templates/page.html b/slider-agent/src/main/python/jinja2/docs/_templates/page.html
new file mode 100644
index 0000000..ee6cad3
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_templates/page.html
@@ -0,0 +1,4 @@
+{% extends 'layout.html' %}
+{% block body %}
+ {{ body }}
+{% endblock %}
diff --git a/slider-agent/src/main/python/jinja2/docs/_templates/search.html b/slider-agent/src/main/python/jinja2/docs/_templates/search.html
new file mode 100644
index 0000000..0c942b7
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/_templates/search.html
@@ -0,0 +1,35 @@
+{% extends "layout.html" %}
+{% set title = 'Search' %}
+{% block extrahead %}
+ <script type="text/javascript" src="{{ pathto('_static/searchtools.js', 1) }}"></script>
+{% endblock %}
+{% block body %}
+ <h1 id="search-documentation">Search</h1>
+ <p>
+ From here you can search these documents. Enter your search
+ words into the box below and click "search". Note that the search
+ function will automatically search for all of the words. Pages
+ containing less words won't appear in the result list.
+ </p>
+ <form action="" method="get"><p>
+ <input type="text" name="q" value="">
+ <input type="submit" value="search">
+ </p></form>
+ {% if search_performed %}
+ <h2>Search Results</h2>
+ {% if not search_results %}
+ <p>Your search did not match any results.</p>
+ {% endif %}
+ {% endif %}
+ <div id="search-results">
+ {% if search_results %}
+ <ul>
+ {% for href, caption, context in search_results %}
+ <li><a href="{{ pathto(item.href) }}">{{ caption }}</a>
+ <div class="context">{{ context|e }}</div>
+ </li>
+ {% endfor %}
+ </ul>
+ {% endif %}
+ </div>
+{% endblock %}
diff --git a/slider-agent/src/main/python/jinja2/docs/api.rst b/slider-agent/src/main/python/jinja2/docs/api.rst
new file mode 100644
index 0000000..3bf8a94
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/api.rst
@@ -0,0 +1,787 @@
+API
+===
+
+.. module:: jinja2
+ :synopsis: public Jinja2 API
+
+This document describes the API to Jinja2 and not the template language. It
+will be most useful as reference to those implementing the template interface
+to the application and not those who are creating Jinja2 templates.
+
+Basics
+------
+
+Jinja2 uses a central object called the template :class:`Environment`.
+Instances of this class are used to store the configuration, global objects
+and are used to load templates from the file system or other locations.
+Even if you are creating templates from strings by using the constructor of
+:class:`Template` class, an environment is created automatically for you,
+albeit a shared one.
+
+Most applications will create one :class:`Environment` object on application
+initialization and use that to load templates. In some cases it's however
+useful to have multiple environments side by side, if different configurations
+are in use.
+
+The simplest way to configure Jinja2 to load templates for your application
+looks roughly like this::
+
+ from jinja2 import Environment, PackageLoader
+ env = Environment(loader=PackageLoader('yourapplication', 'templates'))
+
+This will create a template environment with the default settings and a
+loader that looks up the templates in the `templates` folder inside the
+`yourapplication` python package. Different loaders are available
+and you can also write your own if you want to load templates from a
+database or other resources.
+
+To load a template from this environment you just have to call the
+:meth:`get_template` method which then returns the loaded :class:`Template`::
+
+ template = env.get_template('mytemplate.html')
+
+To render it with some variables, just call the :meth:`render` method::
+
+ print template.render(the='variables', go='here')
+
+Using a template loader rather then passing strings to :class:`Template`
+or :meth:`Environment.from_string` has multiple advantages. Besides being
+a lot easier to use it also enables template inheritance.
+
+
+Unicode
+-------
+
+Jinja2 is using Unicode internally which means that you have to pass Unicode
+objects to the render function or bytestrings that only consist of ASCII
+characters. Additionally newlines are normalized to one end of line
+sequence which is per default UNIX style (``\n``).
+
+Python 2.x supports two ways of representing string objects. One is the
+`str` type and the other is the `unicode` type, both of which extend a type
+called `basestring`. Unfortunately the default is `str` which should not
+be used to store text based information unless only ASCII characters are
+used. With Python 2.6 it is possible to make `unicode` the default on a per
+module level and with Python 3 it will be the default.
+
+To explicitly use a Unicode string you have to prefix the string literal
+with a `u`: ``u'Hänsel und Gretel sagen Hallo'``. That way Python will
+store the string as Unicode by decoding the string with the character
+encoding from the current Python module. If no encoding is specified this
+defaults to 'ASCII' which means that you can't use any non ASCII identifier.
+
+To set a better module encoding add the following comment to the first or
+second line of the Python module using the Unicode literal::
+
+ # -*- coding: utf-8 -*-
+
+We recommend utf-8 as Encoding for Python modules and templates as it's
+possible to represent every Unicode character in utf-8 and because it's
+backwards compatible to ASCII. For Jinja2 the default encoding of templates
+is assumed to be utf-8.
+
+It is not possible to use Jinja2 to process non-Unicode data. The reason
+for this is that Jinja2 uses Unicode already on the language level. For
+example Jinja2 treats the non-breaking space as valid whitespace inside
+expressions which requires knowledge of the encoding or operating on an
+Unicode string.
+
+For more details about Unicode in Python have a look at the excellent
+`Unicode documentation`_.
+
+Another important thing is how Jinja2 is handling string literals in
+templates. A naive implementation would be using Unicode strings for
+all string literals but it turned out in the past that this is problematic
+as some libraries are typechecking against `str` explicitly. For example
+`datetime.strftime` does not accept Unicode arguments. To not break it
+completely Jinja2 is returning `str` for strings that fit into ASCII and
+for everything else `unicode`:
+
+>>> m = Template(u"{% set a, b = 'foo', 'föö' %}").module
+>>> m.a
+'foo'
+>>> m.b
+u'f\xf6\xf6'
+
+
+.. _Unicode documentation: http://docs.python.org/dev/howto/unicode.html
+
+High Level API
+--------------
+
+The high-level API is the API you will use in the application to load and
+render Jinja2 templates. The :ref:`low-level-api` on the other side is only
+useful if you want to dig deeper into Jinja2 or :ref:`develop extensions
+<jinja-extensions>`.
+
+.. autoclass:: Environment([options])
+ :members: from_string, get_template, select_template,
+ get_or_select_template, join_path, extend, compile_expression
+
+ .. attribute:: shared
+
+ If a template was created by using the :class:`Template` constructor
+ an environment is created automatically. These environments are
+ created as shared environments which means that multiple templates
+ may have the same anonymous environment. For all shared environments
+ this attribute is `True`, else `False`.
+
+ .. attribute:: sandboxed
+
+ If the environment is sandboxed this attribute is `True`. For the
+ sandbox mode have a look at the documentation for the
+ :class:`~jinja2.sandbox.SandboxedEnvironment`.
+
+ .. attribute:: filters
+
+ A dict of filters for this environment. As long as no template was
+ loaded it's safe to add new filters or remove old. For custom filters
+ see :ref:`writing-filters`. For valid filter names have a look at
+ :ref:`identifier-naming`.
+
+ .. attribute:: tests
+
+ A dict of test functions for this environment. As long as no
+ template was loaded it's safe to modify this dict. For custom tests
+ see :ref:`writing-tests`. For valid test names have a look at
+ :ref:`identifier-naming`.
+
+ .. attribute:: globals
+
+ A dict of global variables. These variables are always available
+ in a template. As long as no template was loaded it's safe
+ to modify this dict. For more details see :ref:`global-namespace`.
+ For valid object names have a look at :ref:`identifier-naming`.
+
+ .. automethod:: overlay([options])
+
+ .. method:: undefined([hint, obj, name, exc])
+
+ Creates a new :class:`Undefined` object for `name`. This is useful
+ for filters or functions that may return undefined objects for
+ some operations. All parameters except of `hint` should be provided
+ as keyword parameters for better readability. The `hint` is used as
+ error message for the exception if provided, otherwise the error
+ message will be generated from `obj` and `name` automatically. The exception
+ provided as `exc` is raised if something with the generated undefined
+ object is done that the undefined object does not allow. The default
+ exception is :exc:`UndefinedError`. If a `hint` is provided the
+ `name` may be ommited.
+
+ The most common way to create an undefined object is by providing
+ a name only::
+
+ return environment.undefined(name='some_name')
+
+ This means that the name `some_name` is not defined. If the name
+ was from an attribute of an object it makes sense to tell the
+ undefined object the holder object to improve the error message::
+
+ if not hasattr(obj, 'attr'):
+ return environment.undefined(obj=obj, name='attr')
+
+ For a more complex example you can provide a hint. For example
+ the :func:`first` filter creates an undefined object that way::
+
+ return environment.undefined('no first item, sequence was empty')
+
+ If it the `name` or `obj` is known (for example because an attribute
+ was accessed) it shold be passed to the undefined object, even if
+ a custom `hint` is provided. This gives undefined objects the
+ possibility to enhance the error message.
+
+.. autoclass:: Template
+ :members: module, make_module
+
+ .. attribute:: globals
+
+ The dict with the globals of that template. It's unsafe to modify
+ this dict as it may be shared with other templates or the environment
+ that loaded the template.
+
+ .. attribute:: name
+
+ The loading name of the template. If the template was loaded from a
+ string this is `None`.
+
+ .. attribute:: filename
+
+ The filename of the template on the file system if it was loaded from
+ there. Otherwise this is `None`.
+
+ .. automethod:: render([context])
+
+ .. automethod:: generate([context])
+
+ .. automethod:: stream([context])
+
+
+.. autoclass:: jinja2.environment.TemplateStream()
+ :members: disable_buffering, enable_buffering, dump
+
+
+Autoescaping
+------------
+
+.. versionadded:: 2.4
+
+As of Jinja 2.4 the preferred way to do autoescaping is to enable the
+:ref:`autoescape-extension` and to configure a sensible default for
+autoescaping. This makes it possible to enable and disable autoescaping
+on a per-template basis (HTML versus text for instance).
+
+Here a recommended setup that enables autoescaping for templates ending
+in ``'.html'``, ``'.htm'`` and ``'.xml'`` and disabling it by default
+for all other extensions::
+
+ def guess_autoescape(template_name):
+ if template_name is None or '.' not in template_name:
+ return False
+ ext = template_name.rsplit('.', 1)[1]
+ return ext in ('html', 'htm', 'xml')
+
+ env = Environment(autoescape=guess_autoescape,
+ loader=PackageLoader('mypackage'),
+ extensions=['jinja2.ext.autoescape'])
+
+When implementing a guessing autoescape function, make sure you also
+accept `None` as valid template name. This will be passed when generating
+templates from strings.
+
+Inside the templates the behaviour can be temporarily changed by using
+the `autoescape` block (see :ref:`autoescape-overrides`).
+
+
+.. _identifier-naming:
+
+Notes on Identifiers
+--------------------
+
+Jinja2 uses the regular Python 2.x naming rules. Valid identifiers have to
+match ``[a-zA-Z_][a-zA-Z0-9_]*``. As a matter of fact non ASCII characters
+are currently not allowed. This limitation will probably go away as soon as
+unicode identifiers are fully specified for Python 3.
+
+Filters and tests are looked up in separate namespaces and have slightly
+modified identifier syntax. Filters and tests may contain dots to group
+filters and tests by topic. For example it's perfectly valid to add a
+function into the filter dict and call it `to.unicode`. The regular
+expression for filter and test identifiers is
+``[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*```.
+
+
+Undefined Types
+---------------
+
+These classes can be used as undefined types. The :class:`Environment`
+constructor takes an `undefined` parameter that can be one of those classes
+or a custom subclass of :class:`Undefined`. Whenever the template engine is
+unable to look up a name or access an attribute one of those objects is
+created and returned. Some operations on undefined values are then allowed,
+others fail.
+
+The closest to regular Python behavior is the `StrictUndefined` which
+disallows all operations beside testing if it's an undefined object.
+
+.. autoclass:: jinja2.Undefined()
+
+ .. attribute:: _undefined_hint
+
+ Either `None` or an unicode string with the error message for
+ the undefined object.
+
+ .. attribute:: _undefined_obj
+
+ Either `None` or the owner object that caused the undefined object
+ to be created (for example because an attribute does not exist).
+
+ .. attribute:: _undefined_name
+
+ The name for the undefined variable / attribute or just `None`
+ if no such information exists.
+
+ .. attribute:: _undefined_exception
+
+ The exception that the undefined object wants to raise. This
+ is usually one of :exc:`UndefinedError` or :exc:`SecurityError`.
+
+ .. method:: _fail_with_undefined_error(\*args, \**kwargs)
+
+ When called with any arguments this method raises
+ :attr:`_undefined_exception` with an error message generated
+ from the undefined hints stored on the undefined object.
+
+.. autoclass:: jinja2.DebugUndefined()
+
+.. autoclass:: jinja2.StrictUndefined()
+
+Undefined objects are created by calling :attr:`undefined`.
+
+.. admonition:: Implementation
+
+ :class:`Undefined` objects are implemented by overriding the special
+ `__underscore__` methods. For example the default :class:`Undefined`
+ class implements `__unicode__` in a way that it returns an empty
+ string, however `__int__` and others still fail with an exception. To
+ allow conversion to int by returning ``0`` you can implement your own::
+
+ class NullUndefined(Undefined):
+ def __int__(self):
+ return 0
+ def __float__(self):
+ return 0.0
+
+ To disallow a method, just override it and raise
+ :attr:`~Undefined._undefined_exception`. Because this is a very common
+ idom in undefined objects there is the helper method
+ :meth:`~Undefined._fail_with_undefined_error` that does the error raising
+ automatically. Here a class that works like the regular :class:`Undefined`
+ but chokes on iteration::
+
+ class NonIterableUndefined(Undefined):
+ __iter__ = Undefined._fail_with_undefined_error
+
+
+The Context
+-----------
+
+.. autoclass:: jinja2.runtime.Context()
+ :members: resolve, get_exported, get_all
+
+ .. attribute:: parent
+
+ A dict of read only, global variables the template looks up. These
+ can either come from another :class:`Context`, from the
+ :attr:`Environment.globals` or :attr:`Template.globals` or points
+ to a dict created by combining the globals with the variables
+ passed to the render function. It must not be altered.
+
+ .. attribute:: vars
+
+ The template local variables. This list contains environment and
+ context functions from the :attr:`parent` scope as well as local
+ modifications and exported variables from the template. The template
+ will modify this dict during template evaluation but filters and
+ context functions are not allowed to modify it.
+
+ .. attribute:: environment
+
+ The environment that loaded the template.
+
+ .. attribute:: exported_vars
+
+ This set contains all the names the template exports. The values for
+ the names are in the :attr:`vars` dict. In order to get a copy of the
+ exported variables as dict, :meth:`get_exported` can be used.
+
+ .. attribute:: name
+
+ The load name of the template owning this context.
+
+ .. attribute:: blocks
+
+ A dict with the current mapping of blocks in the template. The keys
+ in this dict are the names of the blocks, and the values a list of
+ blocks registered. The last item in each list is the current active
+ block (latest in the inheritance chain).
+
+ .. attribute:: eval_ctx
+
+ The current :ref:`eval-context`.
+
+ .. automethod:: jinja2.runtime.Context.call(callable, \*args, \**kwargs)
+
+
+.. admonition:: Implementation
+
+ Context is immutable for the same reason Python's frame locals are
+ immutable inside functions. Both Jinja2 and Python are not using the
+ context / frame locals as data storage for variables but only as primary
+ data source.
+
+ When a template accesses a variable the template does not define, Jinja2
+ looks up the variable in the context, after that the variable is treated
+ as if it was defined in the template.
+
+
+.. _loaders:
+
+Loaders
+-------
+
+Loaders are responsible for loading templates from a resource such as the
+file system. The environment will keep the compiled modules in memory like
+Python's `sys.modules`. Unlike `sys.modules` however this cache is limited in
+size by default and templates are automatically reloaded.
+All loaders are subclasses of :class:`BaseLoader`. If you want to create your
+own loader, subclass :class:`BaseLoader` and override `get_source`.
+
+.. autoclass:: jinja2.BaseLoader
+ :members: get_source, load
+
+Here a list of the builtin loaders Jinja2 provides:
+
+.. autoclass:: jinja2.FileSystemLoader
+
+.. autoclass:: jinja2.PackageLoader
+
+.. autoclass:: jinja2.DictLoader
+
+.. autoclass:: jinja2.FunctionLoader
+
+.. autoclass:: jinja2.PrefixLoader
+
+.. autoclass:: jinja2.ChoiceLoader
+
+
+.. _bytecode-cache:
+
+Bytecode Cache
+--------------
+
+Jinja 2.1 and higher support external bytecode caching. Bytecode caches make
+it possible to store the generated bytecode on the file system or a different
+location to avoid parsing the templates on first use.
+
+This is especially useful if you have a web application that is initialized on
+the first request and Jinja compiles many templates at once which slows down
+the application.
+
+To use a bytecode cache, instanciate it and pass it to the :class:`Environment`.
+
+.. autoclass:: jinja2.BytecodeCache
+ :members: load_bytecode, dump_bytecode, clear
+
+.. autoclass:: jinja2.bccache.Bucket
+ :members: write_bytecode, load_bytecode, bytecode_from_string,
+ bytecode_to_string, reset
+
+ .. attribute:: environment
+
+ The :class:`Environment` that created the bucket.
+
+ .. attribute:: key
+
+ The unique cache key for this bucket
+
+ .. attribute:: code
+
+ The bytecode if it's loaded, otherwise `None`.
+
+
+Builtin bytecode caches:
+
+.. autoclass:: jinja2.FileSystemBytecodeCache
+
+.. autoclass:: jinja2.MemcachedBytecodeCache
+
+
+Utilities
+---------
+
+These helper functions and classes are useful if you add custom filters or
+functions to a Jinja2 environment.
+
+.. autofunction:: jinja2.environmentfilter
+
+.. autofunction:: jinja2.contextfilter
+
+.. autofunction:: jinja2.evalcontextfilter
+
+.. autofunction:: jinja2.environmentfunction
+
+.. autofunction:: jinja2.contextfunction
+
+.. autofunction:: jinja2.evalcontextfunction
+
+.. function:: escape(s)
+
+ Convert the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in string `s`
+ to HTML-safe sequences. Use this if you need to display text that might
+ contain such characters in HTML. This function will not escaped objects
+ that do have an HTML representation such as already escaped data.
+
+ The return value is a :class:`Markup` string.
+
+.. autofunction:: jinja2.clear_caches
+
+.. autofunction:: jinja2.is_undefined
+
+.. autoclass:: jinja2.Markup([string])
+ :members: escape, unescape, striptags
+
+.. admonition:: Note
+
+ The Jinja2 :class:`Markup` class is compatible with at least Pylons and
+ Genshi. It's expected that more template engines and framework will pick
+ up the `__html__` concept soon.
+
+
+Exceptions
+----------
+
+.. autoexception:: jinja2.TemplateError
+
+.. autoexception:: jinja2.UndefinedError
+
+.. autoexception:: jinja2.TemplateNotFound
+
+.. autoexception:: jinja2.TemplatesNotFound
+
+.. autoexception:: jinja2.TemplateSyntaxError
+
+ .. attribute:: message
+
+ The error message as utf-8 bytestring.
+
+ .. attribute:: lineno
+
+ The line number where the error occurred
+
+ .. attribute:: name
+
+ The load name for the template as unicode string.
+
+ .. attribute:: filename
+
+ The filename that loaded the template as bytestring in the encoding
+ of the file system (most likely utf-8 or mbcs on Windows systems).
+
+ The reason why the filename and error message are bytestrings and not
+ unicode strings is that Python 2.x is not using unicode for exceptions
+ and tracebacks as well as the compiler. This will change with Python 3.
+
+.. autoexception:: jinja2.TemplateAssertionError
+
+
+.. _writing-filters:
+
+Custom Filters
+--------------
+
+Custom filters are just regular Python functions that take the left side of
+the filter as first argument and the the arguments passed to the filter as
+extra arguments or keyword arguments.
+
+For example in the filter ``{{ 42|myfilter(23) }}`` the function would be
+called with ``myfilter(42, 23)``. Here for example a simple filter that can
+be applied to datetime objects to format them::
+
+ def datetimeformat(value, format='%H:%M / %d-%m-%Y'):
+ return value.strftime(format)
+
+You can register it on the template environment by updating the
+:attr:`~Environment.filters` dict on the environment::
+
+ environment.filters['datetimeformat'] = datetimeformat
+
+Inside the template it can then be used as follows:
+
+.. sourcecode:: jinja
+
+ written on: {{ article.pub_date|datetimeformat }}
+ publication date: {{ article.pub_date|datetimeformat('%d-%m-%Y') }}
+
+Filters can also be passed the current template context or environment. This
+is useful if a filter wants to return an undefined value or check the current
+:attr:`~Environment.autoescape` setting. For this purpose three decorators
+exist: :func:`environmentfilter`, :func:`contextfilter` and
+:func:`evalcontextfilter`.
+
+Here a small example filter that breaks a text into HTML line breaks and
+paragraphs and marks the return value as safe HTML string if autoescaping is
+enabled::
+
+ import re
+ from jinja2 import environmentfilter, Markup, escape
+
+ _paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}')
+
+ @evalcontextfilter
+ def nl2br(eval_ctx, value):
+ result = u'\n\n'.join(u'<p>%s</p>' % p.replace('\n', '<br>\n')
+ for p in _paragraph_re.split(escape(value)))
+ if eval_ctx.autoescape:
+ result = Markup(result)
+ return result
+
+Context filters work the same just that the first argument is the current
+active :class:`Context` rather then the environment.
+
+
+.. _eval-context:
+
+Evaluation Context
+------------------
+
+The evaluation context (short eval context or eval ctx) is a new object
+introducted in Jinja 2.4 that makes it possible to activate and deactivate
+compiled features at runtime.
+
+Currently it is only used to enable and disable the automatic escaping but
+can be used for extensions as well.
+
+In previous Jinja versions filters and functions were marked as
+environment callables in order to check for the autoescape status from the
+environment. In new versions it's encouraged to check the setting from the
+evaluation context instead.
+
+Previous versions::
+
+ @environmentfilter
+ def filter(env, value):
+ result = do_something(value)
+ if env.autoescape:
+ result = Markup(result)
+ return result
+
+In new versions you can either use a :func:`contextfilter` and access the
+evaluation context from the actual context, or use a
+:func:`evalcontextfilter` which directly passes the evaluation context to
+the function::
+
+ @contextfilter
+ def filter(context, value):
+ result = do_something(value)
+ if context.eval_ctx.autoescape:
+ result = Markup(result)
+ return result
+
+ @evalcontextfilter
+ def filter(eval_ctx, value):
+ result = do_something(value)
+ if eval_ctx.autoescape:
+ result = Markup(result)
+ return result
+
+The evaluation context must not be modified at runtime. Modifications
+must only happen with a :class:`nodes.EvalContextModifier` and
+:class:`nodes.ScopedEvalContextModifier` from an extension, not on the
+eval context object itself.
+
+.. autoclass:: jinja2.nodes.EvalContext
+
+ .. attribute:: autoescape
+
+ `True` or `False` depending on if autoescaping is active or not.
+
+ .. attribute:: volatile
+
+ `True` if the compiler cannot evaluate some expressions at compile
+ time. At runtime this should always be `False`.
+
+
+.. _writing-tests:
+
+Custom Tests
+------------
+
+Tests work like filters just that there is no way for a test to get access
+to the environment or context and that they can't be chained. The return
+value of a test should be `True` or `False`. The purpose of a test is to
+give the template designers the possibility to perform type and conformability
+checks.
+
+Here a simple test that checks if a variable is a prime number::
+
+ import math
+
+ def is_prime(n):
+ if n == 2:
+ return True
+ for i in xrange(2, int(math.ceil(math.sqrt(n))) + 1):
+ if n % i == 0:
+ return False
+ return True
+
+
+You can register it on the template environment by updating the
+:attr:`~Environment.tests` dict on the environment::
+
+ environment.tests['prime'] = is_prime
+
+A template designer can then use the test like this:
+
+.. sourcecode:: jinja
+
+ {% if 42 is prime %}
+ 42 is a prime number
+ {% else %}
+ 42 is not a prime number
+ {% endif %}
+
+
+.. _global-namespace:
+
+The Global Namespace
+--------------------
+
+Variables stored in the :attr:`Environment.globals` dict are special as they
+are available for imported templates too, even if they are imported without
+context. This is the place where you can put variables and functions
+that should be available all the time. Additionally :attr:`Template.globals`
+exist that are variables available to a specific template that are available
+to all :meth:`~Template.render` calls.
+
+
+.. _low-level-api:
+
+Low Level API
+-------------
+
+The low level API exposes functionality that can be useful to understand some
+implementation details, debugging purposes or advanced :ref:`extension
+<jinja-extensions>` techniques. Unless you know exactly what you are doing we
+don't recommend using any of those.
+
+.. automethod:: Environment.lex
+
+.. automethod:: Environment.parse
+
+.. automethod:: Environment.preprocess
+
+.. automethod:: Template.new_context
+
+.. method:: Template.root_render_func(context)
+
+ This is the low level render function. It's passed a :class:`Context`
+ that has to be created by :meth:`new_context` of the same template or
+ a compatible template. This render function is generated by the
+ compiler from the template code and returns a generator that yields
+ unicode strings.
+
+ If an exception in the template code happens the template engine will
+ not rewrite the exception but pass through the original one. As a
+ matter of fact this function should only be called from within a
+ :meth:`render` / :meth:`generate` / :meth:`stream` call.
+
+.. attribute:: Template.blocks
+
+ A dict of block render functions. Each of these functions works exactly
+ like the :meth:`root_render_func` with the same limitations.
+
+.. attribute:: Template.is_up_to_date
+
+ This attribute is `False` if there is a newer version of the template
+ available, otherwise `True`.
+
+.. admonition:: Note
+
+ The low-level API is fragile. Future Jinja2 versions will try not to
+ change it in a backwards incompatible way but modifications in the Jinja2
+ core may shine through. For example if Jinja2 introduces a new AST node
+ in later versions that may be returned by :meth:`~Environment.parse`.
+
+The Meta API
+------------
+
+.. versionadded:: 2.2
+
+The meta API returns some information about abstract syntax trees that
+could help applications to implement more advanced template concepts. All
+the functions of the meta API operate on an abstract syntax tree as
+returned by the :meth:`Environment.parse` method.
+
+.. autofunction:: jinja2.meta.find_undeclared_variables
+
+.. autofunction:: jinja2.meta.find_referenced_templates
diff --git a/slider-agent/src/main/python/jinja2/docs/cache_extension.py b/slider-agent/src/main/python/jinja2/docs/cache_extension.py
new file mode 100644
index 0000000..8fdefb5
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/cache_extension.py
@@ -0,0 +1,56 @@
+from jinja2 import nodes
+from jinja2.ext import Extension
+
+
+class FragmentCacheExtension(Extension):
+ # a set of names that trigger the extension.
+ tags = set(['cache'])
+
+ def __init__(self, environment):
+ super(FragmentCacheExtension, self).__init__(environment)
+
+ # add the defaults to the environment
+ environment.extend(
+ fragment_cache_prefix='',
+ fragment_cache=None
+ )
+
+ def parse(self, parser):
+ # the first token is the token that started the tag. In our case
+ # we only listen to ``'cache'`` so this will be a name token with
+ # `cache` as value. We get the line number so that we can give
+ # that line number to the nodes we create by hand.
+ lineno = parser.stream.next().lineno
+
+ # now we parse a single expression that is used as cache key.
+ args = [parser.parse_expression()]
+
+ # if there is a comma, the user provided a timeout. If not use
+ # None as second parameter.
+ if parser.stream.skip_if('comma'):
+ args.append(parser.parse_expression())
+ else:
+ args.append(nodes.Const(None))
+
+ # now we parse the body of the cache block up to `endcache` and
+ # drop the needle (which would always be `endcache` in that case)
+ body = parser.parse_statements(['name:endcache'], drop_needle=True)
+
+ # now return a `CallBlock` node that calls our _cache_support
+ # helper method on this extension.
+ return nodes.CallBlock(self.call_method('_cache_support', args),
+ [], [], body).set_lineno(lineno)
+
+ def _cache_support(self, name, timeout, caller):
+ """Helper callback."""
+ key = self.environment.fragment_cache_prefix + name
+
+ # try to load the block from the cache
+ # if there is no fragment in the cache, render it and store
+ # it in the cache.
+ rv = self.environment.fragment_cache.get(key)
+ if rv is not None:
+ return rv
+ rv = caller()
+ self.environment.fragment_cache.add(key, rv, timeout)
+ return rv
diff --git a/slider-agent/src/main/python/jinja2/docs/changelog.rst b/slider-agent/src/main/python/jinja2/docs/changelog.rst
new file mode 100644
index 0000000..9f11484
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/changelog.rst
@@ -0,0 +1,3 @@
+.. module:: jinja2
+
+.. include:: ../CHANGES
diff --git a/slider-agent/src/main/python/jinja2/docs/conf.py b/slider-agent/src/main/python/jinja2/docs/conf.py
new file mode 100644
index 0000000..ba90c49
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/conf.py
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+#
+# Jinja2 documentation build configuration file, created by
+# sphinx-quickstart on Sun Apr 27 21:42:41 2008.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# The contents of this file are pickled, so don't put values in the namespace
+# that aren't pickleable (module imports are okay, they're removed automatically).
+#
+# All configuration values have a default value; values that are commented out
+# serve to show the default value.
+
+import sys, os
+
+# If your extensions are in another directory, add it here. If the directory
+# is relative to the documentation root, use os.path.abspath to make it
+# absolute, like shown here.
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+# General configuration
+# ---------------------
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'jinjaext']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General substitutions.
+project = 'Jinja2'
+copyright = '2008, Armin Ronacher'
+
+# The default replacements for |version| and |release|, also used in various
+# other places throughout the built documents.
+#
+# The short X.Y version.
+version = '2.0'
+# The full version, including alpha/beta/rc tags.
+release = '2.0'
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+today_fmt = '%B %d, %Y'
+
+# List of documents that shouldn't be included in the build.
+#unused_docs = []
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'jinjaext.JinjaStyle'
+
+
+# Options for HTML output
+# -----------------------
+
+# The style sheet to use for HTML and HTML Help pages. A file of that name
+# must exist either in Sphinx' static/ path, or in one of the custom paths
+# given in html_static_path.
+html_style = 'style.css'
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# no modindex
+html_use_modindex = False
+
+# If true, the reST sources are included in the HTML build as _sources/<name>.
+#html_copy_source = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.
+#html_use_opensearch = False
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Jinja2doc'
+
+
+# Options for LaTeX output
+# ------------------------
+
+# The paper size ('letter' or 'a4').
+latex_paper_size = 'a4'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, document class [howto/manual]).
+latex_documents = [
+ ('index', 'Jinja2.tex', 'Jinja2 Documentation', 'Armin Ronacher', 'manual', 'toctree_only'),
+]
+
+# Additional stuff for the LaTeX preamble.
+latex_preamble = '''
+\usepackage{palatino}
+\definecolor{TitleColor}{rgb}{0.7,0,0}
+\definecolor{InnerLinkColor}{rgb}{0.7,0,0}
+\definecolor{OuterLinkColor}{rgb}{0.8,0,0}
+\definecolor{VerbatimColor}{rgb}{0.985,0.985,0.985}
+\definecolor{VerbatimBorderColor}{rgb}{0.8,0.8,0.8}
+'''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+latex_use_modindex = False
diff --git a/slider-agent/src/main/python/jinja2/docs/extensions.rst b/slider-agent/src/main/python/jinja2/docs/extensions.rst
new file mode 100644
index 0000000..c6b6ec9
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/extensions.rst
@@ -0,0 +1,347 @@
+.. _jinja-extensions:
+
+Extensions
+==========
+
+Jinja2 supports extensions that can add extra filters, tests, globals or even
+extend the parser. The main motivation of extensions is it to move often used
+code into a reusable class like adding support for internationalization.
+
+
+Adding Extensions
+-----------------
+
+Extensions are added to the Jinja2 environment at creation time. Once the
+environment is created additional extensions cannot be added. To add an
+extension pass a list of extension classes or import paths to the
+`environment` parameter of the :class:`Environment` constructor. The following
+example creates a Jinja2 environment with the i18n extension loaded::
+
+ jinja_env = Environment(extensions=['jinja2.ext.i18n'])
+
+
+.. _i18n-extension:
+
+i18n Extension
+--------------
+
+**Import name:** `jinja2.ext.i18n`
+
+Jinja2 currently comes with one extension, the i18n extension. It can be
+used in combination with `gettext`_ or `babel`_. If the i18n extension is
+enabled Jinja2 provides a `trans` statement that marks the wrapped string as
+translatable and calls `gettext`.
+
+After enabling dummy `_` function that forwards calls to `gettext` is added
+to the environment globals. An internationalized application then has to
+provide at least an `gettext` and optoinally a `ngettext` function into the
+namespace. Either globally or for each rendering.
+
+Environment Methods
+~~~~~~~~~~~~~~~~~~~
+
+After enabling of the extension the environment provides the following
+additional methods:
+
+.. method:: jinja2.Environment.install_gettext_translations(translations, newstyle=False)
+
+ Installs a translation globally for that environment. The tranlations
+ object provided must implement at least `ugettext` and `ungettext`.
+ The `gettext.NullTranslations` and `gettext.GNUTranslations` classes
+ as well as `Babel`_\s `Translations` class are supported.
+
+ .. versionchanged:: 2.5 newstyle gettext added
+
+.. method:: jinja2.Environment.install_null_translations(newstyle=False)
+
+ Install dummy gettext functions. This is useful if you want to prepare
+ the application for internationalization but don't want to implement the
+ full internationalization system yet.
+
+ .. versionchanged:: 2.5 newstyle gettext added
+
+.. method:: jinja2.Environment.install_gettext_callables(gettext, ngettext, newstyle=False)
+
+ Installs the given `gettext` and `ngettext` callables into the
+ environment as globals. They are supposed to behave exactly like the
+ standard library's :func:`gettext.ugettext` and
+ :func:`gettext.ungettext` functions.
+
+ If `newstyle` is activated, the callables are wrapped to work like
+ newstyle callables. See :ref:`newstyle-gettext` for more information.
+
+ .. versionadded:: 2.5
+
+.. method:: jinja2.Environment.uninstall_gettext_translations()
+
+ Uninstall the translations again.
+
+.. method:: jinja2.Environment.extract_translations(source)
+
+ Extract localizable strings from the given template node or source.
+
+ For every string found this function yields a ``(lineno, function,
+ message)`` tuple, where:
+
+ * `lineno` is the number of the line on which the string was found,
+ * `function` is the name of the `gettext` function used (if the
+ string was extracted from embedded Python code), and
+ * `message` is the string itself (a `unicode` object, or a tuple
+ of `unicode` objects for functions with multiple string arguments).
+
+ If `Babel`_ is installed :ref:`the babel integration <babel-integration>`
+ can be used to extract strings for babel.
+
+For a web application that is available in multiple languages but gives all
+the users the same language (for example a multilingual forum software
+installed for a French community) may load the translations once and add the
+translation methods to the environment at environment generation time::
+
+ translations = get_gettext_translations()
+ env = Environment(extensions=['jinja2.ext.i18n'])
+ env.install_gettext_translations(translations)
+
+The `get_gettext_translations` function would return the translator for the
+current configuration. (For example by using `gettext.find`)
+
+The usage of the `i18n` extension for template designers is covered as part
+:ref:`of the template documentation <i18n-in-templates>`.
+
+.. _gettext: http://docs.python.org/dev/library/gettext
+.. _Babel: http://babel.edgewall.org/
+
+.. _newstyle-gettext:
+
+Newstyle Gettext
+~~~~~~~~~~~~~~~~
+
+.. versionadded:: 2.5
+
+Starting with version 2.5 you can use newstyle gettext calls. These are
+inspired by trac's internal gettext functions and are fully supported by
+the babel extraction tool. They might not work as expected by other
+extraction tools in case you are not using Babel's.
+
+What's the big difference between standard and newstyle gettext calls? In
+general they are less to type and less error prone. Also if they are used
+in an autoescaping environment they better support automatic escaping.
+Here some common differences between old and new calls:
+
+standard gettext:
+
+.. sourcecode:: html+jinja
+
+ {{ gettext('Hello World!') }}
+ {{ gettext('Hello %(name)s!')|format(name='World') }}
+ {{ ngettext('%(num)d apple', '%(num)d apples', apples|count)|format(
+ num=apples|count
+ )}}
+
+newstyle gettext looks like this instead:
+
+.. sourcecode:: html+jinja
+
+ {{ gettext('Hello World!') }}
+ {{ gettext('Hello %(name)s!', name='World') }}
+ {{ ngettext('%(num)d apple', '%(num)d apples', apples|count) }}
+
+The advantages of newstyle gettext is that you have less to type and that
+named placeholders become mandatory. The latter sounds like a
+disadvantage but solves a lot of troubles translators are often facing
+when they are unable to switch the positions of two placeholder. With
+newstyle gettext, all format strings look the same.
+
+Furthermore with newstyle gettext, string formatting is also used if no
+placeholders are used which makes all strings behave exactly the same.
+Last but not least are newstyle gettext calls able to properly mark
+strings for autoescaping which solves lots of escaping related issues many
+templates are experiencing over time when using autoescaping.
+
+Expression Statement
+--------------------
+
+**Import name:** `jinja2.ext.do`
+
+The "do" aka expression-statement extension adds a simple `do` tag to the
+template engine that works like a variable expression but ignores the
+return value.
+
+.. _loopcontrols-extension:
+
+Loop Controls
+-------------
+
+**Import name:** `jinja2.ext.loopcontrols`
+
+This extension adds support for `break` and `continue` in loops. After
+enabling Jinja2 provides those two keywords which work exactly like in
+Python.
+
+.. _with-extension:
+
+With Statement
+--------------
+
+**Import name:** `jinja2.ext.with_`
+
+.. versionadded:: 2.3
+
+This extension adds support for the with keyword. Using this keyword it
+is possible to enforce a nested scope in a template. Variables can be
+declared directly in the opening block of the with statement or using a
+standard `set` statement directly within.
+
+.. _autoescape-extension:
+
+Autoescape Extension
+--------------------
+
+**Import name:** `jinja2.ext.autoescape`
+
+.. versionadded:: 2.4
+
+The autoescape extension allows you to toggle the autoescape feature from
+within the template. If the environment's :attr:`~Environment.autoescape`
+setting is set to `False` it can be activated, if it's `True` it can be
+deactivated. The setting overriding is scoped.
+
+
+.. _writing-extensions:
+
+Writing Extensions
+------------------
+
+.. module:: jinja2.ext
+
+By writing extensions you can add custom tags to Jinja2. This is a non trival
+task and usually not needed as the default tags and expressions cover all
+common use cases. The i18n extension is a good example of why extensions are
+useful, another one would be fragment caching.
+
+When writing extensions you have to keep in mind that you are working with the
+Jinja2 template compiler which does not validate the node tree you are possing
+to it. If the AST is malformed you will get all kinds of compiler or runtime
+errors that are horrible to debug. Always make sure you are using the nodes
+you create correctly. The API documentation below shows which nodes exist and
+how to use them.
+
+Example Extension
+~~~~~~~~~~~~~~~~~
+
+The following example implements a `cache` tag for Jinja2 by using the
+`Werkzeug`_ caching contrib module:
+
+.. literalinclude:: cache_extension.py
+ :language: python
+
+And here is how you use it in an environment::
+
+ from jinja2 import Environment
+ from werkzeug.contrib.cache import SimpleCache
+
+ env = Environment(extensions=[FragmentCacheExtension])
+ env.fragment_cache = SimpleCache()
+
+Inside the template it's then possible to mark blocks as cacheable. The
+following example caches a sidebar for 300 seconds:
+
+.. sourcecode:: html+jinja
+
+ {% cache 'sidebar', 300 %}
+ <div class="sidebar">
+ ...
+ </div>
+ {% endcache %}
+
+.. _Werkzeug: http://werkzeug.pocoo.org/
+
+Extension API
+~~~~~~~~~~~~~
+
+Extensions always have to extend the :class:`jinja2.ext.Extension` class:
+
+.. autoclass:: Extension
+ :members: preprocess, filter_stream, parse, attr, call_method
+
+ .. attribute:: identifier
+
+ The identifier of the extension. This is always the true import name
+ of the extension class and must not be changed.
+
+ .. attribute:: tags
+
+ If the extension implements custom tags this is a set of tag names
+ the extension is listening for.
+
+Parser API
+~~~~~~~~~~
+
+The parser passed to :meth:`Extension.parse` provides ways to parse
+expressions of different types. The following methods may be used by
+extensions:
+
+.. autoclass:: jinja2.parser.Parser
+ :members: parse_expression, parse_tuple, parse_assign_target,
+ parse_statements, free_identifier, fail
+
+ .. attribute:: filename
+
+ The filename of the template the parser processes. This is **not**
+ the load name of the template. For the load name see :attr:`name`.
+ For templates that were not loaded form the file system this is
+ `None`.
+
+ .. attribute:: name
+
+ The load name of the template.
+
+ .. attribute:: stream
+
+ The current :class:`~jinja2.lexer.TokenStream`
+
+.. autoclass:: jinja2.lexer.TokenStream
+ :members: push, look, eos, skip, next, next_if, skip_if, expect
+
+ .. attribute:: current
+
+ The current :class:`~jinja2.lexer.Token`.
+
+.. autoclass:: jinja2.lexer.Token
+ :members: test, test_any
+
+ .. attribute:: lineno
+
+ The line number of the token
+
+ .. attribute:: type
+
+ The type of the token. This string is interned so you may compare
+ it with arbitrary strings using the `is` operator.
+
+ .. attribute:: value
+
+ The value of the token.
+
+There is also a utility function in the lexer module that can count newline
+characters in strings:
+
+.. autofunction:: jinja2.lexer.count_newlines
+
+AST
+~~~
+
+The AST (Abstract Syntax Tree) is used to represent a template after parsing.
+It's build of nodes that the compiler then converts into executable Python
+code objects. Extensions that provide custom statements can return nodes to
+execute custom Python code.
+
+The list below describes all nodes that are currently available. The AST may
+change between Jinja2 versions but will stay backwards compatible.
+
+For more information have a look at the repr of :meth:`jinja2.Environment.parse`.
+
+.. module:: jinja2.nodes
+
+.. jinjanodes::
+
+.. autoexception:: Impossible
diff --git a/slider-agent/src/main/python/jinja2/docs/faq.rst b/slider-agent/src/main/python/jinja2/docs/faq.rst
new file mode 100644
index 0000000..89186b1
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/faq.rst
@@ -0,0 +1,191 @@
+Frequently Asked Questions
+==========================
+
+This page answers some of the often asked questions about Jinja.
+
+.. highlight:: html+jinja
+
+Why is it called Jinja?
+-----------------------
+
+The name Jinja was chosen because it's the name of a Japanese temple and
+temple and template share a similar pronunciation. It is not named after
+the capital city of Uganda.
+
+How fast is it?
+---------------
+
+We really hate benchmarks especially since they don't reflect much. The
+performance of a template depends on many factors and you would have to
+benchmark different engines in different situations. The benchmarks from the
+testsuite show that Jinja2 has a similar performance to `Mako`_ and is between
+10 and 20 times faster than Django's template engine or Genshi. These numbers
+should be taken with tons of salt as the benchmarks that took these numbers
+only test a few performance related situations such as looping. Generally
+speaking the performance of a template engine doesn't matter much as the
+usual bottleneck in a web application is either the database or the application
+code.
+
+.. _Mako: http://www.makotemplates.org/
+
+How Compatible is Jinja2 with Django?
+-------------------------------------
+
+The default syntax of Jinja2 matches Django syntax in many ways. However
+this similarity doesn't mean that you can use a Django template unmodified
+in Jinja2. For example filter arguments use a function call syntax rather
+than a colon to separate filter name and arguments. Additionally the
+extension interface in Jinja is fundamentally different from the Django one
+which means that your custom tags won't work any longer.
+
+Generally speaking you will use much less custom extensions as the Jinja
+template system allows you to use a certain subset of Python expressions
+which can replace most Django extensions. For example instead of using
+something like this::
+
+ {% load comments %}
+ {% get_latest_comments 10 as latest_comments %}
+ {% for comment in latest_comments %}
+ ...
+ {% endfor %}
+
+You will most likely provide an object with attributes to retrieve
+comments from the database::
+
+ {% for comment in models.comments.latest(10) %}
+ ...
+ {% endfor %}
+
+Or directly provide the model for quick testing::
+
+ {% for comment in Comment.objects.order_by('-pub_date')[:10] %}
+ ...
+ {% endfor %}
+
+Please keep in mind that even though you may put such things into templates
+it still isn't a good idea. Queries should go into the view code and not
+the template!
+
+Isn't it a terrible idea to put Logic into Templates?
+-----------------------------------------------------
+
+Without a doubt you should try to remove as much logic from templates as
+possible. But templates without any logic mean that you have to do all
+the processing in the code which is boring and stupid. A template engine
+that does that is shipped with Python and called `string.Template`. Comes
+without loops and if conditions and is by far the fastest template engine
+you can get for Python.
+
+So some amount of logic is required in templates to keep everyone happy.
+And Jinja leaves it pretty much to you how much logic you want to put into
+templates. There are some restrictions in what you can do and what not.
+
+Jinja2 neither allows you to put arbitrary Python code into templates nor
+does it allow all Python expressions. The operators are limited to the
+most common ones and more advanced expressions such as list comprehensions
+and generator expressions are not supported. This keeps the template engine
+easier to maintain and templates more readable.
+
+Why is Autoescaping not the Default?
+------------------------------------
+
+There are multiple reasons why automatic escaping is not the default mode
+and also not the recommended one. While automatic escaping of variables
+means that you will less likely have an XSS problem it also causes a huge
+amount of extra processing in the template engine which can cause serious
+performance problems. As Python doesn't provide a way to mark strings as
+unsafe Jinja has to hack around that limitation by providing a custom
+string class (the :class:`Markup` string) that safely interacts with safe
+and unsafe strings.
+
+With explicit escaping however the template engine doesn't have to perform
+any safety checks on variables. Also a human knows not to escape integers
+or strings that may never contain characters one has to escape or already
+HTML markup. For example when iterating over a list over a table of
+integers and floats for a table of statistics the template designer can
+omit the escaping because he knows that integers or floats don't contain
+any unsafe parameters.
+
+Additionally Jinja2 is a general purpose template engine and not only used
+for HTML/XML generation. For example you may generate LaTeX, emails,
+CSS, JavaScript, or configuration files.
+
+Why is the Context immutable?
+-----------------------------
+
+When writing a :func:`contextfunction` or something similar you may have
+noticed that the context tries to stop you from modifying it. If you have
+managed to modify the context by using an internal context API you may
+have noticed that changes in the context don't seem to be visible in the
+template. The reason for this is that Jinja uses the context only as
+primary data source for template variables for performance reasons.
+
+If you want to modify the context write a function that returns a variable
+instead that one can assign to a variable by using set::
+
+ {% set comments = get_latest_comments() %}
+
+What is the speedups module and why is it missing?
+--------------------------------------------------
+
+To achieve a good performance with automatic escaping enabled, the escaping
+function was also implemented in pure C in older Jinja2 releases and used if
+Jinja2 was installed with the speedups module.
+
+Because this feature itself is very useful for non-template engines as
+well it was moved into a separate project on PyPI called `MarkupSafe`_.
+
+Jinja2 no longer ships with a C implementation of it but only the pure
+Python implementation. It will however check if MarkupSafe is available
+and installed, and if it is, use the Markup class from MarkupSafe.
+
+So if you want the speedups, just import MarkupSafe.
+
+.. _MarkupSafe: http://pypi.python.org/pypi/MarkupSafe
+
+My tracebacks look weird. What's happening?
+--------------------------------------------
+
+If the debugsupport module is not compiled and you are using a Python
+installation without ctypes (Python 2.4 without ctypes, Jython or Google's
+AppEngine) Jinja2 is unable to provide correct debugging information and
+the traceback may be incomplete. There is currently no good workaround
+for Jython or the AppEngine as ctypes is unavailable there and it's not
+possible to use the debugsupport extension.
+
+Why is there no Python 2.3 support?
+-----------------------------------
+
+Python 2.3 is missing a lot of features that are used heavily in Jinja2. This
+decision was made as with the upcoming Python 2.6 and 3.0 versions it becomes
+harder to maintain the code for older Python versions. If you really need
+Python 2.3 support you either have to use `Jinja 1`_ or other templating
+engines that still support 2.3.
+
+My Macros are overriden by something
+------------------------------------
+
+In some situations the Jinja scoping appears arbitrary:
+
+layout.tmpl:
+
+.. sourcecode:: jinja
+
+ {% macro foo() %}LAYOUT{% endmacro %}
+ {% block body %}{% endblock %}
+
+child.tmpl:
+
+.. sourcecode:: jinja
+
+ {% extends 'layout.tmpl' %}
+ {% macro foo() %}CHILD{% endmacro %}
+ {% block body %}{{ foo() }}{% endblock %}
+
+This will print ``LAYOUT`` in Jinja2. This is a side effect of having
+the parent template evaluated after the child one. This allows child
+templates passing information to the parent template. To avoid this
+issue rename the macro or variable in the parent template to have an
+uncommon prefix.
+
+.. _Jinja 1: http://jinja.pocoo.org/1/
diff --git a/slider-agent/src/main/python/jinja2/docs/index.rst b/slider-agent/src/main/python/jinja2/docs/index.rst
new file mode 100644
index 0000000..27bee23
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/index.rst
@@ -0,0 +1,27 @@
+Jinja2 Documentation
+====================
+
+This is the documentation for the Jinja2 general purpose templating language.
+Jinja2 is a library for Python 2.4 and onwards that is designed to be flexible,
+fast and secure.
+
+.. toctree::
+ :maxdepth: 2
+
+ intro
+ api
+ sandbox
+ templates
+ extensions
+ integration
+ switching
+ tricks
+
+ faq
+ changelog
+
+If you can't find the information you're looking for, have a look at the
+index of try to find it using the search function:
+
+* :ref:`genindex`
+* :ref:`search`
diff --git a/slider-agent/src/main/python/jinja2/docs/integration.rst b/slider-agent/src/main/python/jinja2/docs/integration.rst
new file mode 100644
index 0000000..1875711
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/integration.rst
@@ -0,0 +1,88 @@
+Integration
+===========
+
+Jinja2 provides some code for integration into other tools such as frameworks,
+the `Babel`_ library or your favourite editor for fancy code highlighting.
+This is a brief description of whats included.
+
+.. _babel-integration:
+
+Babel Integration
+-----------------
+
+Jinja provides support for extracting gettext messages from templates via a
+`Babel`_ extractor entry point called `jinja2.ext.babel_extract`. The Babel
+support is implemented as part of the :ref:`i18n-extension` extension.
+
+Gettext messages extracted from both `trans` tags and code expressions.
+
+To extract gettext messages from templates, the project needs a Jinja2 section
+in its Babel extraction method `mapping file`_:
+
+.. sourcecode:: ini
+
+ [jinja2: **/templates/**.html]
+ encoding = utf-8
+
+The syntax related options of the :class:`Environment` are also available as
+configuration values in the mapping file. For example to tell the extraction
+that templates use ``%`` as `line_statement_prefix` you can use this code:
+
+.. sourcecode:: ini
+
+ [jinja2: **/templates/**.html]
+ encoding = utf-8
+ line_statement_prefix = %
+
+:ref:`jinja-extensions` may also be defined by passing a comma separated list
+of import paths as `extensions` value. The i18n extension is added
+automatically.
+
+.. _mapping file: http://babel.edgewall.org/wiki/Documentation/messages.html#extraction-method-mapping-and-configuration
+
+Pylons
+------
+
+With `Pylons`_ 0.9.7 onwards it's incredible easy to integrate Jinja into a
+Pylons powered application.
+
+The template engine is configured in `config/environment.py`. The configuration
+for Jinja2 looks something like that::
+
+ from jinja2 import Environment, PackageLoader
+ config['pylons.app_globals'].jinja_env = Environment(
+ loader=PackageLoader('yourapplication', 'templates')
+ )
+
+After that you can render Jinja templates by using the `render_jinja` function
+from the `pylons.templating` module.
+
+Additionally it's a good idea to set the Pylons' `c` object into strict mode.
+Per default any attribute to not existing attributes on the `c` object return
+an empty string and not an undefined object. To change this just use this
+snippet and add it into your `config/environment.py`::
+
+ config['pylons.strict_c'] = True
+
+.. _Pylons: http://www.pylonshq.com/
+
+TextMate
+--------
+
+Inside the `ext` folder of Jinja2 there is a bundle for TextMate that supports
+syntax highlighting for Jinja1 and Jinja2 for text based templates as well as
+HTML. It also contains a few often used snippets.
+
+Vim
+---
+
+A syntax plugin for `Vim`_ exists in the Vim-scripts directory as well as the
+ext folder of Jinja2. `The script <http://www.vim.org/scripts/script.php?script_id=1856>`_
+supports Jinja1 and Jinja2. Once installed two file types are available `jinja`
+and `htmljinja`. The first one for text based templates, the latter for HTML
+templates.
+
+Copy the files into your `syntax` folder.
+
+.. _Babel: http://babel.edgewall.org/
+.. _Vim: http://www.vim.org/
diff --git a/slider-agent/src/main/python/jinja2/docs/intro.rst b/slider-agent/src/main/python/jinja2/docs/intro.rst
new file mode 100644
index 0000000..0800615
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/intro.rst
@@ -0,0 +1,168 @@
+Introduction
+============
+
+This is the documentation for the Jinja2 general purpose templating language.
+Jinja2 is a library for Python 2.4 and onwards that is designed to be flexible,
+fast and secure.
+
+If you have any exposure to other text-based template languages, such as Smarty or
+Django, you should feel right at home with Jinja2. It's both designer and
+developer friendly by sticking to Python's principles and adding functionality
+useful for templating environments.
+
+The key-features are...
+
+- ... **configurable syntax**. If you are generating LaTeX or other formats
+ with Jinja2 you can change the delimiters to something that integrates better
+ into the LaTeX markup.
+
+- ... **fast**. While performance is not the primarily target of Jinja2 it's
+ surprisingly fast. The overhead compared to regular Python code was reduced
+ to the very minimum.
+
+- ... **easy to debug**. Jinja2 integrates directly into the python traceback
+ system which allows you to debug Jinja2 templates with regular python
+ debugging helpers.
+
+- ... **secure**. It's possible to evaluate untrusted template code if the
+ optional sandbox is enabled. This allows Jinja2 to be used as templating
+ language for applications where users may modify the template design.
+
+
+Prerequisites
+-------------
+
+Jinja2 needs at least **Python 2.4** to run. Additionally a working C-compiler
+that can create python extensions should be installed for the debugger if you
+are using Python 2.4.
+
+If you don't have a working C-compiler and you are trying to install the source
+release with the debugsupport you will get a compiler error.
+
+.. _ctypes: http://python.net/crew/theller/ctypes/
+
+
+Installation
+------------
+
+You have multiple ways to install Jinja2. If you are unsure what to do, go
+with the Python egg or tarball.
+
+As a Python egg (via easy_install)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can install the most recent Jinja2 version using `easy_install`_ or `pip`_::
+
+ easy_install Jinja2
+ pip install Jinja2
+
+This will install a Jinja2 egg in your Python installation's site-packages
+directory.
+
+(If you are installing from the windows command line omit the `sudo` and make
+sure to run the command as user with administrator rights)
+
+From the tarball release
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+1. Download the most recent tarball from the `download page`_
+2. Unpack the tarball
+3. ``sudo python setup.py install``
+
+Note that you either have to have setuptools or `distribute`_ installed,
+the latter is preferred.
+
+This will install Jinja2 into your Python installation's site-packages directory.
+
+.. _distribute: http://pypi.python.org/pypi/distribute
+
+Installing the development version
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+1. Install `git`_
+2. ``git clone git://github.com/mitsuhiko/jinja2.git``
+3. ``cd jinja2``
+4. ``ln -s jinja2 /usr/lib/python2.X/site-packages``
+
+As an alternative to steps 4 you can also do ``python setup.py develop``
+which will install the package via distribute in development mode. This also
+has the advantage that the C extensions are compiled.
+
+.. _download page: http://pypi.python.org/pypi/Jinja2
+.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools
+.. _easy_install: http://peak.telecommunity.com/DevCenter/EasyInstall
+.. _pip: http://pypi.python.org/pypi/pip
+.. _git: http://git-scm.org/
+
+
+More Speed with MarkupSafe
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As of version 2.5.1 Jinja2 will check for an installed `MarkupSafe`_
+module. If it can find it, it will use the Markup class of that module
+instead of the one that comes with Jinja2. `MarkupSafe` replaces the
+older speedups module that came with Jinja2 and has the advantage that is
+has a better setup script and will automatically attempt to install the C
+version and nicely fall back to a pure Python implementation if that is
+not possible.
+
+The C implementation of MarkupSafe is much faster and recommended when
+using Jinja2 with autoescaping.
+
+.. _MarkupSafe: http://pypi.python.org/pypi/MarkupSafe
+
+
+Enable the debug support Module
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default Jinja2 will not compile the debug support module. Enabling this
+will fail if you don't have the Python headers or a working compiler. This
+is often the case if you are installing Jinja2 from a windows machine.
+
+Because the debug support is only necessary for Python 2.4 you will not
+have to do this unless you run 2.4::
+
+ sudo python setup.py --with-debugsupport install
+
+
+Basic API Usage
+---------------
+
+This section gives you a brief introduction to the Python API for Jinja2
+templates.
+
+The most basic way to create a template and render it is through
+:class:`~jinja2.Template`. This however is not the recommended way to
+work with it if your templates are not loaded from strings but the file
+system or another data source:
+
+>>> from jinja2 import Template
+>>> template = Template('Hello {{ name }}!')
+>>> template.render(name='John Doe')
+u'Hello John Doe!'
+
+By creating an instance of :class:`~jinja2.Template` you get back a new template
+object that provides a method called :meth:`~jinja2.Template.render` which when
+called with a dict or keyword arguments expands the template. The dict
+or keywords arguments passed to the template are the so-called "context"
+of the template.
+
+What you can see here is that Jinja2 is using unicode internally and the
+return value is an unicode string. So make sure that your application is
+indeed using unicode internally.
+
+
+Experimental Python 3 Support
+-----------------------------
+
+Jinja 2.3 brings experimental support for Python 3. It means that all
+unittests pass on the new version, but there might still be small bugs in
+there and behavior might be inconsistent. If you notice any bugs, please
+provide feedback in the `Jinja bug tracker`_.
+
+Also please keep in mind that the documentation is written with Python 2
+in mind, you will have to adapt the shown code examples to Python 3 syntax
+for yourself.
+
+
+.. _Jinja bug tracker: http://github.com/mitsuhiko/jinja2/issues
diff --git a/slider-agent/src/main/python/jinja2/docs/jinjaext.py b/slider-agent/src/main/python/jinja2/docs/jinjaext.py
new file mode 100644
index 0000000..66f4ba1
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/jinjaext.py
@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+"""
+ Jinja Documentation Extensions
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Support for automatically documenting filters and tests.
+
+ :copyright: Copyright 2008 by Armin Ronacher.
+ :license: BSD.
+"""
+import os
+import re
+import inspect
+import jinja2
+from itertools import islice
+from types import BuiltinFunctionType
+from docutils import nodes
+from docutils.statemachine import ViewList
+from sphinx.ext.autodoc import prepare_docstring
+from sphinx.application import TemplateBridge
+from pygments.style import Style
+from pygments.token import Keyword, Name, Comment, String, Error, \
+ Number, Operator, Generic
+from jinja2 import Environment, FileSystemLoader
+
+
+def parse_rst(state, content_offset, doc):
+ node = nodes.section()
+ # hack around title style bookkeeping
+ surrounding_title_styles = state.memo.title_styles
+ surrounding_section_level = state.memo.section_level
+ state.memo.title_styles = []
+ state.memo.section_level = 0
+ state.nested_parse(doc, content_offset, node, match_titles=1)
+ state.memo.title_styles = surrounding_title_styles
+ state.memo.section_level = surrounding_section_level
+ return node.children
+
+
+class JinjaStyle(Style):
+ title = 'Jinja Style'
+ default_style = ""
+ styles = {
+ Comment: 'italic #aaaaaa',
+ Comment.Preproc: 'noitalic #B11414',
+ Comment.Special: 'italic #505050',
+
+ Keyword: 'bold #B80000',
+ Keyword.Type: '#808080',
+
+ Operator.Word: 'bold #B80000',
+
+ Name.Builtin: '#333333',
+ Name.Function: '#333333',
+ Name.Class: 'bold #333333',
+ Name.Namespace: 'bold #333333',
+ Name.Entity: 'bold #363636',
+ Name.Attribute: '#686868',
+ Name.Tag: 'bold #686868',
+ Name.Decorator: '#686868',
+
+ String: '#AA891C',
+ Number: '#444444',
+
+ Generic.Heading: 'bold #000080',
+ Generic.Subheading: 'bold #800080',
+ Generic.Deleted: '#aa0000',
+ Generic.Inserted: '#00aa00',
+ Generic.Error: '#aa0000',
+ Generic.Emph: 'italic',
+ Generic.Strong: 'bold',
+ Generic.Prompt: '#555555',
+ Generic.Output: '#888888',
+ Generic.Traceback: '#aa0000',
+
+ Error: '#F00 bg:#FAA'
+ }
+
+
+_sig_re = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*(\(.*?\))')
+
+
+def format_function(name, aliases, func):
+ lines = inspect.getdoc(func).splitlines()
+ signature = '()'
+ if isinstance(func, BuiltinFunctionType):
+ match = _sig_re.match(lines[0])
+ if match is not None:
+ del lines[:1 + bool(lines and not lines[0])]
+ signature = match.group(1)
+ else:
+ try:
+ argspec = inspect.getargspec(func)
+ if getattr(func, 'environmentfilter', False) or \
+ getattr(func, 'contextfilter', False):
+ del argspec[0][0]
+ signature = inspect.formatargspec(*argspec)
+ except:
+ pass
+ result = ['.. function:: %s%s' % (name, signature), '']
+ result.extend(' ' + line for line in lines)
+ if aliases:
+ result.extend(('', ' :aliases: %s' % ', '.join(
+ '``%s``' % x for x in sorted(aliases))))
+ return result
+
+
+def dump_functions(mapping):
+ def directive(dirname, arguments, options, content, lineno,
+ content_offset, block_text, state, state_machine):
+ reverse_mapping = {}
+ for name, func in mapping.iteritems():
+ reverse_mapping.setdefault(func, []).append(name)
+ filters = []
+ for func, names in reverse_mapping.iteritems():
+ aliases = sorted(names, key=lambda x: len(x))
+ name = aliases.pop()
+ filters.append((name, aliases, func))
+ filters.sort()
+
+ result = ViewList()
+ for name, aliases, func in filters:
+ for item in format_function(name, aliases, func):
+ result.append(item, '<jinjaext>')
+
+ node = nodes.paragraph()
+ state.nested_parse(result, content_offset, node)
+ return node.children
+ return directive
+
+
+from jinja2.defaults import DEFAULT_FILTERS, DEFAULT_TESTS
+jinja_filters = dump_functions(DEFAULT_FILTERS)
+jinja_tests = dump_functions(DEFAULT_TESTS)
+
+
+def jinja_nodes(dirname, arguments, options, content, lineno,
+ content_offset, block_text, state, state_machine):
+ from jinja2.nodes import Node
+ doc = ViewList()
+ def walk(node, indent):
+ p = ' ' * indent
+ sig = ', '.join(node.fields)
+ doc.append(p + '.. autoclass:: %s(%s)' % (node.__name__, sig), '')
+ if node.abstract:
+ members = []
+ for key, name in node.__dict__.iteritems():
+ if not key.startswith('_') and \
+ not hasattr(node.__base__, key) and callable(name):
+ members.append(key)
+ if members:
+ members.sort()
+ doc.append('%s :members: %s' % (p, ', '.join(members)), '')
+ if node.__base__ != object:
+ doc.append('', '')
+ doc.append('%s :Node type: :class:`%s`' %
+ (p, node.__base__.__name__), '')
+ doc.append('', '')
+ children = node.__subclasses__()
+ children.sort(key=lambda x: x.__name__.lower())
+ for child in children:
+ walk(child, indent)
+ walk(Node, 0)
+ return parse_rst(state, content_offset, doc)
+
+
+def inject_toc(app, doctree, docname):
+ titleiter = iter(doctree.traverse(nodes.title))
+ try:
+ # skip first title, we are not interested in that one
+ titleiter.next()
+ title = titleiter.next()
+ # and check if there is at least another title
+ titleiter.next()
+ except StopIteration:
+ return
+ tocnode = nodes.section('')
+ tocnode['classes'].append('toc')
+ toctitle = nodes.section('')
+ toctitle['classes'].append('toctitle')
+ toctitle.append(nodes.title(text='Table Of Contents'))
+ tocnode.append(toctitle)
+ tocnode += doctree.document.settings.env.get_toc_for(docname)[0][1]
+ title.parent.insert(title.parent.children.index(title), tocnode)
+
+
+def setup(app):
+ app.add_directive('jinjafilters', jinja_filters, 0, (0, 0, 0))
+ app.add_directive('jinjatests', jinja_tests, 0, (0, 0, 0))
+ app.add_directive('jinjanodes', jinja_nodes, 0, (0, 0, 0))
+ # uncomment for inline toc. links are broken unfortunately
+ ##app.connect('doctree-resolved', inject_toc)
diff --git a/slider-agent/src/main/python/jinja2/docs/sandbox.rst b/slider-agent/src/main/python/jinja2/docs/sandbox.rst
new file mode 100644
index 0000000..bb0ca9f
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/sandbox.rst
@@ -0,0 +1,46 @@
+Sandbox
+=======
+
+The Jinja2 sandbox can be used to evaluate untrusted code. Access to unsafe
+attributes and methods is prohibited.
+
+Assuming `env` is a :class:`SandboxedEnvironment` in the default configuration
+the following piece of code shows how it works:
+
+>>> env.from_string("{{ func.func_code }}").render(func=lambda:None)
+u''
+>>> env.from_string("{{ func.func_code.do_something }}").render(func=lambda:None)
+Traceback (most recent call last):
+ ...
+SecurityError: access to attribute 'func_code' of 'function' object is unsafe.
+
+
+.. module:: jinja2.sandbox
+
+.. autoclass:: SandboxedEnvironment([options])
+ :members: is_safe_attribute, is_safe_callable
+
+.. autoclass:: ImmutableSandboxedEnvironment([options])
+
+.. autoexception:: SecurityError
+
+.. autofunction:: unsafe
+
+.. autofunction:: is_internal_attribute
+
+.. autofunction:: modifies_known_mutable
+
+.. admonition:: Note
+
+ The Jinja2 sandbox alone is no solution for perfect security. Especially
+ for web applications you have to keep in mind that users may create
+ templates with arbitrary HTML in so it's crucial to ensure that (if you
+ are running multiple users on the same server) they can't harm each other
+ via JavaScript insertions and much more.
+
+ Also the sandbox is only as good as the configuration. We stronly
+ recommend only passing non-shared resources to the template and use
+ some sort of whitelisting for attributes.
+
+ Also keep in mind that templates may raise runtime or compile time errors,
+ so make sure to catch them.
diff --git a/slider-agent/src/main/python/jinja2/docs/switching.rst b/slider-agent/src/main/python/jinja2/docs/switching.rst
new file mode 100644
index 0000000..ba3cfb1
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/switching.rst
@@ -0,0 +1,242 @@
+Switching from other Template Engines
+=====================================
+
+.. highlight:: html+jinja
+
+If you have used a different template engine in the past and want to swtich
+to Jinja2 here is a small guide that shows the basic syntatic and semantic
+changes between some common, similar text template engines for Python.
+
+Jinja1
+------
+
+Jinja2 is mostly compatible with Jinja1 in terms of API usage and template
+syntax. The differences between Jinja1 and 2 are explained in the following
+list.
+
+API
+~~~
+
+Loaders
+ Jinja2 uses a different loader API. Because the internal representation
+ of templates changed there is no longer support for external caching
+ systems such as memcached. The memory consumed by templates is comparable
+ with regular Python modules now and external caching doesn't give any
+ advantage. If you have used a custom loader in the past have a look at
+ the new :ref:`loader API <loaders>`.
+
+Loading templates from strings
+ In the past it was possible to generate templates from a string with the
+ default environment configuration by using `jinja.from_string`. Jinja2
+ provides a :class:`Template` class that can be used to do the same, but
+ with optional additional configuration.
+
+Automatic unicode conversion
+ Jinja1 performed automatic conversion of bytestrings in a given encoding
+ into unicode objects. This conversion is no longer implemented as it
+ was inconsistent as most libraries are using the regular Python ASCII
+ bytestring to Unicode conversion. An application powered by Jinja2
+ *has to* use unicode internally everywhere or make sure that Jinja2 only
+ gets unicode strings passed.
+
+i18n
+ Jinja1 used custom translators for internationalization. i18n is now
+ available as Jinja2 extension and uses a simpler, more gettext friendly
+ interface and has support for babel. For more details see
+ :ref:`i18n-extension`.
+
+Internal methods
+ Jinja1 exposed a few internal methods on the environment object such
+ as `call_function`, `get_attribute` and others. While they were marked
+ as being an internal method it was possible to override them. Jinja2
+ doesn't have equivalent methods.
+
+Sandbox
+ Jinja1 was running sandbox mode by default. Few applications actually
+ used that feature so it became optional in Jinja2. For more details
+ about the sandboxed execution see :class:`SandboxedEnvironment`.
+
+Context
+ Jinja1 had a stacked context as storage for variables passed to the
+ environment. In Jinja2 a similar object exists but it doesn't allow
+ modifications nor is it a singleton. As inheritance is dynamic now
+ multiple context objects may exist during template evaluation.
+
+Filters and Tests
+ Filters and tests are regular functions now. It's no longer necessary
+ and allowed to use factory functions.
+
+
+Templates
+~~~~~~~~~
+
+Jinja2 has mostly the same syntax as Jinja1. What's different is that
+macros require parentheses around the argument list now.
+
+Additionally Jinja2 allows dynamic inheritance now and dynamic includes.
+The old helper function `rendertemplate` is gone now, `include` can be used
+instead. Includes no longer import macros and variable assignments, for
+that the new `import` tag is used. This concept is explained in the
+:ref:`import` documentation.
+
+Another small change happened in the `for`-tag. The special loop variable
+doesn't have a `parent` attribute, instead you have to alias the loop
+yourself. See :ref:`accessing-the-parent-loop` for more details.
+
+
+Django
+------
+
+If you have previously worked with Django templates, you should find
+Jinja2 very familiar. In fact, most of the syntax elements look and
+work the same.
+
+However, Jinja2 provides some more syntax elements covered in the
+documentation and some work a bit different.
+
+This section covers the template changes. As the API is fundamentally
+different we won't cover it here.
+
+Method Calls
+~~~~~~~~~~~~
+
+In Django method calls work implicitly. With Jinja2 you have to specify that
+you want to call an object. Thus this Django code::
+
+ {% for page in user.get_created_pages %}
+ ...
+ {% endfor %}
+
+will look like this in Jinja::
+
+ {% for page in user.get_created_pages() %}
+ ...
+ {% endfor %}
+
+This allows you to pass variables to the function which is also used for macros
+which is not possible in Django.
+
+Conditions
+~~~~~~~~~~
+
+In Django you can use the following constructs to check for equality::
+
+ {% ifequal foo "bar" %}
+ ...
+ {% else %}
+ ...
+ {% endifequal %}
+
+In Jinja2 you can use the normal if statement in combination with operators::
+
+ {% if foo == 'bar' %}
+ ...
+ {% else %}
+ ...
+ {% endif %}
+
+You can also have multiple elif branches in your template::
+
+ {% if something %}
+ ...
+ {% elif otherthing %}
+ ...
+ {% elif foothing %}
+ ...
+ {% else %}
+ ...
+ {% endif %}
+
+Filter Arguments
+~~~~~~~~~~~~~~~~
+
+Jinja2 provides more than one argument for filters. Also the syntax for
+argument passing is different. A template that looks like this in Django::
+
+ {{ items|join:", " }}
+
+looks like this in Jinja2::
+
+ {{ items|join(', ') }}
+
+In fact it's a bit more verbose but it allows different types of arguments -
+including variables - and more than one of them.
+
+Tests
+~~~~~
+
+In addition to filters there also are tests you can perform using the is
+operator. Here are some examples::
+
+ {% if user.user_id is odd %}
+ {{ user.username|e }} is odd
+ {% else %}
+ hmm. {{ user.username|e }} looks pretty normal
+ {% endif %}
+
+Loops
+~~~~~
+
+For loops work very similar to Django, the only incompatibility is that in
+Jinja2 the special variable for the loop context is called `loop` and not
+`forloop` like in Django.
+
+Cycle
+~~~~~
+
+The ``{% cycle %}`` tag does not exist in Jinja because of it's implicit
+nature. However you can achieve mostly the same by using the `cycle`
+method on a loop object.
+
+The following Django template::
+
+ {% for user in users %}
+ <li class="{% cycle 'odd' 'even' %}">{{ user }}</li>
+ {% endfor %}
+
+Would look like this in Jinja::
+
+ {% for user in users %}
+ <li class="{{ loop.cycle('odd', 'even') }}">{{ user }}</li>
+ {% endfor %}
+
+There is no equivalent of ``{% cycle ... as variable %}``.
+
+
+Mako
+----
+
+.. highlight:: html+mako
+
+If you have used Mako so far and want to switch to Jinja2 you can configure
+Jinja2 to look more like Mako:
+
+.. sourcecode:: python
+
+ env = Environment('<%', '%>', '${', '}', '%')
+
+Once the environment is configure like that Jinja2 should be able to interpret
+a small subset of Mako templates. Jinja2 does not support embedded Python code
+so you would have to move that out of the template. The syntax for defs (in
+Jinja2 defs are called macros) and template inheritance is different too. The
+following Mako template::
+
+ <%inherit file="layout.html" />
+ <%def name="title()">Page Title</%def>
+ <ul>
+ % for item in list:
+ <li>${item}</li>
+ % endfor
+ </ul>
+
+Looks like this in Jinja2 with the above configuration::
+
+ <% extends "layout.html" %>
+ <% block title %>Page Title<% endblock %>
+ <% block body %>
+ <ul>
+ % for item in list:
+ <li>${item}</li>
+ % endfor
+ </ul>
+ <% endblock %>
diff --git a/slider-agent/src/main/python/jinja2/docs/templates.rst b/slider-agent/src/main/python/jinja2/docs/templates.rst
new file mode 100644
index 0000000..4a1f6ff
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/templates.rst
@@ -0,0 +1,1365 @@
+Template Designer Documentation
+===============================
+
+.. highlight:: html+jinja
+
+This document describes the syntax and semantics of the template engine and
+will be most useful as reference to those creating Jinja templates. As the
+template engine is very flexible the configuration from the application might
+be slightly different from here in terms of delimiters and behavior of
+undefined values.
+
+
+Synopsis
+--------
+
+A template is simply a text file. It can generate any text-based format
+(HTML, XML, CSV, LaTeX, etc.). It doesn't have a specific extension,
+``.html`` or ``.xml`` are just fine.
+
+A template contains **variables** or **expressions**, which get replaced with
+values when the template is evaluated, and tags, which control the logic of
+the template. The template syntax is heavily inspired by Django and Python.
+
+Below is a minimal template that illustrates a few basics. We will cover
+the details later in that document::
+
+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+ <html lang="en">
+ <head>
+ <title>My Webpage</title>
+ </head>
+ <body>
+ <ul id="navigation">
+ {% for item in navigation %}
+ <li><a href="{{ item.href }}">{{ item.caption }}</a></li>
+ {% endfor %}
+ </ul>
+
+ <h1>My Webpage</h1>
+ {{ a_variable }}
+ </body>
+ </html>
+
+This covers the default settings. The application developer might have
+changed the syntax from ``{% foo %}`` to ``<% foo %>`` or something similar.
+
+There are two kinds of delimiers. ``{% ... %}`` and ``{{ ... }}``. The first
+one is used to execute statements such as for-loops or assign values, the
+latter prints the result of the expression to the template.
+
+.. _variables:
+
+Variables
+---------
+
+The application passes variables to the templates you can mess around in the
+template. Variables may have attributes or elements on them you can access
+too. How a variable looks like, heavily depends on the application providing
+those.
+
+You can use a dot (``.``) to access attributes of a variable, alternative the
+so-called "subscript" syntax (``[]``) can be used. The following lines do
+the same::
+
+ {{ foo.bar }}
+ {{ foo['bar'] }}
+
+It's important to know that the curly braces are *not* part of the variable
+but the print statement. If you access variables inside tags don't put the
+braces around.
+
+If a variable or attribute does not exist you will get back an undefined
+value. What you can do with that kind of value depends on the application
+configuration, the default behavior is that it evaluates to an empty string
+if printed and that you can iterate over it, but every other operation fails.
+
+.. _notes-on-subscriptions:
+
+.. admonition:: Implementation
+
+ For convenience sake ``foo.bar`` in Jinja2 does the following things on
+ the Python layer:
+
+ - check if there is an attribute called `bar` on `foo`.
+ - if there is not, check if there is an item ``'bar'`` in `foo`.
+ - if there is not, return an undefined object.
+
+ ``foo['bar']`` on the other hand works mostly the same with the a small
+ difference in the order:
+
+ - check if there is an item ``'bar'`` in `foo`.
+ - if there is not, check if there is an attribute called `bar` on `foo`.
+ - if there is not, return an undefined object.
+
+ This is important if an object has an item or attribute with the same
+ name. Additionally there is the :func:`attr` filter that just looks up
+ attributes.
+
+.. _filters:
+
+Filters
+-------
+
+Variables can by modified by **filters**. Filters are separated from the
+variable by a pipe symbol (``|``) and may have optional arguments in
+parentheses. Multiple filters can be chained. The output of one filter is
+applied to the next.
+
+``{{ name|striptags|title }}`` for example will remove all HTML Tags from the
+`name` and title-cases it. Filters that accept arguments have parentheses
+around the arguments, like a function call. This example will join a list
+by commas: ``{{ list|join(', ') }}``.
+
+The :ref:`builtin-filters` below describes all the builtin filters.
+
+.. _tests:
+
+Tests
+-----
+
+Beside filters there are also so called "tests" available. Tests can be used
+to test a variable against a common expression. To test a variable or
+expression you add `is` plus the name of the test after the variable. For
+example to find out if a variable is defined you can do ``name is defined``
+which will then return true or false depending on if `name` is defined.
+
+Tests can accept arguments too. If the test only takes one argument you can
+leave out the parentheses to group them. For example the following two
+expressions do the same::
+
+ {% if loop.index is divisibleby 3 %}
+ {% if loop.index is divisibleby(3) %}
+
+The :ref:`builtin-tests` below describes all the builtin tests.
+
+
+Comments
+--------
+
+To comment-out part of a line in a template, use the comment syntax which is
+by default set to ``{# ... #}``. This is useful to comment out parts of the
+template for debugging or to add information for other template designers or
+yourself::
+
+ {# note: disabled template because we no longer use this
+ {% for user in users %}
+ ...
+ {% endfor %}
+ #}
+
+
+Whitespace Control
+------------------
+
+In the default configuration whitespace is not further modified by the
+template engine, so each whitespace (spaces, tabs, newlines etc.) is returned
+unchanged. If the application configures Jinja to `trim_blocks` the first
+newline after a a template tag is removed automatically (like in PHP).
+
+But you can also strip whitespace in templates by hand. If you put an minus
+sign (``-``) to the start or end of an block (for example a for tag), a
+comment or variable expression you can remove the whitespaces after or before
+that block::
+
+ {% for item in seq -%}
+ {{ item }}
+ {%- endfor %}
+
+This will yield all elements without whitespace between them. If `seq` was
+a list of numbers from ``1`` to ``9`` the output would be ``123456789``.
+
+If :ref:`line-statements` are enabled they strip leading whitespace
+automatically up to the beginning of the line.
+
+.. admonition:: Note
+
+ You must not use a whitespace between the tag and the minus sign.
+
+ **valid**::
+
+ {%- if foo -%}...{% endif %}
+
+ **invalid**::
+
+ {% - if foo - %}...{% endif %}
+
+
+Escaping
+--------
+
+It is sometimes desirable or even necessary to have Jinja ignore parts it
+would otherwise handle as variables or blocks. For example if the default
+syntax is used and you want to use ``{{`` as raw string in the template and
+not start a variable you have to use a trick.
+
+The easiest way is to output the variable delimiter (``{{``) by using a
+variable expression::
+
+ {{ '{{' }}
+
+For bigger sections it makes sense to mark a block `raw`. For example to
+put Jinja syntax as example into a template you can use this snippet::
+
+ {% raw %}
+ <ul>
+ {% for item in seq %}
+ <li>{{ item }}</li>
+ {% endfor %}
+ </ul>
+ {% endraw %}
+
+
+.. _line-statements:
+
+Line Statements
+---------------
+
+If line statements are enabled by the application it's possible to mark a
+line as a statement. For example if the line statement prefix is configured
+to ``#`` the following two examples are equivalent::
+
+ <ul>
+ # for item in seq
+ <li>{{ item }}</li>
+ # endfor
+ </ul>
+
+ <ul>
+ {% for item in seq %}
+ <li>{{ item }}</li>
+ {% endfor %}
+ </ul>
+
+The line statement prefix can appear anywhere on the line as long as no text
+precedes it. For better readability statements that start a block (such as
+`for`, `if`, `elif` etc.) may end with a colon::
+
+ # for item in seq:
+ ...
+ # endfor
+
+
+.. admonition:: Note
+
+ Line statements can span multiple lines if there are open parentheses,
+ braces or brackets::
+
+ <ul>
+ # for href, caption in [('index.html', 'Index'),
+ ('about.html', 'About')]:
+ <li><a href="{{ href }}">{{ caption }}</a></li>
+ # endfor
+ </ul>
+
+Since Jinja 2.2 line-based comments are available as well. For example if
+the line-comment prefix is configured to be ``##`` everything from ``##`` to
+the end of the line is ignored (excluding the newline sign)::
+
+ # for item in seq:
+ <li>{{ item }}</li> ## this comment is ignored
+ # endfor
+
+
+.. _template-inheritance:
+
+Template Inheritance
+--------------------
+
+The most powerful part of Jinja is template inheritance. Template inheritance
+allows you to build a base "skeleton" template that contains all the common
+elements of your site and defines **blocks** that child templates can override.
+
+Sounds complicated but is very basic. It's easiest to understand it by starting
+with an example.
+
+
+Base Template
+~~~~~~~~~~~~~
+
+This template, which we'll call ``base.html``, defines a simple HTML skeleton
+document that you might use for a simple two-column page. It's the job of
+"child" templates to fill the empty blocks with content::
+
+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+ <html lang="en">
+ <html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ {% block head %}
+ <link rel="stylesheet" href="style.css" />
+ <title>{% block title %}{% endblock %} - My Webpage</title>
+ {% endblock %}
+ </head>
+ <body>
+ <div id="content">{% block content %}{% endblock %}</div>
+ <div id="footer">
+ {% block footer %}
+ © Copyright 2008 by <a href="http://domain.invalid/">you</a>.
+ {% endblock %}
+ </div>
+ </body>
+
+In this example, the ``{% block %}`` tags define four blocks that child templates
+can fill in. All the `block` tag does is to tell the template engine that a
+child template may override those portions of the template.
+
+Child Template
+~~~~~~~~~~~~~~
+
+A child template might look like this::
+
+ {% extends "base.html" %}
+ {% block title %}Index{% endblock %}
+ {% block head %}
+ {{ super() }}
+ <style type="text/css">
+ .important { color: #336699; }
+ </style>
+ {% endblock %}
+ {% block content %}
+ <h1>Index</h1>
+ <p class="important">
+ Welcome on my awesome homepage.
+ </p>
+ {% endblock %}
+
+The ``{% extends %}`` tag is the key here. It tells the template engine that
+this template "extends" another template. When the template system evaluates
+this template, first it locates the parent. The extends tag should be the
+first tag in the template. Everything before it is printed out normally and
+may cause confusion. For details about this behavior and how to take
+advantage of it, see :ref:`null-master-fallback`.
+
+The filename of the template depends on the template loader. For example the
+:class:`FileSystemLoader` allows you to access other templates by giving the
+filename. You can access templates in subdirectories with an slash::
+
+ {% extends "layout/default.html" %}
+
+But this behavior can depend on the application embedding Jinja. Note that
+since the child template doesn't define the ``footer`` block, the value from
+the parent template is used instead.
+
+You can't define multiple ``{% block %}`` tags with the same name in the
+same template. This limitation exists because a block tag works in "both"
+directions. That is, a block tag doesn't just provide a hole to fill - it
+also defines the content that fills the hole in the *parent*. If there
+were two similarly-named ``{% block %}`` tags in a template, that template's
+parent wouldn't know which one of the blocks' content to use.
+
+If you want to print a block multiple times you can however use the special
+`self` variable and call the block with that name::
+
+ <title>{% block title %}{% endblock %}</title>
+ <h1>{{ self.title() }}</h1>
+ {% block body %}{% endblock %}
+
+
+Super Blocks
+~~~~~~~~~~~~
+
+It's possible to render the contents of the parent block by calling `super`.
+This gives back the results of the parent block::
+
+ {% block sidebar %}
+ <h3>Table Of Contents</h3>
+ ...
+ {{ super() }}
+ {% endblock %}
+
+
+Named Block End-Tags
+~~~~~~~~~~~~~~~~~~~~
+
+Jinja2 allows you to put the name of the block after the end tag for better
+readability::
+
+ {% block sidebar %}
+ {% block inner_sidebar %}
+ ...
+ {% endblock inner_sidebar %}
+ {% endblock sidebar %}
+
+However the name after the `endblock` word must match the block name.
+
+
+Block Nesting and Scope
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Blocks can be nested for more complex layouts. However per default blocks
+may not access variables from outer scopes::
+
+ {% for item in seq %}
+ <li>{% block loop_item %}{{ item }}{% endblock %}</li>
+ {% endfor %}
+
+This example would output empty ``<li>`` items because `item` is unavailable
+inside the block. The reason for this is that if the block is replaced by
+a child template a variable would appear that was not defined in the block or
+passed to the context.
+
+Starting with Jinja 2.2 you can explicitly specify that variables are
+available in a block by setting the block to "scoped" by adding the `scoped`
+modifier to a block declaration::
+
+ {% for item in seq %}
+ <li>{% block loop_item scoped %}{{ item }}{% endblock %}</li>
+ {% endfor %}
+
+When overriding a block the `scoped` modifier does not have to be provided.
+
+
+Template Objects
+~~~~~~~~~~~~~~~~
+
+.. versionchanged:: 2.4
+
+If a template object was passed to the template context you can
+extend from that object as well. Assuming the calling code passes
+a layout template as `layout_template` to the environment, this
+code works::
+
+ {% extends layout_template %}
+
+Previously the `layout_template` variable had to be a string with
+the layout template's filename for this to work.
+
+
+HTML Escaping
+-------------
+
+When generating HTML from templates, there's always a risk that a variable will
+include characters that affect the resulting HTML. There are two approaches:
+manually escaping each variable or automatically escaping everything by default.
+
+Jinja supports both, but what is used depends on the application configuration.
+The default configuaration is no automatic escaping for various reasons:
+
+- escaping everything except of safe values will also mean that Jinja is
+ escaping variables known to not include HTML such as numbers which is
+ a huge performance hit.
+
+- The information about the safety of a variable is very fragile. It could
+ happen that by coercing safe and unsafe values the return value is double
+ escaped HTML.
+
+Working with Manual Escaping
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If manual escaping is enabled it's **your** responsibility to escape
+variables if needed. What to escape? If you have a variable that *may*
+include any of the following chars (``>``, ``<``, ``&``, or ``"``) you
+**have to** escape it unless the variable contains well-formed and trusted
+HTML. Escaping works by piping the variable through the ``|e`` filter:
+``{{ user.username|e }}``.
+
+Working with Automatic Escaping
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When automatic escaping is enabled everything is escaped by default except
+for values explicitly marked as safe. Those can either be marked by the
+application or in the template by using the `|safe` filter. The main
+problem with this approach is that Python itself doesn't have the concept
+of tainted values so the information if a value is safe or unsafe can get
+lost. If the information is lost escaping will take place which means that
+you could end up with double escaped contents.
+
+Double escaping is easy to avoid however, just rely on the tools Jinja2
+provides and don't use builtin Python constructs such as the string modulo
+operator.
+
+Functions returning template data (macros, `super`, `self.BLOCKNAME`) return
+safe markup always.
+
+String literals in templates with automatic escaping are considered unsafe
+too. The reason for this is that the safe string is an extension to Python
+and not every library will work properly with it.
+
+
+List of Control Structures
+--------------------------
+
+A control structure refers to all those things that control the flow of a
+program - conditionals (i.e. if/elif/else), for-loops, as well as things like
+macros and blocks. Control structures appear inside ``{% ... %}`` blocks
+in the default syntax.
+
+For
+~~~
+
+Loop over each item in a sequence. For example, to display a list of users
+provided in a variable called `users`::
+
+ <h1>Members</h1>
+ <ul>
+ {% for user in users %}
+ <li>{{ user.username|e }}</li>
+ {% endfor %}
+ </ul>
+
+Inside of a for loop block you can access some special variables:
+
++-----------------------+---------------------------------------------------+
+| Variable | Description |
++=======================+===================================================+
+| `loop.index` | The current iteration of the loop. (1 indexed) |
++-----------------------+---------------------------------------------------+
+| `loop.index0` | The current iteration of the loop. (0 indexed) |
++-----------------------+---------------------------------------------------+
+| `loop.revindex` | The number of iterations from the end of the loop |
+| | (1 indexed) |
++-----------------------+---------------------------------------------------+
+| `loop.revindex0` | The number of iterations from the end of the loop |
+| | (0 indexed) |
++-----------------------+---------------------------------------------------+
+| `loop.first` | True if first iteration. |
++-----------------------+---------------------------------------------------+
+| `loop.last` | True if last iteration. |
++-----------------------+---------------------------------------------------+
+| `loop.length` | The number of items in the sequence. |
++-----------------------+---------------------------------------------------+
+| `loop.cycle` | A helper function to cycle between a list of |
+| | sequences. See the explanation below. |
++-----------------------+---------------------------------------------------+
+
+Within a for-loop, it's possible to cycle among a list of strings/variables
+each time through the loop by using the special `loop.cycle` helper::
+
+ {% for row in rows %}
+ <li class="{{ loop.cycle('odd', 'even') }}">{{ row }}</li>
+ {% endfor %}
+
+With Jinja 2.1 an extra `cycle` helper exists that allows loop-unbound
+cycling. For more information have a look at the :ref:`builtin-globals`.
+
+.. _loop-filtering:
+
+Unlike in Python it's not possible to `break` or `continue` in a loop. You
+can however filter the sequence during iteration which allows you to skip
+items. The following example skips all the users which are hidden::
+
+ {% for user in users if not user.hidden %}
+ <li>{{ user.username|e }}</li>
+ {% endfor %}
+
+The advantage is that the special `loop` variable will count correctly thus
+not counting the users not iterated over.
+
+If no iteration took place because the sequence was empty or the filtering
+removed all the items from the sequence you can render a replacement block
+by using `else`::
+
+ <ul>
+ {% for user in users %}
+ <li>{{ user.username|e }}</li>
+ {% else %}
+ <li><em>no users found</em></li>
+ {% endfor %}
+ </ul>
+
+It is also possible to use loops recursively. This is useful if you are
+dealing with recursive data such as sitemaps. To use loops recursively you
+basically have to add the `recursive` modifier to the loop definition and
+call the `loop` variable with the new iterable where you want to recurse.
+
+The following example implements a sitemap with recursive loops::
+
+ <ul class="sitemap">
+ {%- for item in sitemap recursive %}
+ <li><a href="{{ item.href|e }}">{{ item.title }}</a>
+ {%- if item.children -%}
+ <ul class="submenu">{{ loop(item.children) }}</ul>
+ {%- endif %}</li>
+ {%- endfor %}
+ </ul>
+
+
+If
+~~
+
+The `if` statement in Jinja is comparable with the if statements of Python.
+In the simplest form you can use it to test if a variable is defined, not
+empty or not false::
+
+ {% if users %}
+ <ul>
+ {% for user in users %}
+ <li>{{ user.username|e }}</li>
+ {% endfor %}
+ </ul>
+ {% endif %}
+
+For multiple branches `elif` and `else` can be used like in Python. You can
+use more complex :ref:`expressions` there too::
+
+ {% if kenny.sick %}
+ Kenny is sick.
+ {% elif kenny.dead %}
+ You killed Kenny! You bastard!!!
+ {% else %}
+ Kenny looks okay --- so far
+ {% endif %}
+
+If can also be used as :ref:`inline expression <if-expression>` and for
+:ref:`loop filtering <loop-filtering>`.
+
+
+Macros
+~~~~~~
+
+Macros are comparable with functions in regular programming languages. They
+are useful to put often used idioms into reusable functions to not repeat
+yourself.
+
+Here a small example of a macro that renders a form element::
+
+ {% macro input(name, value='', type='text', size=20) -%}
+ <input type="{{ type }}" name="{{ name }}" value="{{
+ value|e }}" size="{{ size }}">
+ {%- endmacro %}
+
+The macro can then be called like a function in the namespace::
+
+ <p>{{ input('username') }}</p>
+ <p>{{ input('password', type='password') }}</p>
+
+If the macro was defined in a different template you have to
+:ref:`import <import>` it first.
+
+Inside macros you have access to three special variables:
+
+`varargs`
+ If more positional arguments are passed to the macro than accepted by the
+ macro they end up in the special `varargs` variable as list of values.
+
+`kwargs`
+ Like `varargs` but for keyword arguments. All unconsumed keyword
+ arguments are stored in this special variable.
+
+`caller`
+ If the macro was called from a :ref:`call<call>` tag the caller is stored
+ in this variable as macro which can be called.
+
+Macros also expose some of their internal details. The following attributes
+are available on a macro object:
+
+`name`
+ The name of the macro. ``{{ input.name }}`` will print ``input``.
+
+`arguments`
+ A tuple of the names of arguments the macro accepts.
+
+`defaults`
+ A tuple of default values.
+
+`catch_kwargs`
+ This is `true` if the macro accepts extra keyword arguments (ie: accesses
+ the special `kwargs` variable).
+
+`catch_varargs`
+ This is `true` if the macro accepts extra positional arguments (ie:
+ accesses the special `varargs` variable).
+
+`caller`
+ This is `true` if the macro accesses the special `caller` variable and may
+ be called from a :ref:`call<call>` tag.
+
+If a macro name starts with an underscore it's not exported and can't
+be imported.
+
+
+.. _call:
+
+Call
+~~~~
+
+In some cases it can be useful to pass a macro to another macro. For this
+purpose you can use the special `call` block. The following example shows
+a macro that takes advantage of the call functionality and how it can be
+used::
+
+ {% macro render_dialog(title, class='dialog') -%}
+ <div class="{{ class }}">
+ <h2>{{ title }}</h2>
+ <div class="contents">
+ {{ caller() }}
+ </div>
+ </div>
+ {%- endmacro %}
+
+ {% call render_dialog('Hello World') %}
+ This is a simple dialog rendered by using a macro and
+ a call block.
+ {% endcall %}
+
+It's also possible to pass arguments back to the call block. This makes it
+useful as replacement for loops. Generally speaking a call block works
+exactly like an macro, just that it doesn't have a name.
+
+Here an example of how a call block can be used with arguments::
+
+ {% macro dump_users(users) -%}
+ <ul>
+ {%- for user in users %}
+ <li><p>{{ user.username|e }}</p>{{ caller(user) }}</li>
+ {%- endfor %}
+ </ul>
+ {%- endmacro %}
+
+ {% call(user) dump_users(list_of_user) %}
+ <dl>
+ <dl>Realname</dl>
+ <dd>{{ user.realname|e }}</dd>
+ <dl>Description</dl>
+ <dd>{{ user.description }}</dd>
+ </dl>
+ {% endcall %}
+
+
+Filters
+~~~~~~~
+
+Filter sections allow you to apply regular Jinja2 filters on a block of
+template data. Just wrap the code in the special `filter` section::
+
+ {% filter upper %}
+ This text becomes uppercase
+ {% endfilter %}
+
+
+Assignments
+~~~~~~~~~~~
+
+Inside code blocks you can also assign values to variables. Assignments at
+top level (outside of blocks, macros or loops) are exported from the template
+like top level macros and can be imported by other templates.
+
+Assignments use the `set` tag and can have multiple targets::
+
+ {% set navigation = [('index.html', 'Index'), ('about.html', 'About')] %}
+ {% set key, value = call_something() %}
+
+
+Extends
+~~~~~~~
+
+The `extends` tag can be used to extend a template from another one. You
+can have multiple of them in a file but only one of them may be executed
+at the time. See the section about :ref:`template-inheritance` above.
+
+
+Block
+~~~~~
+
+Blocks are used for inheritance and act as placeholders and replacements
+at the same time. They are documented in detail as part of the section
+about :ref:`template-inheritance`.
+
+
+Include
+~~~~~~~
+
+The `include` statement is useful to include a template and return the
+rendered contents of that file into the current namespace::
+
+ {% include 'header.html' %}
+ Body
+ {% include 'footer.html' %}
+
+Included templates have access to the variables of the active context by
+default. For more details about context behavior of imports and includes
+see :ref:`import-visibility`.
+
+From Jinja 2.2 onwards you can mark an include with ``ignore missing`` in
+which case Jinja will ignore the statement if the template to be ignored
+does not exist. When combined with ``with`` or ``without context`` it has
+to be placed *before* the context visibility statement. Here some valid
+examples::
+
+ {% include "sidebar.html" ignore missing %}
+ {% include "sidebar.html" ignore missing with context %}
+ {% include "sidebar.html" ignore missing without context %}
+
+.. versionadded:: 2.2
+
+You can also provide a list of templates that are checked for existence
+before inclusion. The first template that exists will be included. If
+`ignore missing` is given, it will fall back to rendering nothing if
+none of the templates exist, otherwise it will raise an exception.
+
+Example::
+
+ {% include ['page_detailed.html', 'page.html'] %}
+ {% include ['special_sidebar.html', 'sidebar.html'] ignore missing %}
+
+.. versionchanged:: 2.4
+ If a template object was passed to the template context you can
+ include that object using `include`.
+
+.. _import:
+
+Import
+~~~~~~
+
+Jinja2 supports putting often used code into macros. These macros can go into
+different templates and get imported from there. This works similar to the
+import statements in Python. It's important to know that imports are cached
+and imported templates don't have access to the current template variables,
+just the globals by defualt. For more details about context behavior of
+imports and includes see :ref:`import-visibility`.
+
+There are two ways to import templates. You can import the complete template
+into a variable or request specific macros / exported variables from it.
+
+Imagine we have a helper module that renders forms (called `forms.html`)::
+
+ {% macro input(name, value='', type='text') -%}
+ <input type="{{ type }}" value="{{ value|e }}" name="{{ name }}">
+ {%- endmacro %}
+
+ {%- macro textarea(name, value='', rows=10, cols=40) -%}
+ <textarea name="{{ name }}" rows="{{ rows }}" cols="{{ cols
+ }}">{{ value|e }}</textarea>
+ {%- endmacro %}
+
+The easiest and most flexible is importing the whole module into a variable.
+That way you can access the attributes::
+
+ {% import 'forms.html' as forms %}
+ <dl>
+ <dt>Username</dt>
+ <dd>{{ forms.input('username') }}</dd>
+ <dt>Password</dt>
+ <dd>{{ forms.input('password', type='password') }}</dd>
+ </dl>
+ <p>{{ forms.textarea('comment') }}</p>
+
+
+Alternatively you can import names from the template into the current
+namespace::
+
+ {% from 'forms.html' import input as input_field, textarea %}
+ <dl>
+ <dt>Username</dt>
+ <dd>{{ input_field('username') }}</dd>
+ <dt>Password</dt>
+ <dd>{{ input_field('password', type='password') }}</dd>
+ </dl>
+ <p>{{ textarea('comment') }}</p>
+
+Macros and variables starting with one ore more underscores are private and
+cannot be imported.
+
+.. versionchanged:: 2.4
+ If a template object was passed to the template context you can
+ import from that object.
+
+
+.. _import-visibility:
+
+Import Context Behavior
+-----------------------
+
+Per default included templates are passed the current context and imported
+templates not. The reason for this is that imports unlike includes are
+cached as imports are often used just as a module that holds macros.
+
+This however can be changed of course explicitly. By adding `with context`
+or `without context` to the import/include directive the current context
+can be passed to the template and caching is disabled automatically.
+
+Here two examples::
+
+ {% from 'forms.html' import input with context %}
+ {% include 'header.html' without context %}
+
+.. admonition:: Note
+
+ In Jinja 2.0 the context that was passed to the included template
+ did not include variables defined in the template. As a matter of
+ fact this did not work::
+
+ {% for box in boxes %}
+ {% include "render_box.html" %}
+ {% endfor %}
+
+ The included template ``render_box.html`` is not able to access
+ `box` in Jinja 2.0, but in Jinja 2.1.
+
+
+.. _expressions:
+
+Expressions
+-----------
+
+Jinja allows basic expressions everywhere. These work very similar to regular
+Python and even if you're not working with Python you should feel comfortable
+with it.
+
+Literals
+~~~~~~~~
+
+The simplest form of expressions are literals. Literals are representations
+for Python objects such as strings and numbers. The following literals exist:
+
+"Hello World":
+ Everything between two double or single quotes is a string. They are
+ useful whenever you need a string in the template (for example as
+ arguments to function calls, filters or just to extend or include a
+ template).
+
+42 / 42.23:
+ Integers and floating point numbers are created by just writing the
+ number down. If a dot is present the number is a float, otherwise an
+ integer. Keep in mind that for Python ``42`` and ``42.0`` is something
+ different.
+
+['list', 'of', 'objects']:
+ Everything between two brackets is a list. Lists are useful to store
+ sequential data in or to iterate over them. For example you can easily
+ create a list of links using lists and tuples with a for loop::
+
+ <ul>
+ {% for href, caption in [('index.html', 'Index'), ('about.html', 'About'),
+ ('downloads.html', 'Downloads')] %}
+ <li><a href="{{ href }}">{{ caption }}</a></li>
+ {% endfor %}
+ </ul>
+
+('tuple', 'of', 'values'):
+ Tuples are like lists, just that you can't modify them. If the tuple
+ only has one item you have to end it with a comma. Tuples are usually
+ used to represent items of two or more elements. See the example above
+ for more details.
+
+{'dict': 'of', 'key': 'and', 'value': 'pairs'}:
+ A dict in Python is a structure that combines keys and values. Keys must
+ be unique and always have exactly one value. Dicts are rarely used in
+ templates, they are useful in some rare cases such as the :func:`xmlattr`
+ filter.
+
+true / false:
+ true is always true and false is always false.
+
+.. admonition:: Note
+
+ The special constants `true`, `false` and `none` are indeed lowercase.
+ Because that caused confusion in the past, when writing `True` expands
+ to an undefined variable that is considered false, all three of them can
+ be written in title case too (`True`, `False`, and `None`). However for
+ consistency (all Jinja identifiers are lowercase) you should use the
+ lowercase versions.
+
+Math
+~~~~
+
+Jinja allows you to calculate with values. This is rarely useful in templates
+but exists for completeness' sake. The following operators are supported:
+
+\+
+ Adds two objects together. Usually the objects are numbers but if both are
+ strings or lists you can concatenate them this way. This however is not
+ the preferred way to concatenate strings! For string concatenation have
+ a look at the ``~`` operator. ``{{ 1 + 1 }}`` is ``2``.
+
+\-
+ Substract the second number from the first one. ``{{ 3 - 2 }}`` is ``1``.
+
+/
+ Divide two numbers. The return value will be a floating point number.
+ ``{{ 1 / 2 }}`` is ``{{ 0.5 }}``.
+
+//
+ Divide two numbers and return the truncated integer result.
+ ``{{ 20 / 7 }}`` is ``2``.
+
+%
+ Calculate the remainder of an integer division. ``{{ 11 % 7 }}`` is ``4``.
+
+\*
+ Multiply the left operand with the right one. ``{{ 2 * 2 }}`` would
+ return ``4``. This can also be used to repeat a string multiple times.
+ ``{{ '=' * 80 }}`` would print a bar of 80 equal signs.
+
+\**
+ Raise the left operand to the power of the right operand. ``{{ 2**3 }}``
+ would return ``8``.
+
+Comparisons
+~~~~~~~~~~~
+
+==
+ Compares two objects for equality.
+
+!=
+ Compares two objects for inequality.
+
+>
+ `true` if the left hand side is greater than the right hand side.
+
+>=
+ `true` if the left hand side is greater or equal to the right hand side.
+
+<
+ `true` if the left hand side is lower than the right hand side.
+
+<=
+ `true` if the left hand side is lower or equal to the right hand side.
+
+Logic
+~~~~~
+
+For `if` statements, `for` filtering or `if` expressions it can be useful to
+combine multiple expressions:
+
+and
+ Return true if the left and the right operand is true.
+
+or
+ Return true if the left or the right operand is true.
+
+not
+ negate a statement (see below).
+
+(expr)
+ group an expression.
+
+.. admonition:: Note
+
+ The ``is`` and ``in`` operators support negation using an infix notation
+ too: ``foo is not bar`` and ``foo not in bar`` instead of ``not foo is bar``
+ and ``not foo in bar``. All other expressions require a prefix notation:
+ ``not (foo and bar).``
+
+
+Other Operators
+~~~~~~~~~~~~~~~
+
+The following operators are very useful but don't fit into any of the other
+two categories:
+
+in
+ Perform sequence / mapping containment test. Returns true if the left
+ operand is contained in the right. ``{{ 1 in [1, 2, 3] }}`` would for
+ example return true.
+
+is
+ Performs a :ref:`test <tests>`.
+
+\|
+ Applies a :ref:`filter <filters>`.
+
+~
+ Converts all operands into strings and concatenates them.
+ ``{{ "Hello " ~ name ~ "!" }}`` would return (assuming `name` is
+ ``'John'``) ``Hello John!``.
+
+()
+ Call a callable: ``{{ post.render() }}``. Inside of the parentheses you
+ can use positional arguments and keyword arguments like in python:
+ ``{{ post.render(user, full=true) }}``.
+
+. / []
+ Get an attribute of an object. (See :ref:`variables`)
+
+
+.. _if-expression:
+
+If Expression
+~~~~~~~~~~~~~
+
+It is also possible to use inline `if` expressions. These are useful in some
+situations. For example you can use this to extend from one template if a
+variable is defined, otherwise from the default layout template::
+
+ {% extends layout_template if layout_template is defined else 'master.html' %}
+
+The general syntax is ``<do something> if <something is true> else <do
+something else>``.
+
+The `else` part is optional. If not provided the else block implicitly
+evaluates into an undefined object::
+
+ {{ '[%s]' % page.title if page.title }}
+
+
+.. _builtin-filters:
+
+List of Builtin Filters
+-----------------------
+
+.. jinjafilters::
+
+
+.. _builtin-tests:
+
+List of Builtin Tests
+---------------------
+
+.. jinjatests::
+
+.. _builtin-globals:
+
+List of Global Functions
+------------------------
+
+The following functions are available in the global scope by default:
+
+.. function:: range([start,] stop[, step])
+
+ Return a list containing an arithmetic progression of integers.
+ range(i, j) returns [i, i+1, i+2, ..., j-1]; start (!) defaults to 0.
+ When step is given, it specifies the increment (or decrement).
+ For example, range(4) returns [0, 1, 2, 3]. The end point is omitted!
+ These are exactly the valid indices for a list of 4 elements.
+
+ This is useful to repeat a template block multiple times for example
+ to fill a list. Imagine you have 7 users in the list but you want to
+ render three empty items to enforce a height with CSS::
+
+ <ul>
+ {% for user in users %}
+ <li>{{ user.username }}</li>
+ {% endfor %}
+ {% for number in range(10 - users|count) %}
+ <li class="empty"><span>...</span></li>
+ {% endfor %}
+ </ul>
+
+.. function:: lipsum(n=5, html=True, min=20, max=100)
+
+ Generates some lorem ipsum for the template. Per default five paragraphs
+ with HTML are generated each paragraph between 20 and 100 words. If html
+ is disabled regular text is returned. This is useful to generate simple
+ contents for layout testing.
+
+.. function:: dict(\**items)
+
+ A convenient alternative to dict literals. ``{'foo': 'bar'}`` is the same
+ as ``dict(foo='bar')``.
+
+.. class:: cycler(\*items)
+
+ The cycler allows you to cycle among values similar to how `loop.cycle`
+ works. Unlike `loop.cycle` however you can use this cycler outside of
+ loops or over multiple loops.
+
+ This is for example very useful if you want to show a list of folders and
+ files, with the folders on top, but both in the same list with alternating
+ row colors.
+
+ The following example shows how `cycler` can be used::
+
+ {% set row_class = cycler('odd', 'even') %}
+ <ul class="browser">
+ {% for folder in folders %}
+ <li class="folder {{ row_class.next() }}">{{ folder|e }}</li>
+ {% endfor %}
+ {% for filename in files %}
+ <li class="file {{ row_class.next() }}">{{ filename|e }}</li>
+ {% endfor %}
+ </ul>
+
+ A cycler has the following attributes and methods:
+
+ .. method:: reset()
+
+ Resets the cycle to the first item.
+
+ .. method:: next()
+
+ Goes one item a head and returns the then current item.
+
+ .. attribute:: current
+
+ Returns the current item.
+
+ **new in Jinja 2.1**
+
+.. class:: joiner(sep=', ')
+
+ A tiny helper that can be use to "join" multiple sections. A joiner is
+ passed a string and will return that string every time it's calld, except
+ the first time in which situation it returns an empty string. You can
+ use this to join things::
+
+ {% set pipe = joiner("|") %}
+ {% if categories %} {{ pipe() }}
+ Categories: {{ categories|join(", ") }}
+ {% endif %}
+ {% if author %} {{ pipe() }}
+ Author: {{ author() }}
+ {% endif %}
+ {% if can_edit %} {{ pipe() }}
+ <a href="?action=edit">Edit</a>
+ {% endif %}
+
+ **new in Jinja 2.1**
+
+
+Extensions
+----------
+
+The following sections cover the built-in Jinja2 extensions that may be
+enabled by the application. The application could also provide further
+extensions not covered by this documentation. In that case there should
+be a separate document explaining the extensions.
+
+.. _i18n-in-templates:
+
+i18n
+~~~~
+
+If the i18n extension is enabled it's possible to mark parts in the template
+as translatable. To mark a section as translatable you can use `trans`::
+
+ <p>{% trans %}Hello {{ user }}!{% endtrans %}</p>
+
+To translate a template expression --- say, using template filters or just
+accessing an attribute of an object --- you need to bind the expression to a
+name for use within the translation block::
+
+ <p>{% trans user=user.username %}Hello {{ user }}!{% endtrans %}</p>
+
+If you need to bind more than one expression inside a `trans` tag, separate
+the pieces with a comma (``,``)::
+
+ {% trans book_title=book.title, author=author.name %}
+ This is {{ book_title }} by {{ author }}
+ {% endtrans %}
+
+Inside trans tags no statements are allowed, only variable tags are.
+
+To pluralize, specify both the singular and plural forms with the `pluralize`
+tag, which appears between `trans` and `endtrans`::
+
+ {% trans count=list|length %}
+ There is {{ count }} {{ name }} object.
+ {% pluralize %}
+ There are {{ count }} {{ name }} objects.
+ {% endtrans %}
+
+Per default the first variable in a block is used to determine the correct
+singular or plural form. If that doesn't work out you can specify the name
+which should be used for pluralizing by adding it as parameter to `pluralize`::
+
+ {% trans ..., user_count=users|length %}...
+ {% pluralize user_count %}...{% endtrans %}
+
+It's also possible to translate strings in expressions. For that purpose
+three functions exist:
+
+_ `gettext`: translate a single string
+- `ngettext`: translate a pluralizable string
+- `_`: alias for `gettext`
+
+For example you can print a translated string easily this way::
+
+ {{ _('Hello World!') }}
+
+To use placeholders you can use the `format` filter::
+
+ {{ _('Hello %(user)s!')|format(user=user.username) }}
+
+For multiple placeholders always use keyword arguments to `format` as other
+languages may not use the words in the same order.
+
+.. versionchanged:: 2.5
+
+If newstyle gettext calls are activated (:ref:`newstyle-gettext`), using
+placeholders is a lot easier:
+
+.. sourcecode:: html+jinja
+
+ {{ gettext('Hello World!') }}
+ {{ gettext('Hello %(name)s!', name='World') }}
+ {{ ngettext('%(num)d apple', '%(num)d apples', apples|count) }}
+
+Note that the `ngettext` function's format string automatically recieves
+the count as `num` parameter additionally to the regular parameters.
+
+
+Expression Statement
+~~~~~~~~~~~~~~~~~~~~
+
+If the expression-statement extension is loaded a tag called `do` is available
+that works exactly like the regular variable expression (``{{ ... }}``) just
+that it doesn't print anything. This can be used to modify lists::
+
+ {% do navigation.append('a string') %}
+
+
+Loop Controls
+~~~~~~~~~~~~~
+
+If the application enables the :ref:`loopcontrols-extension` it's possible to
+use `break` and `continue` in loops. When `break` is reached, the loop is
+terminated, if `continue` is eached the processing is stopped and continues
+with the next iteration.
+
+Here a loop that skips every second item::
+
+ {% for user in users %}
+ {%- if loop.index is even %}{% continue %}{% endif %}
+ ...
+ {% endfor %}
+
+Likewise a look that stops processing after the 10th iteration::
+
+ {% for user in users %}
+ {%- if loop.index >= 10 %}{% break %}{% endif %}
+ {%- endfor %}
+
+
+With Statement
+~~~~~~~~~~~~~~
+
+.. versionadded:: 2.3
+
+If the application enables the :ref:`with-extension` it is possible to
+use the `with` keyword in templates. This makes it possible to create
+a new inner scope. Variables set within this scope are not visible
+outside of the scope.
+
+With in a nutshell::
+
+ {% with %}
+ {% set foo = 42 %}
+ {{ foo }} foo is 42 here
+ {% endwith %}
+ foo is not visible here any longer
+
+Because it is common to set variables at the beginning of the scope
+you can do that within the with statement. The following two examples
+are equivalent::
+
+ {% with foo = 42 %}
+ {{ foo }}
+ {% endwith %}
+
+ {% with %}
+ {% set foo = 42 %}
+ {{ foo }}
+ {% endwith %}
+
+.. _autoescape-overrides:
+
+Autoescape Extension
+--------------------
+
+.. versionadded:: 2.4
+
+If the application enables the :ref:`autoescape-extension` one can
+activate and deactivate the autoescaping from within the templates.
+
+Example::
+
+ {% autoescape true %}
+ Autoescaping is active within this block
+ {% endautoescape %}
+
+ {% autoescape false %}
+ Autoescaping is inactive within this block
+ {% endautoescape %}
+
+After the `endautoescape` the behavior is reverted to what it was before.
diff --git a/slider-agent/src/main/python/jinja2/docs/tricks.rst b/slider-agent/src/main/python/jinja2/docs/tricks.rst
new file mode 100644
index 0000000..566575e
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/docs/tricks.rst
@@ -0,0 +1,100 @@
+Tips and Tricks
+===============
+
+.. highlight:: html+jinja
+
+This part of the documentation shows some tips and tricks for Jinja2
+templates.
+
+
+.. _null-master-fallback:
+
+Null-Master Fallback
+--------------------
+
+Jinja2 supports dynamic inheritance and does not distinguish between parent
+and child template as long as no `extends` tag is visited. While this leads
+to the surprising behavior that everything before the first `extends` tag
+including whitespace is printed out instead of being igored, it can be used
+for a neat trick.
+
+Usually child templates extend from one template that adds a basic HTML
+skeleton. However it's possible put the `extends` tag into an `if` tag to
+only extend from the layout template if the `standalone` variable evaluates
+to false which it does per default if it's not defined. Additionally a very
+basic skeleton is added to the file so that if it's indeed rendered with
+`standalone` set to `True` a very basic HTML skeleton is added::
+
+ {% if not standalone %}{% extends 'master.html' %}{% endif -%}
+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+ <title>{% block title %}The Page Title{% endblock %}</title>
+ <link rel="stylesheet" href="style.css" type="text/css">
+ {% block body %}
+ <p>This is the page body.</p>
+ {% endblock %}
+
+
+Alternating Rows
+----------------
+
+If you want to have different styles for each row of a table or
+list you can use the `cycle` method on the `loop` object::
+
+ <ul>
+ {% for row in rows %}
+ <li class="{{ loop.cycle('odd', 'even') }}">{{ row }}</li>
+ {% endfor %}
+ </ul>
+
+`cycle` can take an unlimited amount of strings. Each time this
+tag is encountered the next item from the list is rendered.
+
+
+Highlighting Active Menu Items
+------------------------------
+
+Often you want to have a navigation bar with an active navigation
+item. This is really simple to achieve. Because assignments outside
+of `block`\s in child templates are global and executed before the layout
+template is evaluated it's possible to define the active menu item in the
+child template::
+
+ {% extends "layout.html" %}
+ {% set active_page = "index" %}
+
+The layout template can then access `active_page`. Additionally it makes
+sense to defined a default for that variable::
+
+ {% set navigation_bar = [
+ ('/', 'index', 'Index'),
+ ('/downloads/', 'downloads', 'Downloads'),
+ ('/about/', 'about', 'About')
+ ] -%}
+ {% set active_page = active_page|default('index') -%}
+ ...
+ <ul id="navigation">
+ {% for href, id, caption in navigation_bar %}
+ <li{% if id == active_page %} class="active"{% endif
+ %}><a href="{{ href|e }}">{{ caption|e }}</a>/li>
+ {% endfor %}
+ </ul>
+ ...
+
+.. _accessing-the-parent-loop:
+
+Accessing the parent Loop
+-------------------------
+
+The special `loop` variable always points to the innermost loop. If it's
+desired to have access to an outer loop it's possible to alias it::
+
+ <table>
+ {% for row in table %}
+ <tr>
+ {% set rowloop = loop %}
+ {% for cell in row %}
+ <td id="cell-{{ rowloop.index }}-{{ loop.index }}>{{ cell }}</td>
+ {% endfor %}
+ </tr>
+ {% endfor %}
+ </table>
diff --git a/slider-agent/src/main/python/jinja2/examples/basic/cycle.py b/slider-agent/src/main/python/jinja2/examples/basic/cycle.py
new file mode 100644
index 0000000..73dd632
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/basic/cycle.py
@@ -0,0 +1,13 @@
+from jinja2 import Environment
+
+
+env = Environment(line_statement_prefix="#", variable_start_string="${", variable_end_string="}")
+
+
+print env.from_string("""\
+<ul>
+# for item in range(10)
+ <li class="${loop.cycle('odd', 'even')}">${item}</li>
+# endfor
+</ul>\
+""").render()
diff --git a/slider-agent/src/main/python/jinja2/examples/basic/debugger.py b/slider-agent/src/main/python/jinja2/examples/basic/debugger.py
new file mode 100644
index 0000000..4291ff7
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/basic/debugger.py
@@ -0,0 +1,7 @@
+from jinja2 import Environment
+from jinja2.loaders import FileSystemLoader
+
+env = Environment(loader=FileSystemLoader('templates'))
+
+tmpl = env.get_template('broken.html')
+print tmpl.render(seq=[3, 2, 4, 5, 3, 2, 0, 2, 1])
diff --git a/slider-agent/src/main/python/jinja2/examples/basic/inheritance.py b/slider-agent/src/main/python/jinja2/examples/basic/inheritance.py
new file mode 100644
index 0000000..aa687c8
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/basic/inheritance.py
@@ -0,0 +1,12 @@
+from jinja2 import Environment
+from jinja2.loaders import DictLoader
+
+
+env = Environment(loader=DictLoader({
+'a': '''[A[{% block body %}{% endblock %}]]''',
+'b': '''{% extends 'a' %}{% block body %}[B]{% endblock %}''',
+'c': '''{% extends 'b' %}{% block body %}###{{ super() }}###{% endblock %}'''
+}))
+
+
+print env.get_template('c').render()
diff --git a/slider-agent/src/main/python/jinja2/examples/basic/templates/broken.html b/slider-agent/src/main/python/jinja2/examples/basic/templates/broken.html
new file mode 100644
index 0000000..294d5c9
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/basic/templates/broken.html
@@ -0,0 +1,6 @@
+{% from 'subbroken.html' import may_break %}
+<ul>
+{% for item in seq %}
+ <li>{{ may_break(item) }}</li>
+{% endfor %}
+</ul>
diff --git a/slider-agent/src/main/python/jinja2/examples/basic/templates/subbroken.html b/slider-agent/src/main/python/jinja2/examples/basic/templates/subbroken.html
new file mode 100644
index 0000000..245eb7e
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/basic/templates/subbroken.html
@@ -0,0 +1,3 @@
+{% macro may_break(item) -%}
+ [{{ item / 0 }}]
+{%- endmacro %}
diff --git a/slider-agent/src/main/python/jinja2/examples/basic/test.py b/slider-agent/src/main/python/jinja2/examples/basic/test.py
new file mode 100644
index 0000000..b62c84f
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/basic/test.py
@@ -0,0 +1,27 @@
+from jinja2 import Environment
+from jinja2.loaders import DictLoader
+
+env = Environment(loader=DictLoader({
+'child.html': u'''\
+{% extends master_layout or 'master.html' %}
+{% include helpers = 'helpers.html' %}
+{% macro get_the_answer() %}42{% endmacro %}
+{% title = 'Hello World' %}
+{% block body %}
+ {{ get_the_answer() }}
+ {{ helpers.conspirate() }}
+{% endblock %}
+''',
+'master.html': u'''\
+<!doctype html>
+<title>{{ title }}</title>
+{% block body %}{% endblock %}
+''',
+'helpers.html': u'''\
+{% macro conspirate() %}23{% endmacro %}
+'''
+}))
+
+
+tmpl = env.get_template("child.html")
+print tmpl.render()
diff --git a/slider-agent/src/main/python/jinja2/examples/basic/test_filter_and_linestatements.py b/slider-agent/src/main/python/jinja2/examples/basic/test_filter_and_linestatements.py
new file mode 100644
index 0000000..c9e8f95
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/basic/test_filter_and_linestatements.py
@@ -0,0 +1,25 @@
+from jinja2 import Environment
+
+
+env = Environment(line_statement_prefix='%', variable_start_string="${", variable_end_string="}")
+tmpl = env.from_string("""\
+% macro foo()
+ ${caller(42)}
+% endmacro
+<ul>
+% for item in seq
+ <li>${item}</li>
+% endfor
+</ul>
+% call(var) foo()
+ [${var}]
+% endcall
+% filter escape
+ <hello world>
+ % for item in [1, 2, 3]
+ - ${item}
+ % endfor
+% endfilter
+""")
+
+print tmpl.render(seq=range(10))
diff --git a/slider-agent/src/main/python/jinja2/examples/basic/test_loop_filter.py b/slider-agent/src/main/python/jinja2/examples/basic/test_loop_filter.py
new file mode 100644
index 0000000..49c2efc
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/basic/test_loop_filter.py
@@ -0,0 +1,12 @@
+from jinja2 import Environment
+
+tmpl = Environment().from_string("""\
+<ul>
+{%- for item in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] if item % 2 == 0 %}
+ <li>{{ loop.index }} / {{ loop.length }}: {{ item }}</li>
+{%- endfor %}
+</ul>
+if condition: {{ 1 if foo else 0 }}
+""")
+
+print tmpl.render(foo=True)
diff --git a/slider-agent/src/main/python/jinja2/examples/basic/translate.py b/slider-agent/src/main/python/jinja2/examples/basic/translate.py
new file mode 100644
index 0000000..3358765
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/basic/translate.py
@@ -0,0 +1,6 @@
+from jinja2 import Environment
+
+print Environment(extensions=['jinja2.i18n.TransExtension']).from_string("""\
+{% trans %}Hello {{ user }}!{% endtrans %}
+{% trans count=users|count %}{{ count }} user{% pluralize %}{{ count }} users{% endtrans %}
+""").render(user="someone")
diff --git a/slider-agent/src/main/python/jinja2/examples/bench.py b/slider-agent/src/main/python/jinja2/examples/bench.py
new file mode 100644
index 0000000..c648dc6
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/bench.py
@@ -0,0 +1,433 @@
+"""\
+ This benchmark compares some python templating engines with Jinja 2 so
+ that we get a picture of how fast Jinja 2 is for a semi real world
+ template. If a template engine is not installed the test is skipped.\
+"""
+import sys
+import cgi
+from timeit import Timer
+from jinja2 import Environment as JinjaEnvironment
+
+context = {
+ 'page_title': 'mitsuhiko\'s benchmark',
+ 'table': [dict(a=1,b=2,c=3,d=4,e=5,f=6,g=7,h=8,i=9,j=10) for x in range(1000)]
+}
+
+jinja_template = JinjaEnvironment(
+ line_statement_prefix='%',
+ variable_start_string="${",
+ variable_end_string="}"
+).from_string("""\
+<!doctype html>
+<html>
+ <head>
+ <title>${page_title|e}</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1>${page_title|e}</h1>
+ </div>
+ <ul class="navigation">
+ % for href, caption in [
+ ('index.html', 'Index'),
+ ('downloads.html', 'Downloads'),
+ ('products.html', 'Products')
+ ]
+ <li><a href="${href|e}">${caption|e}</a></li>
+ % endfor
+ </ul>
+ <div class="table">
+ <table>
+ % for row in table
+ <tr>
+ % for cell in row
+ <td>${cell}</td>
+ % endfor
+ </tr>
+ % endfor
+ </table>
+ </div>
+ </body>
+</html>\
+""")
+
+def test_jinja():
+ jinja_template.render(context)
+
+try:
+ from tornado.template import Template
+except ImportError:
+ test_tornado = None
+else:
+ tornado_template = Template("""\
+<!doctype html>
+<html>
+ <head>
+ <title>{{ page_title }}</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1>{{ page_title }}</h1>
+ </div>
+ <ul class="navigation">
+ {% for href, caption in [ \
+ ('index.html', 'Index'), \
+ ('downloads.html', 'Downloads'), \
+ ('products.html', 'Products') \
+ ] %}
+ <li><a href="{{ href }}">{{ caption }}</a></li>
+ {% end %}
+ </ul>
+ <div class="table">
+ <table>
+ {% for row in table %}
+ <tr>
+ {% for cell in row %}
+ <td>{{ cell }}</td>
+ {% end %}
+ </tr>
+ {% end %}
+ </table>
+ </div>
+ </body>
+</html>\
+""")
+
+ def test_tornado():
+ tornado_template.generate(**context)
+
+try:
+ from django.conf import settings
+ settings.configure()
+ from django.template import Template as DjangoTemplate, Context as DjangoContext
+except ImportError:
+ test_django = None
+else:
+ django_template = DjangoTemplate("""\
+<!doctype html>
+<html>
+ <head>
+ <title>{{ page_title }}</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1>{{ page_title }}</h1>
+ </div>
+ <ul class="navigation">
+ {% for href, caption in navigation %}
+ <li><a href="{{ href }}">{{ caption }}</a></li>
+ {% endfor %}
+ </ul>
+ <div class="table">
+ <table>
+ {% for row in table %}
+ <tr>
+ {% for cell in row %}
+ <td>{{ cell }}</td>
+ {% endfor %}
+ </tr>
+ {% endfor %}
+ </table>
+ </div>
+ </body>
+</html>\
+""")
+
+ def test_django():
+ c = DjangoContext(context)
+ c['navigation'] = [('index.html', 'Index'), ('downloads.html', 'Downloads'),
+ ('products.html', 'Products')]
+ django_template.render(c)
+
+try:
+ from mako.template import Template as MakoTemplate
+except ImportError:
+ test_mako = None
+else:
+ mako_template = MakoTemplate("""\
+<!doctype html>
+<html>
+ <head>
+ <title>${page_title|h}</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1>${page_title|h}</h1>
+ </div>
+ <ul class="navigation">
+ % for href, caption in [('index.html', 'Index'), ('downloads.html', 'Downloads'), ('products.html', 'Products')]:
+ <li><a href="${href|h}">${caption|h}</a></li>
+ % endfor
+ </ul>
+ <div class="table">
+ <table>
+ % for row in table:
+ <tr>
+ % for cell in row:
+ <td>${cell}</td>
+ % endfor
+ </tr>
+ % endfor
+ </table>
+ </div>
+ </body>
+</html>\
+""")
+
+ def test_mako():
+ mako_template.render(**context)
+
+try:
+ from genshi.template import MarkupTemplate as GenshiTemplate
+except ImportError:
+ test_genshi = None
+else:
+ genshi_template = GenshiTemplate("""\
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:py="http://genshi.edgewall.org/">
+ <head>
+ <title>${page_title}</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1>${page_title}</h1>
+ </div>
+ <ul class="navigation">
+ <li py:for="href, caption in [
+ ('index.html', 'Index'),
+ ('downloads.html', 'Downloads'),
+ ('products.html', 'Products')]"><a href="${href}">${caption}</a></li>
+ </ul>
+ <div class="table">
+ <table>
+ <tr py:for="row in table">
+ <td py:for="cell in row">${cell}</td>
+ </tr>
+ </table>
+ </div>
+ </body>
+</html>\
+""")
+
+ def test_genshi():
+ genshi_template.generate(**context).render('html', strip_whitespace=False)
+
+try:
+ from Cheetah.Template import Template as CheetahTemplate
+except ImportError:
+ test_cheetah = None
+else:
+ cheetah_template = CheetahTemplate("""\
+#import cgi
+<!doctype html>
+<html>
+ <head>
+ <title>$cgi.escape($page_title)</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1>$cgi.escape($page_title)</h1>
+ </div>
+ <ul class="navigation">
+ #for $href, $caption in [('index.html', 'Index'), ('downloads.html', 'Downloads'), ('products.html', 'Products')]:
+ <li><a href="$cgi.escape($href)">$cgi.escape($caption)</a></li>
+ #end for
+ </ul>
+ <div class="table">
+ <table>
+ #for $row in $table:
+ <tr>
+ #for $cell in $row:
+ <td>$cell</td>
+ #end for
+ </tr>
+ #end for
+ </table>
+ </div>
+ </body>
+</html>\
+""", searchList=[dict(context)])
+
+ def test_cheetah():
+ unicode(cheetah_template)
+
+try:
+ import tenjin
+except ImportError:
+ test_tenjin = None
+else:
+ tenjin_template = tenjin.Template()
+ tenjin_template.convert("""\
+<!doctype html>
+<html>
+ <head>
+ <title>${page_title}</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1>${page_title}</h1>
+ </div>
+ <ul class="navigation">
+<?py for href, caption in [('index.html', 'Index'), ('downloads.html', 'Downloads'), ('products.html', 'Products')]: ?>
+ <li><a href="${href}">${caption}</a></li>
+<?py #end ?>
+ </ul>
+ <div class="table">
+ <table>
+<?py for row in table: ?>
+ <tr>
+<?py for cell in row: ?>
+ <td>#{cell}</td>
+<?py #end ?>
+ </tr>
+<?py #end ?>
+ </table>
+ </div>
+ </body>
+</html>\
+""")
+
+ def test_tenjin():
+ from tenjin.helpers import escape, to_str
+ tenjin_template.render(context, locals())
+
+try:
+ from spitfire.compiler import util as SpitfireTemplate
+ from spitfire.compiler.analyzer import o2_options as spitfire_optimizer
+except ImportError:
+ test_spitfire = None
+else:
+ spitfire_template = SpitfireTemplate.load_template("""\
+<!doctype html>
+<html>
+ <head>
+ <title>$cgi.escape($page_title)</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1>$cgi.escape($page_title)</h1>
+ </div>
+ <ul class="navigation">
+ #for $href, $caption in [('index.html', 'Index'), ('downloads.html', 'Downloads'), ('products.html', 'Products')]
+ <li><a href="$cgi.escape($href)">$cgi.escape($caption)</a></li>
+ #end for
+ </ul>
+ <div class="table">
+ <table>
+ #for $row in $table
+ <tr>
+ #for $cell in $row
+ <td>$cell</td>
+ #end for
+ </tr>
+ #end for
+ </table>
+ </div>
+ </body>
+</html>\
+""", 'spitfire_tmpl', spitfire_optimizer, {'enable_filters': False})
+ spitfire_context = dict(context, **{'cgi': cgi})
+
+ def test_spitfire():
+ spitfire_template(search_list=[spitfire_context]).main()
+
+
+try:
+ from chameleon.zpt.template import PageTemplate
+except ImportError:
+ test_chameleon = None
+else:
+ chameleon_template = PageTemplate("""\
+<html xmlns:tal="http://xml.zope.org/namespaces/tal">
+ <head>
+ <title tal:content="page_title">Page Title</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1 tal:content="page_title">Page Title</h1>
+ </div>
+ <ul class="navigation">
+ <li tal:repeat="item sections"><a tal:attributes="href item[0]" tal:content="item[1]">caption</a></li>
+ </ul>
+ <div class="table">
+ <table>
+ <tr tal:repeat="row table">
+ <td tal:repeat="cell row" tal:content="row[cell]">cell</td>
+ </tr>
+ </table>
+ </div>
+ </body>
+</html>\
+""")
+ chameleon_context = dict(context)
+ chameleon_context['sections'] = [
+ ('index.html', 'Index'),
+ ('downloads.html', 'Downloads'),
+ ('products.html', 'Products')
+ ]
+ def test_chameleon():
+ chameleon_template.render(**chameleon_context)
+
+try:
+ from chameleon.zpt.template import PageTemplate
+ from chameleon.genshi import language
+except ImportError:
+ test_chameleon_genshi = None
+else:
+ chameleon_genshi_template = PageTemplate("""\
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:py="http://genshi.edgewall.org/">
+ <head>
+ <title>${page_title}</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1>${page_title}</h1>
+ </div>
+ <ul class="navigation">
+ <li py:for="info in sections"><a href="${info[0]}">${info[1]}</a></li>
+ </ul>
+ <div class="table">
+ <table>
+ <tr py:for="row in table">
+ <td py:for="cell in row">${row[cell]}</td>
+ </tr>
+ </table>
+ </div>
+ </body>
+</html>\
+""", parser=language.Parser())
+ chameleon_genshi_context = dict(context)
+ chameleon_genshi_context['sections'] = [
+ ('index.html', 'Index'),
+ ('downloads.html', 'Downloads'),
+ ('products.html', 'Products')
+ ]
+ def test_chameleon_genshi():
+ chameleon_genshi_template.render(**chameleon_genshi_context)
+
+
+sys.stdout.write('\r' + '\n'.join((
+ '=' * 80,
+ 'Template Engine BigTable Benchmark'.center(80),
+ '=' * 80,
+ __doc__,
+ '-' * 80
+)) + '\n')
+
+
+for test in 'jinja', 'mako', 'tornado', 'tenjin', 'spitfire', 'django', 'genshi', 'cheetah', 'chameleon', 'chameleon_genshi':
+ if locals()['test_' + test] is None:
+ sys.stdout.write(' %-20s*not installed*\n' % test)
+ continue
+ t = Timer(setup='from __main__ import test_%s as bench' % test,
+ stmt='bench()')
+ sys.stdout.write(' >> %-20s<running>' % test)
+ sys.stdout.flush()
+ sys.stdout.write('\r %-20s%.4f seconds\n' % (test, t.timeit(number=50) / 50))
+sys.stdout.write('-' * 80 + '\n')
+sys.stdout.write('''\
+ WARNING: The results of this benchmark are useless to compare the
+ performance of template engines and should not be taken seriously in any
+ way. It's testing the performance of simple loops and has no real-world
+ usefulnes. It only used to check if changes on the Jinja code affect
+ performance in a good or bad way and how it roughly compares to others.
+''' + '=' * 80 + '\n')
diff --git a/slider-agent/src/main/python/jinja2/examples/profile.py b/slider-agent/src/main/python/jinja2/examples/profile.py
new file mode 100644
index 0000000..0c907ae
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/profile.py
@@ -0,0 +1,52 @@
+try:
+ from cProfile import Profile
+except ImportError:
+ from profile import Profile
+from pstats import Stats
+from jinja2 import Environment as JinjaEnvironment
+
+context = {
+ 'page_title': 'mitsuhiko\'s benchmark',
+ 'table': [dict(a=1,b=2,c=3,d=4,e=5,f=6,g=7,h=8,i=9,j=10) for x in range(1000)]
+}
+
+source = """\
+% macro testmacro(x)
+ <span>${x}</span>
+% endmacro
+<!doctype html>
+<html>
+ <head>
+ <title>${page_title|e}</title>
+ </head>
+ <body>
+ <div class="header">
+ <h1>${page_title|e}</h1>
+ </div>
+ <div class="table">
+ <table>
+ % for row in table
+ <tr>
+ % for cell in row
+ <td>${testmacro(cell)}</td>
+ % endfor
+ </tr>
+ % endfor
+ </table>
+ </div>
+ </body>
+</html>\
+"""
+jinja_template = JinjaEnvironment(
+ line_statement_prefix='%',
+ variable_start_string="${",
+ variable_end_string="}"
+).from_string(source)
+print jinja_template.environment.compile(source, raw=True)
+
+
+p = Profile()
+p.runcall(lambda: jinja_template.render(context))
+stats = Stats(p)
+stats.sort_stats('time', 'calls')
+stats.print_stats()
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/django/_form.html b/slider-agent/src/main/python/jinja2/examples/rwbench/django/_form.html
new file mode 100644
index 0000000..9c4f710
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/django/_form.html
@@ -0,0 +1 @@
+<form action="{{ action }}" method="{{ method }}">{{ body }}</form>
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/django/_input_field.html b/slider-agent/src/main/python/jinja2/examples/rwbench/django/_input_field.html
new file mode 100644
index 0000000..290fdbd
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/django/_input_field.html
@@ -0,0 +1 @@
+<input type="{{ type }}" value="{{ value }}" name="{{ name }}">
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/django/_textarea.html b/slider-agent/src/main/python/jinja2/examples/rwbench/django/_textarea.html
new file mode 100644
index 0000000..7f10424
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/django/_textarea.html
@@ -0,0 +1 @@
+<textarea name="{{ name }}" rows="{{ rows }}" cols="{{ cols }}">{{ value }}</textarea>
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/django/index.html b/slider-agent/src/main/python/jinja2/examples/rwbench/django/index.html
new file mode 100644
index 0000000..6f620bb
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/django/index.html
@@ -0,0 +1,29 @@
+{% extends "layout.html" %}
+{% block page_title %}Index Page{% endblock %}
+{% block body %}
+ {% for article in articles %}
+ {% if article.published %}
+ <div class="article">
+ <h2><a href="{{ article.href }}">{{ article.title }}</a></h2>
+ <p class="meta">written by <a href="{{ article.user.href }}">{{ article.user.username }}</a> on {{ article.pub_date|dateformat }}</p>
+ <div class="text">{{ article.body|safe }}</div>
+ </div>
+ {% endif %}
+ {% endfor %}
+ {% form %}
+ <dl>
+ <dt>Name</dt>
+ <dd>{% input_field 'name' %}</dd>
+ <dt>E-Mail</dt>
+ <dd>{% input_field 'email' %}</dd>
+ <dt>URL</dt>
+ <dd>{% input_field 'url' %}</dd>
+ <dt>Comment</dt>
+ <dd>{% textarea 'comment' %}</dd>
+ <dt>Captcha</dt>
+ <dd>{% input_field 'captcha' %}</dd>
+ </dl>
+ {% input_field '' 'submit' 'Submit' %}
+ {% input_field 'cancel' 'submit' 'Cancel' %}
+ {% endform %}
+{% endblock %}
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/django/layout.html b/slider-agent/src/main/python/jinja2/examples/rwbench/django/layout.html
new file mode 100644
index 0000000..60039ce
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/django/layout.html
@@ -0,0 +1,29 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+<head>
+ <title>{% block page_title %}{% endblock %} | RealWorld Benchmark</title>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+</head>
+<body>
+ <div class="contents">
+ <div class="header">
+ <h1>RealWorld Benchmark</h1>
+ <blockquote><p>
+ A less stupid benchmark for Mako and Jinja2 to get an impression how
+ code changes affect runtime performance.
+ </p></blockquote>
+ </div>
+ <ul class="navigation">
+ {% for href, caption in page_navigation %}
+ <li><a href="{{ href }}">{{ caption }}</a></li>
+ {% endfor %}
+ </ul>
+ <div class="body">
+ {% block body %}{% endblock %}
+ </div>
+ <div class="footer">
+ © Copyright 2008 by I don't know who.
+ </div>
+ </div>
+</body>
+</html>
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/djangoext.py b/slider-agent/src/main/python/jinja2/examples/rwbench/djangoext.py
new file mode 100644
index 0000000..9e9fa6c
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/djangoext.py
@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+from rwbench import ROOT
+from os.path import join
+from django.conf import settings
+settings.configure(
+ TEMPLATE_DIRS=(join(ROOT, 'django'),),
+ TEMPLATE_LOADERS=(
+ ('django.template.loaders.cached.Loader', (
+ 'django.template.loaders.filesystem.Loader',
+ )),
+ )
+)
+from django.template import loader as django_loader, Context as DjangoContext, \
+ Node, NodeList, Variable, TokenParser
+from django import template as django_template_module
+from django.template import Library
+
+
+# for django extensions. We monkey patch our extensions in so that
+# we don't have to initialize a more complex django setup.
+django_extensions = django_template_module.Library()
+django_template_module.builtins.append(django_extensions)
+
+
+from rwbench import dateformat
+django_extensions.filter(dateformat)
+
+
+def var_or_none(x):
+ if x is not None:
+ return Variable(x)
+
+
+# and more django extensions
+@django_extensions.tag
+def input_field(parser, token):
+ p = TokenParser(token.contents)
+ args = [p.value()]
+ while p.more():
+ args.append(p.value())
+ return InputFieldNode(*args)
+
+
+@django_extensions.tag
+def textarea(parser, token):
+ p = TokenParser(token.contents)
+ args = [p.value()]
+ while p.more():
+ args.append(p.value())
+ return TextareaNode(*args)
+
+
+@django_extensions.tag
+def form(parser, token):
+ p = TokenParser(token.contents)
+ args = []
+ while p.more():
+ args.append(p.value())
+ body = parser.parse(('endform',))
+ parser.delete_first_token()
+ return FormNode(body, *args)
+
+
+class InputFieldNode(Node):
+
+ def __init__(self, name, type=None, value=None):
+ self.name = var_or_none(name)
+ self.type = var_or_none(type)
+ self.value = var_or_none(value)
+
+ def render(self, context):
+ name = self.name.resolve(context)
+ type = 'text'
+ value = ''
+ if self.type is not None:
+ type = self.type.resolve(context)
+ if self.value is not None:
+ value = self.value.resolve(context)
+ tmpl = django_loader.get_template('_input_field.html')
+ return tmpl.render(DjangoContext({
+ 'name': name,
+ 'type': type,
+ 'value': value
+ }))
+
+
+class TextareaNode(Node):
+
+ def __init__(self, name, rows=None, cols=None, value=None):
+ self.name = var_or_none(name)
+ self.rows = var_or_none(rows)
+ self.cols = var_or_none(cols)
+ self.value = var_or_none(value)
+
+ def render(self, context):
+ name = self.name.resolve(context)
+ rows = 10
+ cols = 40
+ value = ''
+ if self.rows is not None:
+ rows = int(self.rows.resolve(context))
+ if self.cols is not None:
+ cols = int(self.cols.resolve(context))
+ if self.value is not None:
+ value = self.value.resolve(context)
+ tmpl = django_loader.get_template('_textarea.html')
+ return tmpl.render(DjangoContext({
+ 'name': name,
+ 'rows': rows,
+ 'cols': cols,
+ 'value': value
+ }))
+
+
+class FormNode(Node):
+
+ def __init__(self, body, action=None, method=None):
+ self.body = body
+ self.action = action
+ self.method = method
+
+ def render(self, context):
+ body = self.body.render(context)
+ action = ''
+ method = 'post'
+ if self.action is not None:
+ action = self.action.resolve(context)
+ if self.method is not None:
+ method = self.method.resolve(context)
+ tmpl = django_loader.get_template('_form.html')
+ return tmpl.render(DjangoContext({
+ 'body': body,
+ 'action': action,
+ 'method': method
+ }))
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/genshi/helpers.html b/slider-agent/src/main/python/jinja2/examples/rwbench/genshi/helpers.html
new file mode 100644
index 0000000..ecc6dc4
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/genshi/helpers.html
@@ -0,0 +1,12 @@
+<div xmlns="http://www.w3.org/1999/xhtml" xmlns:py="http://genshi.edgewall.org/"
+ py:strip="">
+
+ <py:def function="input_field(name='', value='', type='text')">
+ <input type="$type" value="$value" name="$name" />
+ </py:def>
+
+ <py:def function="textarea(name, value='', rows=10, cols=40)">
+ <textarea name="$name" rows="$rows" cols="cols">$value</textarea>
+ </py:def>
+
+</div>
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/genshi/index.html b/slider-agent/src/main/python/jinja2/examples/rwbench/genshi/index.html
new file mode 100644
index 0000000..70f697d
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/genshi/index.html
@@ -0,0 +1,41 @@
+<?python
+ from rwbench import dateformat
+?>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xi="http://www.w3.org/2001/XInclude"
+ xmlns:py="http://genshi.edgewall.org/">
+ <xi:include href="layout.html" />
+ <xi:include href="helpers.html" />
+ <head><title>Index Page</title></head>
+ <body>
+ <div class="article" py:for="article in articles">
+ <py:if test="article.published">
+ <h2><a href="${article.href}">${article.title}</a></h2>
+ <p class="meta">written by <a href="${article.user.href}"
+ >${article.user.username}</a> on ${dateformat(article.pub_date)}</p>
+ <div class="text">${Markup(article.body)}</div>
+ </py:if>
+ </div>
+ <!--
+ For a fair and balanced comparison we would have to use a def here
+ that wraps the form data but I don't know what would be the best
+ Genshi equivalent for that. Quite frankly I doubt that this makes
+ sense in Genshi anyways.
+ -->
+ <form action="" method="post">
+ <dl>
+ <dt>Name</dt>
+ <dd>${input_field('name')}</dd>
+ <dt>E-Mail</dt>
+ <dd>${input_field('email')}</dd>
+ <dt>URL</dt>
+ <dd>${input_field('url')}</dd>
+ <dt>Comment</dt>
+ <dd>${textarea('comment')}</dd>
+ <dt>Captcha</dt>
+ <dd>${input_field('captcha')}</dd>
+ </dl>
+ ${input_field(type='submit', value='Submit')}
+ ${input_field(name='cancel', type='submit', value='Cancel')}
+ </form>
+ </body>
+</html>
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/genshi/layout.html b/slider-agent/src/main/python/jinja2/examples/rwbench/genshi/layout.html
new file mode 100644
index 0000000..b12aec4
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/genshi/layout.html
@@ -0,0 +1,30 @@
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:py="http://genshi.edgewall.org/" >
+ <py:match path="head" once="true">
+ <head>
+ <title>${select('title/text()')} | RealWorld Benchmark</title>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+ </head>
+ </py:match>
+ <py:match path="body" once="true">
+ <body>
+ <div class="contents">
+ <div class="header">
+ <h1>RealWorld Benchmark</h1>
+ <blockquote><p>
+ A less stupid benchmark for Mako and Jinja2 to get an impression how
+ code changes affect runtime performance.
+ </p></blockquote>
+ </div>
+ <ul class="navigation">
+ <li py:for="href, caption in page_navigation"><a href="$href">$caption</a></li>
+ </ul>
+ <div class="body">
+ ${select('*|text()')}
+ </div>
+ <div class="footer">
+ © Copyright 2008 by I don't know who.
+ </div>
+ </div>
+ </body>
+ </py:match>
+</html>
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/jinja/helpers.html b/slider-agent/src/main/python/jinja2/examples/rwbench/jinja/helpers.html
new file mode 100644
index 0000000..89976aa
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/jinja/helpers.html
@@ -0,0 +1,12 @@
+{% macro input_field(name, value='', type='text') -%}
+ <input type="{{ type }}" value="{{ value|e }}" name="{{ name }}">
+{%- endmacro %}
+
+{% macro textarea(name, value='', rows=10, cols=40) -%}
+ <textarea name="{{ name }}" rows="{{ rows }}" cols="{{ cols }}">{{
+ value|e }}</textarea>
+{%- endmacro %}
+
+{% macro form(action='', method='post') -%}
+ <form action="{{ action|e }}" method="{{ method }}">{{ caller() }}</form>
+{%- endmacro %}
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/jinja/index.html b/slider-agent/src/main/python/jinja2/examples/rwbench/jinja/index.html
new file mode 100644
index 0000000..b006d05
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/jinja/index.html
@@ -0,0 +1,29 @@
+{% extends "layout.html" %}
+{% from "helpers.html" import input_field, textarea, form %}
+{% block page_title %}Index Page{% endblock %}
+{% block body %}
+ {%- for article in articles if article.published %}
+ <div class="article">
+ <h2><a href="{{ article.href|e }}">{{ article.title|e }}</a></h2>
+ <p class="meta">written by <a href="{{ article.user.href|e
+ }}">{{ article.user.username|e }}</a> on {{ article.pub_date|dateformat }}</p>
+ <div class="text">{{ article.body }}</div>
+ </div>
+ {%- endfor %}
+ {%- call form() %}
+ <dl>
+ <dt>Name</dt>
+ <dd>{{ input_field('name') }}</dd>
+ <dt>E-Mail</dt>
+ <dd>{{ input_field('email') }}</dd>
+ <dt>URL</dt>
+ <dd>{{ input_field('url') }}</dd>
+ <dt>Comment</dt>
+ <dd>{{ textarea('comment') }}</dd>
+ <dt>Captcha</dt>
+ <dd>{{ input_field('captcha') }}</dd>
+ </dl>
+ {{ input_field(type='submit', value='Submit') }}
+ {{ input_field('cancel', type='submit', value='Cancel') }}
+ {%- endcall %}
+{% endblock %}
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/jinja/layout.html b/slider-agent/src/main/python/jinja2/examples/rwbench/jinja/layout.html
new file mode 100644
index 0000000..755789e
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/jinja/layout.html
@@ -0,0 +1,29 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+<head>
+ <title>{% block page_title %}{% endblock %} | RealWorld Benchmark</title>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+</head>
+<body>
+ <div class="contents">
+ <div class="header">
+ <h1>RealWorld Benchmark</h1>
+ <blockquote><p>
+ A less stupid benchmark for Mako and Jinja2 to get an impression how
+ code changes affect runtime performance.
+ </p></blockquote>
+ </div>
+ <ul class="navigation">
+ {%- for href, caption in page_navigation %}
+ <li><a href="{{ href|e }}">{{ caption }}</a></li>
+ {%- endfor %}
+ </ul>
+ <div class="body">
+ {% block body %}{% endblock %}
+ </div>
+ <div class="footer">
+ © Copyright 2008 by I don't know who.
+ </div>
+ </div>
+</body>
+</html>
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/mako/helpers.html b/slider-agent/src/main/python/jinja2/examples/rwbench/mako/helpers.html
new file mode 100644
index 0000000..a0290eb
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/mako/helpers.html
@@ -0,0 +1,11 @@
+<%def name="input_field(name='', value='', type='text')">
+ <input type="${type}" value="${value|h}" name="${name}">
+</%def>
+
+<%def name="textarea(name, value='', rows=10, cols=40)">
+ <textarea name="${name}" rows="${rows}" cols="${cols}">${value|h}</textarea>
+</%def>
+
+<%def name="form(action='', method='post')">
+ <form action="${action|h}" method="${method}">${caller.body()}</form>
+</%def>
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/mako/index.html b/slider-agent/src/main/python/jinja2/examples/rwbench/mako/index.html
new file mode 100644
index 0000000..c4c6303
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/mako/index.html
@@ -0,0 +1,31 @@
+<%!
+ from rwbench import dateformat
+%>
+<%inherit file="layout.html" />
+<%namespace file="helpers.html" import="input_field, textarea, form" />
+<%def name="page_title()">Index Page</%def>
+% for article in articles:
+ <% if not article.published: continue %>
+<div class="article">
+ <h2><a href="${article.href|h}">${article.title|h}</a></h2>
+ <p class="meta">written by <a href="${article.user.href|h
+ }">${article.user.username|h}</a> on ${dateformat(article.pub_date)}</p>
+ <div class="text">${article.body}</div>
+</div>
+% endfor
+<%call expr="form()">
+ <dl>
+ <dt>Name</dt>
+ <dd>${input_field('name')}</dd>
+ <dt>E-Mail</dt>
+ <dd>${input_field('email')}</dd>
+ <dt>URL</dt>
+ <dd>${input_field('url')}</dd>
+ <dt>Comment</dt>
+ <dd>${textarea('comment')}</dd>
+ <dt>Captcha</dt>
+ <dd>${input_field('captcha')}</dd>
+ </dl>
+ ${input_field(type='submit', value='Submit')}
+ ${input_field(name='cancel', type='submit', value='Cancel')}
+</%call>
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/mako/layout.html b/slider-agent/src/main/python/jinja2/examples/rwbench/mako/layout.html
new file mode 100644
index 0000000..a9c353e
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/mako/layout.html
@@ -0,0 +1,30 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+<head>
+ <title>${self.page_title()} | RealWorld Benchmark</title>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+</head>
+<body>
+ <div class="contents">
+ <div class="header">
+ <h1>RealWorld Benchmark</h1>
+ <blockquote><p>
+ A less stupid benchmark for Mako and Jinja2 to get an impression how
+ code changes affect runtime performance.
+ </p></blockquote>
+ </div>
+ <ul class="navigation">
+ % for href, caption in page_navigation:
+ <li><a href="${href|h}">${caption}</a></li>
+ % endfor
+ </ul>
+ <div class="body">
+ ${self.body()}
+ </div>
+ <div class="footer">
+ © Copyright 2008 by I don't know who.
+ </div>
+ </div>
+</body>
+</html>
+<%def name="page_title()"></%def>
diff --git a/slider-agent/src/main/python/jinja2/examples/rwbench/rwbench.py b/slider-agent/src/main/python/jinja2/examples/rwbench/rwbench.py
new file mode 100644
index 0000000..813dd56
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/examples/rwbench/rwbench.py
@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+"""
+ RealWorldish Benchmark
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ A more real-world benchmark of Jinja2. Like the other benchmark in the
+ Jinja2 repository this has no real-world usefulnes (despite the name).
+ Just go away and ignore it. NOW!
+
+ :copyright: (c) 2009 by the Jinja Team.
+ :license: BSD.
+"""
+import sys
+from os.path import join, dirname, abspath
+try:
+ from cProfile import Profile
+except ImportError:
+ from profile import Profile
+from pstats import Stats
+ROOT = abspath(dirname(__file__))
+
+from random import choice, randrange
+from datetime import datetime
+from timeit import Timer
+from jinja2 import Environment, FileSystemLoader
+from jinja2.utils import generate_lorem_ipsum
+from mako.lookup import TemplateLookup
+from genshi.template import TemplateLoader as GenshiTemplateLoader
+
+
+def dateformat(x):
+ return x.strftime('%Y-%m-%d')
+
+
+jinja_env = Environment(loader=FileSystemLoader(join(ROOT, 'jinja')))
+jinja_env.filters['dateformat'] = dateformat
+mako_lookup = TemplateLookup(directories=[join(ROOT, 'mako')])
+genshi_loader = GenshiTemplateLoader([join(ROOT, 'genshi')])
+
+class Article(object):
+
+ def __init__(self, id):
+ self.id = id
+ self.href = '/article/%d' % self.id
+ self.title = generate_lorem_ipsum(1, False, 5, 10)
+ self.user = choice(users)
+ self.body = generate_lorem_ipsum()
+ self.pub_date = datetime.utcfromtimestamp(randrange(10 ** 9, 2 * 10 ** 9))
+ self.published = True
+
+
+class User(object):
+
+ def __init__(self, username):
+ self.href = '/user/%s' % username
+ self.username = username
+
+
+users = map(User, [u'John Doe', u'Jane Doe', u'Peter Somewhat'])
+articles = map(Article, range(20))
+navigation = [
+ ('index', 'Index'),
+ ('about', 'About'),
+ ('foo?bar=1', 'Foo with Bar'),
+ ('foo?bar=2&s=x', 'Foo with X'),
+ ('blah', 'Blub Blah'),
+ ('hehe', 'Haha'),
+] * 5
+
+context = dict(users=users, articles=articles, page_navigation=navigation)
+
+
+jinja_template = jinja_env.get_template('index.html')
+mako_template = mako_lookup.get_template('index.html')
+genshi_template = genshi_loader.load('index.html')
+
+
+def test_jinja():
+ jinja_template.render(context)
+
+def test_mako():
+ mako_template.render_unicode(**context)
+
+
+from djangoext import django_loader, DjangoContext
+def test_django():
+ # not cached because django is not thread safe and does
+ # not cache by itself so it would be unfair to cache it here.
+ django_template = django_loader.get_template('index.html')
+ django_template.render(DjangoContext(context))
+
+
+def test_genshi():
+ genshi_template.generate(**context).render('html', doctype='html')
+
+
+if __name__ == '__main__':
+ sys.stdout.write('Realworldish Benchmark:\n')
+ for test in 'jinja', 'mako', 'django', 'genshi':
+ t = Timer(setup='from __main__ import test_%s as bench' % test,
+ stmt='bench()')
+ sys.stdout.write(' >> %-20s<running>' % test)
+ sys.stdout.flush()
+ sys.stdout.write('\r %-20s%.4f seconds\n' % (test, t.timeit(number=200) / 200))
+
+ if '-p' in sys.argv:
+ print 'Jinja profile'
+ p = Profile()
+ p.runcall(test_jinja)
+ stats = Stats(p)
+ stats.sort_stats('time', 'calls')
+ stats.print_stats()
diff --git a/slider-agent/src/main/python/jinja2/ext/Vim/htmljinja.vim b/slider-agent/src/main/python/jinja2/ext/Vim/htmljinja.vim
new file mode 100644
index 0000000..3f9cba4
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/ext/Vim/htmljinja.vim
@@ -0,0 +1,27 @@
+" Vim syntax file
+" Language: Jinja HTML template
+" Maintainer: Armin Ronacher <armin.ronacher@active-4.com>
+" Last Change: 2007 Apr 8
+
+" For version 5.x: Clear all syntax items
+" For version 6.x: Quit when a syntax file was already loaded
+if version < 600
+ syntax clear
+elseif exists("b:current_syntax")
+ finish
+endif
+
+if !exists("main_syntax")
+ let main_syntax = 'html'
+endif
+
+if version < 600
+ so <sfile>:p:h/jinja.vim
+ so <sfile>:p:h/html.vim
+else
+ runtime! syntax/jinja.vim
+ runtime! syntax/html.vim
+ unlet b:current_syntax
+endif
+
+let b:current_syntax = "htmljinja"
diff --git a/slider-agent/src/main/python/jinja2/ext/Vim/jinja.vim b/slider-agent/src/main/python/jinja2/ext/Vim/jinja.vim
new file mode 100644
index 0000000..919954b
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/ext/Vim/jinja.vim
@@ -0,0 +1,113 @@
+" Vim syntax file
+" Language: Jinja template
+" Maintainer: Armin Ronacher <armin.ronacher@active-4.com>
+" Last Change: 2008 May 9
+" Version: 1.1
+"
+" Known Bugs:
+" because of odd limitations dicts and the modulo operator
+" appear wrong in the template.
+"
+" Changes:
+"
+" 2008 May 9: Added support for Jinja2 changes (new keyword rules)
+
+" For version 5.x: Clear all syntax items
+" For version 6.x: Quit when a syntax file was already loaded
+if version < 600
+ syntax clear
+elseif exists("b:current_syntax")
+ finish
+endif
+
+syntax case match
+
+" Jinja template built-in tags and parameters (without filter, macro, is and raw, they
+" have special threatment)
+syn keyword jinjaStatement containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained and if else in not or recursive as import
+
+syn keyword jinjaStatement containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained is filter skipwhite nextgroup=jinjaFilter
+syn keyword jinjaStatement containedin=jinjaTagBlock contained macro skipwhite nextgroup=jinjaFunction
+syn keyword jinjaStatement containedin=jinjaTagBlock contained block skipwhite nextgroup=jinjaBlockName
+
+" Variable Names
+syn match jinjaVariable containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/
+syn keyword jinjaSpecial containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained false true none False True None loop super caller varargs kwargs
+
+" Filters
+syn match jinjaOperator "|" containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained nextgroup=jinjaFilter
+syn match jinjaFilter contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/
+syn match jinjaFunction contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/
+syn match jinjaBlockName contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/
+
+" Jinja template constants
+syn region jinjaString containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained start=/"/ skip=/\\"/ end=/"/
+syn region jinjaString containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained start=/'/ skip=/\\'/ end=/'/
+syn match jinjaNumber containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[0-9]\+\(\.[0-9]\+\)\?/
+
+" Operators
+syn match jinjaOperator containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[+\-*\/<>=!,:]/
+syn match jinjaPunctuation containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[()\[\]]/
+syn match jinjaOperator containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /\./ nextgroup=jinjaAttribute
+syn match jinjaAttribute contained /[a-zA-Z_][a-zA-Z0-9_]*/
+
+" Jinja template tag and variable blocks
+syn region jinjaNested matchgroup=jinjaOperator start="(" end=")" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained
+syn region jinjaNested matchgroup=jinjaOperator start="\[" end="\]" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained
+syn region jinjaNested matchgroup=jinjaOperator start="{" end="}" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained
+syn region jinjaTagBlock matchgroup=jinjaTagDelim start=/{%-\?/ end=/-\?%}/ skipwhite containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment
+
+syn region jinjaVarBlock matchgroup=jinjaVarDelim start=/{{-\?/ end=/-\?}}/ containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment
+
+" Jinja template 'raw' tag
+syn region jinjaRaw matchgroup=jinjaRawDelim start="{%\s*raw\s*%}" end="{%\s*endraw\s*%}" containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaString,jinjaComment
+
+" Jinja comments
+syn region jinjaComment matchgroup=jinjaCommentDelim start="{#" end="#}" containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaString
+
+" Block start keywords. A bit tricker. We only highlight at the start of a
+" tag block and only if the name is not followed by a comma or equals sign
+" which usually means that we have to deal with an assignment.
+syn match jinjaStatement containedin=jinjaTagBlock contained skipwhite /\({%-\?\s*\)\@<=\<[a-zA-Z_][a-zA-Z0-9_]*\>\(\s*[,=]\)\@!/
+
+" and context modifiers
+syn match jinjaStatement containedin=jinjaTagBlock contained /\<with\(out\)\?\s\+context\>/ skipwhite
+
+
+" Define the default highlighting.
+" For version 5.7 and earlier: only when not done already
+" For version 5.8 and later: only when an item doesn't have highlighting yet
+if version >= 508 || !exists("did_jinja_syn_inits")
+ if version < 508
+ let did_jinja_syn_inits = 1
+ command -nargs=+ HiLink hi link <args>
+ else
+ command -nargs=+ HiLink hi def link <args>
+ endif
+
+ HiLink jinjaPunctuation jinjaOperator
+ HiLink jinjaAttribute jinjaVariable
+ HiLink jinjaFunction jinjaFilter
+
+ HiLink jinjaTagDelim jinjaTagBlock
+ HiLink jinjaVarDelim jinjaVarBlock
+ HiLink jinjaCommentDelim jinjaComment
+ HiLink jinjaRawDelim jinja
+
+ HiLink jinjaSpecial Special
+ HiLink jinjaOperator Normal
+ HiLink jinjaRaw Normal
+ HiLink jinjaTagBlock PreProc
+ HiLink jinjaVarBlock PreProc
+ HiLink jinjaStatement Statement
+ HiLink jinjaFilter Function
+ HiLink jinjaBlockName Function
+ HiLink jinjaVariable Identifier
+ HiLink jinjaString Constant
+ HiLink jinjaNumber Constant
+ HiLink jinjaComment Comment
+
+ delcommand HiLink
+endif
+
+let b:current_syntax = "jinja"
diff --git a/slider-agent/src/main/python/jinja2/ext/django2jinja/django2jinja.py b/slider-agent/src/main/python/jinja2/ext/django2jinja/django2jinja.py
new file mode 100644
index 0000000..6d9e76c
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/ext/django2jinja/django2jinja.py
@@ -0,0 +1,768 @@
+# -*- coding: utf-8 -*-
+"""
+ Django to Jinja
+ ~~~~~~~~~~~~~~~
+
+ Helper module that can convert django templates into Jinja2 templates.
+
+ This file is not intended to be used as stand alone application but to
+ be used as library. To convert templates you basically create your own
+ writer, add extra conversion logic for your custom template tags,
+ configure your django environment and run the `convert_templates`
+ function.
+
+ Here a simple example::
+
+ # configure django (or use settings.configure)
+ import os
+ os.environ['DJANGO_SETTINGS_MODULE'] = 'yourapplication.settings'
+ from yourapplication.foo.templatetags.bar import MyNode
+
+ from django2jinja import Writer, convert_templates
+
+ def write_my_node(writer, node):
+ writer.start_variable()
+ writer.write('myfunc(')
+ for idx, arg in enumerate(node.args):
+ if idx:
+ writer.write(', ')
+ writer.node(arg)
+ writer.write(')')
+ writer.end_variable()
+
+ writer = Writer()
+ writer.node_handlers[MyNode] = write_my_node
+ convert_templates('/path/to/output/folder', writer=writer)
+
+ Here is an example hos to automatically translate your django
+ variables to jinja2::
+
+ import re
+ # List of tuple (Match pattern, Replace pattern, Exclusion pattern)
+
+ var_re = ((re.compile(r"(u|user)\.is_authenticated"), r"\1.is_authenticated()", None),
+ (re.compile(r"\.non_field_errors"), r".non_field_errors()", None),
+ (re.compile(r"\.label_tag"), r".label_tag()", None),
+ (re.compile(r"\.as_dl"), r".as_dl()", None),
+ (re.compile(r"\.as_table"), r".as_table()", None),
+ (re.compile(r"\.as_widget"), r".as_widget()", None),
+ (re.compile(r"\.as_hidden"), r".as_hidden()", None),
+
+ (re.compile(r"\.get_([0-9_\w]+)_url"), r".get_\1_url()", None),
+ (re.compile(r"\.url"), r".url()", re.compile(r"(form|calendar).url")),
+ (re.compile(r"\.get_([0-9_\w]+)_display"), r".get_\1_display()", None),
+ (re.compile(r"loop\.counter"), r"loop.index", None),
+ (re.compile(r"loop\.revcounter"), r"loop.revindex", None),
+ (re.compile(r"request\.GET\.([0-9_\w]+)"), r"request.GET.get('\1', '')", None),
+ (re.compile(r"request\.get_host"), r"request.get_host()", None),
+
+ (re.compile(r"\.all(?!_)"), r".all()", None),
+ (re.compile(r"\.all\.0"), r".all()[0]", None),
+ (re.compile(r"\.([0-9])($|\s+)"), r"[\1]\2", None),
+ (re.compile(r"\.items"), r".items()", None),
+ )
+ writer = Writer(var_re=var_re)
+
+ For details about the writing process have a look at the module code.
+
+ :copyright: (c) 2009 by the Jinja Team.
+ :license: BSD.
+"""
+import re
+import os
+import sys
+from jinja2.defaults import *
+from django.conf import settings
+from django.template import defaulttags as core_tags, loader, TextNode, \
+ FilterExpression, libraries, Variable, loader_tags, TOKEN_TEXT, \
+ TOKEN_VAR
+from django.template.debug import DebugVariableNode as VariableNode
+from django.templatetags import i18n as i18n_tags
+from StringIO import StringIO
+
+
+_node_handlers = {}
+_resolved_filters = {}
+_newline_re = re.compile(r'(?:\r\n|\r|\n)')
+
+
+# Django stores an itertools object on the cycle node. Not only is this
+# thread unsafe but also a problem for the converter which needs the raw
+# string values passed to the constructor to create a jinja loop.cycle()
+# call from it.
+_old_cycle_init = core_tags.CycleNode.__init__
+def _fixed_cycle_init(self, cyclevars, variable_name=None):
+ self.raw_cycle_vars = map(Variable, cyclevars)
+ _old_cycle_init(self, cyclevars, variable_name)
+core_tags.CycleNode.__init__ = _fixed_cycle_init
+
+
+def node(cls):
+ def proxy(f):
+ _node_handlers[cls] = f
+ return f
+ return proxy
+
+
+def convert_templates(output_dir, extensions=('.html', '.txt'), writer=None,
+ callback=None):
+ """Iterates over all templates in the template dirs configured and
+ translates them and writes the new templates into the output directory.
+ """
+ if writer is None:
+ writer = Writer()
+
+ def filter_templates(files):
+ for filename in files:
+ ifilename = filename.lower()
+ for extension in extensions:
+ if ifilename.endswith(extension):
+ yield filename
+
+ def translate(f, loadname):
+ template = loader.get_template(loadname)
+ original = writer.stream
+ writer.stream = f
+ writer.body(template.nodelist)
+ writer.stream = original
+
+ if callback is None:
+ def callback(template):
+ print template
+
+ for directory in settings.TEMPLATE_DIRS:
+ for dirname, _, files in os.walk(directory):
+ dirname = dirname[len(directory) + 1:]
+ for filename in filter_templates(files):
+ source = os.path.normpath(os.path.join(dirname, filename))
+ target = os.path.join(output_dir, dirname, filename)
+ basetarget = os.path.dirname(target)
+ if not os.path.exists(basetarget):
+ os.makedirs(basetarget)
+ callback(source)
+ f = file(target, 'w')
+ try:
+ translate(f, source)
+ finally:
+ f.close()
+
+
+class Writer(object):
+ """The core writer class."""
+
+ def __init__(self, stream=None, error_stream=None,
+ block_start_string=BLOCK_START_STRING,
+ block_end_string=BLOCK_END_STRING,
+ variable_start_string=VARIABLE_START_STRING,
+ variable_end_string=VARIABLE_END_STRING,
+ comment_start_string=COMMENT_START_STRING,
+ comment_end_string=COMMENT_END_STRING,
+ initial_autoescape=True,
+ use_jinja_autoescape=False,
+ custom_node_handlers=None,
+ var_re=[],
+ env=None):
+ if stream is None:
+ stream = sys.stdout
+ if error_stream is None:
+ error_stream = sys.stderr
+ self.stream = stream
+ self.error_stream = error_stream
+ self.block_start_string = block_start_string
+ self.block_end_string = block_end_string
+ self.variable_start_string = variable_start_string
+ self.variable_end_string = variable_end_string
+ self.comment_start_string = comment_start_string
+ self.comment_end_string = comment_end_string
+ self.autoescape = initial_autoescape
+ self.spaceless = False
+ self.use_jinja_autoescape = use_jinja_autoescape
+ self.node_handlers = dict(_node_handlers,
+ **(custom_node_handlers or {}))
+ self._loop_depth = 0
+ self._filters_warned = set()
+ self.var_re = var_re
+ self.env = env
+
+ def enter_loop(self):
+ """Increments the loop depth so that write functions know if they
+ are in a loop.
+ """
+ self._loop_depth += 1
+
+ def leave_loop(self):
+ """Reverse of enter_loop."""
+ self._loop_depth -= 1
+
+ @property
+ def in_loop(self):
+ """True if we are in a loop."""
+ return self._loop_depth > 0
+
+ def write(self, s):
+ """Writes stuff to the stream."""
+ self.stream.write(s.encode(settings.FILE_CHARSET))
+
+ def print_expr(self, expr):
+ """Open a variable tag, write to the string to the stream and close."""
+ self.start_variable()
+ self.write(expr)
+ self.end_variable()
+
+ def _post_open(self):
+ if self.spaceless:
+ self.write('- ')
+ else:
+ self.write(' ')
+
+ def _pre_close(self):
+ if self.spaceless:
+ self.write(' -')
+ else:
+ self.write(' ')
+
+ def start_variable(self):
+ """Start a variable."""
+ self.write(self.variable_start_string)
+ self._post_open()
+
+ def end_variable(self, always_safe=False):
+ """End a variable."""
+ if not always_safe and self.autoescape and \
+ not self.use_jinja_autoescape:
+ self.write('|e')
+ self._pre_close()
+ self.write(self.variable_end_string)
+
+ def start_block(self):
+ """Starts a block."""
+ self.write(self.block_start_string)
+ self._post_open()
+
+ def end_block(self):
+ """Ends a block."""
+ self._pre_close()
+ self.write(self.block_end_string)
+
+ def tag(self, name):
+ """Like `print_expr` just for blocks."""
+ self.start_block()
+ self.write(name)
+ self.end_block()
+
+ def variable(self, name):
+ """Prints a variable. This performs variable name transformation."""
+ self.write(self.translate_variable_name(name))
+
+ def literal(self, value):
+ """Writes a value as literal."""
+ value = repr(value)
+ if value[:2] in ('u"', "u'"):
+ value = value[1:]
+ self.write(value)
+
+ def filters(self, filters, is_block=False):
+ """Dumps a list of filters."""
+ want_pipe = not is_block
+ for filter, args in filters:
+ name = self.get_filter_name(filter)
+ if name is None:
+ self.warn('Could not find filter %s' % name)
+ continue
+ if name not in DEFAULT_FILTERS and \
+ name not in self._filters_warned:
+ self._filters_warned.add(name)
+ self.warn('Filter %s probably doesn\'t exist in Jinja' %
+ name)
+ if not want_pipe:
+ want_pipe = True
+ else:
+ self.write('|')
+ self.write(name)
+ if args:
+ self.write('(')
+ for idx, (is_var, value) in enumerate(args):
+ if idx:
+ self.write(', ')
+ if is_var:
+ self.node(value)
+ else:
+ self.literal(value)
+ self.write(')')
+
+ def get_location(self, origin, position):
+ """Returns the location for an origin and position tuple as name
+ and lineno.
+ """
+ if hasattr(origin, 'source'):
+ source = origin.source
+ name = '<unknown source>'
+ else:
+ source = origin.loader(origin.loadname, origin.dirs)[0]
+ name = origin.loadname
+ lineno = len(_newline_re.findall(source[:position[0]])) + 1
+ return name, lineno
+
+ def warn(self, message, node=None):
+ """Prints a warning to the error stream."""
+ if node is not None and hasattr(node, 'source'):
+ filename, lineno = self.get_location(*node.source)
+ message = '[%s:%d] %s' % (filename, lineno, message)
+ print >> self.error_stream, message
+
+ def translate_variable_name(self, var):
+ """Performs variable name translation."""
+ if self.in_loop and var == 'forloop' or var.startswith('forloop.'):
+ var = var[3:]
+
+ for reg, rep, unless in self.var_re:
+ no_unless = unless and unless.search(var) or True
+ if reg.search(var) and no_unless:
+ var = reg.sub(rep, var)
+ break
+ return var
+
+ def get_filter_name(self, filter):
+ """Returns the filter name for a filter function or `None` if there
+ is no such filter.
+ """
+ if filter not in _resolved_filters:
+ for library in libraries.values():
+ for key, value in library.filters.iteritems():
+ _resolved_filters[value] = key
+ return _resolved_filters.get(filter, None)
+
+ def node(self, node):
+ """Invokes the node handler for a node."""
+ for cls, handler in self.node_handlers.iteritems():
+ if type(node) is cls or type(node).__name__ == cls:
+ handler(self, node)
+ break
+ else:
+ self.warn('Untranslatable node %s.%s found' % (
+ node.__module__,
+ node.__class__.__name__
+ ), node)
+
+ def body(self, nodes):
+ """Calls node() for every node in the iterable passed."""
+ for node in nodes:
+ self.node(node)
+
+
+@node(TextNode)
+def text_node(writer, node):
+ writer.write(node.s)
+
+
+@node(Variable)
+def variable(writer, node):
+ if node.translate:
+ writer.warn('i18n system used, make sure to install translations', node)
+ writer.write('_(')
+ if node.literal is not None:
+ writer.literal(node.literal)
+ else:
+ writer.variable(node.var)
+ if node.translate:
+ writer.write(')')
+
+
+@node(VariableNode)
+def variable_node(writer, node):
+ writer.start_variable()
+ if node.filter_expression.var.var == 'block.super' \
+ and not node.filter_expression.filters:
+ writer.write('super()')
+ else:
+ writer.node(node.filter_expression)
+ writer.end_variable()
+
+
+@node(FilterExpression)
+def filter_expression(writer, node):
+ writer.node(node.var)
+ writer.filters(node.filters)
+
+
+@node(core_tags.CommentNode)
+def comment_tag(writer, node):
+ pass
+
+
+@node(core_tags.DebugNode)
+def comment_tag(writer, node):
+ writer.warn('Debug tag detected. Make sure to add a global function '
+ 'called debug to the namespace.', node=node)
+ writer.print_expr('debug()')
+
+
+@node(core_tags.ForNode)
+def for_loop(writer, node):
+ writer.start_block()
+ writer.write('for ')
+ for idx, var in enumerate(node.loopvars):
+ if idx:
+ writer.write(', ')
+ writer.variable(var)
+ writer.write(' in ')
+ if node.is_reversed:
+ writer.write('(')
+ writer.node(node.sequence)
+ if node.is_reversed:
+ writer.write(')|reverse')
+ writer.end_block()
+ writer.enter_loop()
+ writer.body(node.nodelist_loop)
+ writer.leave_loop()
+ writer.tag('endfor')
+
+
+@node(core_tags.IfNode)
+def if_condition(writer, node):
+ writer.start_block()
+ writer.write('if ')
+ join_with = 'and'
+ if node.link_type == core_tags.IfNode.LinkTypes.or_:
+ join_with = 'or'
+
+ for idx, (ifnot, expr) in enumerate(node.bool_exprs):
+ if idx:
+ writer.write(' %s ' % join_with)
+ if ifnot:
+ writer.write('not ')
+ writer.node(expr)
+ writer.end_block()
+ writer.body(node.nodelist_true)
+ if node.nodelist_false:
+ writer.tag('else')
+ writer.body(node.nodelist_false)
+ writer.tag('endif')
+
+
+@node(core_tags.IfEqualNode)
+def if_equal(writer, node):
+ writer.start_block()
+ writer.write('if ')
+ writer.node(node.var1)
+ if node.negate:
+ writer.write(' != ')
+ else:
+ writer.write(' == ')
+ writer.node(node.var2)
+ writer.end_block()
+ writer.body(node.nodelist_true)
+ if node.nodelist_false:
+ writer.tag('else')
+ writer.body(node.nodelist_false)
+ writer.tag('endif')
+
+
+@node(loader_tags.BlockNode)
+def block(writer, node):
+ writer.tag('block ' + node.name.replace('-', '_').rstrip('_'))
+ node = node
+ while node.parent is not None:
+ node = node.parent
+ writer.body(node.nodelist)
+ writer.tag('endblock')
+
+
+@node(loader_tags.ExtendsNode)
+def extends(writer, node):
+ writer.start_block()
+ writer.write('extends ')
+ if node.parent_name_expr:
+ writer.node(node.parent_name_expr)
+ else:
+ writer.literal(node.parent_name)
+ writer.end_block()
+ writer.body(node.nodelist)
+
+
+@node(loader_tags.ConstantIncludeNode)
+@node(loader_tags.IncludeNode)
+def include(writer, node):
+ writer.start_block()
+ writer.write('include ')
+ if hasattr(node, 'template'):
+ writer.literal(node.template.name)
+ else:
+ writer.node(node.template_name)
+ writer.end_block()
+
+
+@node(core_tags.CycleNode)
+def cycle(writer, node):
+ if not writer.in_loop:
+ writer.warn('Untranslatable free cycle (cycle outside loop)', node=node)
+ return
+ if node.variable_name is not None:
+ writer.start_block()
+ writer.write('set %s = ' % node.variable_name)
+ else:
+ writer.start_variable()
+ writer.write('loop.cycle(')
+ for idx, var in enumerate(node.raw_cycle_vars):
+ if idx:
+ writer.write(', ')
+ writer.node(var)
+ writer.write(')')
+ if node.variable_name is not None:
+ writer.end_block()
+ else:
+ writer.end_variable()
+
+
+@node(core_tags.FilterNode)
+def filter(writer, node):
+ writer.start_block()
+ writer.write('filter ')
+ writer.filters(node.filter_expr.filters, True)
+ writer.end_block()
+ writer.body(node.nodelist)
+ writer.tag('endfilter')
+
+
+@node(core_tags.AutoEscapeControlNode)
+def autoescape_control(writer, node):
+ original = writer.autoescape
+ writer.autoescape = node.setting
+ writer.body(node.nodelist)
+ writer.autoescape = original
+
+
+@node(core_tags.SpacelessNode)
+def spaceless(writer, node):
+ original = writer.spaceless
+ writer.spaceless = True
+ writer.warn('entering spaceless mode with different semantics', node)
+ # do the initial stripping
+ nodelist = list(node.nodelist)
+ if nodelist:
+ if isinstance(nodelist[0], TextNode):
+ nodelist[0] = TextNode(nodelist[0].s.lstrip())
+ if isinstance(nodelist[-1], TextNode):
+ nodelist[-1] = TextNode(nodelist[-1].s.rstrip())
+ writer.body(nodelist)
+ writer.spaceless = original
+
+
+@node(core_tags.TemplateTagNode)
+def template_tag(writer, node):
+ tag = {
+ 'openblock': writer.block_start_string,
+ 'closeblock': writer.block_end_string,
+ 'openvariable': writer.variable_start_string,
+ 'closevariable': writer.variable_end_string,
+ 'opencomment': writer.comment_start_string,
+ 'closecomment': writer.comment_end_string,
+ 'openbrace': '{',
+ 'closebrace': '}'
+ }.get(node.tagtype)
+ if tag:
+ writer.start_variable()
+ writer.literal(tag)
+ writer.end_variable()
+
+
+@node(core_tags.URLNode)
+def url_tag(writer, node):
+ writer.warn('url node used. make sure to provide a proper url() '
+ 'function', node)
+ if node.asvar:
+ writer.start_block()
+ writer.write('set %s = ' % node.asvar)
+ else:
+ writer.start_variable()
+ autoescape = writer.autoescape
+ writer.write('url(')
+ writer.literal(node.view_name)
+ for arg in node.args:
+ writer.write(', ')
+ writer.node(arg)
+ for key, arg in node.kwargs.items():
+ writer.write(', %s=' % key)
+ writer.node(arg)
+ writer.write(')')
+ if node.asvar:
+ writer.end_block()
+ else:
+ writer.end_variable()
+
+
+@node(core_tags.WidthRatioNode)
+def width_ratio(writer, node):
+ writer.warn('widthratio expanded into formula. You may want to provide '
+ 'a helper function for this calculation', node)
+ writer.start_variable()
+ writer.write('(')
+ writer.node(node.val_expr)
+ writer.write(' / ')
+ writer.node(node.max_expr)
+ writer.write(' * ')
+ writer.write(str(int(node.max_width)))
+ writer.write(')|round|int')
+ writer.end_variable(always_safe=True)
+
+
+@node(core_tags.WithNode)
+def with_block(writer, node):
+ writer.warn('with block expanded into set statement. This could cause '
+ 'variables following that block to be overriden.', node)
+ writer.start_block()
+ writer.write('set %s = ' % node.name)
+ writer.node(node.var)
+ writer.end_block()
+ writer.body(node.nodelist)
+
+
+@node(core_tags.RegroupNode)
+def regroup(writer, node):
+ if node.expression.var.literal:
+ writer.warn('literal in groupby filter used. Behavior in that '
+ 'situation is undefined and translation is skipped.', node)
+ return
+ elif node.expression.filters:
+ writer.warn('filters in groupby filter used. Behavior in that '
+ 'situation is undefined which is most likely a bug '
+ 'in your code. Filters were ignored.', node)
+ writer.start_block()
+ writer.write('set %s = ' % node.var_name)
+ writer.node(node.target)
+ writer.write('|groupby(')
+ writer.literal(node.expression.var.var)
+ writer.write(')')
+ writer.end_block()
+
+
+@node(core_tags.LoadNode)
+def warn_load(writer, node):
+ writer.warn('load statement used which was ignored on conversion', node)
+
+
+@node(i18n_tags.GetAvailableLanguagesNode)
+def get_available_languages(writer, node):
+ writer.warn('make sure to provide a get_available_languages function', node)
+ writer.tag('set %s = get_available_languages()' %
+ writer.translate_variable_name(node.variable))
+
+
+@node(i18n_tags.GetCurrentLanguageNode)
+def get_current_language(writer, node):
+ writer.warn('make sure to provide a get_current_language function', node)
+ writer.tag('set %s = get_current_language()' %
+ writer.translate_variable_name(node.variable))
+
+
+@node(i18n_tags.GetCurrentLanguageBidiNode)
+def get_current_language_bidi(writer, node):
+ writer.warn('make sure to provide a get_current_language_bidi function', node)
+ writer.tag('set %s = get_current_language_bidi()' %
+ writer.translate_variable_name(node.variable))
+
+
+@node(i18n_tags.TranslateNode)
+def simple_gettext(writer, node):
+ writer.warn('i18n system used, make sure to install translations', node)
+ writer.start_variable()
+ writer.write('_(')
+ writer.node(node.value)
+ writer.write(')')
+ writer.end_variable()
+
+
+@node(i18n_tags.BlockTranslateNode)
+def translate_block(writer, node):
+ first_var = []
+ variables = set()
+
+ def touch_var(name):
+ variables.add(name)
+ if not first_var:
+ first_var.append(name)
+
+ def dump_token_list(tokens):
+ for token in tokens:
+ if token.token_type == TOKEN_TEXT:
+ writer.write(token.contents)
+ elif token.token_type == TOKEN_VAR:
+ writer.print_expr(token.contents)
+ touch_var(token.contents)
+
+ writer.warn('i18n system used, make sure to install translations', node)
+ writer.start_block()
+ writer.write('trans')
+ idx = -1
+ for idx, (key, var) in enumerate(node.extra_context.items()):
+ if idx:
+ writer.write(',')
+ writer.write(' %s=' % key)
+ touch_var(key)
+ writer.node(var.filter_expression)
+
+ have_plural = False
+ plural_var = None
+ if node.plural and node.countervar and node.counter:
+ have_plural = True
+ plural_var = node.countervar
+ if plural_var not in variables:
+ if idx > -1:
+ writer.write(',')
+ touch_var(plural_var)
+ writer.write(' %s=' % plural_var)
+ writer.node(node.counter)
+
+ writer.end_block()
+ dump_token_list(node.singular)
+ if node.plural and node.countervar and node.counter:
+ writer.start_block()
+ writer.write('pluralize')
+ if node.countervar != first_var[0]:
+ writer.write(' ' + node.countervar)
+ writer.end_block()
+ dump_token_list(node.plural)
+ writer.tag('endtrans')
+
+@node("SimpleNode")
+def simple_tag(writer, node):
+ """Check if the simple tag exist as a filter in """
+ name = node.tag_name
+ if writer.env and \
+ name not in writer.env.filters and \
+ name not in writer._filters_warned:
+ writer._filters_warned.add(name)
+ writer.warn('Filter %s probably doesn\'t exist in Jinja' %
+ name)
+
+ if not node.vars_to_resolve:
+ # No argument, pass the request
+ writer.start_variable()
+ writer.write('request|')
+ writer.write(name)
+ writer.end_variable()
+ return
+
+ first_var = node.vars_to_resolve[0]
+ args = node.vars_to_resolve[1:]
+ writer.start_variable()
+
+ # Copied from Writer.filters()
+ writer.node(first_var)
+
+ writer.write('|')
+ writer.write(name)
+ if args:
+ writer.write('(')
+ for idx, var in enumerate(args):
+ if idx:
+ writer.write(', ')
+ if var.var:
+ writer.node(var)
+ else:
+ writer.literal(var.literal)
+ writer.write(')')
+ writer.end_variable()
+
+# get rid of node now, it shouldn't be used normally
+del node
diff --git a/slider-agent/src/main/python/jinja2/ext/django2jinja/example.py b/slider-agent/src/main/python/jinja2/ext/django2jinja/example.py
new file mode 100644
index 0000000..2d4ab9a
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/ext/django2jinja/example.py
@@ -0,0 +1,7 @@
+from django.conf import settings
+settings.configure(TEMPLATE_DIRS=['templates'], TEMPLATE_DEBUG=True)
+
+from django2jinja import convert_templates, Writer
+
+writer = Writer(use_jinja_autoescape=True)
+convert_templates('converted', writer=writer)
diff --git a/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/index.html b/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/index.html
new file mode 100644
index 0000000..d0fbe38
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/index.html
@@ -0,0 +1,58 @@
+{% extends "layout.html" %}
+{% load i18n %}
+{% block title %}Foo{% endblock %}
+{% block page-body %}
+ {{ block.super }}
+ Hello {{ name|cut:"d" }}!
+
+ {% for item in seq reversed %}
+ {% if forloop.index|divisibleby:2 %}
+ <li class="{% cycle 'a' 'b' %}">{{ item }}</li>
+ {% endif %}
+ {% endfor %}
+ {% ifequal foo bar %}
+ haha
+ {% else %}
+ hmm
+ {% endifequal %}
+ {% filter upper %}
+ {% include "subtemplate.html" %}
+ {% include foo %}
+ {% endfilter %}
+ {% spaceless %}
+ Hello World
+ {{ foo }}
+ Hmm
+ {% endspaceless %}
+ {% templatetag opencomment %}...{% templatetag closecomment %}
+ {% url foo a, b, c=d %}
+ {% url foo a, b, c=d as hmm %}
+
+ {% with object.value as value %}
+ <img src='bar.gif' height='10' width='{% widthratio value 200 100 %}'>
+ {% endwith %}
+
+ <pre>{% debug %}</pre>
+
+ {% blocktrans with book|title as book_t and author|title as author_t %}
+ This is {{ book_t }} by {{ author_t }}
+ {% endblocktrans %}
+
+ {% blocktrans count list|length as counter %}
+ There is only one {{ name }} object.
+ {% plural %}
+ There are {{ counter }} {{ name }} objects.
+ {% endblocktrans %}
+
+ {% blocktrans with name|escape as name count list|length as counter %}
+ There is only one {{ name }} object.
+ {% plural %}
+ There are {{ counter }} {{ name }} objects.
+ {% endblocktrans %}
+
+ {% blocktrans %}This string will have {{ value }} inside.{% endblocktrans %}
+
+ <p>{% trans "This is the title." %}</p>
+
+ {% regroup people by gender as grouped %}
+{% endblock %}
diff --git a/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/layout.html b/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/layout.html
new file mode 100644
index 0000000..3f21a12
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/layout.html
@@ -0,0 +1,4 @@
+<title>{% block title %}{% endblock %}</title>
+<div class="body">
+ {% block page-body %}{% endblock %}
+</div>
diff --git a/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/subtemplate.html b/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/subtemplate.html
new file mode 100644
index 0000000..980a0d5
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/ext/django2jinja/templates/subtemplate.html
@@ -0,0 +1 @@
+Hello World!
diff --git a/slider-agent/src/main/python/jinja2/ext/djangojinja2.py b/slider-agent/src/main/python/jinja2/ext/djangojinja2.py
new file mode 100644
index 0000000..d24d164
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/ext/djangojinja2.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+"""
+ djangojinja2
+ ~~~~~~~~~~~~
+
+ Adds support for Jinja2 to Django.
+
+ Configuration variables:
+
+ ======================= =============================================
+ Key Description
+ ======================= =============================================
+ `JINJA2_TEMPLATE_DIRS` List of template folders
+ `JINJA2_EXTENSIONS` List of Jinja2 extensions to use
+ `JINJA2_CACHE_SIZE` The size of the Jinja2 template cache.
+ ======================= =============================================
+
+ :copyright: (c) 2009 by the Jinja Team.
+ :license: BSD.
+"""
+from itertools import chain
+from django.conf import settings
+from django.http import HttpResponse
+from django.core.exceptions import ImproperlyConfigured
+from django.template.context import get_standard_processors
+from django.template import TemplateDoesNotExist
+from jinja2 import Environment, FileSystemLoader, TemplateNotFound
+from jinja2.defaults import DEFAULT_NAMESPACE
+
+
+# the environment is unconfigured until the first template is loaded.
+_jinja_env = None
+
+
+def get_env():
+ """Get the Jinja2 env and initialize it if necessary."""
+ global _jinja_env
+ if _jinja_env is None:
+ _jinja_env = create_env()
+ return _jinja_env
+
+
+def create_env():
+ """Create a new Jinja2 environment."""
+ searchpath = list(settings.JINJA2_TEMPLATE_DIRS)
+ return Environment(loader=FileSystemLoader(searchpath),
+ auto_reload=settings.TEMPLATE_DEBUG,
+ cache_size=getattr(settings, 'JINJA2_CACHE_SIZE', 50),
+ extensions=getattr(settings, 'JINJA2_EXTENSIONS', ()))
+
+
+def get_template(template_name, globals=None):
+ """Load a template."""
+ try:
+ return get_env().get_template(template_name, globals=globals)
+ except TemplateNotFound, e:
+ raise TemplateDoesNotExist(str(e))
+
+
+def select_template(templates, globals=None):
+ """Try to load one of the given templates."""
+ env = get_env()
+ for template in templates:
+ try:
+ return env.get_template(template, globals=globals)
+ except TemplateNotFound:
+ continue
+ raise TemplateDoesNotExist(', '.join(templates))
+
+
+def render_to_string(template_name, context=None, request=None,
+ processors=None):
+ """Render a template into a string."""
+ context = dict(context or {})
+ if request is not None:
+ context['request'] = request
+ for processor in chain(get_standard_processors(), processors or ()):
+ context.update(processor(request))
+ return get_template(template_name).render(context)
+
+
+def render_to_response(template_name, context=None, request=None,
+ processors=None, mimetype=None):
+ """Render a template into a response object."""
+ return HttpResponse(render_to_string(template_name, context, request,
+ processors), mimetype=mimetype)
diff --git a/slider-agent/src/main/python/jinja2/ext/inlinegettext.py b/slider-agent/src/main/python/jinja2/ext/inlinegettext.py
new file mode 100644
index 0000000..cf4ed5e
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/ext/inlinegettext.py
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""
+ Inline Gettext
+ ~~~~~~~~~~~~~~
+
+ An example extension for Jinja2 that supports inline gettext calls.
+ Requires the i18n extension to be loaded.
+
+ :copyright: (c) 2009 by the Jinja Team.
+ :license: BSD.
+"""
+import re
+from jinja2.ext import Extension
+from jinja2.lexer import Token, count_newlines
+from jinja2.exceptions import TemplateSyntaxError
+
+
+_outside_re = re.compile(r'\\?(gettext|_)\(')
+_inside_re = re.compile(r'\\?[()]')
+
+
+class InlineGettext(Extension):
+ """This extension implements support for inline gettext blocks::
+
+ <h1>_(Welcome)</h1>
+ <p>_(This is a paragraph)</p>
+
+ Requires the i18n extension to be loaded and configured.
+ """
+
+ def filter_stream(self, stream):
+ paren_stack = 0
+
+ for token in stream:
+ if token.type is not 'data':
+ yield token
+ continue
+
+ pos = 0
+ lineno = token.lineno
+
+ while 1:
+ if not paren_stack:
+ match = _outside_re.search(token.value, pos)
+ else:
+ match = _inside_re.search(token.value, pos)
+ if match is None:
+ break
+ new_pos = match.start()
+ if new_pos > pos:
+ preval = token.value[pos:new_pos]
+ yield Token(lineno, 'data', preval)
+ lineno += count_newlines(preval)
+ gtok = match.group()
+ if gtok[0] == '\\':
+ yield Token(lineno, 'data', gtok[1:])
+ elif not paren_stack:
+ yield Token(lineno, 'block_begin', None)
+ yield Token(lineno, 'name', 'trans')
+ yield Token(lineno, 'block_end', None)
+ paren_stack = 1
+ else:
+ if gtok == '(' or paren_stack > 1:
+ yield Token(lineno, 'data', gtok)
+ paren_stack += gtok == ')' and -1 or 1
+ if not paren_stack:
+ yield Token(lineno, 'block_begin', None)
+ yield Token(lineno, 'name', 'endtrans')
+ yield Token(lineno, 'block_end', None)
+ pos = match.end()
+
+ if pos < len(token.value):
+ yield Token(lineno, 'data', token.value[pos:])
+
+ if paren_stack:
+ raise TemplateSyntaxError('unclosed gettext expression',
+ token.lineno, stream.name,
+ stream.filename)
diff --git a/slider-agent/src/main/python/jinja2/ext/jinja.el b/slider-agent/src/main/python/jinja2/ext/jinja.el
new file mode 100644
index 0000000..401ba29
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/ext/jinja.el
@@ -0,0 +1,213 @@
+;;; jinja.el --- Jinja mode highlighting
+;;
+;; Author: Georg Brandl
+;; Copyright: (c) 2009 by the Jinja Team
+;; Last modified: 2008-05-22 23:04 by gbr
+;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;;; Commentary:
+;;
+;; Mostly ripped off django-mode by Lennart Borgman.
+;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; This program is free software; you can redistribute it and/or
+;; modify it under the terms of the GNU General Public License as
+;; published by the Free Software Foundation; either version 2, or
+;; (at your option) any later version.
+;;
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+;; General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with this program; see the file COPYING. If not, write to
+;; the Free Software Foundation, Inc., 51 Franklin Street, Fifth
+;; Floor, Boston, MA 02110-1301, USA.
+;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;;; Code:
+
+(defconst jinja-font-lock-keywords
+ (list
+; (cons (rx "{% comment %}" (submatch (0+ anything))
+; "{% endcomment %}") (list 1 font-lock-comment-face))
+ '("{# ?\\(.*?\\) ?#}" . (1 font-lock-comment-face))
+ '("{%-?\\|-?%}\\|{{\\|}}" . font-lock-preprocessor-face)
+ '("{#\\|#}" . font-lock-comment-delimiter-face)
+ ;; first word in a block is a command
+ '("{%-?[ \t\n]*\\([a-zA-Z_]+\\)" . (1 font-lock-keyword-face))
+ ;; variables
+ '("\\({{ ?\\)\\([^|]*?\\)\\(|.*?\\)? ?}}" . (1 font-lock-variable-name-face))
+ ;; keywords and builtins
+ (cons (rx word-start
+ (or "in" "as" "recursive" "not" "and" "or" "if" "else"
+ "import" "with" "without" "context")
+ word-end)
+ font-lock-keyword-face)
+ (cons (rx word-start
+ (or "true" "false" "none" "loop" "self" "super")
+ word-end)
+ font-lock-builtin-face)
+ ;; tests
+ '("\\(is\\)[ \t]*\\(not\\)[ \t]*\\([a-zA-Z_]+\\)"
+ (1 font-lock-keyword-face) (2 font-lock-keyword-face)
+ (3 font-lock-function-name-face))
+ ;; builtin filters
+ (cons (rx
+ "|" (* space)
+ (submatch
+ (or "abs" "batch" "capitalize" "capture" "center" "count" "default"
+ "dformat" "dictsort" "e" "escape" "filesizeformat" "first"
+ "float" "format" "getattribute" "getitem" "groupby" "indent"
+ "int" "join" "jsonencode" "last" "length" "lower" "markdown"
+ "pprint" "random" "replace" "reverse" "round" "rst" "slice"
+ "sort" "string" "striptags" "sum" "textile" "title" "trim"
+ "truncate" "upper" "urlencode" "urlize" "wordcount" "wordwrap"
+ "xmlattr")))
+ (list 1 font-lock-builtin-face))
+ )
+ "Minimal highlighting expressions for Jinja mode")
+
+(define-derived-mode jinja-mode nil "Jinja"
+ "Simple Jinja mode for use with `mumamo-mode'.
+This mode only provides syntax highlighting."
+ ;;(set (make-local-variable 'comment-start) "{#")
+ ;;(set (make-local-variable 'comment-end) "#}")
+ (setq font-lock-defaults '(jinja-font-lock-keywords)))
+
+;; mumamo stuff
+
+(when (require 'mumamo nil t)
+
+ (defun mumamo-chunk-jinja3(pos min max)
+ "Find {# ... #}. Return range and `jinja-mode'.
+See `mumamo-find-possible-chunk' for POS, MIN and MAX."
+ (mumamo-find-possible-chunk pos min max
+ 'mumamo-search-bw-exc-start-jinja3
+ 'mumamo-search-bw-exc-end-jinja3
+ 'mumamo-search-fw-exc-start-jinja3
+ 'mumamo-search-fw-exc-end-jinja3))
+
+ (defun mumamo-chunk-jinja2(pos min max)
+ "Find {{ ... }}. Return range and `jinja-mode'.
+See `mumamo-find-possible-chunk' for POS, MIN and MAX."
+ (mumamo-find-possible-chunk pos min max
+ 'mumamo-search-bw-exc-start-jinja2
+ 'mumamo-search-bw-exc-end-jinja2
+ 'mumamo-search-fw-exc-start-jinja2
+ 'mumamo-search-fw-exc-end-jinja2))
+
+ (defun mumamo-chunk-jinja (pos min max)
+ "Find {% ... %}. Return range and `jinja-mode'.
+See `mumamo-find-possible-chunk' for POS, MIN and MAX."
+ (mumamo-find-possible-chunk pos min max
+ 'mumamo-search-bw-exc-start-jinja
+ 'mumamo-search-bw-exc-end-jinja
+ 'mumamo-search-fw-exc-start-jinja
+ 'mumamo-search-fw-exc-end-jinja))
+
+ (defun mumamo-search-bw-exc-start-jinja (pos min)
+ "Helper for `mumamo-chunk-jinja'.
+POS is where to start search and MIN is where to stop."
+ (let ((exc-start (mumamo-chunk-start-bw-str-inc pos min "{%")))
+ (and exc-start
+ (<= exc-start pos)
+ (cons exc-start 'jinja-mode))))
+
+ (defun mumamo-search-bw-exc-start-jinja2(pos min)
+ "Helper for `mumamo-chunk-jinja2'.
+POS is where to start search and MIN is where to stop."
+ (let ((exc-start (mumamo-chunk-start-bw-str-inc pos min "{{")))
+ (and exc-start
+ (<= exc-start pos)
+ (cons exc-start 'jinja-mode))))
+
+ (defun mumamo-search-bw-exc-start-jinja3(pos min)
+ "Helper for `mumamo-chunk-jinja3'.
+POS is where to start search and MIN is where to stop."
+ (let ((exc-start (mumamo-chunk-start-bw-str-inc pos min "{#")))
+ (and exc-start
+ (<= exc-start pos)
+ (cons exc-start 'jinja-mode))))
+
+ (defun mumamo-search-bw-exc-end-jinja (pos min)
+ "Helper for `mumamo-chunk-jinja'.
+POS is where to start search and MIN is where to stop."
+ (mumamo-chunk-end-bw-str-inc pos min "%}"))
+
+ (defun mumamo-search-bw-exc-end-jinja2(pos min)
+ "Helper for `mumamo-chunk-jinja2'.
+POS is where to start search and MIN is where to stop."
+ (mumamo-chunk-end-bw-str-inc pos min "}}"))
+
+ (defun mumamo-search-bw-exc-end-jinja3(pos min)
+ "Helper for `mumamo-chunk-jinja3'.
+POS is where to start search and MIN is where to stop."
+ (mumamo-chunk-end-bw-str-inc pos min "#}"))
+
+ (defun mumamo-search-fw-exc-start-jinja (pos max)
+ "Helper for `mumamo-chunk-jinja'.
+POS is where to start search and MAX is where to stop."
+ (mumamo-chunk-start-fw-str-inc pos max "{%"))
+
+ (defun mumamo-search-fw-exc-start-jinja2(pos max)
+ "Helper for `mumamo-chunk-jinja2'.
+POS is where to start search and MAX is where to stop."
+ (mumamo-chunk-start-fw-str-inc pos max "{{"))
+
+ (defun mumamo-search-fw-exc-start-jinja3(pos max)
+ "Helper for `mumamo-chunk-jinja3'.
+POS is where to start search and MAX is where to stop."
+ (mumamo-chunk-start-fw-str-inc pos max "{#"))
+
+ (defun mumamo-search-fw-exc-end-jinja (pos max)
+ "Helper for `mumamo-chunk-jinja'.
+POS is where to start search and MAX is where to stop."
+ (mumamo-chunk-end-fw-str-inc pos max "%}"))
+
+ (defun mumamo-search-fw-exc-end-jinja2(pos max)
+ "Helper for `mumamo-chunk-jinja2'.
+POS is where to start search and MAX is where to stop."
+ (mumamo-chunk-end-fw-str-inc pos max "}}"))
+
+ (defun mumamo-search-fw-exc-end-jinja3(pos max)
+ "Helper for `mumamo-chunk-jinja3'.
+POS is where to start search and MAX is where to stop."
+ (mumamo-chunk-end-fw-str-inc pos max "#}"))
+
+;;;###autoload
+ (define-mumamo-multi-major-mode jinja-html-mumamo
+ "Turn on multiple major modes for Jinja with main mode `html-mode'.
+This also covers inlined style and javascript."
+ ("Jinja HTML Family" html-mode
+ (mumamo-chunk-jinja
+ mumamo-chunk-jinja2
+ mumamo-chunk-jinja3
+ mumamo-chunk-inlined-style
+ mumamo-chunk-inlined-script
+ mumamo-chunk-style=
+ mumamo-chunk-onjs=
+ )))
+
+;;;###autoload
+ (define-mumamo-multi-major-mode jinja-nxhtml-mumamo
+ "Turn on multiple major modes for Jinja with main mode `nxhtml-mode'.
+This also covers inlined style and javascript."
+ ("Jinja nXhtml Family" nxhtml-mode
+ (mumamo-chunk-jinja
+ mumamo-chunk-jinja2
+ mumamo-chunk-jinja3
+ mumamo-chunk-inlined-style
+ mumamo-chunk-inlined-script
+ mumamo-chunk-style=
+ mumamo-chunk-onjs=
+ )))
+ )
+
+(provide 'jinja)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;; jinja.el ends here
diff --git a/slider-agent/src/main/python/jinja2/jinja2-debug.py b/slider-agent/src/main/python/jinja2/jinja2-debug.py
new file mode 100644
index 0000000..c9c482f
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2-debug.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ Jinja2 Debug Interface
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Helper script for internal Jinja2 debugging. Requires Werkzeug.
+
+ :copyright: Copyright 2010 by Armin Ronacher.
+ :license: BSD.
+"""
+import sys
+import jinja2
+from werkzeug import script
+
+env = jinja2.Environment(extensions=['jinja2.ext.i18n', 'jinja2.ext.do',
+ 'jinja2.ext.loopcontrols'])
+
+def shell_init_func():
+ def _compile(x):
+ print env.compile(x, raw=True)
+ result = {
+ 'e': env,
+ 'c': _compile,
+ 't': env.from_string,
+ 'p': env.parse
+ }
+ for key in jinja2.__all__:
+ result[key] = getattr(jinja2, key)
+ return result
+
+
+def action_compile():
+ print env.compile(sys.stdin.read(), raw=True)
+
+action_shell = script.make_shell(shell_init_func)
+
+
+if __name__ == '__main__':
+ script.run()
diff --git a/slider-agent/src/main/python/jinja2/jinja2/__init__.py b/slider-agent/src/main/python/jinja2/jinja2/__init__.py
new file mode 100644
index 0000000..f944e11
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/__init__.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2
+ ~~~~~~
+
+ Jinja2 is a template engine written in pure Python. It provides a
+ Django inspired non-XML syntax but supports inline expressions and
+ an optional sandboxed environment.
+
+ Nutshell
+ --------
+
+ Here a small example of a Jinja2 template::
+
+ {% extends 'base.html' %}
+ {% block title %}Memberlist{% endblock %}
+ {% block content %}
+ <ul>
+ {% for user in users %}
+ <li><a href="{{ user.url }}">{{ user.username }}</a></li>
+ {% endfor %}
+ </ul>
+ {% endblock %}
+
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+__docformat__ = 'restructuredtext en'
+try:
+ __version__ = __import__('pkg_resources') \
+ .get_distribution('Jinja2').version
+except:
+ __version__ = 'unknown'
+
+# high level interface
+from jinja2.environment import Environment, Template
+
+# loaders
+from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \
+ DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \
+ ModuleLoader
+
+# bytecode caches
+from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \
+ MemcachedBytecodeCache
+
+# undefined types
+from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined
+
+# exceptions
+from jinja2.exceptions import TemplateError, UndefinedError, \
+ TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \
+ TemplateAssertionError
+
+# decorators and public utilities
+from jinja2.filters import environmentfilter, contextfilter, \
+ evalcontextfilter
+from jinja2.utils import Markup, escape, clear_caches, \
+ environmentfunction, evalcontextfunction, contextfunction, \
+ is_undefined
+
+__all__ = [
+ 'Environment', 'Template', 'BaseLoader', 'FileSystemLoader',
+ 'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader',
+ 'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache',
+ 'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined',
+ 'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound',
+ 'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError',
+ 'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape',
+ 'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined',
+ 'evalcontextfilter', 'evalcontextfunction'
+]
diff --git a/slider-agent/src/main/python/jinja2/jinja2/_debugsupport.c b/slider-agent/src/main/python/jinja2/jinja2/_debugsupport.c
new file mode 100644
index 0000000..e756d8e
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/_debugsupport.c
@@ -0,0 +1,78 @@
+/**
+ * jinja2._debugsupport
+ * ~~~~~~~~~~~~~~~~~~~~
+ *
+ * C implementation of `tb_set_next`.
+ *
+ * :copyright: (c) 2010 by the Jinja Team.
+ * :license: BSD.
+ */
+
+#include <Python.h>
+
+
+static PyObject*
+tb_set_next(PyObject *self, PyObject *args)
+{
+ PyTracebackObject *tb, *old;
+ PyObject *next;
+
+ if (!PyArg_ParseTuple(args, "O!O:tb_set_next", &PyTraceBack_Type, &tb, &next))
+ return NULL;
+ if (next == Py_None)
+ next = NULL;
+ else if (!PyTraceBack_Check(next)) {
+ PyErr_SetString(PyExc_TypeError,
+ "tb_set_next arg 2 must be traceback or None");
+ return NULL;
+ }
+ else
+ Py_INCREF(next);
+
+ old = tb->tb_next;
+ tb->tb_next = (PyTracebackObject*)next;
+ Py_XDECREF(old);
+
+ Py_INCREF(Py_None);
+ return Py_None;
+}
+
+static PyMethodDef module_methods[] = {
+ {"tb_set_next", (PyCFunction)tb_set_next, METH_VARARGS,
+ "Set the tb_next member of a traceback object."},
+ {NULL, NULL, 0, NULL} /* Sentinel */
+};
+
+
+#if PY_MAJOR_VERSION < 3
+
+#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
+#define PyMODINIT_FUNC void
+#endif
+PyMODINIT_FUNC
+init_debugsupport(void)
+{
+ Py_InitModule3("jinja2._debugsupport", module_methods, "");
+}
+
+#else /* Python 3.x module initialization */
+
+static struct PyModuleDef module_definition = {
+ PyModuleDef_HEAD_INIT,
+ "jinja2._debugsupport",
+ NULL,
+ -1,
+ module_methods,
+ NULL,
+ NULL,
+ NULL,
+ NULL
+};
+
+PyMODINIT_FUNC
+PyInit__debugsupport(void)
+{
+ return PyModule_Create(&module_definition);
+}
+
+#endif
diff --git a/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/__init__.py b/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/__init__.py
new file mode 100644
index 0000000..ec7bd57
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/__init__.py
@@ -0,0 +1,225 @@
+# -*- coding: utf-8 -*-
+"""
+ markupsafe
+ ~~~~~~~~~~
+
+ Implements a Markup string.
+
+ :copyright: (c) 2010 by Armin Ronacher.
+ :license: BSD, see LICENSE for more details.
+"""
+import re
+from itertools import imap
+
+
+__all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent']
+
+
+_striptags_re = re.compile(r'(<!--.*?-->|<[^>]*>)')
+_entity_re = re.compile(r'&([^;]+);')
+
+
+class Markup(unicode):
+ r"""Marks a string as being safe for inclusion in HTML/XML output without
+ needing to be escaped. This implements the `__html__` interface a couple
+ of frameworks and web applications use. :class:`Markup` is a direct
+ subclass of `unicode` and provides all the methods of `unicode` just that
+ it escapes arguments passed and always returns `Markup`.
+
+ The `escape` function returns markup objects so that double escaping can't
+ happen.
+
+ The constructor of the :class:`Markup` class can be used for three
+ different things: When passed an unicode object it's assumed to be safe,
+ when passed an object with an HTML representation (has an `__html__`
+ method) that representation is used, otherwise the object passed is
+ converted into a unicode string and then assumed to be safe:
+
+ >>> Markup("Hello <em>World</em>!")
+ Markup(u'Hello <em>World</em>!')
+ >>> class Foo(object):
+ ... def __html__(self):
+ ... return '<a href="#">foo</a>'
+ ...
+ >>> Markup(Foo())
+ Markup(u'<a href="#">foo</a>')
+
+ If you want object passed being always treated as unsafe you can use the
+ :meth:`escape` classmethod to create a :class:`Markup` object:
+
+ >>> Markup.escape("Hello <em>World</em>!")
+ Markup(u'Hello <em>World</em>!')
+
+ Operations on a markup string are markup aware which means that all
+ arguments are passed through the :func:`escape` function:
+
+ >>> em = Markup("<em>%s</em>")
+ >>> em % "foo & bar"
+ Markup(u'<em>foo & bar</em>')
+ >>> strong = Markup("<strong>%(text)s</strong>")
+ >>> strong % {'text': '<blink>hacker here</blink>'}
+ Markup(u'<strong><blink>hacker here</blink></strong>')
+ >>> Markup("<em>Hello</em> ") + "<foo>"
+ Markup(u'<em>Hello</em> <foo>')
+ """
+ __slots__ = ()
+
+ def __new__(cls, base=u'', encoding=None, errors='strict'):
+ if hasattr(base, '__html__'):
+ base = base.__html__()
+ if encoding is None:
+ return unicode.__new__(cls, base)
+ return unicode.__new__(cls, base, encoding, errors)
+
+ def __html__(self):
+ return self
+
+ def __add__(self, other):
+ if hasattr(other, '__html__') or isinstance(other, basestring):
+ return self.__class__(unicode(self) + unicode(escape(other)))
+ return NotImplemented
+
+ def __radd__(self, other):
+ if hasattr(other, '__html__') or isinstance(other, basestring):
+ return self.__class__(unicode(escape(other)) + unicode(self))
+ return NotImplemented
+
+ def __mul__(self, num):
+ if isinstance(num, (int, long)):
+ return self.__class__(unicode.__mul__(self, num))
+ return NotImplemented
+ __rmul__ = __mul__
+
+ def __mod__(self, arg):
+ if isinstance(arg, tuple):
+ arg = tuple(imap(_MarkupEscapeHelper, arg))
+ else:
+ arg = _MarkupEscapeHelper(arg)
+ return self.__class__(unicode.__mod__(self, arg))
+
+ def __repr__(self):
+ return '%s(%s)' % (
+ self.__class__.__name__,
+ unicode.__repr__(self)
+ )
+
+ def join(self, seq):
+ return self.__class__(unicode.join(self, imap(escape, seq)))
+ join.__doc__ = unicode.join.__doc__
+
+ def split(self, *args, **kwargs):
+ return map(self.__class__, unicode.split(self, *args, **kwargs))
+ split.__doc__ = unicode.split.__doc__
+
+ def rsplit(self, *args, **kwargs):
+ return map(self.__class__, unicode.rsplit(self, *args, **kwargs))
+ rsplit.__doc__ = unicode.rsplit.__doc__
+
+ def splitlines(self, *args, **kwargs):
+ return map(self.__class__, unicode.splitlines(self, *args, **kwargs))
+ splitlines.__doc__ = unicode.splitlines.__doc__
+
+ def unescape(self):
+ r"""Unescape markup again into an unicode string. This also resolves
+ known HTML4 and XHTML entities:
+
+ >>> Markup("Main » <em>About</em>").unescape()
+ u'Main \xbb <em>About</em>'
+ """
+ from jinja2._markupsafe._constants import HTML_ENTITIES
+ def handle_match(m):
+ name = m.group(1)
+ if name in HTML_ENTITIES:
+ return unichr(HTML_ENTITIES[name])
+ try:
+ if name[:2] in ('#x', '#X'):
+ return unichr(int(name[2:], 16))
+ elif name.startswith('#'):
+ return unichr(int(name[1:]))
+ except ValueError:
+ pass
+ return u''
+ return _entity_re.sub(handle_match, unicode(self))
+
+ def striptags(self):
+ r"""Unescape markup into an unicode string and strip all tags. This
+ also resolves known HTML4 and XHTML entities. Whitespace is
+ normalized to one:
+
+ >>> Markup("Main » <em>About</em>").striptags()
+ u'Main \xbb About'
+ """
+ stripped = u' '.join(_striptags_re.sub('', self).split())
+ return Markup(stripped).unescape()
+
+ @classmethod
+ def escape(cls, s):
+ """Escape the string. Works like :func:`escape` with the difference
+ that for subclasses of :class:`Markup` this function would return the
+ correct subclass.
+ """
+ rv = escape(s)
+ if rv.__class__ is not cls:
+ return cls(rv)
+ return rv
+
+ def make_wrapper(name):
+ orig = getattr(unicode, name)
+ def func(self, *args, **kwargs):
+ args = _escape_argspec(list(args), enumerate(args))
+ _escape_argspec(kwargs, kwargs.iteritems())
+ return self.__class__(orig(self, *args, **kwargs))
+ func.__name__ = orig.__name__
+ func.__doc__ = orig.__doc__
+ return func
+
+ for method in '__getitem__', 'capitalize', \
+ 'title', 'lower', 'upper', 'replace', 'ljust', \
+ 'rjust', 'lstrip', 'rstrip', 'center', 'strip', \
+ 'translate', 'expandtabs', 'swapcase', 'zfill':
+ locals()[method] = make_wrapper(method)
+
+ # new in python 2.5
+ if hasattr(unicode, 'partition'):
+ partition = make_wrapper('partition'),
+ rpartition = make_wrapper('rpartition')
+
+ # new in python 2.6
+ if hasattr(unicode, 'format'):
+ format = make_wrapper('format')
+
+ # not in python 3
+ if hasattr(unicode, '__getslice__'):
+ __getslice__ = make_wrapper('__getslice__')
+
+ del method, make_wrapper
+
+
+def _escape_argspec(obj, iterable):
+ """Helper for various string-wrapped functions."""
+ for key, value in iterable:
+ if hasattr(value, '__html__') or isinstance(value, basestring):
+ obj[key] = escape(value)
+ return obj
+
+
+class _MarkupEscapeHelper(object):
+ """Helper for Markup.__mod__"""
+
+ def __init__(self, obj):
+ self.obj = obj
+
+ __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x])
+ __str__ = lambda s: str(escape(s.obj))
+ __unicode__ = lambda s: unicode(escape(s.obj))
+ __repr__ = lambda s: str(escape(repr(s.obj)))
+ __int__ = lambda s: int(s.obj)
+ __float__ = lambda s: float(s.obj)
+
+
+# we have to import it down here as the speedups and native
+# modules imports the markup type which is define above.
+try:
+ from jinja2._markupsafe._speedups import escape, escape_silent, soft_unicode
+except ImportError:
+ from jinja2._markupsafe._native import escape, escape_silent, soft_unicode
diff --git a/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/_bundle.py b/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/_bundle.py
new file mode 100644
index 0000000..e694faf
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/_bundle.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2._markupsafe._bundle
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ This script pulls in markupsafe from a source folder and
+ bundles it with Jinja2. It does not pull in the speedups
+ module though.
+
+ :copyright: Copyright 2010 by the Jinja team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+import sys
+import os
+import re
+
+
+def rewrite_imports(lines):
+ for idx, line in enumerate(lines):
+ new_line = re.sub(r'(import|from)\s+markupsafe\b',
+ r'\1 jinja2._markupsafe', line)
+ if new_line != line:
+ lines[idx] = new_line
+
+
+def main():
+ if len(sys.argv) != 2:
+ print 'error: only argument is path to markupsafe'
+ sys.exit(1)
+ basedir = os.path.dirname(__file__)
+ markupdir = sys.argv[1]
+ for filename in os.listdir(markupdir):
+ if filename.endswith('.py'):
+ f = open(os.path.join(markupdir, filename))
+ try:
+ lines = list(f)
+ finally:
+ f.close()
+ rewrite_imports(lines)
+ f = open(os.path.join(basedir, filename), 'w')
+ try:
+ for line in lines:
+ f.write(line)
+ finally:
+ f.close()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/_constants.py b/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/_constants.py
new file mode 100644
index 0000000..919bf03
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/_constants.py
@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+"""
+ markupsafe._constants
+ ~~~~~~~~~~~~~~~~~~~~~
+
+ Highlevel implementation of the Markup string.
+
+ :copyright: (c) 2010 by Armin Ronacher.
+ :license: BSD, see LICENSE for more details.
+"""
+
+
+HTML_ENTITIES = {
+ 'AElig': 198,
+ 'Aacute': 193,
+ 'Acirc': 194,
+ 'Agrave': 192,
+ 'Alpha': 913,
+ 'Aring': 197,
+ 'Atilde': 195,
+ 'Auml': 196,
+ 'Beta': 914,
+ 'Ccedil': 199,
+ 'Chi': 935,
+ 'Dagger': 8225,
+ 'Delta': 916,
+ 'ETH': 208,
+ 'Eacute': 201,
+ 'Ecirc': 202,
+ 'Egrave': 200,
+ 'Epsilon': 917,
+ 'Eta': 919,
+ 'Euml': 203,
+ 'Gamma': 915,
+ 'Iacute': 205,
+ 'Icirc': 206,
+ 'Igrave': 204,
+ 'Iota': 921,
+ 'Iuml': 207,
+ 'Kappa': 922,
+ 'Lambda': 923,
+ 'Mu': 924,
+ 'Ntilde': 209,
+ 'Nu': 925,
+ 'OElig': 338,
+ 'Oacute': 211,
+ 'Ocirc': 212,
+ 'Ograve': 210,
+ 'Omega': 937,
+ 'Omicron': 927,
+ 'Oslash': 216,
+ 'Otilde': 213,
+ 'Ouml': 214,
+ 'Phi': 934,
+ 'Pi': 928,
+ 'Prime': 8243,
+ 'Psi': 936,
+ 'Rho': 929,
+ 'Scaron': 352,
+ 'Sigma': 931,
+ 'THORN': 222,
+ 'Tau': 932,
+ 'Theta': 920,
+ 'Uacute': 218,
+ 'Ucirc': 219,
+ 'Ugrave': 217,
+ 'Upsilon': 933,
+ 'Uuml': 220,
+ 'Xi': 926,
+ 'Yacute': 221,
+ 'Yuml': 376,
+ 'Zeta': 918,
+ 'aacute': 225,
+ 'acirc': 226,
+ 'acute': 180,
+ 'aelig': 230,
+ 'agrave': 224,
+ 'alefsym': 8501,
+ 'alpha': 945,
+ 'amp': 38,
+ 'and': 8743,
+ 'ang': 8736,
+ 'apos': 39,
+ 'aring': 229,
+ 'asymp': 8776,
+ 'atilde': 227,
+ 'auml': 228,
+ 'bdquo': 8222,
+ 'beta': 946,
+ 'brvbar': 166,
+ 'bull': 8226,
+ 'cap': 8745,
+ 'ccedil': 231,
+ 'cedil': 184,
+ 'cent': 162,
+ 'chi': 967,
+ 'circ': 710,
+ 'clubs': 9827,
+ 'cong': 8773,
+ 'copy': 169,
+ 'crarr': 8629,
+ 'cup': 8746,
+ 'curren': 164,
+ 'dArr': 8659,
+ 'dagger': 8224,
+ 'darr': 8595,
+ 'deg': 176,
+ 'delta': 948,
+ 'diams': 9830,
+ 'divide': 247,
+ 'eacute': 233,
+ 'ecirc': 234,
+ 'egrave': 232,
+ 'empty': 8709,
+ 'emsp': 8195,
+ 'ensp': 8194,
+ 'epsilon': 949,
+ 'equiv': 8801,
+ 'eta': 951,
+ 'eth': 240,
+ 'euml': 235,
+ 'euro': 8364,
+ 'exist': 8707,
+ 'fnof': 402,
+ 'forall': 8704,
+ 'frac12': 189,
+ 'frac14': 188,
+ 'frac34': 190,
+ 'frasl': 8260,
+ 'gamma': 947,
+ 'ge': 8805,
+ 'gt': 62,
+ 'hArr': 8660,
+ 'harr': 8596,
+ 'hearts': 9829,
+ 'hellip': 8230,
+ 'iacute': 237,
+ 'icirc': 238,
+ 'iexcl': 161,
+ 'igrave': 236,
+ 'image': 8465,
+ 'infin': 8734,
+ 'int': 8747,
+ 'iota': 953,
+ 'iquest': 191,
+ 'isin': 8712,
+ 'iuml': 239,
+ 'kappa': 954,
+ 'lArr': 8656,
+ 'lambda': 955,
+ 'lang': 9001,
+ 'laquo': 171,
+ 'larr': 8592,
+ 'lceil': 8968,
+ 'ldquo': 8220,
+ 'le': 8804,
+ 'lfloor': 8970,
+ 'lowast': 8727,
+ 'loz': 9674,
+ 'lrm': 8206,
+ 'lsaquo': 8249,
+ 'lsquo': 8216,
+ 'lt': 60,
+ 'macr': 175,
+ 'mdash': 8212,
+ 'micro': 181,
+ 'middot': 183,
+ 'minus': 8722,
+ 'mu': 956,
+ 'nabla': 8711,
+ 'nbsp': 160,
+ 'ndash': 8211,
+ 'ne': 8800,
+ 'ni': 8715,
+ 'not': 172,
+ 'notin': 8713,
+ 'nsub': 8836,
+ 'ntilde': 241,
+ 'nu': 957,
+ 'oacute': 243,
+ 'ocirc': 244,
+ 'oelig': 339,
+ 'ograve': 242,
+ 'oline': 8254,
+ 'omega': 969,
+ 'omicron': 959,
+ 'oplus': 8853,
+ 'or': 8744,
+ 'ordf': 170,
+ 'ordm': 186,
+ 'oslash': 248,
+ 'otilde': 245,
+ 'otimes': 8855,
+ 'ouml': 246,
+ 'para': 182,
+ 'part': 8706,
+ 'permil': 8240,
+ 'perp': 8869,
+ 'phi': 966,
+ 'pi': 960,
+ 'piv': 982,
+ 'plusmn': 177,
+ 'pound': 163,
+ 'prime': 8242,
+ 'prod': 8719,
+ 'prop': 8733,
+ 'psi': 968,
+ 'quot': 34,
+ 'rArr': 8658,
+ 'radic': 8730,
+ 'rang': 9002,
+ 'raquo': 187,
+ 'rarr': 8594,
+ 'rceil': 8969,
+ 'rdquo': 8221,
+ 'real': 8476,
+ 'reg': 174,
+ 'rfloor': 8971,
+ 'rho': 961,
+ 'rlm': 8207,
+ 'rsaquo': 8250,
+ 'rsquo': 8217,
+ 'sbquo': 8218,
+ 'scaron': 353,
+ 'sdot': 8901,
+ 'sect': 167,
+ 'shy': 173,
+ 'sigma': 963,
+ 'sigmaf': 962,
+ 'sim': 8764,
+ 'spades': 9824,
+ 'sub': 8834,
+ 'sube': 8838,
+ 'sum': 8721,
+ 'sup': 8835,
+ 'sup1': 185,
+ 'sup2': 178,
+ 'sup3': 179,
+ 'supe': 8839,
+ 'szlig': 223,
+ 'tau': 964,
+ 'there4': 8756,
+ 'theta': 952,
+ 'thetasym': 977,
+ 'thinsp': 8201,
+ 'thorn': 254,
+ 'tilde': 732,
+ 'times': 215,
+ 'trade': 8482,
+ 'uArr': 8657,
+ 'uacute': 250,
+ 'uarr': 8593,
+ 'ucirc': 251,
+ 'ugrave': 249,
+ 'uml': 168,
+ 'upsih': 978,
+ 'upsilon': 965,
+ 'uuml': 252,
+ 'weierp': 8472,
+ 'xi': 958,
+ 'yacute': 253,
+ 'yen': 165,
+ 'yuml': 255,
+ 'zeta': 950,
+ 'zwj': 8205,
+ 'zwnj': 8204
+}
diff --git a/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/_native.py b/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/_native.py
new file mode 100644
index 0000000..7b95828
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/_native.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+"""
+ markupsafe._native
+ ~~~~~~~~~~~~~~~~~~
+
+ Native Python implementation the C module is not compiled.
+
+ :copyright: (c) 2010 by Armin Ronacher.
+ :license: BSD, see LICENSE for more details.
+"""
+from jinja2._markupsafe import Markup
+
+
+def escape(s):
+ """Convert the characters &, <, >, ' and " in string s to HTML-safe
+ sequences. Use this if you need to display text that might contain
+ such characters in HTML. Marks return value as markup string.
+ """
+ if hasattr(s, '__html__'):
+ return s.__html__()
+ return Markup(unicode(s)
+ .replace('&', '&')
+ .replace('>', '>')
+ .replace('<', '<')
+ .replace("'", ''')
+ .replace('"', '"')
+ )
+
+
+def escape_silent(s):
+ """Like :func:`escape` but converts `None` into an empty
+ markup string.
+ """
+ if s is None:
+ return Markup()
+ return escape(s)
+
+
+def soft_unicode(s):
+ """Make a string unicode if it isn't already. That way a markup
+ string is not converted back to unicode.
+ """
+ if not isinstance(s, unicode):
+ s = unicode(s)
+ return s
diff --git a/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/tests.py b/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/tests.py
new file mode 100644
index 0000000..c1ce394
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/_markupsafe/tests.py
@@ -0,0 +1,80 @@
+import gc
+import unittest
+from jinja2._markupsafe import Markup, escape, escape_silent
+
+
+class MarkupTestCase(unittest.TestCase):
+
+ def test_markup_operations(self):
+ # adding two strings should escape the unsafe one
+ unsafe = '<script type="application/x-some-script">alert("foo");</script>'
+ safe = Markup('<em>username</em>')
+ assert unsafe + safe == unicode(escape(unsafe)) + unicode(safe)
+
+ # string interpolations are safe to use too
+ assert Markup('<em>%s</em>') % '<bad user>' == \
+ '<em><bad user></em>'
+ assert Markup('<em>%(username)s</em>') % {
+ 'username': '<bad user>'
+ } == '<em><bad user></em>'
+
+ # an escaped object is markup too
+ assert type(Markup('foo') + 'bar') is Markup
+
+ # and it implements __html__ by returning itself
+ x = Markup("foo")
+ assert x.__html__() is x
+
+ # it also knows how to treat __html__ objects
+ class Foo(object):
+ def __html__(self):
+ return '<em>awesome</em>'
+ def __unicode__(self):
+ return 'awesome'
+ assert Markup(Foo()) == '<em>awesome</em>'
+ assert Markup('<strong>%s</strong>') % Foo() == \
+ '<strong><em>awesome</em></strong>'
+
+ # escaping and unescaping
+ assert escape('"<>&\'') == '"<>&''
+ assert Markup("<em>Foo & Bar</em>").striptags() == "Foo & Bar"
+ assert Markup("<test>").unescape() == "<test>"
+
+ def test_all_set(self):
+ import jinja2._markupsafe as markup
+ for item in markup.__all__:
+ getattr(markup, item)
+
+ def test_escape_silent(self):
+ assert escape_silent(None) == Markup()
+ assert escape(None) == Markup(None)
+ assert escape_silent('<foo>') == Markup(u'<foo>')
+
+
+class MarkupLeakTestCase(unittest.TestCase):
+
+ def test_markup_leaks(self):
+ counts = set()
+ for count in xrange(20):
+ for item in xrange(1000):
+ escape("foo")
+ escape("<foo>")
+ escape(u"foo")
+ escape(u"<foo>")
+ counts.add(len(gc.get_objects()))
+ assert len(counts) == 1, 'ouch, c extension seems to leak objects'
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(MarkupTestCase))
+
+ # this test only tests the c extension
+ if not hasattr(escape, 'func_code'):
+ suite.addTest(unittest.makeSuite(MarkupLeakTestCase))
+
+ return suite
+
+
+if __name__ == '__main__':
+ unittest.main(defaultTest='suite')
diff --git a/slider-agent/src/main/python/jinja2/jinja2/_stringdefs.py b/slider-agent/src/main/python/jinja2/jinja2/_stringdefs.py
new file mode 100644
index 0000000..1161b7f
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/_stringdefs.py
@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2._stringdefs
+ ~~~~~~~~~~~~~~~~~~
+
+ Strings of all Unicode characters of a certain category.
+ Used for matching in Unicode-aware languages. Run to regenerate.
+
+ Inspired by chartypes_create.py from the MoinMoin project, original
+ implementation from Pygments.
+
+ :copyright: Copyright 2006-2009 by the Jinja team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+Cc = u'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f'
+
+Cf = u'\xad\u0600\u0601\u0602\u0603\u06dd\u070f\u17b4\u17b5\u200b\u200c\u200d\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2060\u2061\u2062\u2063\u206a\u206b\u206c\u206d\u206e\u206f\ufeff\ufff9\ufffa\ufffb'
+
+Cn = u'\u0242\u0243\u0244\u0245\u0246\u0247\u0248\u0249\u024a\u024b\u024c\u024d\u024e\u024f\u0370\u0371\u0372\u0373\u0376\u0377\u0378\u0379\u037b\u037c\u037d\u037f\u0380\u0381\u0382\u0383\u038b\u038d\u03a2\u03cf\u0487\u04cf\u04fa\u04fb\u04fc\u04fd\u04fe\u04ff\u0510\u0511\u0512\u0513\u0514\u0515\u0516\u0517\u0518\u0519\u051a\u051b\u051c\u051d\u051e\u051f\u0520\u0521\u0522\u0523\u0524\u0525\u0526\u0527\u0528\u0529\u052a\u052b\u052c\u052d\u052e\u052f\u0530\u0557\u0558\u0560\u0588\u058b\u058c\u058d\u058e\u058f\u0590\u05ba\u05c8\u05c9\u05ca\u05cb\u05cc\u05cd\u05ce\u05cf\u05eb\u05ec\u05ed\u05ee\u05ef\u05f5\u05f6\u05f7\u05f8\u05f9\u05fa\u05fb\u05fc\u05fd\u05fe\u05ff\u0604\u0605\u0606\u0607\u0608\u0609\u060a\u0616\u0617\u0618\u0619\u061a\u061c\u061d\u0620\u063b\u063c\u063d\u063e\u063f\u065f\u070e\u074b\u074c\u076e\u076f\u0770\u0771\u0772\u0773\u0774\u0775\u0776\u0777\u0778\u0779\u077a\u077b\u077c\u077d\u077e\u077f\u07b2\u07b3\u07b4\u07b5\u07b6\u07b7\u07b8\u07b9\u07ba\u07bb\u07bc\u07bd\u07be\u07bf\u07c0\u07c1\u07c2\u07c3\u07c4\u07c5\u07c6\u07c7\u07c8\u07c9\u07ca\u07cb\u07cc\u07cd\u07ce\u07cf\u07d0\u07d1\u07d2\u07d3\u07d4\u07d5\u07d6\u07d7\u07d8\u07d9\u07da\u07db\u07dc\u07dd\u07de\u07df\u07e0\u07e1\u07e2\u07e3\u07e4\u07e5\u07e6\u07e7\u07e8\u07e9\u07ea\u07eb\u07ec\u07ed\u07ee\u07ef\u07f0\u07f1\u07f2\u07f3\u07f4\u07f5\u07f6\u07f7\u07f8\u07f9\u07fa\u07fb\u07fc\u07fd\u07fe\u07ff\u0800\u0801\u0802\u0803\u0804\u0805\u0806\u0807\u0808\u0809\u080a\u080b\u080c\u080d\u080e\u080f\u0810\u0811\u0812\u0813\u0814\u0815\u0816\u0817\u0818\u0819\u081a\u081b\u081c\u081d\u081e\u081f\u0820\u0821\u0822\u0823\u0824\u0825\u0826\u0827\u0828\u0829\u082a\u082b\u082c\u082d\u082e\u082f\u0830\u0831\u0832\u0833\u0834\u0835\u0836\u0837\u0838\u0839\u083a\u083b\u083c\u083d\u083e\u083f\u0840\u0841\u0842\u0843\u0844\u0845\u0846\u0847\u0848\u0849\u084a\u084b\u084c\u084d\u084e\u084f\u0850\u0851\u0852\u0853\u0854\u0855\u0856\u0857\u0858\u0859\u085a\u085b\u085c\u085d\u085e\u085f\u0860\u0861\u0862\u0863\u0864\u0865\u0866\u0867\u0868\u0869\u086a\u086b\u086c\u086d\u086e\u086f\u0870\u0871\u0872\u0873\u0874\u0875\u0876\u0877\u0878\u0879\u087a\u087b\u087c\u087d\u087e\u087f\u0880\u0881\u0882\u0883\u0884\u0885\u0886\u0887\u0888\u0889\u088a\u088b\u088c\u088d\u088e\u088f\u0890\u0891\u0892\u0893\u0894\u0895\u0896\u0897\u0898\u0899\u089a\u089b\u089c\u089d\u089e\u089f\u08a0\u08a1\u08a2\u08a3\u08a4\u08a5\u08a6\u08a7\u08a8\u08a9\u08aa\u08ab\u08ac\u08ad\u08ae\u08af\u08b0\u08b1\u08b2\u08b3\u08b4\u08b5\u08b6\u08b7\u08b8\u08b9\u08ba\u08bb\u08bc\u08bd\u08be\u08bf\u08c0\u08c1\u08c2\u08c3\u08c4\u08c5\u08c6\u08c7\u08c8\u08c9\u08ca\u08cb\u08cc\u08cd\u08ce\u08cf\u08d0\u08d1\u08d2\u08d3\u08d4\u08d5\u08d6\u08d7\u08d8\u08d9\u08da\u08db\u08dc\u08dd\u08de\u08df\u08e0\u08e1\u08e2\u08e3\u08e4\u08e5\u08e6\u08e7\u08e8\u08e9\u08ea\u08eb\u08ec\u08ed\u08ee\u08ef\u08f0\u08f1\u08f2\u08f3\u08f4\u08f5\u08f6\u08f7\u08f8\u08f9\u08fa\u08fb\u08fc\u08fd\u08fe\u08ff\u0900\u093a\u093b\u094e\u094f\u0955\u0956\u0957\u0971\u0972\u0973\u0974\u0975\u0976\u0977\u0978\u0979\u097a\u097b\u097c\u097e\u097f\u0980\u0984\u098d\u098e\u0991\u0992\u09a9\u09b1\u09b3\u09b4\u09b5\u09ba\u09bb\u09c5\u09c6\u09c9\u09ca\u09cf\u09d0\u09d1\u09d2\u09d3\u09d4\u09d5\u09d6\u09d8\u09d9\u09da\u09db\u09de\u09e4\u09e5\u09fb\u09fc\u09fd\u09fe\u09ff\u0a00\u0a04\u0a0b\u0a0c\u0a0d\u0a0e\u0a11\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a\u0a3b\u0a3d\u0a43\u0a44\u0a45\u0a46\u0a49\u0a4a\u0a4e\u0a4f\u0a50\u0a51\u0a52\u0a53\u0a54\u0a55\u0a56\u0a57\u0a58\u0a5d\u0a5f\u0a60\u0a61\u0a62\u0a63\u0a64\u0a65\u0a75\u0a76\u0a77\u0a78\u0a79\u0a7a\u0a7b\u0a7c\u0a7d\u0a7e\u0a7f\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba\u0abb\u0ac6\u0aca\u0ace\u0acf\u0ad1\u0ad2\u0ad3\u0ad4\u0ad5\u0ad6\u0ad7\u0ad8\u0ad9\u0ada\u0adb\u0adc\u0add\u0ade\u0adf\u0ae4\u0ae5\u0af0\u0af2\u0af3\u0af4\u0af5\u0af6\u0af7\u0af8\u0af9\u0afa\u0afb\u0afc\u0afd\u0afe\u0aff\u0b00\u0b04\u0b0d\u0b0e\u0b11\u0b12\u0b29\u0b31\u0b34\u0b3a\u0b3b\u0b44\u0b45\u0b46\u0b49\u0b4a\u0b4e\u0b4f\u0b50\u0b51\u0b52\u0b53\u0b54\u0b55\u0b58\u0b59\u0b5a\u0b5b\u0b5e\u0b62\u0b63\u0b64\u0b65\u0b72\u0b73\u0b74\u0b75\u0b76\u0b77\u0b78\u0b79\u0b7a\u0b7b\u0b7c\u0b7d\u0b7e\u0b7f\u0b80\u0b81\u0b84\u0b8b\u0b8c\u0b8d\u0b91\u0b96\u0b97\u0b98\u0b9b\u0b9d\u0ba0\u0ba1\u0ba2\u0ba5\u0ba6\u0ba7\u0bab\u0bac\u0bad\u0bba\u0bbb\u0bbc\u0bbd\u0bc3\u0bc4\u0bc5\u0bc9\u0bce\u0bcf\u0bd0\u0bd1\u0bd2\u0bd3\u0bd4\u0bd5\u0bd6\u0bd8\u0bd9\u0bda\u0bdb\u0bdc\u0bdd\u0bde\u0bdf\u0be0\u0be1\u0be2\u0be3\u0be4\u0be5\u0bfb\u0bfc\u0bfd\u0bfe\u0bff\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a\u0c3b\u0c3c\u0c3d\u0c45\u0c49\u0c4e\u0c4f\u0c50\u0c51\u0c52\u0c53\u0c54\u0c57\u0c58\u0c59\u0c5a\u0c5b\u0c5c\u0c5d\u0c5e\u0c5f\u0c62\u0c63\u0c64\u0c65\u0c70\u0c71\u0c72\u0c73\u0c74\u0c75\u0c76\u0c77\u0c78\u0c79\u0c7a\u0c7b\u0c7c\u0c7d\u0c7e\u0c7f\u0c80\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba\u0cbb\u0cc5\u0cc9\u0cce\u0ccf\u0cd0\u0cd1\u0cd2\u0cd3\u0cd4\u0cd7\u0cd8\u0cd9\u0cda\u0cdb\u0cdc\u0cdd\u0cdf\u0ce2\u0ce3\u0ce4\u0ce5\u0cf0\u0cf1\u0cf2\u0cf3\u0cf4\u0cf5\u0cf6\u0cf7\u0cf8\u0cf9\u0cfa\u0cfb\u0cfc\u0cfd\u0cfe\u0cff\u0d00\u0d01\u0d04\u0d0d\u0d11\u0d29\u0d3a\u0d3b\u0d3c\u0d3d\u0d44\u0d45\u0d49\u0d4e\u0d4f\u0d50\u0d51\u0d52\u0d53\u0d54\u0d55\u0d56\u0d58\u0d59\u0d5a\u0d5b\u0d5c\u0d5d\u0d5e\u0d5f\u0d62\u0d63\u0d64\u0d65\u0d70\u0d71\u0d72\u0d73\u0d74\u0d75\u0d76\u0d77\u0d78\u0d79\u0d7a\u0d7b\u0d7c\u0d7d\u0d7e\u0d7f\u0d80\u0d81\u0d84\u0d97\u0d98\u0d99\u0db2\u0dbc\u0dbe\u0dbf\u0dc7\u0dc8\u0dc9\u0dcb\u0dcc\u0dcd\u0dce\u0dd5\u0dd7\u0de0\u0de1\u0de2\u0de3\u0de4\u0de5\u0de6\u0de7\u0de8\u0de9\u0dea\u0deb\u0dec\u0ded\u0dee\u0def\u0df0\u0df1\u0df5\u0df6\u0df7\u0df8\u0df9\u0dfa\u0dfb\u0dfc\u0dfd\u0dfe\u0dff\u0e00\u0e3b\u0e3c\u0e3d\u0e3e\u0e5c\u0e5d\u0e5e\u0e5f\u0e60\u0e61\u0e62\u0e63\u0e64\u0e65\u0e66\u0e67\u0e68\u0e69\u0e6a\u0e6b\u0e6c\u0e6d\u0e6e\u0e6f\u0e70\u0e71\u0e72\u0e73\u0e74\u0e75\u0e76\u0e77\u0e78\u0e79\u0e7a\u0e7b\u0e7c\u0e7d\u0e7e\u0e7f\u0e80\u0e83\u0e85\u0e86\u0e89\u0e8b\u0e8c\u0e8e\u0e8f\u0e90\u0e91\u0e92\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8\u0ea9\u0eac\u0eba\u0ebe\u0ebf\u0ec5\u0ec7\u0ece\u0ecf\u0eda\u0edb\u0ede\u0edf\u0ee0\u0ee1\u0ee2\u0ee3\u0ee4\u0ee5\u0ee6\u0ee7\u0ee8\u0ee9\u0eea\u0eeb\u0eec\u0eed\u0eee\u0eef\u0ef0\u0ef1\u0ef2\u0ef3\u0ef4\u0ef5\u0ef6\u0ef7\u0ef8\u0ef9\u0efa\u0efb\u0efc\u0efd\u0efe\u0eff\u0f48\u0f6b\u0f6c\u0f6d\u0f6e\u0f6f\u0f70\u0f8c\u0f8d\u0f8e\u0f8f\u0f98\u0fbd\u0fcd\u0fce\u0fd2\u0fd3\u0fd4\u0fd5\u0fd6\u0fd7\u0fd8\u0fd9\u0fda\u0fdb\u0fdc\u0fdd\u0fde\u0fdf\u0fe0\u0fe1\u0fe2\u0fe3\u0fe4\u0fe5\u0fe6\u0fe7\u0fe8\u0fe9\u0fea\u0feb\u0fec\u0fed\u0fee\u0fef\u0ff0\u0ff1\u0ff2\u0ff3\u0ff4\u0ff5\u0ff6\u0ff7\u0ff8\u0ff9\u0ffa\u0ffb\u0ffc\u0ffd\u0ffe\u0fff\u1022\u1028\u102b\u1033\u1034\u1035\u103a\u103b\u103c\u103d\u103e\u103f\u105a\u105b\u105c\u105d\u105e\u105f\u1060\u1061\u1062\u1063\u1064\u1065\u1066\u1067\u1068\u1069\u106a\u106b\u106c\u106d\u106e\u106f\u1070\u1071\u1072\u1073\u1074\u1075\u1076\u1077\u1078\u1079\u107a\u107b\u107c\u107d\u107e\u107f\u1080\u1081\u1082\u1083\u1084\u1085\u1086\u1087\u1088\u1089\u108a\u108b\u108c\u108d\u108e\u108f\u1090\u1091\u1092\u1093\u1094\u1095\u1096\u1097\u1098\u1099\u109a\u109b\u109c\u109d\u109e\u109f\u10c6\u10c7\u10c8\u10c9\u10ca\u10cb\u10cc\u10cd\u10ce\u10cf\u10fd\u10fe\u10ff\u115a\u115b\u115c\u115d\u115e\u11a3\u11a4\u11a5\u11a6\u11a7\u11fa\u11fb\u11fc\u11fd\u11fe\u11ff\u1249\u124e\u124f\u1257\u1259\u125e\u125f\u1289\u128e\u128f\u12b1\u12b6\u12b7\u12bf\u12c1\u12c6\u12c7\u12d7\u1311\u1316\u1317\u135b\u135c\u135d\u135e\u137d\u137e\u137f\u139a\u139b\u139c\u139d\u139e\u139f\u13f5\u13f6\u13f7\u13f8\u13f9\u13fa\u13fb\u13fc\u13fd\u13fe\u13ff\u1400\u1677\u1678\u1679\u167a\u167b\u167c\u167d\u167e\u167f\u169d\u169e\u169f\u16f1\u16f2\u16f3\u16f4\u16f5\u16f6\u16f7\u16f8\u16f9\u16fa\u16fb\u16fc\u16fd\u16fe\u16ff\u170d\u1715\u1716\u1717\u1718\u1719\u171a\u171b\u171c\u171d\u171e\u171f\u1737\u1738\u1739\u173a\u173b\u173c\u173d\u173e\u173f\u1754\u1755\u1756\u1757\u1758\u1759\u175a\u175b\u175c\u175d\u175e\u175f\u176d\u1771\u1774\u1775\u1776\u1777\u1778\u1779\u177a\u177b\u177c\u177d\u177e\u177f\u17de\u17df\u17ea\u17eb\u17ec\u17ed\u17ee\u17ef\u17fa\u17fb\u17fc\u17fd\u17fe\u17ff\u180f\u181a\u181b\u181c\u181d\u181e\u181f\u1878\u1879\u187a\u187b\u187c\u187d\u187e\u187f\u18aa\u18ab\u18ac\u18ad\u18ae\u18af\u18b0\u18b1\u18b2\u18b3\u18b4\u18b5\u18b6\u18b7\u18b8\u18b9\u18ba\u18bb\u18bc\u18bd\u18be\u18bf\u18c0\u18c1\u18c2\u18c3\u18c4\u18c5\u18c6\u18c7\u18c8\u18c9\u18ca\u18cb\u18cc\u18cd\u18ce\u18cf\u18d0\u18d1\u18d2\u18d3\u18d4\u18d5\u18d6\u18d7\u18d8\u18d9\u18da\u18db\u18dc\u18dd\u18de\u18df\u18e0\u18e1\u18e2\u18e3\u18e4\u18e5\u18e6\u18e7\u18e8\u18e9\u18ea\u18eb\u18ec\u18ed\u18ee\u18ef\u18f0\u18f1\u18f2\u18f3\u18f4\u18f5\u18f6\u18f7\u18f8\u18f9\u18fa\u18fb\u18fc\u18fd\u18fe\u18ff\u191d\u191e\u191f\u192c\u192d\u192e\u192f\u193c\u193d\u193e\u193f\u1941\u1942\u1943\u196e\u196f\u1975\u1976\u1977\u1978\u1979\u197a\u197b\u197c\u197d\u197e\u197f\u19aa\u19ab\u19ac\u19ad\u19ae\u19af\u19ca\u19cb\u19cc\u19cd\u19ce\u19cf\u19da\u19db\u19dc\u19dd\u1a1c\u1a1d\u1a20\u1a21\u1a22\u1a23\u1a24\u1a25\u1a26\u1a27\u1a28\u1a29\u1a2a\u1a2b\u1a2c\u1a2d\u1a2e\u1a2f\u1a30\u1a31\u1a32\u1a33\u1a34\u1a35\u1a36\u1a37\u1a38\u1a39\u1a3a\u1a3b\u1a3c\u1a3d\u1a3e\u1a3f\u1a40\u1a41\u1a42\u1a43\u1a44\u1a45\u1a46\u1a47\u1a48\u1a49\u1a4a\u1a4b\u1a4c\u1a4d\u1a4e\u1a4f\u1a50\u1a51\u1a52\u1a53\u1a54\u1a55\u1a56\u1a57\u1a58\u1a59\u1a5a\u1a5b\u1a5c\u1a5d\u1a5e\u1a5f\u1a60\u1a61\u1a62\u1a63\u1a64\u1a65\u1a66\u1a67\u1a68\u1a69\u1a6a\u1a6b\u1a6c\u1a6d\u1a6e\u1a6f\u1a70\u1a71\u1a72\u1a73\u1a74\u1a75\u1a76\u1a77\u1a78\u1a79\u1a7a\u1a7b\u1a7c\u1a7d\u1a7e\u1a7f\u1a80\u1a81\u1a82\u1a83\u1a84\u1a85\u1a86\u1a87\u1a88\u1a89\u1a8a\u1a8b\u1a8c\u1a8d\u1a8e\u1a8f\u1a90\u1a91\u1a92\u1a93\u1a94\u1a95\u1a96\u1a97\u1a98\u1a99\u1a9a\u1a9b\u1a9c\u1a9d\u1a9e\u1a9f\u1aa0\u1aa1\u1aa2\u1aa3\u1aa4\u1aa5\u1aa6\u1aa7\u1aa8\u1aa9\u1aaa\u1aab\u1aac\u1aad\u1aae\u1aaf\u1ab0\u1ab1\u1ab2\u1ab3\u1ab4\u1ab5\u1ab6\u1ab7\u1ab8\u1ab9\u1aba\u1abb\u1abc\u1abd\u1abe\u1abf\u1ac0\u1ac1\u1ac2\u1ac3\u1ac4\u1ac5\u1ac6\u1ac7\u1ac8\u1ac9\u1aca\u1acb\u1acc\u1acd\u1ace\u1acf\u1ad0\u1ad1\u1ad2\u1ad3\u1ad4\u1ad5\u1ad6\u1ad7\u1ad8\u1ad9\u1ada\u1adb\u1adc\u1add\u1ade\u1adf\u1ae0\u1ae1\u1ae2\u1ae3\u1ae4\u1ae5\u1ae6\u1ae7\u1ae8\u1ae9\u1aea\u1aeb\u1aec\u1aed\u1aee\u1aef\u1af0\u1af1\u1af2\u1af3\u1af4\u1af5\u1af6\u1af7\u1af8\u1af9\u1afa\u1afb\u1afc\u1afd\u1afe\u1aff\u1b00\u1b01\u1b02\u1b03\u1b04\u1b05\u1b06\u1b07\u1b08\u1b09\u1b0a\u1b0b\u1b0c\u1b0d\u1b0e\u1b0f\u1b10\u1b11\u1b12\u1b13\u1b14\u1b15\u1b16\u1b17\u1b18\u1b19\u1b1a\u1b1b\u1b1c\u1b1d\u1b1e\u1b1f\u1b20\u1b21\u1b22\u1b23\u1b24\u1b25\u1b26\u1b27\u1b28\u1b29\u1b2a\u1b2b\u1b2c\u1b2d\u1b2e\u1b2f\u1b30\u1b31\u1b32\u1b33\u1b34\u1b35\u1b36\u1b37\u1b38\u1b39\u1b3a\u1b3b\u1b3c\u1b3d\u1b3e\u1b3f\u1b40\u1b41\u1b42\u1b43\u1b44\u1b45\u1b46\u1b47\u1b48\u1b49\u1b4a\u1b4b\u1b4c\u1b4d\u1b4e\u1b4f\u1b50\u1b51\u1b52\u1b53\u1b54\u1b55\u1b56\u1b57\u1b58\u1b59\u1b5a\u1b5b\u1b5c\u1b5d\u1b5e\u1b5f\u1b60\u1b61\u1b62\u1b63\u1b64\u1b65\u1b66\u1b67\u1b68\u1b69\u1b6a\u1b6b\u1b6c\u1b6d\u1b6e\u1b6f\u1b70\u1b71\u1b72\u1b73\u1b74\u1b75\u1b76\u1b77\u1b78\u1b79\u1b7a\u1b7b\u1b7c\u1b7d\u1b7e\u1b7f\u1b80\u1b81\u1b82\u1b83\u1b84\u1b85\u1b86\u1b87\u1b88\u1b89\u1b8a\u1b8b\u1b8c\u1b8d\u1b8e\u1b8f\u1b90\u1b91\u1b92\u1b93\u1b94\u1b95\u1b96\u1b97\u1b98\u1b99\u1b9a\u1b9b\u1b9c\u1b9d\u1b9e\u1b9f\u1ba0\u1ba1\u1ba2\u1ba3\u1ba4\u1ba5\u1ba6\u1ba7\u1ba8\u1ba9\u1baa\u1bab\u1bac\u1bad\u1bae\u1baf\u1bb0\u1bb1\u1bb2\u1bb3\u1bb4\u1bb5\u1bb6\u1bb7\u1bb8\u1bb9\u1bba\u1bbb\u1bbc\u1bbd\u1bbe\u1bbf\u1bc0\u1bc1\u1bc2\u1bc3\u1bc4\u1bc5\u1bc6\u1bc7\u1bc8\u1bc9\u1bca\u1bcb\u1bcc\u1bcd\u1bce\u1bcf\u1bd0\u1bd1\u1bd2\u1bd3\u1bd4\u1bd5\u1bd6\u1bd7\u1bd8\u1bd9\u1bda\u1bdb\u1bdc\u1bdd\u1bde\u1bdf\u1be0\u1be1\u1be2\u1be3\u1be4\u1be5\u1be6\u1be7\u1be8\u1be9\u1bea\u1beb\u1bec\u1bed\u1bee\u1bef\u1bf0\u1bf1\u1bf2\u1bf3\u1bf4\u1bf5\u1bf6\u1bf7\u1bf8\u1bf9\u1bfa\u1bfb\u1bfc\u1bfd\u1bfe\u1bff\u1c00\u1c01\u1c02\u1c03\u1c04\u1c05\u1c06\u1c07\u1c08\u1c09\u1c0a\u1c0b\u1c0c\u1c0d\u1c0e\u1c0f\u1c10\u1c11\u1c12\u1c13\u1c14\u1c15\u1c16\u1c17\u1c18\u1c19\u1c1a\u1c1b\u1c1c\u1c1d\u1c1e\u1c1f\u1c20\u1c21\u1c22\u1c23\u1c24\u1c25\u1c26\u1c27\u1c28\u1c29\u1c2a\u1c2b\u1c2c\u1c2d\u1c2e\u1c2f\u1c30\u1c31\u1c32\u1c33\u1c34\u1c35\u1c36\u1c37\u1c38\u1c39\u1c3a\u1c3b\u1c3c\u1c3d\u1c3e\u1c3f\u1c40\u1c41\u1c42\u1c43\u1c44\u1c45\u1c46\u1c47\u1c48\u1c49\u1c4a\u1c4b\u1c4c\u1c4d\u1c4e\u1c4f\u1c50\u1c51\u1c52\u1c53\u1c54\u1c55\u1c56\u1c57\u1c58\u1c59\u1c5a\u1c5b\u1c5c\u1c5d\u1c5e\u1c5f\u1c60\u1c61\u1c62\u1c63\u1c64\u1c65\u1c66\u1c67\u1c68\u1c69\u1c6a\u1c6b\u1c6c\u1c6d\u1c6e\u1c6f\u1c70\u1c71\u1c72\u1c73\u1c74\u1c75\u1c76\u1c77\u1c78\u1c79\u1c7a\u1c7b\u1c7c\u1c7d\u1c7e\u1c7f\u1c80\u1c81\u1c82\u1c83\u1c84\u1c85\u1c86\u1c87\u1c88\u1c89\u1c8a\u1c8b\u1c8c\u1c8d\u1c8e\u1c8f\u1c90\u1c91\u1c92\u1c93\u1c94\u1c95\u1c96\u1c97\u1c98\u1c99\u1c9a\u1c9b\u1c9c\u1c9d\u1c9e\u1c9f\u1ca0\u1ca1\u1ca2\u1ca3\u1ca4\u1ca5\u1ca6\u1ca7\u1ca8\u1ca9\u1caa\u1cab\u1cac\u1cad\u1cae\u1caf\u1cb0\u1cb1\u1cb2\u1cb3\u1cb4\u1cb5\u1cb6\u1cb7\u1cb8\u1cb9\u1cba\u1cbb\u1cbc\u1cbd\u1cbe\u1cbf\u1cc0\u1cc1\u1cc2\u1cc3\u1cc4\u1cc5\u1cc6\u1cc7\u1cc8\u1cc9\u1cca\u1ccb\u1ccc\u1ccd\u1cce\u1ccf\u1cd0\u1cd1\u1cd2\u1cd3\u1cd4\u1cd5\u1cd6\u1cd7\u1cd8\u1cd9\u1cda\u1cdb\u1cdc\u1cdd\u1cde\u1cdf\u1ce0\u1ce1\u1ce2\u1ce3\u1ce4\u1ce5\u1ce6\u1ce7\u1ce8\u1ce9\u1cea\u1ceb\u1cec\u1ced\u1cee\u1cef\u1cf0\u1cf1\u1cf2\u1cf3\u1cf4\u1cf5\u1cf6\u1cf7\u1cf8\u1cf9\u1cfa\u1cfb\u1cfc\u1cfd\u1cfe\u1cff\u1dc4\u1dc5\u1dc6\u1dc7\u1dc8\u1dc9\u1dca\u1dcb\u1dcc\u1dcd\u1dce\u1dcf\u1dd0\u1dd1\u1dd2\u1dd3\u1dd4\u1dd5\u1dd6\u1dd7\u1dd8\u1dd9\u1dda\u1ddb\u1ddc\u1ddd\u1dde\u1ddf\u1de0\u1de1\u1de2\u1de3\u1de4\u1de5\u1de6\u1de7\u1de8\u1de9\u1dea\u1deb\u1dec\u1ded\u1dee\u1def\u1df0\u1df1\u1df2\u1df3\u1df4\u1df5\u1df6\u1df7\u1df8\u1df9\u1dfa\u1dfb\u1dfc\u1dfd\u1dfe\u1dff\u1e9c\u1e9d\u1e9e\u1e9f\u1efa\u1efb\u1efc\u1efd\u1efe\u1eff\u1f16\u1f17\u1f1e\u1f1f\u1f46\u1f47\u1f4e\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e\u1f7f\u1fb5\u1fc5\u1fd4\u1fd5\u1fdc\u1ff0\u1ff1\u1ff5\u1fff\u2064\u2065\u2066\u2067\u2068\u2069\u2072\u2073\u208f\u2095\u2096\u2097\u2098\u2099\u209a\u209b\u209c\u209d\u209e\u209f\u20b6\u20b7\u20b8\u20b9\u20ba\u20bb\u20bc\u20bd\u20be\u20bf\u20c0\u20c1\u20c2\u20c3\u20c4\u20c5\u20c6\u20c7\u20c8\u20c9\u20ca\u20cb\u20cc\u20cd\u20ce\u20cf\u20ec\u20ed\u20ee\u20ef\u20f0\u20f1\u20f2\u20f3\u20f4\u20f5\u20f6\u20f7\u20f8\u20f9\u20fa\u20fb\u20fc\u20fd\u20fe\u20ff\u214d\u214e\u214f\u2150\u2151\u2152\u2184\u2185\u2186\u2187\u2188\u2189\u218a\u218b\u218c\u218d\u218e\u218f\u23dc\u23dd\u23de\u23df\u23e0\u23e1\u23e2\u23e3\u23e4\u23e5\u23e6\u23e7\u23e8\u23e9\u23ea\u23eb\u23ec\u23ed\u23ee\u23ef\u23f0\u23f1\u23f2\u23f3\u23f4\u23f5\u23f6\u23f7\u23f8\u23f9\u23fa\u23fb\u23fc\u23fd\u23fe\u23ff\u2427\u2428\u2429\u242a\u242b\u242c\u242d\u242e\u242f\u2430\u2431\u2432\u2433\u2434\u2435\u2436\u2437\u2438\u2439\u243a\u243b\u243c\u243d\u243e\u243f\u244b\u244c\u244d\u244e\u244f\u2450\u2451\u2452\u2453\u2454\u2455\u2456\u2457\u2458\u2459\u245a\u245b\u245c\u245d\u245e\u245f\u269d\u269e\u269f\u26b2\u26b3\u26b4\u26b5\u26b6\u26b7\u26b8\u26b9\u26ba\u26bb\u26bc\u26bd\u26be\u26bf\u26c0\u26c1\u26c2\u26c3\u26c4\u26c5\u26c6\u26c7\u26c8\u26c9\u26ca\u26cb\u26cc\u26cd\u26ce\u26cf\u26d0\u26d1\u26d2\u26d3\u26d4\u26d5\u26d6\u26d7\u26d8\u26d9\u26da\u26db\u26dc\u26dd\u26de\u26df\u26e0\u26e1\u26e2\u26e3\u26e4\u26e5\u26e6\u26e7\u26e8\u26e9\u26ea\u26eb\u26ec\u26ed\u26ee\u26ef\u26f0\u26f1\u26f2\u26f3\u26f4\u26f5\u26f6\u26f7\u26f8\u26f9\u26fa\u26fb\u26fc\u26fd\u26fe\u26ff\u2700\u2705\u270a\u270b\u2728\u274c\u274e\u2753\u2754\u2755\u2757\u275f\u2760\u2795\u2796\u2797\u27b0\u27bf\u27c7\u27c8\u27c9\u27ca\u27cb\u27cc\u27cd\u27ce\u27cf\u27ec\u27ed\u27ee\u27ef\u2b14\u2b15\u2b16\u2b17\u2b18\u2b19\u2b1a\u2b1b\u2b1c\u2b1d\u2b1e\u2b1f\u2b20\u2b21\u2b22\u2b23\u2b24\u2b25\u2b26\u2b27\u2b28\u2b29\u2b2a\u2b2b\u2b2c\u2b2d\u2b2e\u2b2f\u2b30\u2b31\u2b32\u2b33\u2b34\u2b35\u2b36\u2b37\u2b38\u2b39\u2b3a\u2b3b\u2b3c\u2b3d\u2b3e\u2b3f\u2b40\u2b41\u2b42\u2b43\u2b44\u2b45\u2b46\u2b47\u2b48\u2b49\u2b4a\u2b4b\u2b4c\u2b4d\u2b4e\u2b4f\u2b50\u2b51\u2b52\u2b53\u2b54\u2b55\u2b56\u2b57\u2b58\u2b59\u2b5a\u2b5b\u2b5c\u2b5d\u2b5e\u2b5f\u2b60\u2b61\u2b62\u2b63\u2b64\u2b65\u2b66\u2b67\u2b68\u2b69\u2b6a\u2b6b\u2b6c\u2b6d\u2b6e\u2b6f\u2b70\u2b71\u2b72\u2b73\u2b74\u2b75\u2b76\u2b77\u2b78\u2b79\u2b7a\u2b7b\u2b7c\u2b7d\u2b7e\u2b7f\u2b80\u2b81\u2b82\u2b83\u2b84\u2b85\u2b86\u2b87\u2b88\u2b89\u2b8a\u2b8b\u2b8c\u2b8d\u2b8e\u2b8f\u2b90\u2b91\u2b92\u2b93\u2b94\u2b95\u2b96\u2b97\u2b98\u2b99\u2b9a\u2b9b\u2b9c\u2b9d\u2b9e\u2b9f\u2ba0\u2ba1\u2ba2\u2ba3\u2ba4\u2ba5\u2ba6\u2ba7\u2ba8\u2ba9\u2baa\u2bab\u2bac\u2bad\u2bae\u2baf\u2bb0\u2bb1\u2bb2\u2bb3\u2bb4\u2bb5\u2bb6\u2bb7\u2bb8\u2bb9\u2bba\u2bbb\u2bbc\u2bbd\u2bbe\u2bbf\u2bc0\u2bc1\u2bc2\u2bc3\u2bc4\u2bc5\u2bc6\u2bc7\u2bc8\u2bc9\u2bca\u2bcb\u2bcc\u2bcd\u2bce\u2bcf\u2bd0\u2bd1\u2bd2\u2bd3\u2bd4\u2bd5\u2bd6\u2bd7\u2bd8\u2bd9\u2bda\u2bdb\u2bdc\u2bdd\u2bde\u2bdf\u2be0\u2be1\u2be2\u2be3\u2be4\u2be5\u2be6\u2be7\u2be8\u2be9\u2bea\u2beb\u2bec\u2bed\u2bee\u2bef\u2bf0\u2bf1\u2bf2\u2bf3\u2bf4\u2bf5\u2bf6\u2bf7\u2bf8\u2bf9\u2bfa\u2bfb\u2bfc\u2bfd\u2bfe\u2bff\u2c2f\u2c5f\u2c60\u2c61\u2c62\u2c63\u2c64\u2c65\u2c66\u2c67\u2c68\u2c69\u2c6a\u2c6b\u2c6c\u2c6d\u2c6e\u2c6f\u2c70\u2c71\u2c72\u2c73\u2c74\u2c75\u2c76\u2c77\u2c78\u2c79\u2c7a\u2c7b\u2c7c\u2c7d\u2c7e\u2c7f\u2ceb\u2cec\u2ced\u2cee\u2cef\u2cf0\u2cf1\u2cf2\u2cf3\u2cf4\u2cf5\u2cf6\u2cf7\u2cf8\u2d26\u2d27\u2d28\u2d29\u2d2a\u2d2b\u2d2c\u2d2d\u2d2e\u2d2f\u2d66\u2d67\u2d68\u2d69\u2d6a\u2d6b\u2d6c\u2d6d\u2d6e\u2d70\u2d71\u2d72\u2d73\u2d74\u2d75\u2d76\u2d77\u2d78\u2d79\u2d7a\u2d7b\u2d7c\u2d7d\u2d7e\u2d7f\u2d97\u2d98\u2d99\u2d9a\u2d9b\u2d9c\u2d9d\u2d9e\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2de0\u2de1\u2de2\u2de3\u2de4\u2de5\u2de6\u2de7\u2de8\u2de9\u2dea\u2deb\u2dec\u2ded\u2dee\u2def\u2df0\u2df1\u2df2\u2df3\u2df4\u2df5\u2df6\u2df7\u2df8\u2df9\u2dfa\u2dfb\u2dfc\u2dfd\u2dfe\u2dff\u2e18\u2e19\u2e1a\u2e1b\u2e1e\u2e1f\u2e20\u2e21\u2e22\u2e23\u2e24\u2e25\u2e26\u2e27\u2e28\u2e29\u2e2a\u2e2b\u2e2c\u2e2d\u2e2e\u2e2f\u2e30\u2e31\u2e32\u2e33\u2e34\u2e35\u2e36\u2e37\u2e38\u2e39\u2e3a\u2e3b\u2e3c\u2e3d\u2e3e\u2e3f\u2e40\u2e41\u2e42\u2e43\u2e44\u2e45\u2e46\u2e47\u2e48\u2e49\u2e4a\u2e4b\u2e4c\u2e4d\u2e4e\u2e4f\u2e50\u2e51\u2e52\u2e53\u2e54\u2e55\u2e56\u2e57\u2e58\u2e59\u2e5a\u2e5b\u2e5c\u2e5d\u2e5e\u2e5f\u2e60\u2e61\u2e62\u2e63\u2e64\u2e65\u2e66\u2e67\u2e68\u2e69\u2e6a\u2e6b\u2e6c\u2e6d\u2e6e\u2e6f\u2e70\u2e71\u2e72\u2e73\u2e74\u2e75\u2e76\u2e77\u2e78\u2e79\u2e7a\u2e7b\u2e7c\u2e7d\u2e7e\u2e7f\u2e9a\u2ef4\u2ef5\u2ef6\u2ef7\u2ef8\u2ef9\u2efa\u2efb\u2efc\u2efd\u2efe\u2eff\u2fd6\u2fd7\u2fd8\u2fd9\u2fda\u2fdb\u2fdc\u2fdd\u2fde\u2fdf\u2fe0\u2fe1\u2fe2\u2fe3\u2fe4\u2fe5\u2fe6\u2fe7\u2fe8\u2fe9\u2fea\u2feb\u2fec\u2fed\u2fee\u2fef\u2ffc\u2ffd\u2ffe\u2fff\u3040\u3097\u3098\u3100\u3101\u3102\u3103\u3104\u312d\u312e\u312f\u3130\u318f\u31b8\u31b9\u31ba\u31bb\u31bc\u31bd\u31be\u31bf\u31d0\u31d1\u31d2\u31d3\u31d4\u31d5\u31d6\u31d7\u31d8\u31d9\u31da\u31db\u31dc\u31dd\u31de\u31df\u31e0\u31e1\u31e2\u31e3\u31e4\u31e5\u31e6\u31e7\u31e8\u31e9\u31ea\u31eb\u31ec\u31ed\u31ee\u31ef\u321f\u3244\u3245\u3246\u3247\u3248\u3249\u324a\u324b\u324c\u324d\u324e\u324f\u32ff\u4db6\u4db7\u4db8\u4db9\u4dba\u4dbb\u4dbc\u4dbd\u4dbe\u4dbf\u9fbc\u9fbd\u9fbe\u9fbf\u9fc0\u9fc1\u9fc2\u9fc3\u9fc4\u9fc5\u9fc6\u9fc7\u9fc8\u9fc9\u9fca\u9fcb\u9fcc\u9fcd\u9fce\u9fcf\u9fd0\u9fd1\u9fd2\u9fd3\u9fd4\u9fd5\u9fd6\u9fd7\u9fd8\u9fd9\u9fda\u9fdb\u9fdc\u9fdd\u9fde\u9fdf\u9fe0\u9fe1\u9fe2\u9fe3\u9fe4\u9fe5\u9fe6\u9fe7\u9fe8\u9fe9\u9fea\u9feb\u9fec\u9fed\u9fee\u9fef\u9ff0\u9ff1\u9ff2\u9ff3\u9ff4\u9ff5\u9ff6\u9ff7\u9ff8\u9ff9\u9ffa\u9ffb\u9ffc\u9ffd\u9ffe\u9fff\ua48d\ua48e\ua48f\ua4c7\ua4c8\ua4c9\ua4ca\ua4cb\ua4cc\ua4cd\ua4ce\ua4cf\ua4d0\ua4d1\ua4d2\ua4d3\ua4d4\ua4d5\ua4d6\ua4d7\ua4d8\ua4d9\ua4da\ua4db\ua4dc\ua4dd\ua4de\ua4df\ua4e0\ua4e1\ua4e2\ua4e3\ua4e4\ua4e5\ua4e6\ua4e7\ua4e8\ua4e9\ua4ea\ua4eb\ua4ec\ua4ed\ua4ee\ua4ef\ua4f0\ua4f1\ua4f2\ua4f3\ua4f4\ua4f5\ua4f6\ua4f7\ua4f8\ua4f9\ua4fa\ua4fb\ua4fc\ua4fd\ua4fe\ua4ff\ua500\ua501\ua502\ua503\ua504\ua505\ua506\ua507\ua508\ua509\ua50a\ua50b\ua50c\ua50d\ua50e\ua50f\ua510\ua511\ua512\ua513\ua514\ua515\ua516\ua517\ua518\ua519\ua51a\ua51b\ua51c\ua51d\ua51e\ua51f\ua520\ua521\ua522\ua523\ua524\ua525\ua526\ua527\ua528\ua529\ua52a\ua52b\ua52c\ua52d\ua52e\ua52f\ua530\ua531\ua532\ua533\ua534\ua535\ua536\ua537\ua538\ua539\ua53a\ua53b\ua53c\ua53d\ua53e\ua53f\ua540\ua541\ua542\ua543\ua544\ua545\ua546\ua547\ua548\ua549\ua54a\ua54b\ua54c\ua54d\ua54e\ua54f\ua550\ua551\ua552\ua553\ua554\ua555\ua556\ua557\ua558\ua559\ua55a\ua55b\ua55c\ua55d\ua55e\ua55f\ua560\ua561\ua562\ua563\ua564\ua565\ua566\ua567\ua568\ua569\ua56a\ua56b\ua56c\ua56d\ua56e\ua56f\ua570\ua571\ua572\ua573\ua574\ua575\ua576\ua577\ua578\ua579\ua57a\ua57b\ua57c\ua57d\ua57e\ua57f\ua580\ua581\ua582\ua583\ua584\ua585\ua586\ua587\ua588\ua589\ua58a\ua58b\ua58c\ua58d\ua58e\ua58f\ua590\ua591\ua592\ua593\ua594\ua595\ua596\ua597\ua598\ua599\ua59a\ua59b\ua59c\ua59d\ua59e\ua59f\ua5a0\ua5a1\ua5a2\ua5a3\ua5a4\ua5a5\ua5a6\ua5a7\ua5a8\ua5a9\ua5aa\ua5ab\ua5ac\ua5ad\ua5ae\ua5af\ua5b0\ua5b1\ua5b2\ua5b3\ua5b4\ua5b5\ua5b6\ua5b7\ua5b8\ua5b9\ua5ba\ua5bb\ua5bc\ua5bd\ua5be\ua5bf\ua5c0\ua5c1\ua5c2\ua5c3\ua5c4\ua5c5\ua5c6\ua5c7\ua5c8\ua5c9\ua5ca\ua5cb\ua5cc\ua5cd\ua5ce\ua5cf\ua5d0\ua5d1\ua5d2\ua5d3\ua5d4\ua5d5\ua5d6\ua5d7\ua5d8\ua5d9\ua5da\ua5db\ua5dc\ua5dd\ua5de\ua5df\ua5e0\ua5e1\ua5e2\ua5e3\ua5e4\ua5e5\ua5e6\ua5e7\ua5e8\ua5e9\ua5ea\ua5eb\ua5ec\ua5ed\ua5ee\ua5ef\ua5f0\ua5f1\ua5f2\ua5f3\ua5f4\ua5f5\ua5f6\ua5f7\ua5f8\ua5f9\ua5fa\ua5fb\ua5fc\ua5fd\ua5fe\ua5ff\ua600\ua601\ua602\ua603\ua604\ua605\ua606\ua607\ua608\ua609\ua60a\ua60b\ua60c\ua60d\ua60e\ua60f\ua610\ua611\ua612\ua613\ua614\ua615\ua616\ua617\ua618\ua619\ua61a\ua61b\ua61c\ua61d\ua61e\ua61f\ua620\ua621\ua622\ua623\ua624\ua625\ua626\ua627\ua628\ua629\ua62a\ua62b\ua62c\ua62d\ua62e\ua62f\ua630\ua631\ua632\ua633\ua634\ua635\ua636\ua637\ua638\ua639\ua63a\ua63b\ua63c\ua63d\ua63e\ua63f\ua640\ua641\ua642\ua643\ua644\ua645\ua646\ua647\ua648\ua649\ua64a\ua64b\ua64c\ua64d\ua64e\ua64f\ua650\ua651\ua652\ua653\ua654\ua655\ua656\ua657\ua658\ua659\ua65a\ua65b\ua65c\ua65d\ua65e\ua65f\ua660\ua661\ua662\ua663\ua664\ua665\ua666\ua667\ua668\ua669\ua66a\ua66b\ua66c\ua66d\ua66e\ua66f\ua670\ua671\ua672\ua673\ua674\ua675\ua676\ua677\ua678\ua679\ua67a\ua67b\ua67c\ua67d\ua67e\ua67f\ua680\ua681\ua682\ua683\ua684\ua685\ua686\ua687\ua688\ua689\ua68a\ua68b\ua68c\ua68d\ua68e\ua68f\ua690\ua691\ua692\ua693\ua694\ua695\ua696\ua697\ua698\ua699\ua69a\ua69b\ua69c\ua69d\ua69e\ua69f\ua6a0\ua6a1\ua6a2\ua6a3\ua6a4\ua6a5\ua6a6\ua6a7\ua6a8\ua6a9\ua6aa\ua6ab\ua6ac\ua6ad\ua6ae\ua6af\ua6b0\ua6b1\ua6b2\ua6b3\ua6b4\ua6b5\ua6b6\ua6b7\ua6b8\ua6b9\ua6ba\ua6bb\ua6bc\ua6bd\ua6be\ua6bf\ua6c0\ua6c1\ua6c2\ua6c3\ua6c4\ua6c5\ua6c6\ua6c7\ua6c8\ua6c9\ua6ca\ua6cb\ua6cc\ua6cd\ua6ce\ua6cf\ua6d0\ua6d1\ua6d2\ua6d3\ua6d4\ua6d5\ua6d6\ua6d7\ua6d8\ua6d9\ua6da\ua6db\ua6dc\ua6dd\ua6de\ua6df\ua6e0\ua6e1\ua6e2\ua6e3\ua6e4\ua6e5\ua6e6\ua6e7\ua6e8\ua6e9\ua6ea\ua6eb\ua6ec\ua6ed\ua6ee\ua6ef\ua6f0\ua6f1\ua6f2\ua6f3\ua6f4\ua6f5\ua6f6\ua6f7\ua6f8\ua6f9\ua6fa\ua6fb\ua6fc\ua6fd\ua6fe\ua6ff\ua717\ua718\ua719\ua71a\ua71b\ua71c\ua71d\ua71e\ua71f\ua720\ua721\ua722\ua723\ua724\ua725\ua726\ua727\ua728\ua729\ua72a\ua72b\ua72c\ua72d\ua72e\ua72f\ua730\ua731\ua732\ua733\ua734\ua735\ua736\ua737\ua738\ua739\ua73a\ua73b\ua73c\ua73d\ua73e\ua73f\ua740\ua741\ua742\ua743\ua744\ua745\ua746\ua747\ua748\ua749\ua74a\ua74b\ua74c\ua74d\ua74e\ua74f\ua750\ua751\ua752\ua753\ua754\ua755\ua756\ua757\ua758\ua759\ua75a\ua75b\ua75c\ua75d\ua75e\ua75f\ua760\ua761\ua762\ua763\ua764\ua765\ua766\ua767\ua768\ua769\ua76a\ua76b\ua76c\ua76d\ua76e\ua76f\ua770\ua771\ua772\ua773\ua774\ua775\ua776\ua777\ua778\ua779\ua77a\ua77b\ua77c\ua77d\ua77e\ua77f\ua780\ua781\ua782\ua783\ua784\ua785\ua786\ua787\ua788\ua789\ua78a\ua78b\ua78c\ua78d\ua78e\ua78f\ua790\ua791\ua792\ua793\ua794\ua795\ua796\ua797\ua798\ua799\ua79a\ua79b\ua79c\ua79d\ua79e\ua79f\ua7a0\ua7a1\ua7a2\ua7a3\ua7a4\ua7a5\ua7a6\ua7a7\ua7a8\ua7a9\ua7aa\ua7ab\ua7ac\ua7ad\ua7ae\ua7af\ua7b0\ua7b1\ua7b2\ua7b3\ua7b4\ua7b5\ua7b6\ua7b7\ua7b8\ua7b9\ua7ba\ua7bb\ua7bc\ua7bd\ua7be\ua7bf\ua7c0\ua7c1\ua7c2\ua7c3\ua7c4\ua7c5\ua7c6\ua7c7\ua7c8\ua7c9\ua7ca\ua7cb\ua7cc\ua7cd\ua7ce\ua7cf\ua7d0\ua7d1\ua7d2\ua7d3\ua7d4\ua7d5\ua7d6\ua7d7\ua7d8\ua7d9\ua7da\ua7db\ua7dc\ua7dd\ua7de\ua7df\ua7e0\ua7e1\ua7e2\ua7e3\ua7e4\ua7e5\ua7e6\ua7e7\ua7e8\ua7e9\ua7ea\ua7eb\ua7ec\ua7ed\ua7ee\ua7ef\ua7f0\ua7f1\ua7f2\ua7f3\ua7f4\ua7f5\ua7f6\ua7f7\ua7f8\ua7f9\ua7fa\ua7fb\ua7fc\ua7fd\ua7fe\ua7ff\ua82c\ua82d\ua82e\ua82f\ua830\ua831\ua832\ua833\ua834\ua835\ua836\ua837\ua838\ua839\ua83a\ua83b\ua83c\ua83d\ua83e\ua83f\ua840\ua841\ua842\ua843\ua844\ua845\ua846\ua847\ua848\ua849\ua84a\ua84b\ua84c\ua84d\ua84e\ua84f\ua850\ua851\ua852\ua853\ua854\ua855\ua856\ua857\ua858\ua859\ua85a\ua85b\ua85c\ua85d\ua85e\ua85f\ua860\ua861\ua862\ua863\ua864\ua865\ua866\ua867\ua868\ua869\ua86a\ua86b\ua86c\ua86d\ua86e\ua86f\ua870\ua871\ua872\ua873\ua874\ua875\ua876\ua877\ua878\ua879\ua87a\ua87b\ua87c\ua87d\ua87e\ua87f\ua880\ua881\ua882\ua883\ua884\ua885\ua886\ua887\ua888\ua889\ua88a\ua88b\ua88c\ua88d\ua88e\ua88f\ua890\ua891\ua892\ua893\ua894\ua895\ua896\ua897\ua898\ua899\ua89a\ua89b\ua89c\ua89d\ua89e\ua89f\ua8a0\ua8a1\ua8a2\ua8a3\ua8a4\ua8a5\ua8a6\ua8a7\ua8a8\ua8a9\ua8aa\ua8ab\ua8ac\ua8ad\ua8ae\ua8af\ua8b0\ua8b1\ua8b2\ua8b3\ua8b4\ua8b5\ua8b6\ua8b7\ua8b8\ua8b9\ua8ba\ua8bb\ua8bc\ua8bd\ua8be\ua8bf\ua8c0\ua8c1\ua8c2\ua8c3\ua8c4\ua8c5\ua8c6\ua8c7\ua8c8\ua8c9\ua8ca\ua8cb\ua8cc\ua8cd\ua8ce\ua8cf\ua8d0\ua8d1\ua8d2\ua8d3\ua8d4\ua8d5\ua8d6\ua8d7\ua8d8\ua8d9\ua8da\ua8db\ua8dc\ua8dd\ua8de\ua8df\ua8e0\ua8e1\ua8e2\ua8e3\ua8e4\ua8e5\ua8e6\ua8e7\ua8e8\ua8e9\ua8ea\ua8eb\ua8ec\ua8ed\ua8ee\ua8ef\ua8f0\ua8f1\ua8f2\ua8f3\ua8f4\ua8f5\ua8f6\ua8f7\ua8f8\ua8f9\ua8fa\ua8fb\ua8fc\ua8fd\ua8fe\ua8ff\ua900\ua901\ua902\ua903\ua904\ua905\ua906\ua907\ua908\ua909\ua90a\ua90b\ua90c\ua90d\ua90e\ua90f\ua910\ua911\ua912\ua913\ua914\ua915\ua916\ua917\ua918\ua919\ua91a\ua91b\ua91c\ua91d\ua91e\ua91f\ua920\ua921\ua922\ua923\ua924\ua925\ua926\ua927\ua928\ua929\ua92a\ua92b\ua92c\ua92d\ua92e\ua92f\ua930\ua931\ua932\ua933\ua934\ua935\ua936\ua937\ua938\ua939\ua93a\ua93b\ua93c\ua93d\ua93e\ua93f\ua940\ua941\ua942\ua943\ua944\ua945\ua946\ua947\ua948\ua949\ua94a\ua94b\ua94c\ua94d\ua94e\ua94f\ua950\ua951\ua952\ua953\ua954\ua955\ua956\ua957\ua958\ua959\ua95a\ua95b\ua95c\ua95d\ua95e\ua95f\ua960\ua961\ua962\ua963\ua964\ua965\ua966\ua967\ua968\ua969\ua96a\ua96b\ua96c\ua96d\ua96e\ua96f\ua970\ua971\ua972\ua973\ua974\ua975\ua976\ua977\ua978\ua979\ua97a\ua97b\ua97c\ua97d\ua97e\ua97f\ua980\ua981\ua982\ua983\ua984\ua985\ua986\ua987\ua988\ua989\ua98a\ua98b\ua98c\ua98d\ua98e\ua98f\ua990\ua991\ua992\ua993\ua994\ua995\ua996\ua997\ua998\ua999\ua99a\ua99b\ua99c\ua99d\ua99e\ua99f\ua9a0\ua9a1\ua9a2\ua9a3\ua9a4\ua9a5\ua9a6\ua9a7\ua9a8\ua9a9\ua9aa\ua9ab\ua9ac\ua9ad\ua9ae\ua9af\ua9b0\ua9b1\ua9b2\ua9b3\ua9b4\ua9b5\ua9b6\ua9b7\ua9b8\ua9b9\ua9ba\ua9bb\ua9bc\ua9bd\ua9be\ua9bf\ua9c0\ua9c1\ua9c2\ua9c3\ua9c4\ua9c5\ua9c6\ua9c7\ua9c8\ua9c9\ua9ca\ua9cb\ua9cc\ua9cd\ua9ce\ua9cf\ua9d0\ua9d1\ua9d2\ua9d3\ua9d4\ua9d5\ua9d6\ua9d7\ua9d8\ua9d9\ua9da\ua9db\ua9dc\ua9dd\ua9de\ua9df\ua9e0\ua9e1\ua9e2\ua9e3\ua9e4\ua9e5\ua9e6\ua9e7\ua9e8\ua9e9\ua9ea\ua9eb\ua9ec\ua9ed\ua9ee\ua9ef\ua9f0\ua9f1\ua9f2\ua9f3\ua9f4\ua9f5\ua9f6\ua9f7\ua9f8\ua9f9\ua9fa\ua9fb\ua9fc\ua9fd\ua9fe\ua9ff\uaa00\uaa01\uaa02\uaa03\uaa04\uaa05\uaa06\uaa07\uaa08\uaa09\uaa0a\uaa0b\uaa0c\uaa0d\uaa0e\uaa0f\uaa10\uaa11\uaa12\uaa13\uaa14\uaa15\uaa16\uaa17\uaa18\uaa19\uaa1a\uaa1b\uaa1c\uaa1d\uaa1e\uaa1f\uaa20\uaa21\uaa22\uaa23\uaa24\uaa25\uaa26\uaa27\uaa28\uaa29\uaa2a\uaa2b\uaa2c\uaa2d\uaa2e\uaa2f\uaa30\uaa31\uaa32\uaa33\uaa34\uaa35\uaa36\uaa37\uaa38\uaa39\uaa3a\uaa3b\uaa3c\uaa3d\uaa3e\uaa3f\uaa40\uaa41\uaa42\uaa43\uaa44\uaa45\uaa46\uaa47\uaa48\uaa49\uaa4a\uaa4b\uaa4c\uaa4d\uaa4e\uaa4f\uaa50\uaa51\uaa52\uaa53\uaa54\uaa55\uaa56\uaa57\uaa58\uaa59\uaa5a\uaa5b\uaa5c\uaa5d\uaa5e\uaa5f\uaa60\uaa61\uaa62\uaa63\uaa64\uaa65\uaa66\uaa67\uaa68\uaa69\uaa6a\uaa6b\uaa6c\uaa6d\uaa6e\uaa6f\uaa70\uaa71\uaa72\uaa73\uaa74\uaa75\uaa76\uaa77\uaa78\uaa79\uaa7a\uaa7b\uaa7c\uaa7d\uaa7e\uaa7f\uaa80\uaa81\uaa82\uaa83\uaa84\uaa85\uaa86\uaa87\uaa88\uaa89\uaa8a\uaa8b\uaa8c\uaa8d\uaa8e\uaa8f\uaa90\uaa91\uaa92\uaa93\uaa94\uaa95\uaa96\uaa97\uaa98\uaa99\uaa9a\uaa9b\uaa9c\uaa9d\uaa9e\uaa9f\uaaa0\uaaa1\uaaa2\uaaa3\uaaa4\uaaa5\uaaa6\uaaa7\uaaa8\uaaa9\uaaaa\uaaab\uaaac\uaaad\uaaae\uaaaf\uaab0\uaab1\uaab2\uaab3\uaab4\uaab5\uaab6\uaab7\uaab8\uaab9\uaaba\uaabb\uaabc\uaabd\uaabe\uaabf\uaac0\uaac1\uaac2\uaac3\uaac4\uaac5\uaac6\uaac7\uaac8\uaac9\uaaca\uaacb\uaacc\uaacd\uaace\uaacf\uaad0\uaad1\uaad2\uaad3\uaad4\uaad5\uaad6\uaad7\uaad8\uaad9\uaada\uaadb\uaadc\uaadd\uaade\uaadf\uaae0\uaae1\uaae2\uaae3\uaae4\uaae5\uaae6\uaae7\uaae8\uaae9\uaaea\uaaeb\uaaec\uaaed\uaaee\uaaef\uaaf0\uaaf1\uaaf2\uaaf3\uaaf4\uaaf5\uaaf6\uaaf7\uaaf8\uaaf9\uaafa\uaafb\uaafc\uaafd\uaafe\uaaff\uab00\uab01\uab02\uab03\uab04\uab05\uab06\uab07\uab08\uab09\uab0a\uab0b\uab0c\uab0d\uab0e\uab0f\uab10\uab11\uab12\uab13\uab14\uab15\uab16\uab17\uab18\uab19\uab1a\uab1b\uab1c\uab1d\uab1e\uab1f\uab20\uab21\uab22\uab23\uab24\uab25\uab26\uab27\uab28\uab29\uab2a\uab2b\uab2c\uab2d\uab2e\uab2f\uab30\uab31\uab32\uab33\uab34\uab35\uab36\uab37\uab38\uab39\uab3a\uab3b\uab3c\uab3d\uab3e\uab3f\uab40\uab41\uab42\uab43\uab44\uab45\uab46\uab47\uab48\uab49\uab4a\uab4b\uab4c\uab4d\uab4e\uab4f\uab50\uab51\uab52\uab53\uab54\uab55\uab56\uab57\uab58\uab59\uab5a\uab5b\uab5c\uab5d\uab5e\uab5f\uab60\uab61\uab62\uab63\uab64\uab65\uab66\uab67\uab68\uab69\uab6a\uab6b\uab6c\uab6d\uab6e\uab6f\uab70\uab71\uab72\uab73\uab74\uab75\uab76\uab77\uab78\uab79\uab7a\uab7b\uab7c\uab7d\uab7e\uab7f\uab80\uab81\uab82\uab83\uab84\uab85\uab86\uab87\uab88\uab89\uab8a\uab8b\uab8c\uab8d\uab8e\uab8f\uab90\uab91\uab92\uab93\uab94\uab95\uab96\uab97\uab98\uab99\uab9a\uab9b\uab9c\uab9d\uab9e\uab9f\uaba0\uaba1\uaba2\uaba3\uaba4\uaba5\uaba6\uaba7\uaba8\uaba9\uabaa\uabab\uabac\uabad\uabae\uabaf\uabb0\uabb1\uabb2\uabb3\uabb4\uabb5\uabb6\uabb7\uabb8\uabb9\uabba\uabbb\uabbc\uabbd\uabbe\uabbf\uabc0\uabc1\uabc2\uabc3\uabc4\uabc5\uabc6\uabc7\uabc8\uabc9\uabca\uabcb\uabcc\uabcd\uabce\uabcf\uabd0\uabd1\uabd2\uabd3\uabd4\uabd5\uabd6\uabd7\uabd8\uabd9\uabda\uabdb\uabdc\uabdd\uabde\uabdf\uabe0\uabe1\uabe2\uabe3\uabe4\uabe5\uabe6\uabe7\uabe8\uabe9\uabea\uabeb\uabec\uabed\uabee\uabef\uabf0\uabf1\uabf2\uabf3\uabf4\uabf5\uabf6\uabf7\uabf8\uabf9\uabfa\uabfb\uabfc\uabfd\uabfe\uabff\ud7a4\ud7a5\ud7a6\ud7a7\ud7a8\ud7a9\ud7aa\ud7ab\ud7ac\ud7ad\ud7ae\ud7af\ud7b0\ud7b1\ud7b2\ud7b3\ud7b4\ud7b5\ud7b6\ud7b7\ud7b8\ud7b9\ud7ba\ud7bb\ud7bc\ud7bd\ud7be\ud7bf\ud7c0\ud7c1\ud7c2\ud7c3\ud7c4\ud7c5\ud7c6\ud7c7\ud7c8\ud7c9\ud7ca\ud7cb\ud7cc\ud7cd\ud7ce\ud7cf\ud7d0\ud7d1\ud7d2\ud7d3\ud7d4\ud7d5\ud7d6\ud7d7\ud7d8\ud7d9\ud7da\ud7db\ud7dc\ud7dd\ud7de\ud7df\ud7e0\ud7e1\ud7e2\ud7e3\ud7e4\ud7e5\ud7e6\ud7e7\ud7e8\ud7e9\ud7ea\ud7eb\ud7ec\ud7ed\ud7ee\ud7ef\ud7f0\ud7f1\ud7f2\ud7f3\ud7f4\ud7f5\ud7f6\ud7f7\ud7f8\ud7f9\ud7fa\ud7fb\ud7fc\ud7fd\ud7fe\ud7ff\ufa2e\ufa2f\ufa6b\ufa6c\ufa6d\ufa6e\ufa6f\ufada\ufadb\ufadc\ufadd\ufade\ufadf\ufae0\ufae1\ufae2\ufae3\ufae4\ufae5\ufae6\ufae7\ufae8\ufae9\ufaea\ufaeb\ufaec\ufaed\ufaee\ufaef\ufaf0\ufaf1\ufaf2\ufaf3\ufaf4\ufaf5\ufaf6\ufaf7\ufaf8\ufaf9\ufafa\ufafb\ufafc\ufafd\ufafe\ufaff\ufb07\ufb08\ufb09\ufb0a\ufb0b\ufb0c\ufb0d\ufb0e\ufb0f\ufb10\ufb11\ufb12\ufb18\ufb19\ufb1a\ufb1b\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbb2\ufbb3\ufbb4\ufbb5\ufbb6\ufbb7\ufbb8\ufbb9\ufbba\ufbbb\ufbbc\ufbbd\ufbbe\ufbbf\ufbc0\ufbc1\ufbc2\ufbc3\ufbc4\ufbc5\ufbc6\ufbc7\ufbc8\ufbc9\ufbca\ufbcb\ufbcc\ufbcd\ufbce\ufbcf\ufbd0\ufbd1\ufbd2\ufd40\ufd41\ufd42\ufd43\ufd44\ufd45\ufd46\ufd47\ufd48\ufd49\ufd4a\ufd4b\ufd4c\ufd4d\ufd4e\ufd4f\ufd90\ufd91\ufdc8\ufdc9\ufdca\ufdcb\ufdcc\ufdcd\ufdce\ufdcf\ufdd0\ufdd1\ufdd2\ufdd3\ufdd4\ufdd5\ufdd6\ufdd7\ufdd8\ufdd9\ufdda\ufddb\ufddc\ufddd\ufdde\ufddf\ufde0\ufde1\ufde2\ufde3\ufde4\ufde5\ufde6\ufde7\ufde8\ufde9\ufdea\ufdeb\ufdec\ufded\ufdee\ufdef\ufdfe\ufdff\ufe1a\ufe1b\ufe1c\ufe1d\ufe1e\ufe1f\ufe24\ufe25\ufe26\ufe27\ufe28\ufe29\ufe2a\ufe2b\ufe2c\ufe2d\ufe2e\ufe2f\ufe53\ufe67\ufe6c\ufe6d\ufe6e\ufe6f\ufe75\ufefd\ufefe\uff00\uffbf\uffc0\uffc1\uffc8\uffc9\uffd0\uffd1\uffd8\uffd9\uffdd\uffde\uffdf\uffe7\uffef\ufff0\ufff1\ufff2\ufff3\ufff4\ufff5\ufff6\ufff7\ufff8\ufffe'
+
+Co = u'\ue000\ue001\ue002\ue003\ue004\ue005\ue006\ue007\ue008\ue009\ue00a\ue00b\ue00c\ue00d\ue00e\ue00f\ue010\ue011\ue012\ue013\ue014\ue015\ue016\ue017\ue018\ue019\ue01a\ue01b\ue01c\ue01d\ue01e\ue01f\ue020\ue021\ue022\ue023\ue024\ue025\ue026\ue027\ue028\ue029\ue02a\ue02b\ue02c\ue02d\ue02e\ue02f\ue030\ue031\ue032\ue033\ue034\ue035\ue036\ue037\ue038\ue039\ue03a\ue03b\ue03c\ue03d\ue03e\ue03f\ue040\ue041\ue042\ue043\ue044\ue045\ue046\ue047\ue048\ue049\ue04a\ue04b\ue04c\ue04d\ue04e\ue04f\ue050\ue051\ue052\ue053\ue054\ue055\ue056\ue057\ue058\ue059\ue05a\ue05b\ue05c\ue05d\ue05e\ue05f\ue060\ue061\ue062\ue063\ue064\ue065\ue066\ue067\ue068\ue069\ue06a\ue06b\ue06c\ue06d\ue06e\ue06f\ue070\ue071\ue072\ue073\ue074\ue075\ue076\ue077\ue078\ue079\ue07a\ue07b\ue07c\ue07d\ue07e\ue07f\ue080\ue081\ue082\ue083\ue084\ue085\ue086\ue087\ue088\ue089\ue08a\ue08b\ue08c\ue08d\ue08e\ue08f\ue090\ue091\ue092\ue093\ue094\ue095\ue096\ue097\ue098\ue099\ue09a\ue09b\ue09c\ue09d\ue09e\ue09f\ue0a0\ue0a1\ue0a2\ue0a3\ue0a4\ue0a5\ue0a6\ue0a7\ue0a8\ue0a9\ue0aa\ue0ab\ue0ac\ue0ad\ue0ae\ue0af\ue0b0\ue0b1\ue0b2\ue0b3\ue0b4\ue0b5\ue0b6\ue0b7\ue0b8\ue0b9\ue0ba\ue0bb\ue0bc\ue0bd\ue0be\ue0bf\ue0c0\ue0c1\ue0c2\ue0c3\ue0c4\ue0c5\ue0c6\ue0c7\ue0c8\ue0c9\ue0ca\ue0cb\ue0cc\ue0cd\ue0ce\ue0cf\ue0d0\ue0d1\ue0d2\ue0d3\ue0d4\ue0d5\ue0d6\ue0d7\ue0d8\ue0d9\ue0da\ue0db\ue0dc\ue0dd\ue0de\ue0df\ue0e0\ue0e1\ue0e2\ue0e3\ue0e4\ue0e5\ue0e6\ue0e7\ue0e8\ue0e9\ue0ea\ue0eb\ue0ec\ue0ed\ue0ee\ue0ef\ue0f0\ue0f1\ue0f2\ue0f3\ue0f4\ue0f5\ue0f6\ue0f7\ue0f8\ue0f9\ue0fa\ue0fb\ue0fc\ue0fd\ue0fe\ue0ff\ue100\ue101\ue102\ue103\ue104\ue105\ue106\ue107\ue108\ue109\ue10a\ue10b\ue10c\ue10d\ue10e\ue10f\ue110\ue111\ue112\ue113\ue114\ue115\ue116\ue117\ue118\ue119\ue11a\ue11b\ue11c\ue11d\ue11e\ue11f\ue120\ue121\ue122\ue123\ue124\ue125\ue126\ue127\ue128\ue129\ue12a\ue12b\ue12c\ue12d\ue12e\ue12f\ue130\ue131\ue132\ue133\ue134\ue135\ue136\ue137\ue138\ue139\ue13a\ue13b\ue13c\ue13d\ue13e\ue13f\ue140\ue141\ue142\ue143\ue144\ue145\ue146\ue147\ue148\ue149\ue14a\ue14b\ue14c\ue14d\ue14e\ue14f\ue150\ue151\ue152\ue153\ue154\ue155\ue156\ue157\ue158\ue159\ue15a\ue15b\ue15c\ue15d\ue15e\ue15f\ue160\ue161\ue162\ue163\ue164\ue165\ue166\ue167\ue168\ue169\ue16a\ue16b\ue16c\ue16d\ue16e\ue16f\ue170\ue171\ue172\ue173\ue174\ue175\ue176\ue177\ue178\ue179\ue17a\ue17b\ue17c\ue17d\ue17e\ue17f\ue180\ue181\ue182\ue183\ue184\ue185\ue186\ue187\ue188\ue189\ue18a\ue18b\ue18c\ue18d\ue18e\ue18f\ue190\ue191\ue192\ue193\ue194\ue195\ue196\ue197\ue198\ue199\ue19a\ue19b\ue19c\ue19d\ue19e\ue19f\ue1a0\ue1a1\ue1a2\ue1a3\ue1a4\ue1a5\ue1a6\ue1a7\ue1a8\ue1a9\ue1aa\ue1ab\ue1ac\ue1ad\ue1ae\ue1af\ue1b0\ue1b1\ue1b2\ue1b3\ue1b4\ue1b5\ue1b6\ue1b7\ue1b8\ue1b9\ue1ba\ue1bb\ue1bc\ue1bd\ue1be\ue1bf\ue1c0\ue1c1\ue1c2\ue1c3\ue1c4\ue1c5\ue1c6\ue1c7\ue1c8\ue1c9\ue1ca\ue1cb\ue1cc\ue1cd\ue1ce\ue1cf\ue1d0\ue1d1\ue1d2\ue1d3\ue1d4\ue1d5\ue1d6\ue1d7\ue1d8\ue1d9\ue1da\ue1db\ue1dc\ue1dd\ue1de\ue1df\ue1e0\ue1e1\ue1e2\ue1e3\ue1e4\ue1e5\ue1e6\ue1e7\ue1e8\ue1e9\ue1ea\ue1eb\ue1ec\ue1ed\ue1ee\ue1ef\ue1f0\ue1f1\ue1f2\ue1f3\ue1f4\ue1f5\ue1f6\ue1f7\ue1f8\ue1f9\ue1fa\ue1fb\ue1fc\ue1fd\ue1fe\ue1ff\ue200\ue201\ue202\ue203\ue204\ue205\ue206\ue207\ue208\ue209\ue20a\ue20b\ue20c\ue20d\ue20e\ue20f\ue210\ue211\ue212\ue213\ue214\ue215\ue216\ue217\ue218\ue219\ue21a\ue21b\ue21c\ue21d\ue21e\ue21f\ue220\ue221\ue222\ue223\ue224\ue225\ue226\ue227\ue228\ue229\ue22a\ue22b\ue22c\ue22d\ue22e\ue22f\ue230\ue231\ue232\ue233\ue234\ue235\ue236\ue237\ue238\ue239\ue23a\ue23b\ue23c\ue23d\ue23e\ue23f\ue240\ue241\ue242\ue243\ue244\ue245\ue246\ue247\ue248\ue249\ue24a\ue24b\ue24c\ue24d\ue24e\ue24f\ue250\ue251\ue252\ue253\ue254\ue255\ue256\ue257\ue258\ue259\ue25a\ue25b\ue25c\ue25d\ue25e\ue25f\ue260\ue261\ue262\ue263\ue264\ue265\ue266\ue267\ue268\ue269\ue26a\ue26b\ue26c\ue26d\ue26e\ue26f\ue270\ue271\ue272\ue273\ue274\ue275\ue276\ue277\ue278\ue279\ue27a\ue27b\ue27c\ue27d\ue27e\ue27f\ue280\ue281\ue282\ue283\ue284\ue285\ue286\ue287\ue288\ue289\ue28a\ue28b\ue28c\ue28d\ue28e\ue28f\ue290\ue291\ue292\ue293\ue294\ue295\ue296\ue297\ue298\ue299\ue29a\ue29b\ue29c\ue29d\ue29e\ue29f\ue2a0\ue2a1\ue2a2\ue2a3\ue2a4\ue2a5\ue2a6\ue2a7\ue2a8\ue2a9\ue2aa\ue2ab\ue2ac\ue2ad\ue2ae\ue2af\ue2b0\ue2b1\ue2b2\ue2b3\ue2b4\ue2b5\ue2b6\ue2b7\ue2b8\ue2b9\ue2ba\ue2bb\ue2bc\ue2bd\ue2be\ue2bf\ue2c0\ue2c1\ue2c2\ue2c3\ue2c4\ue2c5\ue2c6\ue2c7\ue2c8\ue2c9\ue2ca\ue2cb\ue2cc\ue2cd\ue2ce\ue2cf\ue2d0\ue2d1\ue2d2\ue2d3\ue2d4\ue2d5\ue2d6\ue2d7\ue2d8\ue2d9\ue2da\ue2db\ue2dc\ue2dd\ue2de\ue2df\ue2e0\ue2e1\ue2e2\ue2e3\ue2e4\ue2e5\ue2e6\ue2e7\ue2e8\ue2e9\ue2ea\ue2eb\ue2ec\ue2ed\ue2ee\ue2ef\ue2f0\ue2f1\ue2f2\ue2f3\ue2f4\ue2f5\ue2f6\ue2f7\ue2f8\ue2f9\ue2fa\ue2fb\ue2fc\ue2fd\ue2fe\ue2ff\ue300\ue301\ue302\ue303\ue304\ue305\ue306\ue307\ue308\ue309\ue30a\ue30b\ue30c\ue30d\ue30e\ue30f\ue310\ue311\ue312\ue313\ue314\ue315\ue316\ue317\ue318\ue319\ue31a\ue31b\ue31c\ue31d\ue31e\ue31f\ue320\ue321\ue322\ue323\ue324\ue325\ue326\ue327\ue328\ue329\ue32a\ue32b\ue32c\ue32d\ue32e\ue32f\ue330\ue331\ue332\ue333\ue334\ue335\ue336\ue337\ue338\ue339\ue33a\ue33b\ue33c\ue33d\ue33e\ue33f\ue340\ue341\ue342\ue343\ue344\ue345\ue346\ue347\ue348\ue349\ue34a\ue34b\ue34c\ue34d\ue34e\ue34f\ue350\ue351\ue352\ue353\ue354\ue355\ue356\ue357\ue358\ue359\ue35a\ue35b\ue35c\ue35d\ue35e\ue35f\ue360\ue361\ue362\ue363\ue364\ue365\ue366\ue367\ue368\ue369\ue36a\ue36b\ue36c\ue36d\ue36e\ue36f\ue370\ue371\ue372\ue373\ue374\ue375\ue376\ue377\ue378\ue379\ue37a\ue37b\ue37c\ue37d\ue37e\ue37f\ue380\ue381\ue382\ue383\ue384\ue385\ue386\ue387\ue388\ue389\ue38a\ue38b\ue38c\ue38d\ue38e\ue38f\ue390\ue391\ue392\ue393\ue394\ue395\ue396\ue397\ue398\ue399\ue39a\ue39b\ue39c\ue39d\ue39e\ue39f\ue3a0\ue3a1\ue3a2\ue3a3\ue3a4\ue3a5\ue3a6\ue3a7\ue3a8\ue3a9\ue3aa\ue3ab\ue3ac\ue3ad\ue3ae\ue3af\ue3b0\ue3b1\ue3b2\ue3b3\ue3b4\ue3b5\ue3b6\ue3b7\ue3b8\ue3b9\ue3ba\ue3bb\ue3bc\ue3bd\ue3be\ue3bf\ue3c0\ue3c1\ue3c2\ue3c3\ue3c4\ue3c5\ue3c6\ue3c7\ue3c8\ue3c9\ue3ca\ue3cb\ue3cc\ue3cd\ue3ce\ue3cf\ue3d0\ue3d1\ue3d2\ue3d3\ue3d4\ue3d5\ue3d6\ue3d7\ue3d8\ue3d9\ue3da\ue3db\ue3dc\ue3dd\ue3de\ue3df\ue3e0\ue3e1\ue3e2\ue3e3\ue3e4\ue3e5\ue3e6\ue3e7\ue3e8\ue3e9\ue3ea\ue3eb\ue3ec\ue3ed\ue3ee\ue3ef\ue3f0\ue3f1\ue3f2\ue3f3\ue3f4\ue3f5\ue3f6\ue3f7\ue3f8\ue3f9\ue3fa\ue3fb\ue3fc\ue3fd\ue3fe\ue3ff\ue400\ue401\ue402\ue403\ue404\ue405\ue406\ue407\ue408\ue409\ue40a\ue40b\ue40c\ue40d\ue40e\ue40f\ue410\ue411\ue412\ue413\ue414\ue415\ue416\ue417\ue418\ue419\ue41a\ue41b\ue41c\ue41d\ue41e\ue41f\ue420\ue421\ue422\ue423\ue424\ue425\ue426\ue427\ue428\ue429\ue42a\ue42b\ue42c\ue42d\ue42e\ue42f\ue430\ue431\ue432\ue433\ue434\ue435\ue436\ue437\ue438\ue439\ue43a\ue43b\ue43c\ue43d\ue43e\ue43f\ue440\ue441\ue442\ue443\ue444\ue445\ue446\ue447\ue448\ue449\ue44a\ue44b\ue44c\ue44d\ue44e\ue44f\ue450\ue451\ue452\ue453\ue454\ue455\ue456\ue457\ue458\ue459\ue45a\ue45b\ue45c\ue45d\ue45e\ue45f\ue460\ue461\ue462\ue463\ue464\ue465\ue466\ue467\ue468\ue469\ue46a\ue46b\ue46c\ue46d\ue46e\ue46f\ue470\ue471\ue472\ue473\ue474\ue475\ue476\ue477\ue478\ue479\ue47a\ue47b\ue47c\ue47d\ue47e\ue47f\ue480\ue481\ue482\ue483\ue484\ue485\ue486\ue487\ue488\ue489\ue48a\ue48b\ue48c\ue48d\ue48e\ue48f\ue490\ue491\ue492\ue493\ue494\ue495\ue496\ue497\ue498\ue499\ue49a\ue49b\ue49c\ue49d\ue49e\ue49f\ue4a0\ue4a1\ue4a2\ue4a3\ue4a4\ue4a5\ue4a6\ue4a7\ue4a8\ue4a9\ue4aa\ue4ab\ue4ac\ue4ad\ue4ae\ue4af\ue4b0\ue4b1\ue4b2\ue4b3\ue4b4\ue4b5\ue4b6\ue4b7\ue4b8\ue4b9\ue4ba\ue4bb\ue4bc\ue4bd\ue4be\ue4bf\ue4c0\ue4c1\ue4c2\ue4c3\ue4c4\ue4c5\ue4c6\ue4c7\ue4c8\ue4c9\ue4ca\ue4cb\ue4cc\ue4cd\ue4ce\ue4cf\ue4d0\ue4d1\ue4d2\ue4d3\ue4d4\ue4d5\ue4d6\ue4d7\ue4d8\ue4d9\ue4da\ue4db\ue4dc\ue4dd\ue4de\ue4df\ue4e0\ue4e1\ue4e2\ue4e3\ue4e4\ue4e5\ue4e6\ue4e7\ue4e8\ue4e9\ue4ea\ue4eb\ue4ec\ue4ed\ue4ee\ue4ef\ue4f0\ue4f1\ue4f2\ue4f3\ue4f4\ue4f5\ue4f6\ue4f7\ue4f8\ue4f9\ue4fa\ue4fb\ue4fc\ue4fd\ue4fe\ue4ff\ue500\ue501\ue502\ue503\ue504\ue505\ue506\ue507\ue508\ue509\ue50a\ue50b\ue50c\ue50d\ue50e\ue50f\ue510\ue511\ue512\ue513\ue514\ue515\ue516\ue517\ue518\ue519\ue51a\ue51b\ue51c\ue51d\ue51e\ue51f\ue520\ue521\ue522\ue523\ue524\ue525\ue526\ue527\ue528\ue529\ue52a\ue52b\ue52c\ue52d\ue52e\ue52f\ue530\ue531\ue532\ue533\ue534\ue535\ue536\ue537\ue538\ue539\ue53a\ue53b\ue53c\ue53d\ue53e\ue53f\ue540\ue541\ue542\ue543\ue544\ue545\ue546\ue547\ue548\ue549\ue54a\ue54b\ue54c\ue54d\ue54e\ue54f\ue550\ue551\ue552\ue553\ue554\ue555\ue556\ue557\ue558\ue559\ue55a\ue55b\ue55c\ue55d\ue55e\ue55f\ue560\ue561\ue562\ue563\ue564\ue565\ue566\ue567\ue568\ue569\ue56a\ue56b\ue56c\ue56d\ue56e\ue56f\ue570\ue571\ue572\ue573\ue574\ue575\ue576\ue577\ue578\ue579\ue57a\ue57b\ue57c\ue57d\ue57e\ue57f\ue580\ue581\ue582\ue583\ue584\ue585\ue586\ue587\ue588\ue589\ue58a\ue58b\ue58c\ue58d\ue58e\ue58f\ue590\ue591\ue592\ue593\ue594\ue595\ue596\ue597\ue598\ue599\ue59a\ue59b\ue59c\ue59d\ue59e\ue59f\ue5a0\ue5a1\ue5a2\ue5a3\ue5a4\ue5a5\ue5a6\ue5a7\ue5a8\ue5a9\ue5aa\ue5ab\ue5ac\ue5ad\ue5ae\ue5af\ue5b0\ue5b1\ue5b2\ue5b3\ue5b4\ue5b5\ue5b6\ue5b7\ue5b8\ue5b9\ue5ba\ue5bb\ue5bc\ue5bd\ue5be\ue5bf\ue5c0\ue5c1\ue5c2\ue5c3\ue5c4\ue5c5\ue5c6\ue5c7\ue5c8\ue5c9\ue5ca\ue5cb\ue5cc\ue5cd\ue5ce\ue5cf\ue5d0\ue5d1\ue5d2\ue5d3\ue5d4\ue5d5\ue5d6\ue5d7\ue5d8\ue5d9\ue5da\ue5db\ue5dc\ue5dd\ue5de\ue5df\ue5e0\ue5e1\ue5e2\ue5e3\ue5e4\ue5e5\ue5e6\ue5e7\ue5e8\ue5e9\ue5ea\ue5eb\ue5ec\ue5ed\ue5ee\ue5ef\ue5f0\ue5f1\ue5f2\ue5f3\ue5f4\ue5f5\ue5f6\ue5f7\ue5f8\ue5f9\ue5fa\ue5fb\ue5fc\ue5fd\ue5fe\ue5ff\ue600\ue601\ue602\ue603\ue604\ue605\ue606\ue607\ue608\ue609\ue60a\ue60b\ue60c\ue60d\ue60e\ue60f\ue610\ue611\ue612\ue613\ue614\ue615\ue616\ue617\ue618\ue619\ue61a\ue61b\ue61c\ue61d\ue61e\ue61f\ue620\ue621\ue622\ue623\ue624\ue625\ue626\ue627\ue628\ue629\ue62a\ue62b\ue62c\ue62d\ue62e\ue62f\ue630\ue631\ue632\ue633\ue634\ue635\ue636\ue637\ue638\ue639\ue63a\ue63b\ue63c\ue63d\ue63e\ue63f\ue640\ue641\ue642\ue643\ue644\ue645\ue646\ue647\ue648\ue649\ue64a\ue64b\ue64c\ue64d\ue64e\ue64f\ue650\ue651\ue652\ue653\ue654\ue655\ue656\ue657\ue658\ue659\ue65a\ue65b\ue65c\ue65d\ue65e\ue65f\ue660\ue661\ue662\ue663\ue664\ue665\ue666\ue667\ue668\ue669\ue66a\ue66b\ue66c\ue66d\ue66e\ue66f\ue670\ue671\ue672\ue673\ue674\ue675\ue676\ue677\ue678\ue679\ue67a\ue67b\ue67c\ue67d\ue67e\ue67f\ue680\ue681\ue682\ue683\ue684\ue685\ue686\ue687\ue688\ue689\ue68a\ue68b\ue68c\ue68d\ue68e\ue68f\ue690\ue691\ue692\ue693\ue694\ue695\ue696\ue697\ue698\ue699\ue69a\ue69b\ue69c\ue69d\ue69e\ue69f\ue6a0\ue6a1\ue6a2\ue6a3\ue6a4\ue6a5\ue6a6\ue6a7\ue6a8\ue6a9\ue6aa\ue6ab\ue6ac\ue6ad\ue6ae\ue6af\ue6b0\ue6b1\ue6b2\ue6b3\ue6b4\ue6b5\ue6b6\ue6b7\ue6b8\ue6b9\ue6ba\ue6bb\ue6bc\ue6bd\ue6be\ue6bf\ue6c0\ue6c1\ue6c2\ue6c3\ue6c4\ue6c5\ue6c6\ue6c7\ue6c8\ue6c9\ue6ca\ue6cb\ue6cc\ue6cd\ue6ce\ue6cf\ue6d0\ue6d1\ue6d2\ue6d3\ue6d4\ue6d5\ue6d6\ue6d7\ue6d8\ue6d9\ue6da\ue6db\ue6dc\ue6dd\ue6de\ue6df\ue6e0\ue6e1\ue6e2\ue6e3\ue6e4\ue6e5\ue6e6\ue6e7\ue6e8\ue6e9\ue6ea\ue6eb\ue6ec\ue6ed\ue6ee\ue6ef\ue6f0\ue6f1\ue6f2\ue6f3\ue6f4\ue6f5\ue6f6\ue6f7\ue6f8\ue6f9\ue6fa\ue6fb\ue6fc\ue6fd\ue6fe\ue6ff\ue700\ue701\ue702\ue703\ue704\ue705\ue706\ue707\ue708\ue709\ue70a\ue70b\ue70c\ue70d\ue70e\ue70f\ue710\ue711\ue712\ue713\ue714\ue715\ue716\ue717\ue718\ue719\ue71a\ue71b\ue71c\ue71d\ue71e\ue71f\ue720\ue721\ue722\ue723\ue724\ue725\ue726\ue727\ue728\ue729\ue72a\ue72b\ue72c\ue72d\ue72e\ue72f\ue730\ue731\ue732\ue733\ue734\ue735\ue736\ue737\ue738\ue739\ue73a\ue73b\ue73c\ue73d\ue73e\ue73f\ue740\ue741\ue742\ue743\ue744\ue745\ue746\ue747\ue748\ue749\ue74a\ue74b\ue74c\ue74d\ue74e\ue74f\ue750\ue751\ue752\ue753\ue754\ue755\ue756\ue757\ue758\ue759\ue75a\ue75b\ue75c\ue75d\ue75e\ue75f\ue760\ue761\ue762\ue763\ue764\ue765\ue766\ue767\ue768\ue769\ue76a\ue76b\ue76c\ue76d\ue76e\ue76f\ue770\ue771\ue772\ue773\ue774\ue775\ue776\ue777\ue778\ue779\ue77a\ue77b\ue77c\ue77d\ue77e\ue77f\ue780\ue781\ue782\ue783\ue784\ue785\ue786\ue787\ue788\ue789\ue78a\ue78b\ue78c\ue78d\ue78e\ue78f\ue790\ue791\ue792\ue793\ue794\ue795\ue796\ue797\ue798\ue799\ue79a\ue79b\ue79c\ue79d\ue79e\ue79f\ue7a0\ue7a1\ue7a2\ue7a3\ue7a4\ue7a5\ue7a6\ue7a7\ue7a8\ue7a9\ue7aa\ue7ab\ue7ac\ue7ad\ue7ae\ue7af\ue7b0\ue7b1\ue7b2\ue7b3\ue7b4\ue7b5\ue7b6\ue7b7\ue7b8\ue7b9\ue7ba\ue7bb\ue7bc\ue7bd\ue7be\ue7bf\ue7c0\ue7c1\ue7c2\ue7c3\ue7c4\ue7c5\ue7c6\ue7c7\ue7c8\ue7c9\ue7ca\ue7cb\ue7cc\ue7cd\ue7ce\ue7cf\ue7d0\ue7d1\ue7d2\ue7d3\ue7d4\ue7d5\ue7d6\ue7d7\ue7d8\ue7d9\ue7da\ue7db\ue7dc\ue7dd\ue7de\ue7df\ue7e0\ue7e1\ue7e2\ue7e3\ue7e4\ue7e5\ue7e6\ue7e7\ue7e8\ue7e9\ue7ea\ue7eb\ue7ec\ue7ed\ue7ee\ue7ef\ue7f0\ue7f1\ue7f2\ue7f3\ue7f4\ue7f5\ue7f6\ue7f7\ue7f8\ue7f9\ue7fa\ue7fb\ue7fc\ue7fd\ue7fe\ue7ff\ue800\ue801\ue802\ue803\ue804\ue805\ue806\ue807\ue808\ue809\ue80a\ue80b\ue80c\ue80d\ue80e\ue80f\ue810\ue811\ue812\ue813\ue814\ue815\ue816\ue817\ue818\ue819\ue81a\ue81b\ue81c\ue81d\ue81e\ue81f\ue820\ue821\ue822\ue823\ue824\ue825\ue826\ue827\ue828\ue829\ue82a\ue82b\ue82c\ue82d\ue82e\ue82f\ue830\ue831\ue832\ue833\ue834\ue835\ue836\ue837\ue838\ue839\ue83a\ue83b\ue83c\ue83d\ue83e\ue83f\ue840\ue841\ue842\ue843\ue844\ue845\ue846\ue847\ue848\ue849\ue84a\ue84b\ue84c\ue84d\ue84e\ue84f\ue850\ue851\ue852\ue853\ue854\ue855\ue856\ue857\ue858\ue859\ue85a\ue85b\ue85c\ue85d\ue85e\ue85f\ue860\ue861\ue862\ue863\ue864\ue865\ue866\ue867\ue868\ue869\ue86a\ue86b\ue86c\ue86d\ue86e\ue86f\ue870\ue871\ue872\ue873\ue874\ue875\ue876\ue877\ue878\ue879\ue87a\ue87b\ue87c\ue87d\ue87e\ue87f\ue880\ue881\ue882\ue883\ue884\ue885\ue886\ue887\ue888\ue889\ue88a\ue88b\ue88c\ue88d\ue88e\ue88f\ue890\ue891\ue892\ue893\ue894\ue895\ue896\ue897\ue898\ue899\ue89a\ue89b\ue89c\ue89d\ue89e\ue89f\ue8a0\ue8a1\ue8a2\ue8a3\ue8a4\ue8a5\ue8a6\ue8a7\ue8a8\ue8a9\ue8aa\ue8ab\ue8ac\ue8ad\ue8ae\ue8af\ue8b0\ue8b1\ue8b2\ue8b3\ue8b4\ue8b5\ue8b6\ue8b7\ue8b8\ue8b9\ue8ba\ue8bb\ue8bc\ue8bd\ue8be\ue8bf\ue8c0\ue8c1\ue8c2\ue8c3\ue8c4\ue8c5\ue8c6\ue8c7\ue8c8\ue8c9\ue8ca\ue8cb\ue8cc\ue8cd\ue8ce\ue8cf\ue8d0\ue8d1\ue8d2\ue8d3\ue8d4\ue8d5\ue8d6\ue8d7\ue8d8\ue8d9\ue8da\ue8db\ue8dc\ue8dd\ue8de\ue8df\ue8e0\ue8e1\ue8e2\ue8e3\ue8e4\ue8e5\ue8e6\ue8e7\ue8e8\ue8e9\ue8ea\ue8eb\ue8ec\ue8ed\ue8ee\ue8ef\ue8f0\ue8f1\ue8f2\ue8f3\ue8f4\ue8f5\ue8f6\ue8f7\ue8f8\ue8f9\ue8fa\ue8fb\ue8fc\ue8fd\ue8fe\ue8ff\ue900\ue901\ue902\ue903\ue904\ue905\ue906\ue907\ue908\ue909\ue90a\ue90b\ue90c\ue90d\ue90e\ue90f\ue910\ue911\ue912\ue913\ue914\ue915\ue916\ue917\ue918\ue919\ue91a\ue91b\ue91c\ue91d\ue91e\ue91f\ue920\ue921\ue922\ue923\ue924\ue925\ue926\ue927\ue928\ue929\ue92a\ue92b\ue92c\ue92d\ue92e\ue92f\ue930\ue931\ue932\ue933\ue934\ue935\ue936\ue937\ue938\ue939\ue93a\ue93b\ue93c\ue93d\ue93e\ue93f\ue940\ue941\ue942\ue943\ue944\ue945\ue946\ue947\ue948\ue949\ue94a\ue94b\ue94c\ue94d\ue94e\ue94f\ue950\ue951\ue952\ue953\ue954\ue955\ue956\ue957\ue958\ue959\ue95a\ue95b\ue95c\ue95d\ue95e\ue95f\ue960\ue961\ue962\ue963\ue964\ue965\ue966\ue967\ue968\ue969\ue96a\ue96b\ue96c\ue96d\ue96e\ue96f\ue970\ue971\ue972\ue973\ue974\ue975\ue976\ue977\ue978\ue979\ue97a\ue97b\ue97c\ue97d\ue97e\ue97f\ue980\ue981\ue982\ue983\ue984\ue985\ue986\ue987\ue988\ue989\ue98a\ue98b\ue98c\ue98d\ue98e\ue98f\ue990\ue991\ue992\ue993\ue994\ue995\ue996\ue997\ue998\ue999\ue99a\ue99b\ue99c\ue99d\ue99e\ue99f\ue9a0\ue9a1\ue9a2\ue9a3\ue9a4\ue9a5\ue9a6\ue9a7\ue9a8\ue9a9\ue9aa\ue9ab\ue9ac\ue9ad\ue9ae\ue9af\ue9b0\ue9b1\ue9b2\ue9b3\ue9b4\ue9b5\ue9b6\ue9b7\ue9b8\ue9b9\ue9ba\ue9bb\ue9bc\ue9bd\ue9be\ue9bf\ue9c0\ue9c1\ue9c2\ue9c3\ue9c4\ue9c5\ue9c6\ue9c7\ue9c8\ue9c9\ue9ca\ue9cb\ue9cc\ue9cd\ue9ce\ue9cf\ue9d0\ue9d1\ue9d2\ue9d3\ue9d4\ue9d5\ue9d6\ue9d7\ue9d8\ue9d9\ue9da\ue9db\ue9dc\ue9dd\ue9de\ue9df\ue9e0\ue9e1\ue9e2\ue9e3\ue9e4\ue9e5\ue9e6\ue9e7\ue9e8\ue9e9\ue9ea\ue9eb\ue9ec\ue9ed\ue9ee\ue9ef\ue9f0\ue9f1\ue9f2\ue9f3\ue9f4\ue9f5\ue9f6\ue9f7\ue9f8\ue9f9\ue9fa\ue9fb\ue9fc\ue9fd\ue9fe\ue9ff\uea00\uea01\uea02\uea03\uea04\uea05\uea06\uea07\uea08\uea09\uea0a\uea0b\uea0c\uea0d\uea0e\uea0f\uea10\uea11\uea12\uea13\uea14\uea15\uea16\uea17\uea18\uea19\uea1a\uea1b\uea1c\uea1d\uea1e\uea1f\uea20\uea21\uea22\uea23\uea24\uea25\uea26\uea27\uea28\uea29\uea2a\uea2b\uea2c\uea2d\uea2e\uea2f\uea30\uea31\uea32\uea33\uea34\uea35\uea36\uea37\uea38\uea39\uea3a\uea3b\uea3c\uea3d\uea3e\uea3f\uea40\uea41\uea42\uea43\uea44\uea45\uea46\uea47\uea48\uea49\uea4a\uea4b\uea4c\uea4d\uea4e\uea4f\uea50\uea51\uea52\uea53\uea54\uea55\uea56\uea57\uea58\uea59\uea5a\uea5b\uea5c\uea5d\uea5e\uea5f\uea60\uea61\uea62\uea63\uea64\uea65\uea66\uea67\uea68\uea69\uea6a\uea6b\uea6c\uea6d\uea6e\uea6f\uea70\uea71\uea72\uea73\uea74\uea75\uea76\uea77\uea78\uea79\uea7a\uea7b\uea7c\uea7d\uea7e\uea7f\uea80\uea81\uea82\uea83\uea84\uea85\uea86\uea87\uea88\uea89\uea8a\uea8b\uea8c\uea8d\uea8e\uea8f\uea90\uea91\uea92\uea93\uea94\uea95\uea96\uea97\uea98\uea99\uea9a\uea9b\uea9c\uea9d\uea9e\uea9f\ueaa0\ueaa1\ueaa2\ueaa3\ueaa4\ueaa5\ueaa6\ueaa7\ueaa8\ueaa9\ueaaa\ueaab\ueaac\ueaad\ueaae\ueaaf\ueab0\ueab1\ueab2\ueab3\ueab4\ueab5\ueab6\ueab7\ueab8\ueab9\ueaba\ueabb\ueabc\ueabd\ueabe\ueabf\ueac0\ueac1\ueac2\ueac3\ueac4\ueac5\ueac6\ueac7\ueac8\ueac9\ueaca\ueacb\ueacc\ueacd\ueace\ueacf\uead0\uead1\uead2\uead3\uead4\uead5\uead6\uead7\uead8\uead9\ueada\ueadb\ueadc\ueadd\ueade\ueadf\ueae0\ueae1\ueae2\ueae3\ueae4\ueae5\ueae6\ueae7\ueae8\ueae9\ueaea\ueaeb\ueaec\ueaed\ueaee\ueaef\ueaf0\ueaf1\ueaf2\ueaf3\ueaf4\ueaf5\ueaf6\ueaf7\ueaf8\ueaf9\ueafa\ueafb\ueafc\ueafd\ueafe\ueaff\ueb00\ueb01\ueb02\ueb03\ueb04\ueb05\ueb06\ueb07\ueb08\ueb09\ueb0a\ueb0b\ueb0c\ueb0d\ueb0e\ueb0f\ueb10\ueb11\ueb12\ueb13\ueb14\ueb15\ueb16\ueb17\ueb18\ueb19\ueb1a\ueb1b\ueb1c\ueb1d\ueb1e\ueb1f\ueb20\ueb21\ueb22\ueb23\ueb24\ueb25\ueb26\ueb27\ueb28\ueb29\ueb2a\ueb2b\ueb2c\ueb2d\ueb2e\ueb2f\ueb30\ueb31\ueb32\ueb33\ueb34\ueb35\ueb36\ueb37\ueb38\ueb39\ueb3a\ueb3b\ueb3c\ueb3d\ueb3e\ueb3f\ueb40\ueb41\ueb42\ueb43\ueb44\ueb45\ueb46\ueb47\ueb48\ueb49\ueb4a\ueb4b\ueb4c\ueb4d\ueb4e\ueb4f\ueb50\ueb51\ueb52\ueb53\ueb54\ueb55\ueb56\ueb57\ueb58\ueb59\ueb5a\ueb5b\ueb5c\ueb5d\ueb5e\ueb5f\ueb60\ueb61\ueb62\ueb63\ueb64\ueb65\ueb66\ueb67\ueb68\ueb69\ueb6a\ueb6b\ueb6c\ueb6d\ueb6e\ueb6f\ueb70\ueb71\ueb72\ueb73\ueb74\ueb75\ueb76\ueb77\ueb78\ueb79\ueb7a\ueb7b\ueb7c\ueb7d\ueb7e\ueb7f\ueb80\ueb81\ueb82\ueb83\ueb84\ueb85\ueb86\ueb87\ueb88\ueb89\ueb8a\ueb8b\ueb8c\ueb8d\ueb8e\ueb8f\ueb90\ueb91\ueb92\ueb93\ueb94\ueb95\ueb96\ueb97\ueb98\ueb99\ueb9a\ueb9b\ueb9c\ueb9d\ueb9e\ueb9f\ueba0\ueba1\ueba2\ueba3\ueba4\ueba5\ueba6\ueba7\ueba8\ueba9\uebaa\uebab\uebac\uebad\uebae\uebaf\uebb0\uebb1\uebb2\uebb3\uebb4\uebb5\uebb6\uebb7\uebb8\uebb9\uebba\uebbb\uebbc\uebbd\uebbe\uebbf\uebc0\uebc1\uebc2\uebc3\uebc4\uebc5\uebc6\uebc7\uebc8\uebc9\uebca\uebcb\uebcc\uebcd\uebce\uebcf\uebd0\uebd1\uebd2\uebd3\uebd4\uebd5\uebd6\uebd7\uebd8\uebd9\uebda\uebdb\uebdc\uebdd\uebde\uebdf\uebe0\uebe1\uebe2\uebe3\uebe4\uebe5\uebe6\uebe7\uebe8\uebe9\uebea\uebeb\uebec\uebed\uebee\uebef\uebf0\uebf1\uebf2\uebf3\uebf4\uebf5\uebf6\uebf7\uebf8\uebf9\uebfa\uebfb\uebfc\uebfd\uebfe\uebff\uec00\uec01\uec02\uec03\uec04\uec05\uec06\uec07\uec08\uec09\uec0a\uec0b\uec0c\uec0d\uec0e\uec0f\uec10\uec11\uec12\uec13\uec14\uec15\uec16\uec17\uec18\uec19\uec1a\uec1b\uec1c\uec1d\uec1e\uec1f\uec20\uec21\uec22\uec23\uec24\uec25\uec26\uec27\uec28\uec29\uec2a\uec2b\uec2c\uec2d\uec2e\uec2f\uec30\uec31\uec32\uec33\uec34\uec35\uec36\uec37\uec38\uec39\uec3a\uec3b\uec3c\uec3d\uec3e\uec3f\uec40\uec41\uec42\uec43\uec44\uec45\uec46\uec47\uec48\uec49\uec4a\uec4b\uec4c\uec4d\uec4e\uec4f\uec50\uec51\uec52\uec53\uec54\uec55\uec56\uec57\uec58\uec59\uec5a\uec5b\uec5c\uec5d\uec5e\uec5f\uec60\uec61\uec62\uec63\uec64\uec65\uec66\uec67\uec68\uec69\uec6a\uec6b\uec6c\uec6d\uec6e\uec6f\uec70\uec71\uec72\uec73\uec74\uec75\uec76\uec77\uec78\uec79\uec7a\uec7b\uec7c\uec7d\uec7e\uec7f\uec80\uec81\uec82\uec83\uec84\uec85\uec86\uec87\uec88\uec89\uec8a\uec8b\uec8c\uec8d\uec8e\uec8f\uec90\uec91\uec92\uec93\uec94\uec95\uec96\uec97\uec98\uec99\uec9a\uec9b\uec9c\uec9d\uec9e\uec9f\ueca0\ueca1\ueca2\ueca3\ueca4\ueca5\ueca6\ueca7\ueca8\ueca9\uecaa\uecab\uecac\uecad\uecae\uecaf\uecb0\uecb1\uecb2\uecb3\uecb4\uecb5\uecb6\uecb7\uecb8\uecb9\uecba\uecbb\uecbc\uecbd\uecbe\uecbf\uecc0\uecc1\uecc2\uecc3\uecc4\uecc5\uecc6\uecc7\uecc8\uecc9\uecca\ueccb\ueccc\ueccd\uecce\ueccf\uecd0\uecd1\uecd2\uecd3\uecd4\uecd5\uecd6\uecd7\uecd8\uecd9\uecda\uecdb\uecdc\uecdd\uecde\uecdf\uece0\uece1\uece2\uece3\uece4\uece5\uece6\uece7\uece8\uece9\uecea\ueceb\uecec\ueced\uecee\uecef\uecf0\uecf1\uecf2\uecf3\uecf4\uecf5\uecf6\uecf7\uecf8\uecf9\uecfa\uecfb\uecfc\uecfd\uecfe\uecff\ued00\ued01\ued02\ued03\ued04\ued05\ued06\ued07\ued08\ued09\ued0a\ued0b\ued0c\ued0d\ued0e\ued0f\ued10\ued11\ued12\ued13\ued14\ued15\ued16\ued17\ued18\ued19\ued1a\ued1b\ued1c\ued1d\ued1e\ued1f\ued20\ued21\ued22\ued23\ued24\ued25\ued26\ued27\ued28\ued29\ued2a\ued2b\ued2c\ued2d\ued2e\ued2f\ued30\ued31\ued32\ued33\ued34\ued35\ued36\ued37\ued38\ued39\ued3a\ued3b\ued3c\ued3d\ued3e\ued3f\ued40\ued41\ued42\ued43\ued44\ued45\ued46\ued47\ued48\ued49\ued4a\ued4b\ued4c\ued4d\ued4e\ued4f\ued50\ued51\ued52\ued53\ued54\ued55\ued56\ued57\ued58\ued59\ued5a\ued5b\ued5c\ued5d\ued5e\ued5f\ued60\ued61\ued62\ued63\ued64\ued65\ued66\ued67\ued68\ued69\ued6a\ued6b\ued6c\ued6d\ued6e\ued6f\ued70\ued71\ued72\ued73\ued74\ued75\ued76\ued77\ued78\ued79\ued7a\ued7b\ued7c\ued7d\ued7e\ued7f\ued80\ued81\ued82\ued83\ued84\ued85\ued86\ued87\ued88\ued89\ued8a\ued8b\ued8c\ued8d\ued8e\ued8f\ued90\ued91\ued92\ued93\ued94\ued95\ued96\ued97\ued98\ued99\ued9a\ued9b\ued9c\ued9d\ued9e\ued9f\ueda0\ueda1\ueda2\ueda3\ueda4\ueda5\ueda6\ueda7\ueda8\ueda9\uedaa\uedab\uedac\uedad\uedae\uedaf\uedb0\uedb1\uedb2\uedb3\uedb4\uedb5\uedb6\uedb7\uedb8\uedb9\uedba\uedbb\uedbc\uedbd\uedbe\uedbf\uedc0\uedc1\uedc2\uedc3\uedc4\uedc5\uedc6\uedc7\uedc8\uedc9\uedca\uedcb\uedcc\uedcd\uedce\uedcf\uedd0\uedd1\uedd2\uedd3\uedd4\uedd5\uedd6\uedd7\uedd8\uedd9\uedda\ueddb\ueddc\ueddd\uedde\ueddf\uede0\uede1\uede2\uede3\uede4\uede5\uede6\uede7\uede8\uede9\uedea\uedeb\uedec\ueded\uedee\uedef\uedf0\uedf1\uedf2\uedf3\uedf4\uedf5\uedf6\uedf7\uedf8\uedf9\uedfa\uedfb\uedfc\uedfd\uedfe\uedff\uee00\uee01\uee02\uee03\uee04\uee05\uee06\uee07\uee08\uee09\uee0a\uee0b\uee0c\uee0d\uee0e\uee0f\uee10\uee11\uee12\uee13\uee14\uee15\uee16\uee17\uee18\uee19\uee1a\uee1b\uee1c\uee1d\uee1e\uee1f\uee20\uee21\uee22\uee23\uee24\uee25\uee26\uee27\uee28\uee29\uee2a\uee2b\uee2c\uee2d\uee2e\uee2f\uee30\uee31\uee32\uee33\uee34\uee35\uee36\uee37\uee38\uee39\uee3a\uee3b\uee3c\uee3d\uee3e\uee3f\uee40\uee41\uee42\uee43\uee44\uee45\uee46\uee47\uee48\uee49\uee4a\uee4b\uee4c\uee4d\uee4e\uee4f\uee50\uee51\uee52\uee53\uee54\uee55\uee56\uee57\uee58\uee59\uee5a\uee5b\uee5c\uee5d\uee5e\uee5f\uee60\uee61\uee62\uee63\uee64\uee65\uee66\uee67\uee68\uee69\uee6a\uee6b\uee6c\uee6d\uee6e\uee6f\uee70\uee71\uee72\uee73\uee74\uee75\uee76\uee77\uee78\uee79\uee7a\uee7b\uee7c\uee7d\uee7e\uee7f\uee80\uee81\uee82\uee83\uee84\uee85\uee86\uee87\uee88\uee89\uee8a\uee8b\uee8c\uee8d\uee8e\uee8f\uee90\uee91\uee92\uee93\uee94\uee95\uee96\uee97\uee98\uee99\uee9a\uee9b\uee9c\uee9d\uee9e\uee9f\ueea0\ueea1\ueea2\ueea3\ueea4\ueea5\ueea6\ueea7\ueea8\ueea9\ueeaa\ueeab\ueeac\ueead\ueeae\ueeaf\ueeb0\ueeb1\ueeb2\ueeb3\ueeb4\ueeb5\ueeb6\ueeb7\ueeb8\ueeb9\ueeba\ueebb\ueebc\ueebd\ueebe\ueebf\ueec0\ueec1\ueec2\ueec3\ueec4\ueec5\ueec6\ueec7\ueec8\ueec9\ueeca\ueecb\ueecc\ueecd\ueece\ueecf\ueed0\ueed1\ueed2\ueed3\ueed4\ueed5\ueed6\ueed7\ueed8\ueed9\ueeda\ueedb\ueedc\ueedd\ueede\ueedf\ueee0\ueee1\ueee2\ueee3\ueee4\ueee5\ueee6\ueee7\ueee8\ueee9\ueeea\ueeeb\ueeec\ueeed\ueeee\ueeef\ueef0\ueef1\ueef2\ueef3\ueef4\ueef5\ueef6\ueef7\ueef8\ueef9\ueefa\ueefb\ueefc\ueefd\ueefe\ueeff\uef00\uef01\uef02\uef03\uef04\uef05\uef06\uef07\uef08\uef09\uef0a\uef0b\uef0c\uef0d\uef0e\uef0f\uef10\uef11\uef12\uef13\uef14\uef15\uef16\uef17\uef18\uef19\uef1a\uef1b\uef1c\uef1d\uef1e\uef1f\uef20\uef21\uef22\uef23\uef24\uef25\uef26\uef27\uef28\uef29\uef2a\uef2b\uef2c\uef2d\uef2e\uef2f\uef30\uef31\uef32\uef33\uef34\uef35\uef36\uef37\uef38\uef39\uef3a\uef3b\uef3c\uef3d\uef3e\uef3f\uef40\uef41\uef42\uef43\uef44\uef45\uef46\uef47\uef48\uef49\uef4a\uef4b\uef4c\uef4d\uef4e\uef4f\uef50\uef51\uef52\uef53\uef54\uef55\uef56\uef57\uef58\uef59\uef5a\uef5b\uef5c\uef5d\uef5e\uef5f\uef60\uef61\uef62\uef63\uef64\uef65\uef66\uef67\uef68\uef69\uef6a\uef6b\uef6c\uef6d\uef6e\uef6f\uef70\uef71\uef72\uef73\uef74\uef75\uef76\uef77\uef78\uef79\uef7a\uef7b\uef7c\uef7d\uef7e\uef7f\uef80\uef81\uef82\uef83\uef84\uef85\uef86\uef87\uef88\uef89\uef8a\uef8b\uef8c\uef8d\uef8e\uef8f\uef90\uef91\uef92\uef93\uef94\uef95\uef96\uef97\uef98\uef99\uef9a\uef9b\uef9c\uef9d\uef9e\uef9f\uefa0\uefa1\uefa2\uefa3\uefa4\uefa5\uefa6\uefa7\uefa8\uefa9\uefaa\uefab\uefac\uefad\uefae\uefaf\uefb0\uefb1\uefb2\uefb3\uefb4\uefb5\uefb6\uefb7\uefb8\uefb9\uefba\uefbb\uefbc\uefbd\uefbe\uefbf\uefc0\uefc1\uefc2\uefc3\uefc4\uefc5\uefc6\uefc7\uefc8\uefc9\uefca\uefcb\uefcc\uefcd\uefce\uefcf\uefd0\uefd1\uefd2\uefd3\uefd4\uefd5\uefd6\uefd7\uefd8\uefd9\uefda\uefdb\uefdc\uefdd\uefde\uefdf\uefe0\uefe1\uefe2\uefe3\uefe4\uefe5\uefe6\uefe7\uefe8\uefe9\uefea\uefeb\uefec\uefed\uefee\uefef\ueff0\ueff1\ueff2\ueff3\ueff4\ueff5\ueff6\ueff7\ueff8\ueff9\ueffa\ueffb\ueffc\ueffd\ueffe\uefff\uf000\uf001\uf002\uf003\uf004\uf005\uf006\uf007\uf008\uf009\uf00a\uf00b\uf00c\uf00d\uf00e\uf00f\uf010\uf011\uf012\uf013\uf014\uf015\uf016\uf017\uf018\uf019\uf01a\uf01b\uf01c\uf01d\uf01e\uf01f\uf020\uf021\uf022\uf023\uf024\uf025\uf026\uf027\uf028\uf029\uf02a\uf02b\uf02c\uf02d\uf02e\uf02f\uf030\uf031\uf032\uf033\uf034\uf035\uf036\uf037\uf038\uf039\uf03a\uf03b\uf03c\uf03d\uf03e\uf03f\uf040\uf041\uf042\uf043\uf044\uf045\uf046\uf047\uf048\uf049\uf04a\uf04b\uf04c\uf04d\uf04e\uf04f\uf050\uf051\uf052\uf053\uf054\uf055\uf056\uf057\uf058\uf059\uf05a\uf05b\uf05c\uf05d\uf05e\uf05f\uf060\uf061\uf062\uf063\uf064\uf065\uf066\uf067\uf068\uf069\uf06a\uf06b\uf06c\uf06d\uf06e\uf06f\uf070\uf071\uf072\uf073\uf074\uf075\uf076\uf077\uf078\uf079\uf07a\uf07b\uf07c\uf07d\uf07e\uf07f\uf080\uf081\uf082\uf083\uf084\uf085\uf086\uf087\uf088\uf089\uf08a\uf08b\uf08c\uf08d\uf08e\uf08f\uf090\uf091\uf092\uf093\uf094\uf095\uf096\uf097\uf098\uf099\uf09a\uf09b\uf09c\uf09d\uf09e\uf09f\uf0a0\uf0a1\uf0a2\uf0a3\uf0a4\uf0a5\uf0a6\uf0a7\uf0a8\uf0a9\uf0aa\uf0ab\uf0ac\uf0ad\uf0ae\uf0af\uf0b0\uf0b1\uf0b2\uf0b3\uf0b4\uf0b5\uf0b6\uf0b7\uf0b8\uf0b9\uf0ba\uf0bb\uf0bc\uf0bd\uf0be\uf0bf\uf0c0\uf0c1\uf0c2\uf0c3\uf0c4\uf0c5\uf0c6\uf0c7\uf0c8\uf0c9\uf0ca\uf0cb\uf0cc\uf0cd\uf0ce\uf0cf\uf0d0\uf0d1\uf0d2\uf0d3\uf0d4\uf0d5\uf0d6\uf0d7\uf0d8\uf0d9\uf0da\uf0db\uf0dc\uf0dd\uf0de\uf0df\uf0e0\uf0e1\uf0e2\uf0e3\uf0e4\uf0e5\uf0e6\uf0e7\uf0e8\uf0e9\uf0ea\uf0eb\uf0ec\uf0ed\uf0ee\uf0ef\uf0f0\uf0f1\uf0f2\uf0f3\uf0f4\uf0f5\uf0f6\uf0f7\uf0f8\uf0f9\uf0fa\uf0fb\uf0fc\uf0fd\uf0fe\uf0ff\uf100\uf101\uf102\uf103\uf104\uf105\uf106\uf107\uf108\uf109\uf10a\uf10b\uf10c\uf10d\uf10e\uf10f\uf110\uf111\uf112\uf113\uf114\uf115\uf116\uf117\uf118\uf119\uf11a\uf11b\uf11c\uf11d\uf11e\uf11f\uf120\uf121\uf122\uf123\uf124\uf125\uf126\uf127\uf128\uf129\uf12a\uf12b\uf12c\uf12d\uf12e\uf12f\uf130\uf131\uf132\uf133\uf134\uf135\uf136\uf137\uf138\uf139\uf13a\uf13b\uf13c\uf13d\uf13e\uf13f\uf140\uf141\uf142\uf143\uf144\uf145\uf146\uf147\uf148\uf149\uf14a\uf14b\uf14c\uf14d\uf14e\uf14f\uf150\uf151\uf152\uf153\uf154\uf155\uf156\uf157\uf158\uf159\uf15a\uf15b\uf15c\uf15d\uf15e\uf15f\uf160\uf161\uf162\uf163\uf164\uf165\uf166\uf167\uf168\uf169\uf16a\uf16b\uf16c\uf16d\uf16e\uf16f\uf170\uf171\uf172\uf173\uf174\uf175\uf176\uf177\uf178\uf179\uf17a\uf17b\uf17c\uf17d\uf17e\uf17f\uf180\uf181\uf182\uf183\uf184\uf185\uf186\uf187\uf188\uf189\uf18a\uf18b\uf18c\uf18d\uf18e\uf18f\uf190\uf191\uf192\uf193\uf194\uf195\uf196\uf197\uf198\uf199\uf19a\uf19b\uf19c\uf19d\uf19e\uf19f\uf1a0\uf1a1\uf1a2\uf1a3\uf1a4\uf1a5\uf1a6\uf1a7\uf1a8\uf1a9\uf1aa\uf1ab\uf1ac\uf1ad\uf1ae\uf1af\uf1b0\uf1b1\uf1b2\uf1b3\uf1b4\uf1b5\uf1b6\uf1b7\uf1b8\uf1b9\uf1ba\uf1bb\uf1bc\uf1bd\uf1be\uf1bf\uf1c0\uf1c1\uf1c2\uf1c3\uf1c4\uf1c5\uf1c6\uf1c7\uf1c8\uf1c9\uf1ca\uf1cb\uf1cc\uf1cd\uf1ce\uf1cf\uf1d0\uf1d1\uf1d2\uf1d3\uf1d4\uf1d5\uf1d6\uf1d7\uf1d8\uf1d9\uf1da\uf1db\uf1dc\uf1dd\uf1de\uf1df\uf1e0\uf1e1\uf1e2\uf1e3\uf1e4\uf1e5\uf1e6\uf1e7\uf1e8\uf1e9\uf1ea\uf1eb\uf1ec\uf1ed\uf1ee\uf1ef\uf1f0\uf1f1\uf1f2\uf1f3\uf1f4\uf1f5\uf1f6\uf1f7\uf1f8\uf1f9\uf1fa\uf1fb\uf1fc\uf1fd\uf1fe\uf1ff\uf200\uf201\uf202\uf203\uf204\uf205\uf206\uf207\uf208\uf209\uf20a\uf20b\uf20c\uf20d\uf20e\uf20f\uf210\uf211\uf212\uf213\uf214\uf215\uf216\uf217\uf218\uf219\uf21a\uf21b\uf21c\uf21d\uf21e\uf21f\uf220\uf221\uf222\uf223\uf224\uf225\uf226\uf227\uf228\uf229\uf22a\uf22b\uf22c\uf22d\uf22e\uf22f\uf230\uf231\uf232\uf233\uf234\uf235\uf236\uf237\uf238\uf239\uf23a\uf23b\uf23c\uf23d\uf23e\uf23f\uf240\uf241\uf242\uf243\uf244\uf245\uf246\uf247\uf248\uf249\uf24a\uf24b\uf24c\uf24d\uf24e\uf24f\uf250\uf251\uf252\uf253\uf254\uf255\uf256\uf257\uf258\uf259\uf25a\uf25b\uf25c\uf25d\uf25e\uf25f\uf260\uf261\uf262\uf263\uf264\uf265\uf266\uf267\uf268\uf269\uf26a\uf26b\uf26c\uf26d\uf26e\uf26f\uf270\uf271\uf272\uf273\uf274\uf275\uf276\uf277\uf278\uf279\uf27a\uf27b\uf27c\uf27d\uf27e\uf27f\uf280\uf281\uf282\uf283\uf284\uf285\uf286\uf287\uf288\uf289\uf28a\uf28b\uf28c\uf28d\uf28e\uf28f\uf290\uf291\uf292\uf293\uf294\uf295\uf296\uf297\uf298\uf299\uf29a\uf29b\uf29c\uf29d\uf29e\uf29f\uf2a0\uf2a1\uf2a2\uf2a3\uf2a4\uf2a5\uf2a6\uf2a7\uf2a8\uf2a9\uf2aa\uf2ab\uf2ac\uf2ad\uf2ae\uf2af\uf2b0\uf2b1\uf2b2\uf2b3\uf2b4\uf2b5\uf2b6\uf2b7\uf2b8\uf2b9\uf2ba\uf2bb\uf2bc\uf2bd\uf2be\uf2bf\uf2c0\uf2c1\uf2c2\uf2c3\uf2c4\uf2c5\uf2c6\uf2c7\uf2c8\uf2c9\uf2ca\uf2cb\uf2cc\uf2cd\uf2ce\uf2cf\uf2d0\uf2d1\uf2d2\uf2d3\uf2d4\uf2d5\uf2d6\uf2d7\uf2d8\uf2d9\uf2da\uf2db\uf2dc\uf2dd\uf2de\uf2df\uf2e0\uf2e1\uf2e2\uf2e3\uf2e4\uf2e5\uf2e6\uf2e7\uf2e8\uf2e9\uf2ea\uf2eb\uf2ec\uf2ed\uf2ee\uf2ef\uf2f0\uf2f1\uf2f2\uf2f3\uf2f4\uf2f5\uf2f6\uf2f7\uf2f8\uf2f9\uf2fa\uf2fb\uf2fc\uf2fd\uf2fe\uf2ff\uf300\uf301\uf302\uf303\uf304\uf305\uf306\uf307\uf308\uf309\uf30a\uf30b\uf30c\uf30d\uf30e\uf30f\uf310\uf311\uf312\uf313\uf314\uf315\uf316\uf317\uf318\uf319\uf31a\uf31b\uf31c\uf31d\uf31e\uf31f\uf320\uf321\uf322\uf323\uf324\uf325\uf326\uf327\uf328\uf329\uf32a\uf32b\uf32c\uf32d\uf32e\uf32f\uf330\uf331\uf332\uf333\uf334\uf335\uf336\uf337\uf338\uf339\uf33a\uf33b\uf33c\uf33d\uf33e\uf33f\uf340\uf341\uf342\uf343\uf344\uf345\uf346\uf347\uf348\uf349\uf34a\uf34b\uf34c\uf34d\uf34e\uf34f\uf350\uf351\uf352\uf353\uf354\uf355\uf356\uf357\uf358\uf359\uf35a\uf35b\uf35c\uf35d\uf35e\uf35f\uf360\uf361\uf362\uf363\uf364\uf365\uf366\uf367\uf368\uf369\uf36a\uf36b\uf36c\uf36d\uf36e\uf36f\uf370\uf371\uf372\uf373\uf374\uf375\uf376\uf377\uf378\uf379\uf37a\uf37b\uf37c\uf37d\uf37e\uf37f\uf380\uf381\uf382\uf383\uf384\uf385\uf386\uf387\uf388\uf389\uf38a\uf38b\uf38c\uf38d\uf38e\uf38f\uf390\uf391\uf392\uf393\uf394\uf395\uf396\uf397\uf398\uf399\uf39a\uf39b\uf39c\uf39d\uf39e\uf39f\uf3a0\uf3a1\uf3a2\uf3a3\uf3a4\uf3a5\uf3a6\uf3a7\uf3a8\uf3a9\uf3aa\uf3ab\uf3ac\uf3ad\uf3ae\uf3af\uf3b0\uf3b1\uf3b2\uf3b3\uf3b4\uf3b5\uf3b6\uf3b7\uf3b8\uf3b9\uf3ba\uf3bb\uf3bc\uf3bd\uf3be\uf3bf\uf3c0\uf3c1\uf3c2\uf3c3\uf3c4\uf3c5\uf3c6\uf3c7\uf3c8\uf3c9\uf3ca\uf3cb\uf3cc\uf3cd\uf3ce\uf3cf\uf3d0\uf3d1\uf3d2\uf3d3\uf3d4\uf3d5\uf3d6\uf3d7\uf3d8\uf3d9\uf3da\uf3db\uf3dc\uf3dd\uf3de\uf3df\uf3e0\uf3e1\uf3e2\uf3e3\uf3e4\uf3e5\uf3e6\uf3e7\uf3e8\uf3e9\uf3ea\uf3eb\uf3ec\uf3ed\uf3ee\uf3ef\uf3f0\uf3f1\uf3f2\uf3f3\uf3f4\uf3f5\uf3f6\uf3f7\uf3f8\uf3f9\uf3fa\uf3fb\uf3fc\uf3fd\uf3fe\uf3ff\uf400\uf401\uf402\uf403\uf404\uf405\uf406\uf407\uf408\uf409\uf40a\uf40b\uf40c\uf40d\uf40e\uf40f\uf410\uf411\uf412\uf413\uf414\uf415\uf416\uf417\uf418\uf419\uf41a\uf41b\uf41c\uf41d\uf41e\uf41f\uf420\uf421\uf422\uf423\uf424\uf425\uf426\uf427\uf428\uf429\uf42a\uf42b\uf42c\uf42d\uf42e\uf42f\uf430\uf431\uf432\uf433\uf434\uf435\uf436\uf437\uf438\uf439\uf43a\uf43b\uf43c\uf43d\uf43e\uf43f\uf440\uf441\uf442\uf443\uf444\uf445\uf446\uf447\uf448\uf449\uf44a\uf44b\uf44c\uf44d\uf44e\uf44f\uf450\uf451\uf452\uf453\uf454\uf455\uf456\uf457\uf458\uf459\uf45a\uf45b\uf45c\uf45d\uf45e\uf45f\uf460\uf461\uf462\uf463\uf464\uf465\uf466\uf467\uf468\uf469\uf46a\uf46b\uf46c\uf46d\uf46e\uf46f\uf470\uf471\uf472\uf473\uf474\uf475\uf476\uf477\uf478\uf479\uf47a\uf47b\uf47c\uf47d\uf47e\uf47f\uf480\uf481\uf482\uf483\uf484\uf485\uf486\uf487\uf488\uf489\uf48a\uf48b\uf48c\uf48d\uf48e\uf48f\uf490\uf491\uf492\uf493\uf494\uf495\uf496\uf497\uf498\uf499\uf49a\uf49b\uf49c\uf49d\uf49e\uf49f\uf4a0\uf4a1\uf4a2\uf4a3\uf4a4\uf4a5\uf4a6\uf4a7\uf4a8\uf4a9\uf4aa\uf4ab\uf4ac\uf4ad\uf4ae\uf4af\uf4b0\uf4b1\uf4b2\uf4b3\uf4b4\uf4b5\uf4b6\uf4b7\uf4b8\uf4b9\uf4ba\uf4bb\uf4bc\uf4bd\uf4be\uf4bf\uf4c0\uf4c1\uf4c2\uf4c3\uf4c4\uf4c5\uf4c6\uf4c7\uf4c8\uf4c9\uf4ca\uf4cb\uf4cc\uf4cd\uf4ce\uf4cf\uf4d0\uf4d1\uf4d2\uf4d3\uf4d4\uf4d5\uf4d6\uf4d7\uf4d8\uf4d9\uf4da\uf4db\uf4dc\uf4dd\uf4de\uf4df\uf4e0\uf4e1\uf4e2\uf4e3\uf4e4\uf4e5\uf4e6\uf4e7\uf4e8\uf4e9\uf4ea\uf4eb\uf4ec\uf4ed\uf4ee\uf4ef\uf4f0\uf4f1\uf4f2\uf4f3\uf4f4\uf4f5\uf4f6\uf4f7\uf4f8\uf4f9\uf4fa\uf4fb\uf4fc\uf4fd\uf4fe\uf4ff\uf500\uf501\uf502\uf503\uf504\uf505\uf506\uf507\uf508\uf509\uf50a\uf50b\uf50c\uf50d\uf50e\uf50f\uf510\uf511\uf512\uf513\uf514\uf515\uf516\uf517\uf518\uf519\uf51a\uf51b\uf51c\uf51d\uf51e\uf51f\uf520\uf521\uf522\uf523\uf524\uf525\uf526\uf527\uf528\uf529\uf52a\uf52b\uf52c\uf52d\uf52e\uf52f\uf530\uf531\uf532\uf533\uf534\uf535\uf536\uf537\uf538\uf539\uf53a\uf53b\uf53c\uf53d\uf53e\uf53f\uf540\uf541\uf542\uf543\uf544\uf545\uf546\uf547\uf548\uf549\uf54a\uf54b\uf54c\uf54d\uf54e\uf54f\uf550\uf551\uf552\uf553\uf554\uf555\uf556\uf557\uf558\uf559\uf55a\uf55b\uf55c\uf55d\uf55e\uf55f\uf560\uf561\uf562\uf563\uf564\uf565\uf566\uf567\uf568\uf569\uf56a\uf56b\uf56c\uf56d\uf56e\uf56f\uf570\uf571\uf572\uf573\uf574\uf575\uf576\uf577\uf578\uf579\uf57a\uf57b\uf57c\uf57d\uf57e\uf57f\uf580\uf581\uf582\uf583\uf584\uf585\uf586\uf587\uf588\uf589\uf58a\uf58b\uf58c\uf58d\uf58e\uf58f\uf590\uf591\uf592\uf593\uf594\uf595\uf596\uf597\uf598\uf599\uf59a\uf59b\uf59c\uf59d\uf59e\uf59f\uf5a0\uf5a1\uf5a2\uf5a3\uf5a4\uf5a5\uf5a6\uf5a7\uf5a8\uf5a9\uf5aa\uf5ab\uf5ac\uf5ad\uf5ae\uf5af\uf5b0\uf5b1\uf5b2\uf5b3\uf5b4\uf5b5\uf5b6\uf5b7\uf5b8\uf5b9\uf5ba\uf5bb\uf5bc\uf5bd\uf5be\uf5bf\uf5c0\uf5c1\uf5c2\uf5c3\uf5c4\uf5c5\uf5c6\uf5c7\uf5c8\uf5c9\uf5ca\uf5cb\uf5cc\uf5cd\uf5ce\uf5cf\uf5d0\uf5d1\uf5d2\uf5d3\uf5d4\uf5d5\uf5d6\uf5d7\uf5d8\uf5d9\uf5da\uf5db\uf5dc\uf5dd\uf5de\uf5df\uf5e0\uf5e1\uf5e2\uf5e3\uf5e4\uf5e5\uf5e6\uf5e7\uf5e8\uf5e9\uf5ea\uf5eb\uf5ec\uf5ed\uf5ee\uf5ef\uf5f0\uf5f1\uf5f2\uf5f3\uf5f4\uf5f5\uf5f6\uf5f7\uf5f8\uf5f9\uf5fa\uf5fb\uf5fc\uf5fd\uf5fe\uf5ff\uf600\uf601\uf602\uf603\uf604\uf605\uf606\uf607\uf608\uf609\uf60a\uf60b\uf60c\uf60d\uf60e\uf60f\uf610\uf611\uf612\uf613\uf614\uf615\uf616\uf617\uf618\uf619\uf61a\uf61b\uf61c\uf61d\uf61e\uf61f\uf620\uf621\uf622\uf623\uf624\uf625\uf626\uf627\uf628\uf629\uf62a\uf62b\uf62c\uf62d\uf62e\uf62f\uf630\uf631\uf632\uf633\uf634\uf635\uf636\uf637\uf638\uf639\uf63a\uf63b\uf63c\uf63d\uf63e\uf63f\uf640\uf641\uf642\uf643\uf644\uf645\uf646\uf647\uf648\uf649\uf64a\uf64b\uf64c\uf64d\uf64e\uf64f\uf650\uf651\uf652\uf653\uf654\uf655\uf656\uf657\uf658\uf659\uf65a\uf65b\uf65c\uf65d\uf65e\uf65f\uf660\uf661\uf662\uf663\uf664\uf665\uf666\uf667\uf668\uf669\uf66a\uf66b\uf66c\uf66d\uf66e\uf66f\uf670\uf671\uf672\uf673\uf674\uf675\uf676\uf677\uf678\uf679\uf67a\uf67b\uf67c\uf67d\uf67e\uf67f\uf680\uf681\uf682\uf683\uf684\uf685\uf686\uf687\uf688\uf689\uf68a\uf68b\uf68c\uf68d\uf68e\uf68f\uf690\uf691\uf692\uf693\uf694\uf695\uf696\uf697\uf698\uf699\uf69a\uf69b\uf69c\uf69d\uf69e\uf69f\uf6a0\uf6a1\uf6a2\uf6a3\uf6a4\uf6a5\uf6a6\uf6a7\uf6a8\uf6a9\uf6aa\uf6ab\uf6ac\uf6ad\uf6ae\uf6af\uf6b0\uf6b1\uf6b2\uf6b3\uf6b4\uf6b5\uf6b6\uf6b7\uf6b8\uf6b9\uf6ba\uf6bb\uf6bc\uf6bd\uf6be\uf6bf\uf6c0\uf6c1\uf6c2\uf6c3\uf6c4\uf6c5\uf6c6\uf6c7\uf6c8\uf6c9\uf6ca\uf6cb\uf6cc\uf6cd\uf6ce\uf6cf\uf6d0\uf6d1\uf6d2\uf6d3\uf6d4\uf6d5\uf6d6\uf6d7\uf6d8\uf6d9\uf6da\uf6db\uf6dc\uf6dd\uf6de\uf6df\uf6e0\uf6e1\uf6e2\uf6e3\uf6e4\uf6e5\uf6e6\uf6e7\uf6e8\uf6e9\uf6ea\uf6eb\uf6ec\uf6ed\uf6ee\uf6ef\uf6f0\uf6f1\uf6f2\uf6f3\uf6f4\uf6f5\uf6f6\uf6f7\uf6f8\uf6f9\uf6fa\uf6fb\uf6fc\uf6fd\uf6fe\uf6ff\uf700\uf701\uf702\uf703\uf704\uf705\uf706\uf707\uf708\uf709\uf70a\uf70b\uf70c\uf70d\uf70e\uf70f\uf710\uf711\uf712\uf713\uf714\uf715\uf716\uf717\uf718\uf719\uf71a\uf71b\uf71c\uf71d\uf71e\uf71f\uf720\uf721\uf722\uf723\uf724\uf725\uf726\uf727\uf728\uf729\uf72a\uf72b\uf72c\uf72d\uf72e\uf72f\uf730\uf731\uf732\uf733\uf734\uf735\uf736\uf737\uf738\uf739\uf73a\uf73b\uf73c\uf73d\uf73e\uf73f\uf740\uf741\uf742\uf743\uf744\uf745\uf746\uf747\uf748\uf749\uf74a\uf74b\uf74c\uf74d\uf74e\uf74f\uf750\uf751\uf752\uf753\uf754\uf755\uf756\uf757\uf758\uf759\uf75a\uf75b\uf75c\uf75d\uf75e\uf75f\uf760\uf761\uf762\uf763\uf764\uf765\uf766\uf767\uf768\uf769\uf76a\uf76b\uf76c\uf76d\uf76e\uf76f\uf770\uf771\uf772\uf773\uf774\uf775\uf776\uf777\uf778\uf779\uf77a\uf77b\uf77c\uf77d\uf77e\uf77f\uf780\uf781\uf782\uf783\uf784\uf785\uf786\uf787\uf788\uf789\uf78a\uf78b\uf78c\uf78d\uf78e\uf78f\uf790\uf791\uf792\uf793\uf794\uf795\uf796\uf797\uf798\uf799\uf79a\uf79b\uf79c\uf79d\uf79e\uf79f\uf7a0\uf7a1\uf7a2\uf7a3\uf7a4\uf7a5\uf7a6\uf7a7\uf7a8\uf7a9\uf7aa\uf7ab\uf7ac\uf7ad\uf7ae\uf7af\uf7b0\uf7b1\uf7b2\uf7b3\uf7b4\uf7b5\uf7b6\uf7b7\uf7b8\uf7b9\uf7ba\uf7bb\uf7bc\uf7bd\uf7be\uf7bf\uf7c0\uf7c1\uf7c2\uf7c3\uf7c4\uf7c5\uf7c6\uf7c7\uf7c8\uf7c9\uf7ca\uf7cb\uf7cc\uf7cd\uf7ce\uf7cf\uf7d0\uf7d1\uf7d2\uf7d3\uf7d4\uf7d5\uf7d6\uf7d7\uf7d8\uf7d9\uf7da\uf7db\uf7dc\uf7dd\uf7de\uf7df\uf7e0\uf7e1\uf7e2\uf7e3\uf7e4\uf7e5\uf7e6\uf7e7\uf7e8\uf7e9\uf7ea\uf7eb\uf7ec\uf7ed\uf7ee\uf7ef\uf7f0\uf7f1\uf7f2\uf7f3\uf7f4\uf7f5\uf7f6\uf7f7\uf7f8\uf7f9\uf7fa\uf7fb\uf7fc\uf7fd\uf7fe\uf7ff\uf800\uf801\uf802\uf803\uf804\uf805\uf806\uf807\uf808\uf809\uf80a\uf80b\uf80c\uf80d\uf80e\uf80f\uf810\uf811\uf812\uf813\uf814\uf815\uf816\uf817\uf818\uf819\uf81a\uf81b\uf81c\uf81d\uf81e\uf81f\uf820\uf821\uf822\uf823\uf824\uf825\uf826\uf827\uf828\uf829\uf82a\uf82b\uf82c\uf82d\uf82e\uf82f\uf830\uf831\uf832\uf833\uf834\uf835\uf836\uf837\uf838\uf839\uf83a\uf83b\uf83c\uf83d\uf83e\uf83f\uf840\uf841\uf842\uf843\uf844\uf845\uf846\uf847\uf848\uf849\uf84a\uf84b\uf84c\uf84d\uf84e\uf84f\uf850\uf851\uf852\uf853\uf854\uf855\uf856\uf857\uf858\uf859\uf85a\uf85b\uf85c\uf85d\uf85e\uf85f\uf860\uf861\uf862\uf863\uf864\uf865\uf866\uf867\uf868\uf869\uf86a\uf86b\uf86c\uf86d\uf86e\uf86f\uf870\uf871\uf872\uf873\uf874\uf875\uf876\uf877\uf878\uf879\uf87a\uf87b\uf87c\uf87d\uf87e\uf87f\uf880\uf881\uf882\uf883\uf884\uf885\uf886\uf887\uf888\uf889\uf88a\uf88b\uf88c\uf88d\uf88e\uf88f\uf890\uf891\uf892\uf893\uf894\uf895\uf896\uf897\uf898\uf899\uf89a\uf89b\uf89c\uf89d\uf89e\uf89f\uf8a0\uf8a1\uf8a2\uf8a3\uf8a4\uf8a5\uf8a6\uf8a7\uf8a8\uf8a9\uf8aa\uf8ab\uf8ac\uf8ad\uf8ae\uf8af\uf8b0\uf8b1\uf8b2\uf8b3\uf8b4\uf8b5\uf8b6\uf8b7\uf8b8\uf8b9\uf8ba\uf8bb\uf8bc\uf8bd\uf8be\uf8bf\uf8c0\uf8c1\uf8c2\uf8c3\uf8c4\uf8c5\uf8c6\uf8c7\uf8c8\uf8c9\uf8ca\uf8cb\uf8cc\uf8cd\uf8ce\uf8cf\uf8d0\uf8d1\uf8d2\uf8d3\uf8d4\uf8d5\uf8d6\uf8d7\uf8d8\uf8d9\uf8da\uf8db\uf8dc\uf8dd\uf8de\uf8df\uf8e0\uf8e1\uf8e2\uf8e3\uf8e4\uf8e5\uf8e6\uf8e7\uf8e8\uf8e9\uf8ea\uf8eb\uf8ec\uf8ed\uf8ee\uf8ef\uf8f0\uf8f1\uf8f2\uf8f3\uf8f4\uf8f5\uf8f6\uf8f7\uf8f8\uf8f9\uf8fa\uf8fb\uf8fc\uf8fd\uf8fe\uf8ff'
+
+try:
+ Cs = eval(r"'\ud800\ud801\ud802\ud803\ud804\ud805\ud806\ud807\ud808\ud809\ud80a\ud80b\ud80c\ud80d\ud80e\ud80f\ud810\ud811\ud812\ud813\ud814\ud815\ud816\ud817\ud818\ud819\ud81a\ud81b\ud81c\ud81d\ud81e\ud81f\ud820\ud821\ud822\ud823\ud824\ud825\ud826\ud827\ud828\ud829\ud82a\ud82b\ud82c\ud82d\ud82e\ud82f\ud830\ud831\ud832\ud833\ud834\ud835\ud836\ud837\ud838\ud839\ud83a\ud83b\ud83c\ud83d\ud83e\ud83f\ud840\ud841\ud842\ud843\ud844\ud845\ud846\ud847\ud848\ud849\ud84a\ud84b\ud84c\ud84d\ud84e\ud84f\ud850\ud851\ud852\ud853\ud854\ud855\ud856\ud857\ud858\ud859\ud85a\ud85b\ud85c\ud85d\ud85e\ud85f\ud860\ud861\ud862\ud863\ud864\ud865\ud866\ud867\ud868\ud869\ud86a\ud86b\ud86c\ud86d\ud86e\ud86f\ud870\ud871\ud872\ud873\ud874\ud875\ud876\ud877\ud878\ud879\ud87a\ud87b\ud87c\ud87d\ud87e\ud87f\ud880\ud881\ud882\ud883\ud884\ud885\ud886\ud887\ud888\ud889\ud88a\ud88b\ud88c\ud88d\ud88e\ud88f\ud890\ud891\ud892\ud893\ud894\ud895\ud896\ud897\ud898\ud899\ud89a\ud89b\ud89c\ud89d\ud89e\ud89f\ud8a0\ud8a1\ud8a2\ud8a3\ud8a4\ud8a5\ud8a6\ud8a7\ud8a8\ud8a9\ud8aa\ud8ab\ud8ac\ud8ad\ud8ae\ud8af\ud8b0\ud8b1\ud8b2\ud8b3\ud8b4\ud8b5\ud8b6\ud8b7\ud8b8\ud8b9\ud8ba\ud8bb\ud8bc\ud8bd\ud8be\ud8bf\ud8c0\ud8c1\ud8c2\ud8c3\ud8c4\ud8c5\ud8c6\ud8c7\ud8c8\ud8c9\ud8ca\ud8cb\ud8cc\ud8cd\ud8ce\ud8cf\ud8d0\ud8d1\ud8d2\ud8d3\ud8d4\ud8d5\ud8d6\ud8d7\ud8d8\ud8d9\ud8da\ud8db\ud8dc\ud8dd\ud8de\ud8df\ud8e0\ud8e1\ud8e2\ud8e3\ud8e4\ud8e5\ud8e6\ud8e7\ud8e8\ud8e9\ud8ea\ud8eb\ud8ec\ud8ed\ud8ee\ud8ef\ud8f0\ud8f1\ud8f2\ud8f3\ud8f4\ud8f5\ud8f6\ud8f7\ud8f8\ud8f9\ud8fa\ud8fb\ud8fc\ud8fd\ud8fe\ud8ff\ud900\ud901\ud902\ud903\ud904\ud905\ud906\ud907\ud908\ud909\ud90a\ud90b\ud90c\ud90d\ud90e\ud90f\ud910\ud911\ud912\ud913\ud914\ud915\ud916\ud917\ud918\ud919\ud91a\ud91b\ud91c\ud91d\ud91e\ud91f\ud920\ud921\ud922\ud923\ud924\ud925\ud926\ud927\ud928\ud929\ud92a\ud92b\ud92c\ud92d\ud92e\ud92f\ud930\ud931\ud932\ud933\ud934\ud935\ud936\ud937\ud938\ud939\ud93a\ud93b\ud93c\ud93d\ud93e\ud93f\ud940\ud941\ud942\ud943\ud944\ud945\ud946\ud947\ud948\ud949\ud94a\ud94b\ud94c\ud94d\ud94e\ud94f\ud950\ud951\ud952\ud953\ud954\ud955\ud956\ud957\ud958\ud959\ud95a\ud95b\ud95c\ud95d\ud95e\ud95f\ud960\ud961\ud962\ud963\ud964\ud965\ud966\ud967\ud968\ud969\ud96a\ud96b\ud96c\ud96d\ud96e\ud96f\ud970\ud971\ud972\ud973\ud974\ud975\ud976\ud977\ud978\ud979\ud97a\ud97b\ud97c\ud97d\ud97e\ud97f\ud980\ud981\ud982\ud983\ud984\ud985\ud986\ud987\ud988\ud989\ud98a\ud98b\ud98c\ud98d\ud98e\ud98f\ud990\ud991\ud992\ud993\ud994\ud995\ud996\ud997\ud998\ud999\ud99a\ud99b\ud99c\ud99d\ud99e\ud99f\ud9a0\ud9a1\ud9a2\ud9a3\ud9a4\ud9a5\ud9a6\ud9a7\ud9a8\ud9a9\ud9aa\ud9ab\ud9ac\ud9ad\ud9ae\ud9af\ud9b0\ud9b1\ud9b2\ud9b3\ud9b4\ud9b5\ud9b6\ud9b7\ud9b8\ud9b9\ud9ba\ud9bb\ud9bc\ud9bd\ud9be\ud9bf\ud9c0\ud9c1\ud9c2\ud9c3\ud9c4\ud9c5\ud9c6\ud9c7\ud9c8\ud9c9\ud9ca\ud9cb\ud9cc\ud9cd\ud9ce\ud9cf\ud9d0\ud9d1\ud9d2\ud9d3\ud9d4\ud9d5\ud9d6\ud9d7\ud9d8\ud9d9\ud9da\ud9db\ud9dc\ud9dd\ud9de\ud9df\ud9e0\ud9e1\ud9e2\ud9e3\ud9e4\ud9e5\ud9e6\ud9e7\ud9e8\ud9e9\ud9ea\ud9eb\ud9ec\ud9ed\ud9ee\ud9ef\ud9f0\ud9f1\ud9f2\ud9f3\ud9f4\ud9f5\ud9f6\ud9f7\ud9f8\ud9f9\ud9fa\ud9fb\ud9fc\ud9fd\ud9fe\ud9ff\uda00\uda01\uda02\uda03\uda04\uda05\uda06\uda07\uda08\uda09\uda0a\uda0b\uda0c\uda0d\uda0e\uda0f\uda10\uda11\uda12\uda13\uda14\uda15\uda16\uda17\uda18\uda19\uda1a\uda1b\uda1c\uda1d\uda1e\uda1f\uda20\uda21\uda22\uda23\uda24\uda25\uda26\uda27\uda28\uda29\uda2a\uda2b\uda2c\uda2d\uda2e\uda2f\uda30\uda31\uda32\uda33\uda34\uda35\uda36\uda37\uda38\uda39\uda3a\uda3b\uda3c\uda3d\uda3e\uda3f\uda40\uda41\uda42\uda43\uda44\uda45\uda46\uda47\uda48\uda49\uda4a\uda4b\uda4c\uda4d\uda4e\uda4f\uda50\uda51\uda52\uda53\uda54\uda55\uda56\uda57\uda58\uda59\uda5a\uda5b\uda5c\uda5d\uda5e\uda5f\uda60\uda61\uda62\uda63\uda64\uda65\uda66\uda67\uda68\uda69\uda6a\uda6b\uda6c\uda6d\uda6e\uda6f\uda70\uda71\uda72\uda73\uda74\uda75\uda76\uda77\uda78\uda79\uda7a\uda7b\uda7c\uda7d\uda7e\uda7f\uda80\uda81\uda82\uda83\uda84\uda85\uda86\uda87\uda88\uda89\uda8a\uda8b\uda8c\uda8d\uda8e\uda8f\uda90\uda91\uda92\uda93\uda94\uda95\uda96\uda97\uda98\uda99\uda9a\uda9b\uda9c\uda9d\uda9e\uda9f\udaa0\udaa1\udaa2\udaa3\udaa4\udaa5\udaa6\udaa7\udaa8\udaa9\udaaa\udaab\udaac\udaad\udaae\udaaf\udab0\udab1\udab2\udab3\udab4\udab5\udab6\udab7\udab8\udab9\udaba\udabb\udabc\udabd\udabe\udabf\udac0\udac1\udac2\udac3\udac4\udac5\udac6\udac7\udac8\udac9\udaca\udacb\udacc\udacd\udace\udacf\udad0\udad1\udad2\udad3\udad4\udad5\udad6\udad7\udad8\udad9\udada\udadb\udadc\udadd\udade\udadf\udae0\udae1\udae2\udae3\udae4\udae5\udae6\udae7\udae8\udae9\udaea\udaeb\udaec\udaed\udaee\udaef\udaf0\udaf1\udaf2\udaf3\udaf4\udaf5\udaf6\udaf7\udaf8\udaf9\udafa\udafb\udafc\udafd\udafe\udaff\udb00\udb01\udb02\udb03\udb04\udb05\udb06\udb07\udb08\udb09\udb0a\udb0b\udb0c\udb0d\udb0e\udb0f\udb10\udb11\udb12\udb13\udb14\udb15\udb16\udb17\udb18\udb19\udb1a\udb1b\udb1c\udb1d\udb1e\udb1f\udb20\udb21\udb22\udb23\udb24\udb25\udb26\udb27\udb28\udb29\udb2a\udb2b\udb2c\udb2d\udb2e\udb2f\udb30\udb31\udb32\udb33\udb34\udb35\udb36\udb37\udb38\udb39\udb3a\udb3b\udb3c\udb3d\udb3e\udb3f\udb40\udb41\udb42\udb43\udb44\udb45\udb46\udb47\udb48\udb49\udb4a\udb4b\udb4c\udb4d\udb4e\udb4f\udb50\udb51\udb52\udb53\udb54\udb55\udb56\udb57\udb58\udb59\udb5a\udb5b\udb5c\udb5d\udb5e\udb5f\udb60\udb61\udb62\udb63\udb64\udb65\udb66\udb67\udb68\udb69\udb6a\udb6b\udb6c\udb6d\udb6e\udb6f\udb70\udb71\udb72\udb73\udb74\udb75\udb76\udb77\udb78\udb79\udb7a\udb7b\udb7c\udb7d\udb7e\udb7f\udb80\udb81\udb82\udb83\udb84\udb85\udb86\udb87\udb88\udb89\udb8a\udb8b\udb8c\udb8d\udb8e\udb8f\udb90\udb91\udb92\udb93\udb94\udb95\udb96\udb97\udb98\udb99\udb9a\udb9b\udb9c\udb9d\udb9e\udb9f\udba0\udba1\udba2\udba3\udba4\udba5\udba6\udba7\udba8\udba9\udbaa\udbab\udbac\udbad\udbae\udbaf\udbb0\udbb1\udbb2\udbb3\udbb4\udbb5\udbb6\udbb7\udbb8\udbb9\udbba\udbbb\udbbc\udbbd\udbbe\udbbf\udbc0\udbc1\udbc2\udbc3\udbc4\udbc5\udbc6\udbc7\udbc8\udbc9\udbca\udbcb\udbcc\udbcd\udbce\udbcf\udbd0\udbd1\udbd2\udbd3\udbd4\udbd5\udbd6\udbd7\udbd8\udbd9\udbda\udbdb\udbdc\udbdd\udbde\udbdf\udbe0\udbe1\udbe2\udbe3\udbe4\udbe5\udbe6\udbe7\udbe8\udbe9\udbea\udbeb\udbec\udbed\udbee\udbef\udbf0\udbf1\udbf2\udbf3\udbf4\udbf5\udbf6\udbf7\udbf8\udbf9\udbfa\udbfb\udbfc\udbfd\udbfe\U0010fc00\udc01\udc02\udc03\udc04\udc05\udc06\udc07\udc08\udc09\udc0a\udc0b\udc0c\udc0d\udc0e\udc0f\udc10\udc11\udc12\udc13\udc14\udc15\udc16\udc17\udc18\udc19\udc1a\udc1b\udc1c\udc1d\udc1e\udc1f\udc20\udc21\udc22\udc23\udc24\udc25\udc26\udc27\udc28\udc29\udc2a\udc2b\udc2c\udc2d\udc2e\udc2f\udc30\udc31\udc32\udc33\udc34\udc35\udc36\udc37\udc38\udc39\udc3a\udc3b\udc3c\udc3d\udc3e\udc3f\udc40\udc41\udc42\udc43\udc44\udc45\udc46\udc47\udc48\udc49\udc4a\udc4b\udc4c\udc4d\udc4e\udc4f\udc50\udc51\udc52\udc53\udc54\udc55\udc56\udc57\udc58\udc59\udc5a\udc5b\udc5c\udc5d\udc5e\udc5f\udc60\udc61\udc62\udc63\udc64\udc65\udc66\udc67\udc68\udc69\udc6a\udc6b\udc6c\udc6d\udc6e\udc6f\udc70\udc71\udc72\udc73\udc74\udc75\udc76\udc77\udc78\udc79\udc7a\udc7b\udc7c\udc7d\udc7e\udc7f\udc80\udc81\udc82\udc83\udc84\udc85\udc86\udc87\udc88\udc89\udc8a\udc8b\udc8c\udc8d\udc8e\udc8f\udc90\udc91\udc92\udc93\udc94\udc95\udc96\udc97\udc98\udc99\udc9a\udc9b\udc9c\udc9d\udc9e\udc9f\udca0\udca1\udca2\udca3\udca4\udca5\udca6\udca7\udca8\udca9\udcaa\udcab\udcac\udcad\udcae\udcaf\udcb0\udcb1\udcb2\udcb3\udcb4\udcb5\udcb6\udcb7\udcb8\udcb9\udcba\udcbb\udcbc\udcbd\udcbe\udcbf\udcc0\udcc1\udcc2\udcc3\udcc4\udcc5\udcc6\udcc7\udcc8\udcc9\udcca\udccb\udccc\udccd\udcce\udccf\udcd0\udcd1\udcd2\udcd3\udcd4\udcd5\udcd6\udcd7\udcd8\udcd9\udcda\udcdb\udcdc\udcdd\udcde\udcdf\udce0\udce1\udce2\udce3\udce4\udce5\udce6\udce7\udce8\udce9\udcea\udceb\udcec\udced\udcee\udcef\udcf0\udcf1\udcf2\udcf3\udcf4\udcf5\udcf6\udcf7\udcf8\udcf9\udcfa\udcfb\udcfc\udcfd\udcfe\udcff\udd00\udd01\udd02\udd03\udd04\udd05\udd06\udd07\udd08\udd09\udd0a\udd0b\udd0c\udd0d\udd0e\udd0f\udd10\udd11\udd12\udd13\udd14\udd15\udd16\udd17\udd18\udd19\udd1a\udd1b\udd1c\udd1d\udd1e\udd1f\udd20\udd21\udd22\udd23\udd24\udd25\udd26\udd27\udd28\udd29\udd2a\udd2b\udd2c\udd2d\udd2e\udd2f\udd30\udd31\udd32\udd33\udd34\udd35\udd36\udd37\udd38\udd39\udd3a\udd3b\udd3c\udd3d\udd3e\udd3f\udd40\udd41\udd42\udd43\udd44\udd45\udd46\udd47\udd48\udd49\udd4a\udd4b\udd4c\udd4d\udd4e\udd4f\udd50\udd51\udd52\udd53\udd54\udd55\udd56\udd57\udd58\udd59\udd5a\udd5b\udd5c\udd5d\udd5e\udd5f\udd60\udd61\udd62\udd63\udd64\udd65\udd66\udd67\udd68\udd69\udd6a\udd6b\udd6c\udd6d\udd6e\udd6f\udd70\udd71\udd72\udd73\udd74\udd75\udd76\udd77\udd78\udd79\udd7a\udd7b\udd7c\udd7d\udd7e\udd7f\udd80\udd81\udd82\udd83\udd84\udd85\udd86\udd87\udd88\udd89\udd8a\udd8b\udd8c\udd8d\udd8e\udd8f\udd90\udd91\udd92\udd93\udd94\udd95\udd96\udd97\udd98\udd99\udd9a\udd9b\udd9c\udd9d\udd9e\udd9f\udda0\udda1\udda2\udda3\udda4\udda5\udda6\udda7\udda8\udda9\uddaa\uddab\uddac\uddad\uddae\uddaf\uddb0\uddb1\uddb2\uddb3\uddb4\uddb5\uddb6\uddb7\uddb8\uddb9\uddba\uddbb\uddbc\uddbd\uddbe\uddbf\uddc0\uddc1\uddc2\uddc3\uddc4\uddc5\uddc6\uddc7\uddc8\uddc9\uddca\uddcb\uddcc\uddcd\uddce\uddcf\uddd0\uddd1\uddd2\uddd3\uddd4\uddd5\uddd6\uddd7\uddd8\uddd9\uddda\udddb\udddc\udddd\uddde\udddf\udde0\udde1\udde2\udde3\udde4\udde5\udde6\udde7\udde8\udde9\uddea\uddeb\uddec\udded\uddee\uddef\uddf0\uddf1\uddf2\uddf3\uddf4\uddf5\uddf6\uddf7\uddf8\uddf9\uddfa\uddfb\uddfc\uddfd\uddfe\uddff\ude00\ude01\ude02\ude03\ude04\ude05\ude06\ude07\ude08\ude09\ude0a\ude0b\ude0c\ude0d\ude0e\ude0f\ude10\ude11\ude12\ude13\ude14\ude15\ude16\ude17\ude18\ude19\ude1a\ude1b\ude1c\ude1d\ude1e\ude1f\ude20\ude21\ude22\ude23\ude24\ude25\ude26\ude27\ude28\ude29\ude2a\ude2b\ude2c\ude2d\ude2e\ude2f\ude30\ude31\ude32\ude33\ude34\ude35\ude36\ude37\ude38\ude39\ude3a\ude3b\ude3c\ude3d\ude3e\ude3f\ude40\ude41\ude42\ude43\ude44\ude45\ude46\ude47\ude48\ude49\ude4a\ude4b\ude4c\ude4d\ude4e\ude4f\ude50\ude51\ude52\ude53\ude54\ude55\ude56\ude57\ude58\ude59\ude5a\ude5b\ude5c\ude5d\ude5e\ude5f\ude60\ude61\ude62\ude63\ude64\ude65\ude66\ude67\ude68\ude69\ude6a\ude6b\ude6c\ude6d\ude6e\ude6f\ude70\ude71\ude72\ude73\ude74\ude75\ude76\ude77\ude78\ude79\ude7a\ude7b\ude7c\ude7d\ude7e\ude7f\ude80\ude81\ude82\ude83\ude84\ude85\ude86\ude87\ude88\ude89\ude8a\ude8b\ude8c\ude8d\ude8e\ude8f\ude90\ude91\ude92\ude93\ude94\ude95\ude96\ude97\ude98\ude99\ude9a\ude9b\ude9c\ude9d\ude9e\ude9f\udea0\udea1\udea2\udea3\udea4\udea5\udea6\udea7\udea8\udea9\udeaa\udeab\udeac\udead\udeae\udeaf\udeb0\udeb1\udeb2\udeb3\udeb4\udeb5\udeb6\udeb7\udeb8\udeb9\udeba\udebb\udebc\udebd\udebe\udebf\udec0\udec1\udec2\udec3\udec4\udec5\udec6\udec7\udec8\udec9\udeca\udecb\udecc\udecd\udece\udecf\uded0\uded1\uded2\uded3\uded4\uded5\uded6\uded7\uded8\uded9\udeda\udedb\udedc\udedd\udede\udedf\udee0\udee1\udee2\udee3\udee4\udee5\udee6\udee7\udee8\udee9\udeea\udeeb\udeec\udeed\udeee\udeef\udef0\udef1\udef2\udef3\udef4\udef5\udef6\udef7\udef8\udef9\udefa\udefb\udefc\udefd\udefe\udeff\udf00\udf01\udf02\udf03\udf04\udf05\udf06\udf07\udf08\udf09\udf0a\udf0b\udf0c\udf0d\udf0e\udf0f\udf10\udf11\udf12\udf13\udf14\udf15\udf16\udf17\udf18\udf19\udf1a\udf1b\udf1c\udf1d\udf1e\udf1f\udf20\udf21\udf22\udf23\udf24\udf25\udf26\udf27\udf28\udf29\udf2a\udf2b\udf2c\udf2d\udf2e\udf2f\udf30\udf31\udf32\udf33\udf34\udf35\udf36\udf37\udf38\udf39\udf3a\udf3b\udf3c\udf3d\udf3e\udf3f\udf40\udf41\udf42\udf43\udf44\udf45\udf46\udf47\udf48\udf49\udf4a\udf4b\udf4c\udf4d\udf4e\udf4f\udf50\udf51\udf52\udf53\udf54\udf55\udf56\udf57\udf58\udf59\udf5a\udf5b\udf5c\udf5d\udf5e\udf5f\udf60\udf61\udf62\udf63\udf64\udf65\udf66\udf67\udf68\udf69\udf6a\udf6b\udf6c\udf6d\udf6e\udf6f\udf70\udf71\udf72\udf73\udf74\udf75\udf76\udf77\udf78\udf79\udf7a\udf7b\udf7c\udf7d\udf7e\udf7f\udf80\udf81\udf82\udf83\udf84\udf85\udf86\udf87\udf88\udf89\udf8a\udf8b\udf8c\udf8d\udf8e\udf8f\udf90\udf91\udf92\udf93\udf94\udf95\udf96\udf97\udf98\udf99\udf9a\udf9b\udf9c\udf9d\udf9e\udf9f\udfa0\udfa1\udfa2\udfa3\udfa4\udfa5\udfa6\udfa7\udfa8\udfa9\udfaa\udfab\udfac\udfad\udfae\udfaf\udfb0\udfb1\udfb2\udfb3\udfb4\udfb5\udfb6\udfb7\udfb8\udfb9\udfba\udfbb\udfbc\udfbd\udfbe\udfbf\udfc0\udfc1\udfc2\udfc3\udfc4\udfc5\udfc6\udfc7\udfc8\udfc9\udfca\udfcb\udfcc\udfcd\udfce\udfcf\udfd0\udfd1\udfd2\udfd3\udfd4\udfd5\udfd6\udfd7\udfd8\udfd9\udfda\udfdb\udfdc\udfdd\udfde\udfdf\udfe0\udfe1\udfe2\udfe3\udfe4\udfe5\udfe6\udfe7\udfe8\udfe9\udfea\udfeb\udfec\udfed\udfee\udfef\udff0\udff1\udff2\udff3\udff4\udff5\udff6\udff7\udff8\udff9\udffa\udffb\udffc\udffd\udffe\udfff'")
+except UnicodeDecodeError:
+ Cs = '' # Jython can't handle isolated surrogates
+
+Ll = u'abcdefghijklmnopqrstuvwxyz\xaa\xb5\xba\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e\u017f\u0180\u0183\u0185\u0188\u018c\u018d\u0192\u0195\u0199\u019a\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9\u01ba\u01bd\u01be\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233\u0234\u0235\u0236\u0237\u0238\u0239\u023c\u023f\u0240\u0250\u0251\u0252\u0253\u0254\u0255\u0256\u0257\u0258\u0259\u025a\u025b\u025c\u025d\u025e\u025f\u0260\u0261\u0262\u0263\u0264\u0265\u0266\u0267\u0268\u0269\u026a\u026b\u026c\u026d\u026e\u026f\u0270\u0271\u0272\u0273\u0274\u0275\u0276\u0277\u0278\u0279\u027a\u027b\u027c\u027d\u027e\u027f\u0280\u0281\u0282\u0283\u0284\u0285\u0286\u0287\u0288\u0289\u028a\u028b\u028c\u028d\u028e\u028f\u0290\u0291\u0292\u0293\u0294\u0295\u0296\u0297\u0298\u0299\u029a\u029b\u029c\u029d\u029e\u029f\u02a0\u02a1\u02a2\u02a3\u02a4\u02a5\u02a6\u02a7\u02a8\u02a9\u02aa\u02ab\u02ac\u02ad\u02ae\u02af\u0390\u03ac\u03ad\u03ae\u03af\u03b0\u03b1\u03b2\u03b3\u03b4\u03b5\u03b6\u03b7\u03b8\u03b9\u03ba\u03bb\u03bc\u03bd\u03be\u03bf\u03c0\u03c1\u03c2\u03c3\u03c4\u03c5\u03c6\u03c7\u03c8\u03c9\u03ca\u03cb\u03cc\u03cd\u03ce\u03d0\u03d1\u03d5\u03d6\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef\u03f0\u03f1\u03f2\u03f3\u03f5\u03f8\u03fb\u03fc\u0430\u0431\u0432\u0433\u0434\u0435\u0436\u0437\u0438\u0439\u043a\u043b\u043c\u043d\u043e\u043f\u0440\u0441\u0442\u0443\u0444\u0445\u0446\u0447\u0448\u0449\u044a\u044b\u044c\u044d\u044e\u044f\u0450\u0451\u0452\u0453\u0454\u0455\u0456\u0457\u0458\u0459\u045a\u045b\u045c\u045d\u045e\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0561\u0562\u0563\u0564\u0565\u0566\u0567\u0568\u0569\u056a\u056b\u056c\u056d\u056e\u056f\u0570\u0571\u0572\u0573\u0574\u0575\u0576\u0577\u0578\u0579\u057a\u057b\u057c\u057d\u057e\u057f\u0580\u0581\u0582\u0583\u0584\u0585\u0586\u0587\u1d00\u1d01\u1d02\u1d03\u1d04\u1d05\u1d06\u1d07\u1d08\u1d09\u1d0a\u1d0b\u1d0c\u1d0d\u1d0e\u1d0f\u1d10\u1d11\u1d12\u1d13\u1d14\u1d15\u1d16\u1d17\u1d18\u1d19\u1d1a\u1d1b\u1d1c\u1d1d\u1d1e\u1d1f\u1d20\u1d21\u1d22\u1d23\u1d24\u1d25\u1d26\u1d27\u1d28\u1d29\u1d2a\u1d2b\u1d62\u1d63\u1d64\u1d65\u1d66\u1d67\u1d68\u1d69\u1d6a\u1d6b\u1d6c\u1d6d\u1d6e\u1d6f\u1d70\u1d71\u1d72\u1d73\u1d74\u1d75\u1d76\u1d77\u1d79\u1d7a\u1d7b\u1d7c\u1d7d\u1d7e\u1d7f\u1d80\u1d81\u1d82\u1d83\u1d84\u1d85\u1d86\u1d87\u1d88\u1d89\u1d8a\u1d8b\u1d8c\u1d8d\u1d8e\u1d8f\u1d90\u1d91\u1d92\u1d93\u1d94\u1d95\u1d96\u1d97\u1d98\u1d99\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95\u1e96\u1e97\u1e98\u1e99\u1e9a\u1e9b\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1f00\u1f01\u1f02\u1f03\u1f04\u1f05\u1f06\u1f07\u1f10\u1f11\u1f12\u1f13\u1f14\u1f15\u1f20\u1f21\u1f22\u1f23\u1f24\u1f25\u1f26\u1f27\u1f30\u1f31\u1f32\u1f33\u1f34\u1f35\u1f36\u1f37\u1f40\u1f41\u1f42\u1f43\u1f44\u1f45\u1f50\u1f51\u1f52\u1f53\u1f54\u1f55\u1f56\u1f57\u1f60\u1f61\u1f62\u1f63\u1f64\u1f65\u1f66\u1f67\u1f70\u1f71\u1f72\u1f73\u1f74\u1f75\u1f76\u1f77\u1f78\u1f79\u1f7a\u1f7b\u1f7c\u1f7d\u1f80\u1f81\u1f82\u1f83\u1f84\u1f85\u1f86\u1f87\u1f90\u1f91\u1f92\u1f93\u1f94\u1f95\u1f96\u1f97\u1fa0\u1fa1\u1fa2\u1fa3\u1fa4\u1fa5\u1fa6\u1fa7\u1fb0\u1fb1\u1fb2\u1fb3\u1fb4\u1fb6\u1fb7\u1fbe\u1fc2\u1fc3\u1fc4\u1fc6\u1fc7\u1fd0\u1fd1\u1fd2\u1fd3\u1fd6\u1fd7\u1fe0\u1fe1\u1fe2\u1fe3\u1fe4\u1fe5\u1fe6\u1fe7\u1ff2\u1ff3\u1ff4\u1ff6\u1ff7\u2071\u207f\u210a\u210e\u210f\u2113\u212f\u2134\u2139\u213c\u213d\u2146\u2147\u2148\u2149\u2c30\u2c31\u2c32\u2c33\u2c34\u2c35\u2c36\u2c37\u2c38\u2c39\u2c3a\u2c3b\u2c3c\u2c3d\u2c3e\u2c3f\u2c40\u2c41\u2c42\u2c43\u2c44\u2c45\u2c46\u2c47\u2c48\u2c49\u2c4a\u2c4b\u2c4c\u2c4d\u2c4e\u2c4f\u2c50\u2c51\u2c52\u2c53\u2c54\u2c55\u2c56\u2c57\u2c58\u2c59\u2c5a\u2c5b\u2c5c\u2c5d\u2c5e\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3\u2ce4\u2d00\u2d01\u2d02\u2d03\u2d04\u2d05\u2d06\u2d07\u2d08\u2d09\u2d0a\u2d0b\u2d0c\u2d0d\u2d0e\u2d0f\u2d10\u2d11\u2d12\u2d13\u2d14\u2d15\u2d16\u2d17\u2d18\u2d19\u2d1a\u2d1b\u2d1c\u2d1d\u2d1e\u2d1f\u2d20\u2d21\u2d22\u2d23\u2d24\u2d25\ufb00\ufb01\ufb02\ufb03\ufb04\ufb05\ufb06\ufb13\ufb14\ufb15\ufb16\ufb17\uff41\uff42\uff43\uff44\uff45\uff46\uff47\uff48\uff49\uff4a\uff4b\uff4c\uff4d\uff4e\uff4f\uff50\uff51\uff52\uff53\uff54\uff55\uff56\uff57\uff58\uff59\uff5a'
+
+Lm = u'\u02b0\u02b1\u02b2\u02b3\u02b4\u02b5\u02b6\u02b7\u02b8\u02b9\u02ba\u02bb\u02bc\u02bd\u02be\u02bf\u02c0\u02c1\u02c6\u02c7\u02c8\u02c9\u02ca\u02cb\u02cc\u02cd\u02ce\u02cf\u02d0\u02d1\u02e0\u02e1\u02e2\u02e3\u02e4\u02ee\u037a\u0559\u0640\u06e5\u06e6\u0e46\u0ec6\u10fc\u17d7\u1843\u1d2c\u1d2d\u1d2e\u1d2f\u1d30\u1d31\u1d32\u1d33\u1d34\u1d35\u1d36\u1d37\u1d38\u1d39\u1d3a\u1d3b\u1d3c\u1d3d\u1d3e\u1d3f\u1d40\u1d41\u1d42\u1d43\u1d44\u1d45\u1d46\u1d47\u1d48\u1d49\u1d4a\u1d4b\u1d4c\u1d4d\u1d4e\u1d4f\u1d50\u1d51\u1d52\u1d53\u1d54\u1d55\u1d56\u1d57\u1d58\u1d59\u1d5a\u1d5b\u1d5c\u1d5d\u1d5e\u1d5f\u1d60\u1d61\u1d78\u1d9b\u1d9c\u1d9d\u1d9e\u1d9f\u1da0\u1da1\u1da2\u1da3\u1da4\u1da5\u1da6\u1da7\u1da8\u1da9\u1daa\u1dab\u1dac\u1dad\u1dae\u1daf\u1db0\u1db1\u1db2\u1db3\u1db4\u1db5\u1db6\u1db7\u1db8\u1db9\u1dba\u1dbb\u1dbc\u1dbd\u1dbe\u1dbf\u2090\u2091\u2092\u2093\u2094\u2d6f\u3005\u3031\u3032\u3033\u3034\u3035\u303b\u309d\u309e\u30fc\u30fd\u30fe\ua015\uff70\uff9e\uff9f'
+
+Lo = u'\u01bb\u01c0\u01c1\u01c2\u01c3\u05d0\u05d1\u05d2\u05d3\u05d4\u05d5\u05d6\u05d7\u05d8\u05d9\u05da\u05db\u05dc\u05dd\u05de\u05df\u05e0\u05e1\u05e2\u05e3\u05e4\u05e5\u05e6\u05e7\u05e8\u05e9\u05ea\u05f0\u05f1\u05f2\u0621\u0622\u0623\u0624\u0625\u0626\u0627\u0628\u0629\u062a\u062b\u062c\u062d\u062e\u062f\u0630\u0631\u0632\u0633\u0634\u0635\u0636\u0637\u0638\u0639\u063a\u0641\u0642\u0643\u0644\u0645\u0646\u0647\u0648\u0649\u064a\u066e\u066f\u0671\u0672\u0673\u0674\u0675\u0676\u0677\u0678\u0679\u067a\u067b\u067c\u067d\u067e\u067f\u0680\u0681\u0682\u0683\u0684\u0685\u0686\u0687\u0688\u0689\u068a\u068b\u068c\u068d\u068e\u068f\u0690\u0691\u0692\u0693\u0694\u0695\u0696\u0697\u0698\u0699\u069a\u069b\u069c\u069d\u069e\u069f\u06a0\u06a1\u06a2\u06a3\u06a4\u06a5\u06a6\u06a7\u06a8\u06a9\u06aa\u06ab\u06ac\u06ad\u06ae\u06af\u06b0\u06b1\u06b2\u06b3\u06b4\u06b5\u06b6\u06b7\u06b8\u06b9\u06ba\u06bb\u06bc\u06bd\u06be\u06bf\u06c0\u06c1\u06c2\u06c3\u06c4\u06c5\u06c6\u06c7\u06c8\u06c9\u06ca\u06cb\u06cc\u06cd\u06ce\u06cf\u06d0\u06d1\u06d2\u06d3\u06d5\u06ee\u06ef\u06fa\u06fb\u06fc\u06ff\u0710\u0712\u0713\u0714\u0715\u0716\u0717\u0718\u0719\u071a\u071b\u071c\u071d\u071e\u071f\u0720\u0721\u0722\u0723\u0724\u0725\u0726\u0727\u0728\u0729\u072a\u072b\u072c\u072d\u072e\u072f\u074d\u074e\u074f\u0750\u0751\u0752\u0753\u0754\u0755\u0756\u0757\u0758\u0759\u075a\u075b\u075c\u075d\u075e\u075f\u0760\u0761\u0762\u0763\u0764\u0765\u0766\u0767\u0768\u0769\u076a\u076b\u076c\u076d\u0780\u0781\u0782\u0783\u0784\u0785\u0786\u0787\u0788\u0789\u078a\u078b\u078c\u078d\u078e\u078f\u0790\u0791\u0792\u0793\u0794\u0795\u0796\u0797\u0798\u0799\u079a\u079b\u079c\u079d\u079e\u079f\u07a0\u07a1\u07a2\u07a3\u07a4\u07a5\u07b1\u0904\u0905\u0906\u0907\u0908\u0909\u090a\u090b\u090c\u090d\u090e\u090f\u0910\u0911\u0912\u0913\u0914\u0915\u0916\u0917\u0918\u0919\u091a\u091b\u091c\u091d\u091e\u091f\u0920\u0921\u0922\u0923\u0924\u0925\u0926\u0927\u0928\u0929\u092a\u092b\u092c\u092d\u092e\u092f\u0930\u0931\u0932\u0933\u0934\u0935\u0936\u0937\u0938\u0939\u093d\u0950\u0958\u0959\u095a\u095b\u095c\u095d\u095e\u095f\u0960\u0961\u097d\u0985\u0986\u0987\u0988\u0989\u098a\u098b\u098c\u098f\u0990\u0993\u0994\u0995\u0996\u0997\u0998\u0999\u099a\u099b\u099c\u099d\u099e\u099f\u09a0\u09a1\u09a2\u09a3\u09a4\u09a5\u09a6\u09a7\u09a8\u09aa\u09ab\u09ac\u09ad\u09ae\u09af\u09b0\u09b2\u09b6\u09b7\u09b8\u09b9\u09bd\u09ce\u09dc\u09dd\u09df\u09e0\u09e1\u09f0\u09f1\u0a05\u0a06\u0a07\u0a08\u0a09\u0a0a\u0a0f\u0a10\u0a13\u0a14\u0a15\u0a16\u0a17\u0a18\u0a19\u0a1a\u0a1b\u0a1c\u0a1d\u0a1e\u0a1f\u0a20\u0a21\u0a22\u0a23\u0a24\u0a25\u0a26\u0a27\u0a28\u0a2a\u0a2b\u0a2c\u0a2d\u0a2e\u0a2f\u0a30\u0a32\u0a33\u0a35\u0a36\u0a38\u0a39\u0a59\u0a5a\u0a5b\u0a5c\u0a5e\u0a72\u0a73\u0a74\u0a85\u0a86\u0a87\u0a88\u0a89\u0a8a\u0a8b\u0a8c\u0a8d\u0a8f\u0a90\u0a91\u0a93\u0a94\u0a95\u0a96\u0a97\u0a98\u0a99\u0a9a\u0a9b\u0a9c\u0a9d\u0a9e\u0a9f\u0aa0\u0aa1\u0aa2\u0aa3\u0aa4\u0aa5\u0aa6\u0aa7\u0aa8\u0aaa\u0aab\u0aac\u0aad\u0aae\u0aaf\u0ab0\u0ab2\u0ab3\u0ab5\u0ab6\u0ab7\u0ab8\u0ab9\u0abd\u0ad0\u0ae0\u0ae1\u0b05\u0b06\u0b07\u0b08\u0b09\u0b0a\u0b0b\u0b0c\u0b0f\u0b10\u0b13\u0b14\u0b15\u0b16\u0b17\u0b18\u0b19\u0b1a\u0b1b\u0b1c\u0b1d\u0b1e\u0b1f\u0b20\u0b21\u0b22\u0b23\u0b24\u0b25\u0b26\u0b27\u0b28\u0b2a\u0b2b\u0b2c\u0b2d\u0b2e\u0b2f\u0b30\u0b32\u0b33\u0b35\u0b36\u0b37\u0b38\u0b39\u0b3d\u0b5c\u0b5d\u0b5f\u0b60\u0b61\u0b71\u0b83\u0b85\u0b86\u0b87\u0b88\u0b89\u0b8a\u0b8e\u0b8f\u0b90\u0b92\u0b93\u0b94\u0b95\u0b99\u0b9a\u0b9c\u0b9e\u0b9f\u0ba3\u0ba4\u0ba8\u0ba9\u0baa\u0bae\u0baf\u0bb0\u0bb1\u0bb2\u0bb3\u0bb4\u0bb5\u0bb6\u0bb7\u0bb8\u0bb9\u0c05\u0c06\u0c07\u0c08\u0c09\u0c0a\u0c0b\u0c0c\u0c0e\u0c0f\u0c10\u0c12\u0c13\u0c14\u0c15\u0c16\u0c17\u0c18\u0c19\u0c1a\u0c1b\u0c1c\u0c1d\u0c1e\u0c1f\u0c20\u0c21\u0c22\u0c23\u0c24\u0c25\u0c26\u0c27\u0c28\u0c2a\u0c2b\u0c2c\u0c2d\u0c2e\u0c2f\u0c30\u0c31\u0c32\u0c33\u0c35\u0c36\u0c37\u0c38\u0c39\u0c60\u0c61\u0c85\u0c86\u0c87\u0c88\u0c89\u0c8a\u0c8b\u0c8c\u0c8e\u0c8f\u0c90\u0c92\u0c93\u0c94\u0c95\u0c96\u0c97\u0c98\u0c99\u0c9a\u0c9b\u0c9c\u0c9d\u0c9e\u0c9f\u0ca0\u0ca1\u0ca2\u0ca3\u0ca4\u0ca5\u0ca6\u0ca7\u0ca8\u0caa\u0cab\u0cac\u0cad\u0cae\u0caf\u0cb0\u0cb1\u0cb2\u0cb3\u0cb5\u0cb6\u0cb7\u0cb8\u0cb9\u0cbd\u0cde\u0ce0\u0ce1\u0d05\u0d06\u0d07\u0d08\u0d09\u0d0a\u0d0b\u0d0c\u0d0e\u0d0f\u0d10\u0d12\u0d13\u0d14\u0d15\u0d16\u0d17\u0d18\u0d19\u0d1a\u0d1b\u0d1c\u0d1d\u0d1e\u0d1f\u0d20\u0d21\u0d22\u0d23\u0d24\u0d25\u0d26\u0d27\u0d28\u0d2a\u0d2b\u0d2c\u0d2d\u0d2e\u0d2f\u0d30\u0d31\u0d32\u0d33\u0d34\u0d35\u0d36\u0d37\u0d38\u0d39\u0d60\u0d61\u0d85\u0d86\u0d87\u0d88\u0d89\u0d8a\u0d8b\u0d8c\u0d8d\u0d8e\u0d8f\u0d90\u0d91\u0d92\u0d93\u0d94\u0d95\u0d96\u0d9a\u0d9b\u0d9c\u0d9d\u0d9e\u0d9f\u0da0\u0da1\u0da2\u0da3\u0da4\u0da5\u0da6\u0da7\u0da8\u0da9\u0daa\u0dab\u0dac\u0dad\u0dae\u0daf\u0db0\u0db1\u0db3\u0db4\u0db5\u0db6\u0db7\u0db8\u0db9\u0dba\u0dbb\u0dbd\u0dc0\u0dc1\u0dc2\u0dc3\u0dc4\u0dc5\u0dc6\u0e01\u0e02\u0e03\u0e04\u0e05\u0e06\u0e07\u0e08\u0e09\u0e0a\u0e0b\u0e0c\u0e0d\u0e0e\u0e0f\u0e10\u0e11\u0e12\u0e13\u0e14\u0e15\u0e16\u0e17\u0e18\u0e19\u0e1a\u0e1b\u0e1c\u0e1d\u0e1e\u0e1f\u0e20\u0e21\u0e22\u0e23\u0e24\u0e25\u0e26\u0e27\u0e28\u0e29\u0e2a\u0e2b\u0e2c\u0e2d\u0e2e\u0e2f\u0e30\u0e32\u0e33\u0e40\u0e41\u0e42\u0e43\u0e44\u0e45\u0e81\u0e82\u0e84\u0e87\u0e88\u0e8a\u0e8d\u0e94\u0e95\u0e96\u0e97\u0e99\u0e9a\u0e9b\u0e9c\u0e9d\u0e9e\u0e9f\u0ea1\u0ea2\u0ea3\u0ea5\u0ea7\u0eaa\u0eab\u0ead\u0eae\u0eaf\u0eb0\u0eb2\u0eb3\u0ebd\u0ec0\u0ec1\u0ec2\u0ec3\u0ec4\u0edc\u0edd\u0f00\u0f40\u0f41\u0f42\u0f43\u0f44\u0f45\u0f46\u0f47\u0f49\u0f4a\u0f4b\u0f4c\u0f4d\u0f4e\u0f4f\u0f50\u0f51\u0f52\u0f53\u0f54\u0f55\u0f56\u0f57\u0f58\u0f59\u0f5a\u0f5b\u0f5c\u0f5d\u0f5e\u0f5f\u0f60\u0f61\u0f62\u0f63\u0f64\u0f65\u0f66\u0f67\u0f68\u0f69\u0f6a\u0f88\u0f89\u0f8a\u0f8b\u1000\u1001\u1002\u1003\u1004\u1005\u1006\u1007\u1008\u1009\u100a\u100b\u100c\u100d\u100e\u100f\u1010\u1011\u1012\u1013\u1014\u1015\u1016\u1017\u1018\u1019\u101a\u101b\u101c\u101d\u101e\u101f\u1020\u1021\u1023\u1024\u1025\u1026\u1027\u1029\u102a\u1050\u1051\u1052\u1053\u1054\u1055\u10d0\u10d1\u10d2\u10d3\u10d4\u10d5\u10d6\u10d7\u10d8\u10d9\u10da\u10db\u10dc\u10dd\u10de\u10df\u10e0\u10e1\u10e2\u10e3\u10e4\u10e5\u10e6\u10e7\u10e8\u10e9\u10ea\u10eb\u10ec\u10ed\u10ee\u10ef\u10f0\u10f1\u10f2\u10f3\u10f4\u10f5\u10f6\u10f7\u10f8\u10f9\u10fa\u1100\u1101\u1102\u1103\u1104\u1105\u1106\u1107\u1108\u1109\u110a\u110b\u110c\u110d\u110e\u110f\u1110\u1111\u1112\u1113\u1114\u1115\u1116\u1117\u1118\u1119\u111a\u111b\u111c\u111d\u111e\u111f\u1120\u1121\u1122\u1123\u1124\u1125\u1126\u1127\u1128\u1129\u112a\u112b\u112c\u112d\u112e\u112f\u1130\u1131\u1132\u1133\u1134\u1135\u1136\u1137\u1138\u1139\u113a\u113b\u113c\u113d\u113e\u113f\u1140\u1141\u1142\u1143\u1144\u1145\u1146\u1147\u1148\u1149\u114a\u114b\u114c\u114d\u114e\u114f\u1150\u1151\u1152\u1153\u1154\u1155\u1156\u1157\u1158\u1159\u115f\u1160\u1161\u1162\u1163\u1164\u1165\u1166\u1167\u1168\u1169\u116a\u116b\u116c\u116d\u116e\u116f\u1170\u1171\u1172\u1173\u1174\u1175\u1176\u1177\u1178\u1179\u117a\u117b\u117c\u117d\u117e\u117f\u1180\u1181\u1182\u1183\u1184\u1185\u1186\u1187\u1188\u1189\u118a\u118b\u118c\u118d\u118e\u118f\u1190\u1191\u1192\u1193\u1194\u1195\u1196\u1197\u1198\u1199\u119a\u119b\u119c\u119d\u119e\u119f\u11a0\u11a1\u11a2\u11a8\u11a9\u11aa\u11ab\u11ac\u11ad\u11ae\u11af\u11b0\u11b1\u11b2\u11b3\u11b4\u11b5\u11b6\u11b7\u11b8\u11b9\u11ba\u11bb\u11bc\u11bd\u11be\u11bf\u11c0\u11c1\u11c2\u11c3\u11c4\u11c5\u11c6\u11c7\u11c8\u11c9\u11ca\u11cb\u11cc\u11cd\u11ce\u11cf\u11d0\u11d1\u11d2\u11d3\u11d4\u11d5\u11d6\u11d7\u11d8\u11d9\u11da\u11db\u11dc\u11dd\u11de\u11df\u11e0\u11e1\u11e2\u11e3\u11e4\u11e5\u11e6\u11e7\u11e8\u11e9\u11ea\u11eb\u11ec\u11ed\u11ee\u11ef\u11f0\u11f1\u11f2\u11f3\u11f4\u11f5\u11f6\u11f7\u11f8\u11f9\u1200\u1201\u1202\u1203\u1204\u1205\u1206\u1207\u1208\u1209\u120a\u120b\u120c\u120d\u120e\u120f\u1210\u1211\u1212\u1213\u1214\u1215\u1216\u1217\u1218\u1219\u121a\u121b\u121c\u121d\u121e\u121f\u1220\u1221\u1222\u1223\u1224\u1225\u1226\u1227\u1228\u1229\u122a\u122b\u122c\u122d\u122e\u122f\u1230\u1231\u1232\u1233\u1234\u1235\u1236\u1237\u1238\u1239\u123a\u123b\u123c\u123d\u123e\u123f\u1240\u1241\u1242\u1243\u1244\u1245\u1246\u1247\u1248\u124a\u124b\u124c\u124d\u1250\u1251\u1252\u1253\u1254\u1255\u1256\u1258\u125a\u125b\u125c\u125d\u1260\u1261\u1262\u1263\u1264\u1265\u1266\u1267\u1268\u1269\u126a\u126b\u126c\u126d\u126e\u126f\u1270\u1271\u1272\u1273\u1274\u1275\u1276\u1277\u1278\u1279\u127a\u127b\u127c\u127d\u127e\u127f\u1280\u1281\u1282\u1283\u1284\u1285\u1286\u1287\u1288\u128a\u128b\u128c\u128d\u1290\u1291\u1292\u1293\u1294\u1295\u1296\u1297\u1298\u1299\u129a\u129b\u129c\u129d\u129e\u129f\u12a0\u12a1\u12a2\u12a3\u12a4\u12a5\u12a6\u12a7\u12a8\u12a9\u12aa\u12ab\u12ac\u12ad\u12ae\u12af\u12b0\u12b2\u12b3\u12b4\u12b5\u12b8\u12b9\u12ba\u12bb\u12bc\u12bd\u12be\u12c0\u12c2\u12c3\u12c4\u12c5\u12c8\u12c9\u12ca\u12cb\u12cc\u12cd\u12ce\u12cf\u12d0\u12d1\u12d2\u12d3\u12d4\u12d5\u12d6\u12d8\u12d9\u12da\u12db\u12dc\u12dd\u12de\u12df\u12e0\u12e1\u12e2\u12e3\u12e4\u12e5\u12e6\u12e7\u12e8\u12e9\u12ea\u12eb\u12ec\u12ed\u12ee\u12ef\u12f0\u12f1\u12f2\u12f3\u12f4\u12f5\u12f6\u12f7\u12f8\u12f9\u12fa\u12fb\u12fc\u12fd\u12fe\u12ff\u1300\u1301\u1302\u1303\u1304\u1305\u1306\u1307\u1308\u1309\u130a\u130b\u130c\u130d\u130e\u130f\u1310\u1312\u1313\u1314\u1315\u1318\u1319\u131a\u131b\u131c\u131d\u131e\u131f\u1320\u1321\u1322\u1323\u1324\u1325\u1326\u1327\u1328\u1329\u132a\u132b\u132c\u132d\u132e\u132f\u1330\u1331\u1332\u1333\u1334\u1335\u1336\u1337\u1338\u1339\u133a\u133b\u133c\u133d\u133e\u133f\u1340\u1341\u1342\u1343\u1344\u1345\u1346\u1347\u1348\u1349\u134a\u134b\u134c\u134d\u134e\u134f\u1350\u1351\u1352\u1353\u1354\u1355\u1356\u1357\u1358\u1359\u135a\u1380\u1381\u1382\u1383\u1384\u1385\u1386\u1387\u1388\u1389\u138a\u138b\u138c\u138d\u138e\u138f\u13a0\u13a1\u13a2\u13a3\u13a4\u13a5\u13a6\u13a7\u13a8\u13a9\u13aa\u13ab\u13ac\u13ad\u13ae\u13af\u13b0\u13b1\u13b2\u13b3\u13b4\u13b5\u13b6\u13b7\u13b8\u13b9\u13ba\u13bb\u13bc\u13bd\u13be\u13bf\u13c0\u13c1\u13c2\u13c3\u13c4\u13c5\u13c6\u13c7\u13c8\u13c9\u13ca\u13cb\u13cc\u13cd\u13ce\u13cf\u13d0\u13d1\u13d2\u13d3\u13d4\u13d5\u13d6\u13d7\u13d8\u13d9\u13da\u13db\u13dc\u13dd\u13de\u13df\u13e0\u13e1\u13e2\u13e3\u13e4\u13e5\u13e6\u13e7\u13e8\u13e9\u13ea\u13eb\u13ec\u13ed\u13ee\u13ef\u13f0\u13f1\u13f2\u13f3\u13f4\u1401\u1402\u1403\u1404\u1405\u1406\u1407\u1408\u1409\u140a\u140b\u140c\u140d\u140e\u140f\u1410\u1411\u1412\u1413\u1414\u1415\u1416\u1417\u1418\u1419\u141a\u141b\u141c\u141d\u141e\u141f\u1420\u1421\u1422\u1423\u1424\u1425\u1426\u1427\u1428\u1429\u142a\u142b\u142c\u142d\u142e\u142f\u1430\u1431\u1432\u1433\u1434\u1435\u1436\u1437\u1438\u1439\u143a\u143b\u143c\u143d\u143e\u143f\u1440\u1441\u1442\u1443\u1444\u1445\u1446\u1447\u1448\u1449\u144a\u144b\u144c\u144d\u144e\u144f\u1450\u1451\u1452\u1453\u1454\u1455\u1456\u1457\u1458\u1459\u145a\u145b\u145c\u145d\u145e\u145f\u1460\u1461\u1462\u1463\u1464\u1465\u1466\u1467\u1468\u1469\u146a\u146b\u146c\u146d\u146e\u146f\u1470\u1471\u1472\u1473\u1474\u1475\u1476\u1477\u1478\u1479\u147a\u147b\u147c\u147d\u147e\u147f\u1480\u1481\u1482\u1483\u1484\u1485\u1486\u1487\u1488\u1489\u148a\u148b\u148c\u148d\u148e\u148f\u1490\u1491\u1492\u1493\u1494\u1495\u1496\u1497\u1498\u1499\u149a\u149b\u149c\u149d\u149e\u149f\u14a0\u14a1\u14a2\u14a3\u14a4\u14a5\u14a6\u14a7\u14a8\u14a9\u14aa\u14ab\u14ac\u14ad\u14ae\u14af\u14b0\u14b1\u14b2\u14b3\u14b4\u14b5\u14b6\u14b7\u14b8\u14b9\u14ba\u14bb\u14bc\u14bd\u14be\u14bf\u14c0\u14c1\u14c2\u14c3\u14c4\u14c5\u14c6\u14c7\u14c8\u14c9\u14ca\u14cb\u14cc\u14cd\u14ce\u14cf\u14d0\u14d1\u14d2\u14d3\u14d4\u14d5\u14d6\u14d7\u14d8\u14d9\u14da\u14db\u14dc\u14dd\u14de\u14df\u14e0\u14e1\u14e2\u14e3\u14e4\u14e5\u14e6\u14e7\u14e8\u14e9\u14ea\u14eb\u14ec\u14ed\u14ee\u14ef\u14f0\u14f1\u14f2\u14f3\u14f4\u14f5\u14f6\u14f7\u14f8\u14f9\u14fa\u14fb\u14fc\u14fd\u14fe\u14ff\u1500\u1501\u1502\u1503\u1504\u1505\u1506\u1507\u1508\u1509\u150a\u150b\u150c\u150d\u150e\u150f\u1510\u1511\u1512\u1513\u1514\u1515\u1516\u1517\u1518\u1519\u151a\u151b\u151c\u151d\u151e\u151f\u1520\u1521\u1522\u1523\u1524\u1525\u1526\u1527\u1528\u1529\u152a\u152b\u152c\u152d\u152e\u152f\u1530\u1531\u1532\u1533\u1534\u1535\u1536\u1537\u1538\u1539\u153a\u153b\u153c\u153d\u153e\u153f\u1540\u1541\u1542\u1543\u1544\u1545\u1546\u1547\u1548\u1549\u154a\u154b\u154c\u154d\u154e\u154f\u1550\u1551\u1552\u1553\u1554\u1555\u1556\u1557\u1558\u1559\u155a\u155b\u155c\u155d\u155e\u155f\u1560\u1561\u1562\u1563\u1564\u1565\u1566\u1567\u1568\u1569\u156a\u156b\u156c\u156d\u156e\u156f\u1570\u1571\u1572\u1573\u1574\u1575\u1576\u1577\u1578\u1579\u157a\u157b\u157c\u157d\u157e\u157f\u1580\u1581\u1582\u1583\u1584\u1585\u1586\u1587\u1588\u1589\u158a\u158b\u158c\u158d\u158e\u158f\u1590\u1591\u1592\u1593\u1594\u1595\u1596\u1597\u1598\u1599\u159a\u159b\u159c\u159d\u159e\u159f\u15a0\u15a1\u15a2\u15a3\u15a4\u15a5\u15a6\u15a7\u15a8\u15a9\u15aa\u15ab\u15ac\u15ad\u15ae\u15af\u15b0\u15b1\u15b2\u15b3\u15b4\u15b5\u15b6\u15b7\u15b8\u15b9\u15ba\u15bb\u15bc\u15bd\u15be\u15bf\u15c0\u15c1\u15c2\u15c3\u15c4\u15c5\u15c6\u15c7\u15c8\u15c9\u15ca\u15cb\u15cc\u15cd\u15ce\u15cf\u15d0\u15d1\u15d2\u15d3\u15d4\u15d5\u15d6\u15d7\u15d8\u15d9\u15da\u15db\u15dc\u15dd\u15de\u15df\u15e0\u15e1\u15e2\u15e3\u15e4\u15e5\u15e6\u15e7\u15e8\u15e9\u15ea\u15eb\u15ec\u15ed\u15ee\u15ef\u15f0\u15f1\u15f2\u15f3\u15f4\u15f5\u15f6\u15f7\u15f8\u15f9\u15fa\u15fb\u15fc\u15fd\u15fe\u15ff\u1600\u1601\u1602\u1603\u1604\u1605\u1606\u1607\u1608\u1609\u160a\u160b\u160c\u160d\u160e\u160f\u1610\u1611\u1612\u1613\u1614\u1615\u1616\u1617\u1618\u1619\u161a\u161b\u161c\u161d\u161e\u161f\u1620\u1621\u1622\u1623\u1624\u1625\u1626\u1627\u1628\u1629\u162a\u162b\u162c\u162d\u162e\u162f\u1630\u1631\u1632\u1633\u1634\u1635\u1636\u1637\u1638\u1639\u163a\u163b\u163c\u163d\u163e\u163f\u1640\u1641\u1642\u1643\u1644\u1645\u1646\u1647\u1648\u1649\u164a\u164b\u164c\u164d\u164e\u164f\u1650\u1651\u1652\u1653\u1654\u1655\u1656\u1657\u1658\u1659\u165a\u165b\u165c\u165d\u165e\u165f\u1660\u1661\u1662\u1663\u1664\u1665\u1666\u1667\u1668\u1669\u166a\u166b\u166c\u166f\u1670\u1671\u1672\u1673\u1674\u1675\u1676\u1681\u1682\u1683\u1684\u1685\u1686\u1687\u1688\u1689\u168a\u168b\u168c\u168d\u168e\u168f\u1690\u1691\u1692\u1693\u1694\u1695\u1696\u1697\u1698\u1699\u169a\u16a0\u16a1\u16a2\u16a3\u16a4\u16a5\u16a6\u16a7\u16a8\u16a9\u16aa\u16ab\u16ac\u16ad\u16ae\u16af\u16b0\u16b1\u16b2\u16b3\u16b4\u16b5\u16b6\u16b7\u16b8\u16b9\u16ba\u16bb\u16bc\u16bd\u16be\u16bf\u16c0\u16c1\u16c2\u16c3\u16c4\u16c5\u16c6\u16c7\u16c8\u16c9\u16ca\u16cb\u16cc\u16cd\u16ce\u16cf\u16d0\u16d1\u16d2\u16d3\u16d4\u16d5\u16d6\u16d7\u16d8\u16d9\u16da\u16db\u16dc\u16dd\u16de\u16df\u16e0\u16e1\u16e2\u16e3\u16e4\u16e5\u16e6\u16e7\u16e8\u16e9\u16ea\u1700\u1701\u1702\u1703\u1704\u1705\u1706\u1707\u1708\u1709\u170a\u170b\u170c\u170e\u170f\u1710\u1711\u1720\u1721\u1722\u1723\u1724\u1725\u1726\u1727\u1728\u1729\u172a\u172b\u172c\u172d\u172e\u172f\u1730\u1731\u1740\u1741\u1742\u1743\u1744\u1745\u1746\u1747\u1748\u1749\u174a\u174b\u174c\u174d\u174e\u174f\u1750\u1751\u1760\u1761\u1762\u1763\u1764\u1765\u1766\u1767\u1768\u1769\u176a\u176b\u176c\u176e\u176f\u1770\u1780\u1781\u1782\u1783\u1784\u1785\u1786\u1787\u1788\u1789\u178a\u178b\u178c\u178d\u178e\u178f\u1790\u1791\u1792\u1793\u1794\u1795\u1796\u1797\u1798\u1799\u179a\u179b\u179c\u179d\u179e\u179f\u17a0\u17a1\u17a2\u17a3\u17a4\u17a5\u17a6\u17a7\u17a8\u17a9\u17aa\u17ab\u17ac\u17ad\u17ae\u17af\u17b0\u17b1\u17b2\u17b3\u17dc\u1820\u1821\u1822\u1823\u1824\u1825\u1826\u1827\u1828\u1829\u182a\u182b\u182c\u182d\u182e\u182f\u1830\u1831\u1832\u1833\u1834\u1835\u1836\u1837\u1838\u1839\u183a\u183b\u183c\u183d\u183e\u183f\u1840\u1841\u1842\u1844\u1845\u1846\u1847\u1848\u1849\u184a\u184b\u184c\u184d\u184e\u184f\u1850\u1851\u1852\u1853\u1854\u1855\u1856\u1857\u1858\u1859\u185a\u185b\u185c\u185d\u185e\u185f\u1860\u1861\u1862\u1863\u1864\u1865\u1866\u1867\u1868\u1869\u186a\u186b\u186c\u186d\u186e\u186f\u1870\u1871\u1872\u1873\u1874\u1875\u1876\u1877\u1880\u1881\u1882\u1883\u1884\u1885\u1886\u1887\u1888\u1889\u188a\u188b\u188c\u188d\u188e\u188f\u1890\u1891\u1892\u1893\u1894\u1895\u1896\u1897\u1898\u1899\u189a\u189b\u189c\u189d\u189e\u189f\u18a0\u18a1\u18a2\u18a3\u18a4\u18a5\u18a6\u18a7\u18a8\u1900\u1901\u1902\u1903\u1904\u1905\u1906\u1907\u1908\u1909\u190a\u190b\u190c\u190d\u190e\u190f\u1910\u1911\u1912\u1913\u1914\u1915\u1916\u1917\u1918\u1919\u191a\u191b\u191c\u1950\u1951\u1952\u1953\u1954\u1955\u1956\u1957\u1958\u1959\u195a\u195b\u195c\u195d\u195e\u195f\u1960\u1961\u1962\u1963\u1964\u1965\u1966\u1967\u1968\u1969\u196a\u196b\u196c\u196d\u1970\u1971\u1972\u1973\u1974\u1980\u1981\u1982\u1983\u1984\u1985\u1986\u1987\u1988\u1989\u198a\u198b\u198c\u198d\u198e\u198f\u1990\u1991\u1992\u1993\u1994\u1995\u1996\u1997\u1998\u1999\u199a\u199b\u199c\u199d\u199e\u199f\u19a0\u19a1\u19a2\u19a3\u19a4\u19a5\u19a6\u19a7\u19a8\u19a9\u19c1\u19c2\u19c3\u19c4\u19c5\u19c6\u19c7\u1a00\u1a01\u1a02\u1a03\u1a04\u1a05\u1a06\u1a07\u1a08\u1a09\u1a0a\u1a0b\u1a0c\u1a0d\u1a0e\u1a0f\u1a10\u1a11\u1a12\u1a13\u1a14\u1a15\u1a16\u2135\u2136\u2137\u2138\u2d30\u2d31\u2d32\u2d33\u2d34\u2d35\u2d36\u2d37\u2d38\u2d39\u2d3a\u2d3b\u2d3c\u2d3d\u2d3e\u2d3f\u2d40\u2d41\u2d42\u2d43\u2d44\u2d45\u2d46\u2d47\u2d48\u2d49\u2d4a\u2d4b\u2d4c\u2d4d\u2d4e\u2d4f\u2d50\u2d51\u2d52\u2d53\u2d54\u2d55\u2d56\u2d57\u2d58\u2d59\u2d5a\u2d5b\u2d5c\u2d5d\u2d5e\u2d5f\u2d60\u2d61\u2d62\u2d63\u2d64\u2d65\u2d80\u2d81\u2d82\u2d83\u2d84\u2d85\u2d86\u2d87\u2d88\u2d89\u2d8a\u2d8b\u2d8c\u2d8d\u2d8e\u2d8f\u2d90\u2d91\u2d92\u2d93\u2d94\u2d95\u2d96\u2da0\u2da1\u2da2\u2da3\u2da4\u2da5\u2da6\u2da8\u2da9\u2daa\u2dab\u2dac\u2dad\u2dae\u2db0\u2db1\u2db2\u2db3\u2db4\u2db5\u2db6\u2db8\u2db9\u2dba\u2dbb\u2dbc\u2dbd\u2dbe\u2dc0\u2dc1\u2dc2\u2dc3\u2dc4\u2dc5\u2dc6\u2dc8\u2dc9\u2dca\u2dcb\u2dcc\u2dcd\u2dce\u2dd0\u2dd1\u2dd2\u2dd3\u2dd4\u2dd5\u2dd6\u2dd8\u2dd9\u2dda\u2ddb\u2ddc\u2ddd\u2dde\u3006\u303c\u3041\u3042\u3043\u3044\u3045\u3046\u3047\u3048\u3049\u304a\u304b\u304c\u304d\u304e\u304f\u3050\u3051\u3052\u3053\u3054\u3055\u3056\u3057\u3058\u3059\u305a\u305b\u305c\u305d\u305e\u305f\u3060\u3061\u3062\u3063\u3064\u3065\u3066\u3067\u3068\u3069\u306a\u306b\u306c\u306d\u306e\u306f\u3070\u3071\u3072\u3073\u3074\u3075\u3076\u3077\u3078\u3079\u307a\u307b\u307c\u307d\u307e\u307f\u3080\u3081\u3082\u3083\u3084\u3085\u3086\u3087\u3088\u3089\u308a\u308b\u308c\u308d\u308e\u308f\u3090\u3091\u3092\u3093\u3094\u3095\u3096\u309f\u30a1\u30a2\u30a3\u30a4\u30a5\u30a6\u30a7\u30a8\u30a9\u30aa\u30ab\u30ac\u30ad\u30ae\u30af\u30b0\u30b1\u30b2\u30b3\u30b4\u30b5\u30b6\u30b7\u30b8\u30b9\u30ba\u30bb\u30bc\u30bd\u30be\u30bf\u30c0\u30c1\u30c2\u30c3\u30c4\u30c5\u30c6\u30c7\u30c8\u30c9\u30ca\u30cb\u30cc\u30cd\u30ce\u30cf\u30d0\u30d1\u30d2\u30d3\u30d4\u30d5\u30d6\u30d7\u30d8\u30d9\u30da\u30db\u30dc\u30dd\u30de\u30df\u30e0\u30e1\u30e2\u30e3\u30e4\u30e5\u30e6\u30e7\u30e8\u30e9\u30ea\u30eb\u30ec\u30ed\u30ee\u30ef\u30f0\u30f1\u30f2\u30f3\u30f4\u30f5\u30f6\u30f7\u30f8\u30f9\u30fa\u30ff\u3105\u3106\u3107\u3108\u3109\u310a\u310b\u310c\u310d\u310e\u310f\u3110\u3111\u3112\u3113\u3114\u3115\u3116\u3117\u3118\u3119\u311a\u311b\u311c\u311d\u311e\u311f\u3120\u3121\u3122\u3123\u3124\u3125\u3126\u3127\u3128\u3129\u312a\u312b\u312c\u3131\u3132\u3133\u3134\u3135\u3136\u3137\u3138\u3139\u313a\u313b\u313c\u313d\u313e\u313f\u3140\u3141\u3142\u3143\u3144\u3145\u3146\u3147\u3148\u3149\u314a\u314b\u314c\u314d\u314e\u314f\u3150\u3151\u3152\u3153\u3154\u3155\u3156\u3157\u3158\u3159\u315a\u315b\u315c\u315d\u315e\u315f\u3160\u3161\u3162\u3163\u3164\u3165\u3166\u3167\u3168\u3169\u316a\u316b\u316c\u316d\u316e\u316f\u3170\u3171\u3172\u3173\u3174\u3175\u3176\u3177\u3178\u3179\u317a\u317b\u317c\u317d\u317e\u317f\u3180\u3181\u3182\u3183\u3184\u3185\u3186\u3187\u3188\u3189\u318a\u318b\u318c\u318d\u318e\u31a0\u31a1\u31a2\u31a3\u31a4\u31a5\u31a6\u31a7\u31a8\u31a9\u31aa\u31ab\u31ac\u31ad\u31ae\u31af\u31b0\u31b1\u31b2\u31b3\u31b4\u31b5\u31b6\u31b7\u31f0\u31f1\u31f2\u31f3\u31f4\u31f5\u31f6\u31f7\u31f8\u31f9\u31fa\u31fb\u31fc\u31fd\u31fe\u31ff\u3400\u3401\u3402\u3403\u3404\u3405\u3406\u3407\u3408\u3409\u340a\u340b\u340c\u340d\u340e\u340f\u3410\u3411\u3412\u3413\u3414\u3415\u3416\u3417\u3418\u3419\u341a\u341b\u341c\u341d\u341e\u341f\u3420\u3421\u3422\u3423\u3424\u3425\u3426\u3427\u3428\u3429\u342a\u342b\u342c\u342d\u342e\u342f\u3430\u3431\u3432\u3433\u3434\u3435\u3436\u3437\u3438\u3439\u343a\u343b\u343c\u343d\u343e\u343f\u3440\u3441\u3442\u3443\u3444\u3445\u3446\u3447\u3448\u3449\u344a\u344b\u344c\u344d\u344e\u344f\u3450\u3451\u3452\u3453\u3454\u3455\u3456\u3457\u3458\u3459\u345a\u345b\u345c\u345d\u345e\u345f\u3460\u3461\u3462\u3463\u3464\u3465\u3466\u3467\u3468\u3469\u346a\u346b\u346c\u346d\u346e\u346f\u3470\u3471\u3472\u3473\u3474\u3475\u3476\u3477\u3478\u3479\u347a\u347b\u347c\u347d\u347e\u347f\u3480\u3481\u3482\u3483\u3484\u3485\u3486\u3487\u3488\u3489\u348a\u348b\u348c\u348d\u348e\u348f\u3490\u3491\u3492\u3493\u3494\u3495\u3496\u3497\u3498\u3499\u349a\u349b\u349c\u349d\u349e\u349f\u34a0\u34a1\u34a2\u34a3\u34a4\u34a5\u34a6\u34a7\u34a8\u34a9\u34aa\u34ab\u34ac\u34ad\u34ae\u34af\u34b0\u34b1\u34b2\u34b3\u34b4\u34b5\u34b6\u34b7\u34b8\u34b9\u34ba\u34bb\u34bc\u34bd\u34be\u34bf\u34c0\u34c1\u34c2\u34c3\u34c4\u34c5\u34c6\u34c7\u34c8\u34c9\u34ca\u34cb\u34cc\u34cd\u34ce\u34cf\u34d0\u34d1\u34d2\u34d3\u34d4\u34d5\u34d6\u34d7\u34d8\u34d9\u34da\u34db\u34dc\u34dd\u34de\u34df\u34e0\u34e1\u34e2\u34e3\u34e4\u34e5\u34e6\u34e7\u34e8\u34e9\u34ea\u34eb\u34ec\u34ed\u34ee\u34ef\u34f0\u34f1\u34f2\u34f3\u34f4\u34f5\u34f6\u34f7\u34f8\u34f9\u34fa\u34fb\u34fc\u34fd\u34fe\u34ff\u3500\u3501\u3502\u3503\u3504\u3505\u3506\u3507\u3508\u3509\u350a\u350b\u350c\u350d\u350e\u350f\u3510\u3511\u3512\u3513\u3514\u3515\u3516\u3517\u3518\u3519\u351a\u351b\u351c\u351d\u351e\u351f\u3520\u3521\u3522\u3523\u3524\u3525\u3526\u3527\u3528\u3529\u352a\u352b\u352c\u352d\u352e\u352f\u3530\u3531\u3532\u3533\u3534\u3535\u3536\u3537\u3538\u3539\u353a\u353b\u353c\u353d\u353e\u353f\u3540\u3541\u3542\u3543\u3544\u3545\u3546\u3547\u3548\u3549\u354a\u354b\u354c\u354d\u354e\u354f\u3550\u3551\u3552\u3553\u3554\u3555\u3556\u3557\u3558\u3559\u355a\u355b\u355c\u355d\u355e\u355f\u3560\u3561\u3562\u3563\u3564\u3565\u3566\u3567\u3568\u3569\u356a\u356b\u356c\u356d\u356e\u356f\u3570\u3571\u3572\u3573\u3574\u3575\u3576\u3577\u3578\u3579\u357a\u357b\u357c\u357d\u357e\u357f\u3580\u3581\u3582\u3583\u3584\u3585\u3586\u3587\u3588\u3589\u358a\u358b\u358c\u358d\u358e\u358f\u3590\u3591\u3592\u3593\u3594\u3595\u3596\u3597\u3598\u3599\u359a\u359b\u359c\u359d\u359e\u359f\u35a0\u35a1\u35a2\u35a3\u35a4\u35a5\u35a6\u35a7\u35a8\u35a9\u35aa\u35ab\u35ac\u35ad\u35ae\u35af\u35b0\u35b1\u35b2\u35b3\u35b4\u35b5\u35b6\u35b7\u35b8\u35b9\u35ba\u35bb\u35bc\u35bd\u35be\u35bf\u35c0\u35c1\u35c2\u35c3\u35c4\u35c5\u35c6\u35c7\u35c8\u35c9\u35ca\u35cb\u35cc\u35cd\u35ce\u35cf\u35d0\u35d1\u35d2\u35d3\u35d4\u35d5\u35d6\u35d7\u35d8\u35d9\u35da\u35db\u35dc\u35dd\u35de\u35df\u35e0\u35e1\u35e2\u35e3\u35e4\u35e5\u35e6\u35e7\u35e8\u35e9\u35ea\u35eb\u35ec\u35ed\u35ee\u35ef\u35f0\u35f1\u35f2\u35f3\u35f4\u35f5\u35f6\u35f7\u35f8\u35f9\u35fa\u35fb\u35fc\u35fd\u35fe\u35ff\u3600\u3601\u3602\u3603\u3604\u3605\u3606\u3607\u3608\u3609\u360a\u360b\u360c\u360d\u360e\u360f\u3610\u3611\u3612\u3613\u3614\u3615\u3616\u3617\u3618\u3619\u361a\u361b\u361c\u361d\u361e\u361f\u3620\u3621\u3622\u3623\u3624\u3625\u3626\u3627\u3628\u3629\u362a\u362b\u362c\u362d\u362e\u362f\u3630\u3631\u3632\u3633\u3634\u3635\u3636\u3637\u3638\u3639\u363a\u363b\u363c\u363d\u363e\u363f\u3640\u3641\u3642\u3643\u3644\u3645\u3646\u3647\u3648\u3649\u364a\u364b\u364c\u364d\u364e\u364f\u3650\u3651\u3652\u3653\u3654\u3655\u3656\u3657\u3658\u3659\u365a\u365b\u365c\u365d\u365e\u365f\u3660\u3661\u3662\u3663\u3664\u3665\u3666\u3667\u3668\u3669\u366a\u366b\u366c\u366d\u366e\u366f\u3670\u3671\u3672\u3673\u3674\u3675\u3676\u3677\u3678\u3679\u367a\u367b\u367c\u367d\u367e\u367f\u3680\u3681\u3682\u3683\u3684\u3685\u3686\u3687\u3688\u3689\u368a\u368b\u368c\u368d\u368e\u368f\u3690\u3691\u3692\u3693\u3694\u3695\u3696\u3697\u3698\u3699\u369a\u369b\u369c\u369d\u369e\u369f\u36a0\u36a1\u36a2\u36a3\u36a4\u36a5\u36a6\u36a7\u36a8\u36a9\u36aa\u36ab\u36ac\u36ad\u36ae\u36af\u36b0\u36b1\u36b2\u36b3\u36b4\u36b5\u36b6\u36b7\u36b8\u36b9\u36ba\u36bb\u36bc\u36bd\u36be\u36bf\u36c0\u36c1\u36c2\u36c3\u36c4\u36c5\u36c6\u36c7\u36c8\u36c9\u36ca\u36cb\u36cc\u36cd\u36ce\u36cf\u36d0\u36d1\u36d2\u36d3\u36d4\u36d5\u36d6\u36d7\u36d8\u36d9\u36da\u36db\u36dc\u36dd\u36de\u36df\u36e0\u36e1\u36e2\u36e3\u36e4\u36e5\u36e6\u36e7\u36e8\u36e9\u36ea\u36eb\u36ec\u36ed\u36ee\u36ef\u36f0\u36f1\u36f2\u36f3\u36f4\u36f5\u36f6\u36f7\u36f8\u36f9\u36fa\u36fb\u36fc\u36fd\u36fe\u36ff\u3700\u3701\u3702\u3703\u3704\u3705\u3706\u3707\u3708\u3709\u370a\u370b\u370c\u370d\u370e\u370f\u3710\u3711\u3712\u3713\u3714\u3715\u3716\u3717\u3718\u3719\u371a\u371b\u371c\u371d\u371e\u371f\u3720\u3721\u3722\u3723\u3724\u3725\u3726\u3727\u3728\u3729\u372a\u372b\u372c\u372d\u372e\u372f\u3730\u3731\u3732\u3733\u3734\u3735\u3736\u3737\u3738\u3739\u373a\u373b\u373c\u373d\u373e\u373f\u3740\u3741\u3742\u3743\u3744\u3745\u3746\u3747\u3748\u3749\u374a\u374b\u374c\u374d\u374e\u374f\u3750\u3751\u3752\u3753\u3754\u3755\u3756\u3757\u3758\u3759\u375a\u375b\u375c\u375d\u375e\u375f\u3760\u3761\u3762\u3763\u3764\u3765\u3766\u3767\u3768\u3769\u376a\u376b\u376c\u376d\u376e\u376f\u3770\u3771\u3772\u3773\u3774\u3775\u3776\u3777\u3778\u3779\u377a\u377b\u377c\u377d\u377e\u377f\u3780\u3781\u3782\u3783\u3784\u3785\u3786\u3787\u3788\u3789\u378a\u378b\u378c\u378d\u378e\u378f\u3790\u3791\u3792\u3793\u3794\u3795\u3796\u3797\u3798\u3799\u379a\u379b\u379c\u379d\u379e\u379f\u37a0\u37a1\u37a2\u37a3\u37a4\u37a5\u37a6\u37a7\u37a8\u37a9\u37aa\u37ab\u37ac\u37ad\u37ae\u37af\u37b0\u37b1\u37b2\u37b3\u37b4\u37b5\u37b6\u37b7\u37b8\u37b9\u37ba\u37bb\u37bc\u37bd\u37be\u37bf\u37c0\u37c1\u37c2\u37c3\u37c4\u37c5\u37c6\u37c7\u37c8\u37c9\u37ca\u37cb\u37cc\u37cd\u37ce\u37cf\u37d0\u37d1\u37d2\u37d3\u37d4\u37d5\u37d6\u37d7\u37d8\u37d9\u37da\u37db\u37dc\u37dd\u37de\u37df\u37e0\u37e1\u37e2\u37e3\u37e4\u37e5\u37e6\u37e7\u37e8\u37e9\u37ea\u37eb\u37ec\u37ed\u37ee\u37ef\u37f0\u37f1\u37f2\u37f3\u37f4\u37f5\u37f6\u37f7\u37f8\u37f9\u37fa\u37fb\u37fc\u37fd\u37fe\u37ff\u3800\u3801\u3802\u3803\u3804\u3805\u3806\u3807\u3808\u3809\u380a\u380b\u380c\u380d\u380e\u380f\u3810\u3811\u3812\u3813\u3814\u3815\u3816\u3817\u3818\u3819\u381a\u381b\u381c\u381d\u381e\u381f\u3820\u3821\u3822\u3823\u3824\u3825\u3826\u3827\u3828\u3829\u382a\u382b\u382c\u382d\u382e\u382f\u3830\u3831\u3832\u3833\u3834\u3835\u3836\u3837\u3838\u3839\u383a\u383b\u383c\u383d\u383e\u383f\u3840\u3841\u3842\u3843\u3844\u3845\u3846\u3847\u3848\u3849\u384a\u384b\u384c\u384d\u384e\u384f\u3850\u3851\u3852\u3853\u3854\u3855\u3856\u3857\u3858\u3859\u385a\u385b\u385c\u385d\u385e\u385f\u3860\u3861\u3862\u3863\u3864\u3865\u3866\u3867\u3868\u3869\u386a\u386b\u386c\u386d\u386e\u386f\u3870\u3871\u3872\u3873\u3874\u3875\u3876\u3877\u3878\u3879\u387a\u387b\u387c\u387d\u387e\u387f\u3880\u3881\u3882\u3883\u3884\u3885\u3886\u3887\u3888\u3889\u388a\u388b\u388c\u388d\u388e\u388f\u3890\u3891\u3892\u3893\u3894\u3895\u3896\u3897\u3898\u3899\u389a\u389b\u389c\u389d\u389e\u389f\u38a0\u38a1\u38a2\u38a3\u38a4\u38a5\u38a6\u38a7\u38a8\u38a9\u38aa\u38ab\u38ac\u38ad\u38ae\u38af\u38b0\u38b1\u38b2\u38b3\u38b4\u38b5\u38b6\u38b7\u38b8\u38b9\u38ba\u38bb\u38bc\u38bd\u38be\u38bf\u38c0\u38c1\u38c2\u38c3\u38c4\u38c5\u38c6\u38c7\u38c8\u38c9\u38ca\u38cb\u38cc\u38cd\u38ce\u38cf\u38d0\u38d1\u38d2\u38d3\u38d4\u38d5\u38d6\u38d7\u38d8\u38d9\u38da\u38db\u38dc\u38dd\u38de\u38df\u38e0\u38e1\u38e2\u38e3\u38e4\u38e5\u38e6\u38e7\u38e8\u38e9\u38ea\u38eb\u38ec\u38ed\u38ee\u38ef\u38f0\u38f1\u38f2\u38f3\u38f4\u38f5\u38f6\u38f7\u38f8\u38f9\u38fa\u38fb\u38fc\u38fd\u38fe\u38ff\u3900\u3901\u3902\u3903\u3904\u3905\u3906\u3907\u3908\u3909\u390a\u390b\u390c\u390d\u390e\u390f\u3910\u3911\u3912\u3913\u3914\u3915\u3916\u3917\u3918\u3919\u391a\u391b\u391c\u391d\u391e\u391f\u3920\u3921\u3922\u3923\u3924\u3925\u3926\u3927\u3928\u3929\u392a\u392b\u392c\u392d\u392e\u392f\u3930\u3931\u3932\u3933\u3934\u3935\u3936\u3937\u3938\u3939\u393a\u393b\u393c\u393d\u393e\u393f\u3940\u3941\u3942\u3943\u3944\u3945\u3946\u3947\u3948\u3949\u394a\u394b\u394c\u394d\u394e\u394f\u3950\u3951\u3952\u3953\u3954\u3955\u3956\u3957\u3958\u3959\u395a\u395b\u395c\u395d\u395e\u395f\u3960\u3961\u3962\u3963\u3964\u3965\u3966\u3967\u3968\u3969\u396a\u396b\u396c\u396d\u396e\u396f\u3970\u3971\u3972\u3973\u3974\u3975\u3976\u3977\u3978\u3979\u397a\u397b\u397c\u397d\u397e\u397f\u3980\u3981\u3982\u3983\u3984\u3985\u3986\u3987\u3988\u3989\u398a\u398b\u398c\u398d\u398e\u398f\u3990\u3991\u3992\u3993\u3994\u3995\u3996\u3997\u3998\u3999\u399a\u399b\u399c\u399d\u399e\u399f\u39a0\u39a1\u39a2\u39a3\u39a4\u39a5\u39a6\u39a7\u39a8\u39a9\u39aa\u39ab\u39ac\u39ad\u39ae\u39af\u39b0\u39b1\u39b2\u39b3\u39b4\u39b5\u39b6\u39b7\u39b8\u39b9\u39ba\u39bb\u39bc\u39bd\u39be\u39bf\u39c0\u39c1\u39c2\u39c3\u39c4\u39c5\u39c6\u39c7\u39c8\u39c9\u39ca\u39cb\u39cc\u39cd\u39ce\u39cf\u39d0\u39d1\u39d2\u39d3\u39d4\u39d5\u39d6\u39d7\u39d8\u39d9\u39da\u39db\u39dc\u39dd\u39de\u39df\u39e0\u39e1\u39e2\u39e3\u39e4\u39e5\u39e6\u39e7\u39e8\u39e9\u39ea\u39eb\u39ec\u39ed\u39ee\u39ef\u39f0\u39f1\u39f2\u39f3\u39f4\u39f5\u39f6\u39f7\u39f8\u39f9\u39fa\u39fb\u39fc\u39fd\u39fe\u39ff\u3a00\u3a01\u3a02\u3a03\u3a04\u3a05\u3a06\u3a07\u3a08\u3a09\u3a0a\u3a0b\u3a0c\u3a0d\u3a0e\u3a0f\u3a10\u3a11\u3a12\u3a13\u3a14\u3a15\u3a16\u3a17\u3a18\u3a19\u3a1a\u3a1b\u3a1c\u3a1d\u3a1e\u3a1f\u3a20\u3a21\u3a22\u3a23\u3a24\u3a25\u3a26\u3a27\u3a28\u3a29\u3a2a\u3a2b\u3a2c\u3a2d\u3a2e\u3a2f\u3a30\u3a31\u3a32\u3a33\u3a34\u3a35\u3a36\u3a37\u3a38\u3a39\u3a3a\u3a3b\u3a3c\u3a3d\u3a3e\u3a3f\u3a40\u3a41\u3a42\u3a43\u3a44\u3a45\u3a46\u3a47\u3a48\u3a49\u3a4a\u3a4b\u3a4c\u3a4d\u3a4e\u3a4f\u3a50\u3a51\u3a52\u3a53\u3a54\u3a55\u3a56\u3a57\u3a58\u3a59\u3a5a\u3a5b\u3a5c\u3a5d\u3a5e\u3a5f\u3a60\u3a61\u3a62\u3a63\u3a64\u3a65\u3a66\u3a67\u3a68\u3a69\u3a6a\u3a6b\u3a6c\u3a6d\u3a6e\u3a6f\u3a70\u3a71\u3a72\u3a73\u3a74\u3a75\u3a76\u3a77\u3a78\u3a79\u3a7a\u3a7b\u3a7c\u3a7d\u3a7e\u3a7f\u3a80\u3a81\u3a82\u3a83\u3a84\u3a85\u3a86\u3a87\u3a88\u3a89\u3a8a\u3a8b\u3a8c\u3a8d\u3a8e\u3a8f\u3a90\u3a91\u3a92\u3a93\u3a94\u3a95\u3a96\u3a97\u3a98\u3a99\u3a9a\u3a9b\u3a9c\u3a9d\u3a9e\u3a9f\u3aa0\u3aa1\u3aa2\u3aa3\u3aa4\u3aa5\u3aa6\u3aa7\u3aa8\u3aa9\u3aaa\u3aab\u3aac\u3aad\u3aae\u3aaf\u3ab0\u3ab1\u3ab2\u3ab3\u3ab4\u3ab5\u3ab6\u3ab7\u3ab8\u3ab9\u3aba\u3abb\u3abc\u3abd\u3abe\u3abf\u3ac0\u3ac1\u3ac2\u3ac3\u3ac4\u3ac5\u3ac6\u3ac7\u3ac8\u3ac9\u3aca\u3acb\u3acc\u3acd\u3ace\u3acf\u3ad0\u3ad1\u3ad2\u3ad3\u3ad4\u3ad5\u3ad6\u3ad7\u3ad8\u3ad9\u3ada\u3adb\u3adc\u3add\u3ade\u3adf\u3ae0\u3ae1\u3ae2\u3ae3\u3ae4\u3ae5\u3ae6\u3ae7\u3ae8\u3ae9\u3aea\u3aeb\u3aec\u3aed\u3aee\u3aef\u3af0\u3af1\u3af2\u3af3\u3af4\u3af5\u3af6\u3af7\u3af8\u3af9\u3afa\u3afb\u3afc\u3afd\u3afe\u3aff\u3b00\u3b01\u3b02\u3b03\u3b04\u3b05\u3b06\u3b07\u3b08\u3b09\u3b0a\u3b0b\u3b0c\u3b0d\u3b0e\u3b0f\u3b10\u3b11\u3b12\u3b13\u3b14\u3b15\u3b16\u3b17\u3b18\u3b19\u3b1a\u3b1b\u3b1c\u3b1d\u3b1e\u3b1f\u3b20\u3b21\u3b22\u3b23\u3b24\u3b25\u3b26\u3b27\u3b28\u3b29\u3b2a\u3b2b\u3b2c\u3b2d\u3b2e\u3b2f\u3b30\u3b31\u3b32\u3b33\u3b34\u3b35\u3b36\u3b37\u3b38\u3b39\u3b3a\u3b3b\u3b3c\u3b3d\u3b3e\u3b3f\u3b40\u3b41\u3b42\u3b43\u3b44\u3b45\u3b46\u3b47\u3b48\u3b49\u3b4a\u3b4b\u3b4c\u3b4d\u3b4e\u3b4f\u3b50\u3b51\u3b52\u3b53\u3b54\u3b55\u3b56\u3b57\u3b58\u3b59\u3b5a\u3b5b\u3b5c\u3b5d\u3b5e\u3b5f\u3b60\u3b61\u3b62\u3b63\u3b64\u3b65\u3b66\u3b67\u3b68\u3b69\u3b6a\u3b6b\u3b6c\u3b6d\u3b6e\u3b6f\u3b70\u3b71\u3b72\u3b73\u3b74\u3b75\u3b76\u3b77\u3b78\u3b79\u3b7a\u3b7b\u3b7c\u3b7d\u3b7e\u3b7f\u3b80\u3b81\u3b82\u3b83\u3b84\u3b85\u3b86\u3b87\u3b88\u3b89\u3b8a\u3b8b\u3b8c\u3b8d\u3b8e\u3b8f\u3b90\u3b91\u3b92\u3b93\u3b94\u3b95\u3b96\u3b97\u3b98\u3b99\u3b9a\u3b9b\u3b9c\u3b9d\u3b9e\u3b9f\u3ba0\u3ba1\u3ba2\u3ba3\u3ba4\u3ba5\u3ba6\u3ba7\u3ba8\u3ba9\u3baa\u3bab\u3bac\u3bad\u3bae\u3baf\u3bb0\u3bb1\u3bb2\u3bb3\u3bb4\u3bb5\u3bb6\u3bb7\u3bb8\u3bb9\u3bba\u3bbb\u3bbc\u3bbd\u3bbe\u3bbf\u3bc0\u3bc1\u3bc2\u3bc3\u3bc4\u3bc5\u3bc6\u3bc7\u3bc8\u3bc9\u3bca\u3bcb\u3bcc\u3bcd\u3bce\u3bcf\u3bd0\u3bd1\u3bd2\u3bd3\u3bd4\u3bd5\u3bd6\u3bd7\u3bd8\u3bd9\u3bda\u3bdb\u3bdc\u3bdd\u3bde\u3bdf\u3be0\u3be1\u3be2\u3be3\u3be4\u3be5\u3be6\u3be7\u3be8\u3be9\u3bea\u3beb\u3bec\u3bed\u3bee\u3bef\u3bf0\u3bf1\u3bf2\u3bf3\u3bf4\u3bf5\u3bf6\u3bf7\u3bf8\u3bf9\u3bfa\u3bfb\u3bfc\u3bfd\u3bfe\u3bff\u3c00\u3c01\u3c02\u3c03\u3c04\u3c05\u3c06\u3c07\u3c08\u3c09\u3c0a\u3c0b\u3c0c\u3c0d\u3c0e\u3c0f\u3c10\u3c11\u3c12\u3c13\u3c14\u3c15\u3c16\u3c17\u3c18\u3c19\u3c1a\u3c1b\u3c1c\u3c1d\u3c1e\u3c1f\u3c20\u3c21\u3c22\u3c23\u3c24\u3c25\u3c26\u3c27\u3c28\u3c29\u3c2a\u3c2b\u3c2c\u3c2d\u3c2e\u3c2f\u3c30\u3c31\u3c32\u3c33\u3c34\u3c35\u3c36\u3c37\u3c38\u3c39\u3c3a\u3c3b\u3c3c\u3c3d\u3c3e\u3c3f\u3c40\u3c41\u3c42\u3c43\u3c44\u3c45\u3c46\u3c47\u3c48\u3c49\u3c4a\u3c4b\u3c4c\u3c4d\u3c4e\u3c4f\u3c50\u3c51\u3c52\u3c53\u3c54\u3c55\u3c56\u3c57\u3c58\u3c59\u3c5a\u3c5b\u3c5c\u3c5d\u3c5e\u3c5f\u3c60\u3c61\u3c62\u3c63\u3c64\u3c65\u3c66\u3c67\u3c68\u3c69\u3c6a\u3c6b\u3c6c\u3c6d\u3c6e\u3c6f\u3c70\u3c71\u3c72\u3c73\u3c74\u3c75\u3c76\u3c77\u3c78\u3c79\u3c7a\u3c7b\u3c7c\u3c7d\u3c7e\u3c7f\u3c80\u3c81\u3c82\u3c83\u3c84\u3c85\u3c86\u3c87\u3c88\u3c89\u3c8a\u3c8b\u3c8c\u3c8d\u3c8e\u3c8f\u3c90\u3c91\u3c92\u3c93\u3c94\u3c95\u3c96\u3c97\u3c98\u3c99\u3c9a\u3c9b\u3c9c\u3c9d\u3c9e\u3c9f\u3ca0\u3ca1\u3ca2\u3ca3\u3ca4\u3ca5\u3ca6\u3ca7\u3ca8\u3ca9\u3caa\u3cab\u3cac\u3cad\u3cae\u3caf\u3cb0\u3cb1\u3cb2\u3cb3\u3cb4\u3cb5\u3cb6\u3cb7\u3cb8\u3cb9\u3cba\u3cbb\u3cbc\u3cbd\u3cbe\u3cbf\u3cc0\u3cc1\u3cc2\u3cc3\u3cc4\u3cc5\u3cc6\u3cc7\u3cc8\u3cc9\u3cca\u3ccb\u3ccc\u3ccd\u3cce\u3ccf\u3cd0\u3cd1\u3cd2\u3cd3\u3cd4\u3cd5\u3cd6\u3cd7\u3cd8\u3cd9\u3cda\u3cdb\u3cdc\u3cdd\u3cde\u3cdf\u3ce0\u3ce1\u3ce2\u3ce3\u3ce4\u3ce5\u3ce6\u3ce7\u3ce8\u3ce9\u3cea\u3ceb\u3cec\u3ced\u3cee\u3cef\u3cf0\u3cf1\u3cf2\u3cf3\u3cf4\u3cf5\u3cf6\u3cf7\u3cf8\u3cf9\u3cfa\u3cfb\u3cfc\u3cfd\u3cfe\u3cff\u3d00\u3d01\u3d02\u3d03\u3d04\u3d05\u3d06\u3d07\u3d08\u3d09\u3d0a\u3d0b\u3d0c\u3d0d\u3d0e\u3d0f\u3d10\u3d11\u3d12\u3d13\u3d14\u3d15\u3d16\u3d17\u3d18\u3d19\u3d1a\u3d1b\u3d1c\u3d1d\u3d1e\u3d1f\u3d20\u3d21\u3d22\u3d23\u3d24\u3d25\u3d26\u3d27\u3d28\u3d29\u3d2a\u3d2b\u3d2c\u3d2d\u3d2e\u3d2f\u3d30\u3d31\u3d32\u3d33\u3d34\u3d35\u3d36\u3d37\u3d38\u3d39\u3d3a\u3d3b\u3d3c\u3d3d\u3d3e\u3d3f\u3d40\u3d41\u3d42\u3d43\u3d44\u3d45\u3d46\u3d47\u3d48\u3d49\u3d4a\u3d4b\u3d4c\u3d4d\u3d4e\u3d4f\u3d50\u3d51\u3d52\u3d53\u3d54\u3d55\u3d56\u3d57\u3d58\u3d59\u3d5a\u3d5b\u3d5c\u3d5d\u3d5e\u3d5f\u3d60\u3d61\u3d62\u3d63\u3d64\u3d65\u3d66\u3d67\u3d68\u3d69\u3d6a\u3d6b\u3d6c\u3d6d\u3d6e\u3d6f\u3d70\u3d71\u3d72\u3d73\u3d74\u3d75\u3d76\u3d77\u3d78\u3d79\u3d7a\u3d7b\u3d7c\u3d7d\u3d7e\u3d7f\u3d80\u3d81\u3d82\u3d83\u3d84\u3d85\u3d86\u3d87\u3d88\u3d89\u3d8a\u3d8b\u3d8c\u3d8d\u3d8e\u3d8f\u3d90\u3d91\u3d92\u3d93\u3d94\u3d95\u3d96\u3d97\u3d98\u3d99\u3d9a\u3d9b\u3d9c\u3d9d\u3d9e\u3d9f\u3da0\u3da1\u3da2\u3da3\u3da4\u3da5\u3da6\u3da7\u3da8\u3da9\u3daa\u3dab\u3dac\u3dad\u3dae\u3daf\u3db0\u3db1\u3db2\u3db3\u3db4\u3db5\u3db6\u3db7\u3db8\u3db9\u3dba\u3dbb\u3dbc\u3dbd\u3dbe\u3dbf\u3dc0\u3dc1\u3dc2\u3dc3\u3dc4\u3dc5\u3dc6\u3dc7\u3dc8\u3dc9\u3dca\u3dcb\u3dcc\u3dcd\u3dce\u3dcf\u3dd0\u3dd1\u3dd2\u3dd3\u3dd4\u3dd5\u3dd6\u3dd7\u3dd8\u3dd9\u3dda\u3ddb\u3ddc\u3ddd\u3dde\u3ddf\u3de0\u3de1\u3de2\u3de3\u3de4\u3de5\u3de6\u3de7\u3de8\u3de9\u3dea\u3deb\u3dec\u3ded\u3dee\u3def\u3df0\u3df1\u3df2\u3df3\u3df4\u3df5\u3df6\u3df7\u3df8\u3df9\u3dfa\u3dfb\u3dfc\u3dfd\u3dfe\u3dff\u3e00\u3e01\u3e02\u3e03\u3e04\u3e05\u3e06\u3e07\u3e08\u3e09\u3e0a\u3e0b\u3e0c\u3e0d\u3e0e\u3e0f\u3e10\u3e11\u3e12\u3e13\u3e14\u3e15\u3e16\u3e17\u3e18\u3e19\u3e1a\u3e1b\u3e1c\u3e1d\u3e1e\u3e1f\u3e20\u3e21\u3e22\u3e23\u3e24\u3e25\u3e26\u3e27\u3e28\u3e29\u3e2a\u3e2b\u3e2c\u3e2d\u3e2e\u3e2f\u3e30\u3e31\u3e32\u3e33\u3e34\u3e35\u3e36\u3e37\u3e38\u3e39\u3e3a\u3e3b\u3e3c\u3e3d\u3e3e\u3e3f\u3e40\u3e41\u3e42\u3e43\u3e44\u3e45\u3e46\u3e47\u3e48\u3e49\u3e4a\u3e4b\u3e4c\u3e4d\u3e4e\u3e4f\u3e50\u3e51\u3e52\u3e53\u3e54\u3e55\u3e56\u3e57\u3e58\u3e59\u3e5a\u3e5b\u3e5c\u3e5d\u3e5e\u3e5f\u3e60\u3e61\u3e62\u3e63\u3e64\u3e65\u3e66\u3e67\u3e68\u3e69\u3e6a\u3e6b\u3e6c\u3e6d\u3e6e\u3e6f\u3e70\u3e71\u3e72\u3e73\u3e74\u3e75\u3e76\u3e77\u3e78\u3e79\u3e7a\u3e7b\u3e7c\u3e7d\u3e7e\u3e7f\u3e80\u3e81\u3e82\u3e83\u3e84\u3e85\u3e86\u3e87\u3e88\u3e89\u3e8a\u3e8b\u3e8c\u3e8d\u3e8e\u3e8f\u3e90\u3e91\u3e92\u3e93\u3e94\u3e95\u3e96\u3e97\u3e98\u3e99\u3e9a\u3e9b\u3e9c\u3e9d\u3e9e\u3e9f\u3ea0\u3ea1\u3ea2\u3ea3\u3ea4\u3ea5\u3ea6\u3ea7\u3ea8\u3ea9\u3eaa\u3eab\u3eac\u3ead\u3eae\u3eaf\u3eb0\u3eb1\u3eb2\u3eb3\u3eb4\u3eb5\u3eb6\u3eb7\u3eb8\u3eb9\u3eba\u3ebb\u3ebc\u3ebd\u3ebe\u3ebf\u3ec0\u3ec1\u3ec2\u3ec3\u3ec4\u3ec5\u3ec6\u3ec7\u3ec8\u3ec9\u3eca\u3ecb\u3ecc\u3ecd\u3ece\u3ecf\u3ed0\u3ed1\u3ed2\u3ed3\u3ed4\u3ed5\u3ed6\u3ed7\u3ed8\u3ed9\u3eda\u3edb\u3edc\u3edd\u3ede\u3edf\u3ee0\u3ee1\u3ee2\u3ee3\u3ee4\u3ee5\u3ee6\u3ee7\u3ee8\u3ee9\u3eea\u3eeb\u3eec\u3eed\u3eee\u3eef\u3ef0\u3ef1\u3ef2\u3ef3\u3ef4\u3ef5\u3ef6\u3ef7\u3ef8\u3ef9\u3efa\u3efb\u3efc\u3efd\u3efe\u3eff\u3f00\u3f01\u3f02\u3f03\u3f04\u3f05\u3f06\u3f07\u3f08\u3f09\u3f0a\u3f0b\u3f0c\u3f0d\u3f0e\u3f0f\u3f10\u3f11\u3f12\u3f13\u3f14\u3f15\u3f16\u3f17\u3f18\u3f19\u3f1a\u3f1b\u3f1c\u3f1d\u3f1e\u3f1f\u3f20\u3f21\u3f22\u3f23\u3f24\u3f25\u3f26\u3f27\u3f28\u3f29\u3f2a\u3f2b\u3f2c\u3f2d\u3f2e\u3f2f\u3f30\u3f31\u3f32\u3f33\u3f34\u3f35\u3f36\u3f37\u3f38\u3f39\u3f3a\u3f3b\u3f3c\u3f3d\u3f3e\u3f3f\u3f40\u3f41\u3f42\u3f43\u3f44\u3f45\u3f46\u3f47\u3f48\u3f49\u3f4a\u3f4b\u3f4c\u3f4d\u3f4e\u3f4f\u3f50\u3f51\u3f52\u3f53\u3f54\u3f55\u3f56\u3f57\u3f58\u3f59\u3f5a\u3f5b\u3f5c\u3f5d\u3f5e\u3f5f\u3f60\u3f61\u3f62\u3f63\u3f64\u3f65\u3f66\u3f67\u3f68\u3f69\u3f6a\u3f6b\u3f6c\u3f6d\u3f6e\u3f6f\u3f70\u3f71\u3f72\u3f73\u3f74\u3f75\u3f76\u3f77\u3f78\u3f79\u3f7a\u3f7b\u3f7c\u3f7d\u3f7e\u3f7f\u3f80\u3f81\u3f82\u3f83\u3f84\u3f85\u3f86\u3f87\u3f88\u3f89\u3f8a\u3f8b\u3f8c\u3f8d\u3f8e\u3f8f\u3f90\u3f91\u3f92\u3f93\u3f94\u3f95\u3f96\u3f97\u3f98\u3f99\u3f9a\u3f9b\u3f9c\u3f9d\u3f9e\u3f9f\u3fa0\u3fa1\u3fa2\u3fa3\u3fa4\u3fa5\u3fa6\u3fa7\u3fa8\u3fa9\u3faa\u3fab\u3fac\u3fad\u3fae\u3faf\u3fb0\u3fb1\u3fb2\u3fb3\u3fb4\u3fb5\u3fb6\u3fb7\u3fb8\u3fb9\u3fba\u3fbb\u3fbc\u3fbd\u3fbe\u3fbf\u3fc0\u3fc1\u3fc2\u3fc3\u3fc4\u3fc5\u3fc6\u3fc7\u3fc8\u3fc9\u3fca\u3fcb\u3fcc\u3fcd\u3fce\u3fcf\u3fd0\u3fd1\u3fd2\u3fd3\u3fd4\u3fd5\u3fd6\u3fd7\u3fd8\u3fd9\u3fda\u3fdb\u3fdc\u3fdd\u3fde\u3fdf\u3fe0\u3fe1\u3fe2\u3fe3\u3fe4\u3fe5\u3fe6\u3fe7\u3fe8\u3fe9\u3fea\u3feb\u3fec\u3fed\u3fee\u3fef\u3ff0\u3ff1\u3ff2\u3ff3\u3ff4\u3ff5\u3ff6\u3ff7\u3ff8\u3ff9\u3ffa\u3ffb\u3ffc\u3ffd\u3ffe\u3fff\u4000\u4001\u4002\u4003\u4004\u4005\u4006\u4007\u4008\u4009\u400a\u400b\u400c\u400d\u400e\u400f\u4010\u4011\u4012\u4013\u4014\u4015\u4016\u4017\u4018\u4019\u401a\u401b\u401c\u401d\u401e\u401f\u4020\u4021\u4022\u4023\u4024\u4025\u4026\u4027\u4028\u4029\u402a\u402b\u402c\u402d\u402e\u402f\u4030\u4031\u4032\u4033\u4034\u4035\u4036\u4037\u4038\u4039\u403a\u403b\u403c\u403d\u403e\u403f\u4040\u4041\u4042\u4043\u4044\u4045\u4046\u4047\u4048\u4049\u404a\u404b\u404c\u404d\u404e\u404f\u4050\u4051\u4052\u4053\u4054\u4055\u4056\u4057\u4058\u4059\u405a\u405b\u405c\u405d\u405e\u405f\u4060\u4061\u4062\u4063\u4064\u4065\u4066\u4067\u4068\u4069\u406a\u406b\u406c\u406d\u406e\u406f\u4070\u4071\u4072\u4073\u4074\u4075\u4076\u4077\u4078\u4079\u407a\u407b\u407c\u407d\u407e\u407f\u4080\u4081\u4082\u4083\u4084\u4085\u4086\u4087\u4088\u4089\u408a\u408b\u408c\u408d\u408e\u408f\u4090\u4091\u4092\u4093\u4094\u4095\u4096\u4097\u4098\u4099\u409a\u409b\u409c\u409d\u409e\u409f\u40a0\u40a1\u40a2\u40a3\u40a4\u40a5\u40a6\u40a7\u40a8\u40a9\u40aa\u40ab\u40ac\u40ad\u40ae\u40af\u40b0\u40b1\u40b2\u40b3\u40b4\u40b5\u40b6\u40b7\u40b8\u40b9\u40ba\u40bb\u40bc\u40bd\u40be\u40bf\u40c0\u40c1\u40c2\u40c3\u40c4\u40c5\u40c6\u40c7\u40c8\u40c9\u40ca\u40cb\u40cc\u40cd\u40ce\u40cf\u40d0\u40d1\u40d2\u40d3\u40d4\u40d5\u40d6\u40d7\u40d8\u40d9\u40da\u40db\u40dc\u40dd\u40de\u40df\u40e0\u40e1\u40e2\u40e3\u40e4\u40e5\u40e6\u40e7\u40e8\u40e9\u40ea\u40eb\u40ec\u40ed\u40ee\u40ef\u40f0\u40f1\u40f2\u40f3\u40f4\u40f5\u40f6\u40f7\u40f8\u40f9\u40fa\u40fb\u40fc\u40fd\u40fe\u40ff\u4100\u4101\u4102\u4103\u4104\u4105\u4106\u4107\u4108\u4109\u410a\u410b\u410c\u410d\u410e\u410f\u4110\u4111\u4112\u4113\u4114\u4115\u4116\u4117\u4118\u4119\u411a\u411b\u411c\u411d\u411e\u411f\u4120\u4121\u4122\u4123\u4124\u4125\u4126\u4127\u4128\u4129\u412a\u412b\u412c\u412d\u412e\u412f\u4130\u4131\u4132\u4133\u4134\u4135\u4136\u4137\u4138\u4139\u413a\u413b\u413c\u413d\u413e\u413f\u4140\u4141\u4142\u4143\u4144\u4145\u4146\u4147\u4148\u4149\u414a\u414b\u414c\u414d\u414e\u414f\u4150\u4151\u4152\u4153\u4154\u4155\u4156\u4157\u4158\u4159\u415a\u415b\u415c\u415d\u415e\u415f\u4160\u4161\u4162\u4163\u4164\u4165\u4166\u4167\u4168\u4169\u416a\u416b\u416c\u416d\u416e\u416f\u4170\u4171\u4172\u4173\u4174\u4175\u4176\u4177\u4178\u4179\u417a\u417b\u417c\u417d\u417e\u417f\u4180\u4181\u4182\u4183\u4184\u4185\u4186\u4187\u4188\u4189\u418a\u418b\u418c\u418d\u418e\u418f\u4190\u4191\u4192\u4193\u4194\u4195\u4196\u4197\u4198\u4199\u419a\u419b\u419c\u419d\u419e\u419f\u41a0\u41a1\u41a2\u41a3\u41a4\u41a5\u41a6\u41a7\u41a8\u41a9\u41aa\u41ab\u41ac\u41ad\u41ae\u41af\u41b0\u41b1\u41b2\u41b3\u41b4\u41b5\u41b6\u41b7\u41b8\u41b9\u41ba\u41bb\u41bc\u41bd\u41be\u41bf\u41c0\u41c1\u41c2\u41c3\u41c4\u41c5\u41c6\u41c7\u41c8\u41c9\u41ca\u41cb\u41cc\u41cd\u41ce\u41cf\u41d0\u41d1\u41d2\u41d3\u41d4\u41d5\u41d6\u41d7\u41d8\u41d9\u41da\u41db\u41dc\u41dd\u41de\u41df\u41e0\u41e1\u41e2\u41e3\u41e4\u41e5\u41e6\u41e7\u41e8\u41e9\u41ea\u41eb\u41ec\u41ed\u41ee\u41ef\u41f0\u41f1\u41f2\u41f3\u41f4\u41f5\u41f6\u41f7\u41f8\u41f9\u41fa\u41fb\u41fc\u41fd\u41fe\u41ff\u4200\u4201\u4202\u4203\u4204\u4205\u4206\u4207\u4208\u4209\u420a\u420b\u420c\u420d\u420e\u420f\u4210\u4211\u4212\u4213\u4214\u4215\u4216\u4217\u4218\u4219\u421a\u421b\u421c\u421d\u421e\u421f\u4220\u4221\u4222\u4223\u4224\u4225\u4226\u4227\u4228\u4229\u422a\u422b\u422c\u422d\u422e\u422f\u4230\u4231\u4232\u4233\u4234\u4235\u4236\u4237\u4238\u4239\u423a\u423b\u423c\u423d\u423e\u423f\u4240\u4241\u4242\u4243\u4244\u4245\u4246\u4247\u4248\u4249\u424a\u424b\u424c\u424d\u424e\u424f\u4250\u4251\u4252\u4253\u4254\u4255\u4256\u4257\u4258\u4259\u425a\u425b\u425c\u425d\u425e\u425f\u4260\u4261\u4262\u4263\u4264\u4265\u4266\u4267\u4268\u4269\u426a\u426b\u426c\u426d\u426e\u426f\u4270\u4271\u4272\u4273\u4274\u4275\u4276\u4277\u4278\u4279\u427a\u427b\u427c\u427d\u427e\u427f\u4280\u4281\u4282\u4283\u4284\u4285\u4286\u4287\u4288\u4289\u428a\u428b\u428c\u428d\u428e\u428f\u4290\u4291\u4292\u4293\u4294\u4295\u4296\u4297\u4298\u4299\u429a\u429b\u429c\u429d\u429e\u429f\u42a0\u42a1\u42a2\u42a3\u42a4\u42a5\u42a6\u42a7\u42a8\u42a9\u42aa\u42ab\u42ac\u42ad\u42ae\u42af\u42b0\u42b1\u42b2\u42b3\u42b4\u42b5\u42b6\u42b7\u42b8\u42b9\u42ba\u42bb\u42bc\u42bd\u42be\u42bf\u42c0\u42c1\u42c2\u42c3\u42c4\u42c5\u42c6\u42c7\u42c8\u42c9\u42ca\u42cb\u42cc\u42cd\u42ce\u42cf\u42d0\u42d1\u42d2\u42d3\u42d4\u42d5\u42d6\u42d7\u42d8\u42d9\u42da\u42db\u42dc\u42dd\u42de\u42df\u42e0\u42e1\u42e2\u42e3\u42e4\u42e5\u42e6\u42e7\u42e8\u42e9\u42ea\u42eb\u42ec\u42ed\u42ee\u42ef\u42f0\u42f1\u42f2\u42f3\u42f4\u42f5\u42f6\u42f7\u42f8\u42f9\u42fa\u42fb\u42fc\u42fd\u42fe\u42ff\u4300\u4301\u4302\u4303\u4304\u4305\u4306\u4307\u4308\u4309\u430a\u430b\u430c\u430d\u430e\u430f\u4310\u4311\u4312\u4313\u4314\u4315\u4316\u4317\u4318\u4319\u431a\u431b\u431c\u431d\u431e\u431f\u4320\u4321\u4322\u4323\u4324\u4325\u4326\u4327\u4328\u4329\u432a\u432b\u432c\u432d\u432e\u432f\u4330\u4331\u4332\u4333\u4334\u4335\u4336\u4337\u4338\u4339\u433a\u433b\u433c\u433d\u433e\u433f\u4340\u4341\u4342\u4343\u4344\u4345\u4346\u4347\u4348\u4349\u434a\u434b\u434c\u434d\u434e\u434f\u4350\u4351\u4352\u4353\u4354\u4355\u4356\u4357\u4358\u4359\u435a\u435b\u435c\u435d\u435e\u435f\u4360\u4361\u4362\u4363\u4364\u4365\u4366\u4367\u4368\u4369\u436a\u436b\u436c\u436d\u436e\u436f\u4370\u4371\u4372\u4373\u4374\u4375\u4376\u4377\u4378\u4379\u437a\u437b\u437c\u437d\u437e\u437f\u4380\u4381\u4382\u4383\u4384\u4385\u4386\u4387\u4388\u4389\u438a\u438b\u438c\u438d\u438e\u438f\u4390\u4391\u4392\u4393\u4394\u4395\u4396\u4397\u4398\u4399\u439a\u439b\u439c\u439d\u439e\u439f\u43a0\u43a1\u43a2\u43a3\u43a4\u43a5\u43a6\u43a7\u43a8\u43a9\u43aa\u43ab\u43ac\u43ad\u43ae\u43af\u43b0\u43b1\u43b2\u43b3\u43b4\u43b5\u43b6\u43b7\u43b8\u43b9\u43ba\u43bb\u43bc\u43bd\u43be\u43bf\u43c0\u43c1\u43c2\u43c3\u43c4\u43c5\u43c6\u43c7\u43c8\u43c9\u43ca\u43cb\u43cc\u43cd\u43ce\u43cf\u43d0\u43d1\u43d2\u43d3\u43d4\u43d5\u43d6\u43d7\u43d8\u43d9\u43da\u43db\u43dc\u43dd\u43de\u43df\u43e0\u43e1\u43e2\u43e3\u43e4\u43e5\u43e6\u43e7\u43e8\u43e9\u43ea\u43eb\u43ec\u43ed\u43ee\u43ef\u43f0\u43f1\u43f2\u43f3\u43f4\u43f5\u43f6\u43f7\u43f8\u43f9\u43fa\u43fb\u43fc\u43fd\u43fe\u43ff\u4400\u4401\u4402\u4403\u4404\u4405\u4406\u4407\u4408\u4409\u440a\u440b\u440c\u440d\u440e\u440f\u4410\u4411\u4412\u4413\u4414\u4415\u4416\u4417\u4418\u4419\u441a\u441b\u441c\u441d\u441e\u441f\u4420\u4421\u4422\u4423\u4424\u4425\u4426\u4427\u4428\u4429\u442a\u442b\u442c\u442d\u442e\u442f\u4430\u4431\u4432\u4433\u4434\u4435\u4436\u4437\u4438\u4439\u443a\u443b\u443c\u443d\u443e\u443f\u4440\u4441\u4442\u4443\u4444\u4445\u4446\u4447\u4448\u4449\u444a\u444b\u444c\u444d\u444e\u444f\u4450\u4451\u4452\u4453\u4454\u4455\u4456\u4457\u4458\u4459\u445a\u445b\u445c\u445d\u445e\u445f\u4460\u4461\u4462\u4463\u4464\u4465\u4466\u4467\u4468\u4469\u446a\u446b\u446c\u446d\u446e\u446f\u4470\u4471\u4472\u4473\u4474\u4475\u4476\u4477\u4478\u4479\u447a\u447b\u447c\u447d\u447e\u447f\u4480\u4481\u4482\u4483\u4484\u4485\u4486\u4487\u4488\u4489\u448a\u448b\u448c\u448d\u448e\u448f\u4490\u4491\u4492\u4493\u4494\u4495\u4496\u4497\u4498\u4499\u449a\u449b\u449c\u449d\u449e\u449f\u44a0\u44a1\u44a2\u44a3\u44a4\u44a5\u44a6\u44a7\u44a8\u44a9\u44aa\u44ab\u44ac\u44ad\u44ae\u44af\u44b0\u44b1\u44b2\u44b3\u44b4\u44b5\u44b6\u44b7\u44b8\u44b9\u44ba\u44bb\u44bc\u44bd\u44be\u44bf\u44c0\u44c1\u44c2\u44c3\u44c4\u44c5\u44c6\u44c7\u44c8\u44c9\u44ca\u44cb\u44cc\u44cd\u44ce\u44cf\u44d0\u44d1\u44d2\u44d3\u44d4\u44d5\u44d6\u44d7\u44d8\u44d9\u44da\u44db\u44dc\u44dd\u44de\u44df\u44e0\u44e1\u44e2\u44e3\u44e4\u44e5\u44e6\u44e7\u44e8\u44e9\u44ea\u44eb\u44ec\u44ed\u44ee\u44ef\u44f0\u44f1\u44f2\u44f3\u44f4\u44f5\u44f6\u44f7\u44f8\u44f9\u44fa\u44fb\u44fc\u44fd\u44fe\u44ff\u4500\u4501\u4502\u4503\u4504\u4505\u4506\u4507\u4508\u4509\u450a\u450b\u450c\u450d\u450e\u450f\u4510\u4511\u4512\u4513\u4514\u4515\u4516\u4517\u4518\u4519\u451a\u451b\u451c\u451d\u451e\u451f\u4520\u4521\u4522\u4523\u4524\u4525\u4526\u4527\u4528\u4529\u452a\u452b\u452c\u452d\u452e\u452f\u4530\u4531\u4532\u4533\u4534\u4535\u4536\u4537\u4538\u4539\u453a\u453b\u453c\u453d\u453e\u453f\u4540\u4541\u4542\u4543\u4544\u4545\u4546\u4547\u4548\u4549\u454a\u454b\u454c\u454d\u454e\u454f\u4550\u4551\u4552\u4553\u4554\u4555\u4556\u4557\u4558\u4559\u455a\u455b\u455c\u455d\u455e\u455f\u4560\u4561\u4562\u4563\u4564\u4565\u4566\u4567\u4568\u4569\u456a\u456b\u456c\u456d\u456e\u456f\u4570\u4571\u4572\u4573\u4574\u4575\u4576\u4577\u4578\u4579\u457a\u457b\u457c\u457d\u457e\u457f\u4580\u4581\u4582\u4583\u4584\u4585\u4586\u4587\u4588\u4589\u458a\u458b\u458c\u458d\u458e\u458f\u4590\u4591\u4592\u4593\u4594\u4595\u4596\u4597\u4598\u4599\u459a\u459b\u459c\u459d\u459e\u459f\u45a0\u45a1\u45a2\u45a3\u45a4\u45a5\u45a6\u45a7\u45a8\u45a9\u45aa\u45ab\u45ac\u45ad\u45ae\u45af\u45b0\u45b1\u45b2\u45b3\u45b4\u45b5\u45b6\u45b7\u45b8\u45b9\u45ba\u45bb\u45bc\u45bd\u45be\u45bf\u45c0\u45c1\u45c2\u45c3\u45c4\u45c5\u45c6\u45c7\u45c8\u45c9\u45ca\u45cb\u45cc\u45cd\u45ce\u45cf\u45d0\u45d1\u45d2\u45d3\u45d4\u45d5\u45d6\u45d7\u45d8\u45d9\u45da\u45db\u45dc\u45dd\u45de\u45df\u45e0\u45e1\u45e2\u45e3\u45e4\u45e5\u45e6\u45e7\u45e8\u45e9\u45ea\u45eb\u45ec\u45ed\u45ee\u45ef\u45f0\u45f1\u45f2\u45f3\u45f4\u45f5\u45f6\u45f7\u45f8\u45f9\u45fa\u45fb\u45fc\u45fd\u45fe\u45ff\u4600\u4601\u4602\u4603\u4604\u4605\u4606\u4607\u4608\u4609\u460a\u460b\u460c\u460d\u460e\u460f\u4610\u4611\u4612\u4613\u4614\u4615\u4616\u4617\u4618\u4619\u461a\u461b\u461c\u461d\u461e\u461f\u4620\u4621\u4622\u4623\u4624\u4625\u4626\u4627\u4628\u4629\u462a\u462b\u462c\u462d\u462e\u462f\u4630\u4631\u4632\u4633\u4634\u4635\u4636\u4637\u4638\u4639\u463a\u463b\u463c\u463d\u463e\u463f\u4640\u4641\u4642\u4643\u4644\u4645\u4646\u4647\u4648\u4649\u464a\u464b\u464c\u464d\u464e\u464f\u4650\u4651\u4652\u4653\u4654\u4655\u4656\u4657\u4658\u4659\u465a\u465b\u465c\u465d\u465e\u465f\u4660\u4661\u4662\u4663\u4664\u4665\u4666\u4667\u4668\u4669\u466a\u466b\u466c\u466d\u466e\u466f\u4670\u4671\u4672\u4673\u4674\u4675\u4676\u4677\u4678\u4679\u467a\u467b\u467c\u467d\u467e\u467f\u4680\u4681\u4682\u4683\u4684\u4685\u4686\u4687\u4688\u4689\u468a\u468b\u468c\u468d\u468e\u468f\u4690\u4691\u4692\u4693\u4694\u4695\u4696\u4697\u4698\u4699\u469a\u469b\u469c\u469d\u469e\u469f\u46a0\u46a1\u46a2\u46a3\u46a4\u46a5\u46a6\u46a7\u46a8\u46a9\u46aa\u46ab\u46ac\u46ad\u46ae\u46af\u46b0\u46b1\u46b2\u46b3\u46b4\u46b5\u46b6\u46b7\u46b8\u46b9\u46ba\u46bb\u46bc\u46bd\u46be\u46bf\u46c0\u46c1\u46c2\u46c3\u46c4\u46c5\u46c6\u46c7\u46c8\u46c9\u46ca\u46cb\u46cc\u46cd\u46ce\u46cf\u46d0\u46d1\u46d2\u46d3\u46d4\u46d5\u46d6\u46d7\u46d8\u46d9\u46da\u46db\u46dc\u46dd\u46de\u46df\u46e0\u46e1\u46e2\u46e3\u46e4\u46e5\u46e6\u46e7\u46e8\u46e9\u46ea\u46eb\u46ec\u46ed\u46ee\u46ef\u46f0\u46f1\u46f2\u46f3\u46f4\u46f5\u46f6\u46f7\u46f8\u46f9\u46fa\u46fb\u46fc\u46fd\u46fe\u46ff\u4700\u4701\u4702\u4703\u4704\u4705\u4706\u4707\u4708\u4709\u470a\u470b\u470c\u470d\u470e\u470f\u4710\u4711\u4712\u4713\u4714\u4715\u4716\u4717\u4718\u4719\u471a\u471b\u471c\u471d\u471e\u471f\u4720\u4721\u4722\u4723\u4724\u4725\u4726\u4727\u4728\u4729\u472a\u472b\u472c\u472d\u472e\u472f\u4730\u4731\u4732\u4733\u4734\u4735\u4736\u4737\u4738\u4739\u473a\u473b\u473c\u473d\u473e\u473f\u4740\u4741\u4742\u4743\u4744\u4745\u4746\u4747\u4748\u4749\u474a\u474b\u474c\u474d\u474e\u474f\u4750\u4751\u4752\u4753\u4754\u4755\u4756\u4757\u4758\u4759\u475a\u475b\u475c\u475d\u475e\u475f\u4760\u4761\u4762\u4763\u4764\u4765\u4766\u4767\u4768\u4769\u476a\u476b\u476c\u476d\u476e\u476f\u4770\u4771\u4772\u4773\u4774\u4775\u4776\u4777\u4778\u4779\u477a\u477b\u477c\u477d\u477e\u477f\u4780\u4781\u4782\u4783\u4784\u4785\u4786\u4787\u4788\u4789\u478a\u478b\u478c\u478d\u478e\u478f\u4790\u4791\u4792\u4793\u4794\u4795\u4796\u4797\u4798\u4799\u479a\u479b\u479c\u479d\u479e\u479f\u47a0\u47a1\u47a2\u47a3\u47a4\u47a5\u47a6\u47a7\u47a8\u47a9\u47aa\u47ab\u47ac\u47ad\u47ae\u47af\u47b0\u47b1\u47b2\u47b3\u47b4\u47b5\u47b6\u47b7\u47b8\u47b9\u47ba\u47bb\u47bc\u47bd\u47be\u47bf\u47c0\u47c1\u47c2\u47c3\u47c4\u47c5\u47c6\u47c7\u47c8\u47c9\u47ca\u47cb\u47cc\u47cd\u47ce\u47cf\u47d0\u47d1\u47d2\u47d3\u47d4\u47d5\u47d6\u47d7\u47d8\u47d9\u47da\u47db\u47dc\u47dd\u47de\u47df\u47e0\u47e1\u47e2\u47e3\u47e4\u47e5\u47e6\u47e7\u47e8\u47e9\u47ea\u47eb\u47ec\u47ed\u47ee\u47ef\u47f0\u47f1\u47f2\u47f3\u47f4\u47f5\u47f6\u47f7\u47f8\u47f9\u47fa\u47fb\u47fc\u47fd\u47fe\u47ff\u4800\u4801\u4802\u4803\u4804\u4805\u4806\u4807\u4808\u4809\u480a\u480b\u480c\u480d\u480e\u480f\u4810\u4811\u4812\u4813\u4814\u4815\u4816\u4817\u4818\u4819\u481a\u481b\u481c\u481d\u481e\u481f\u4820\u4821\u4822\u4823\u4824\u4825\u4826\u4827\u4828\u4829\u482a\u482b\u482c\u482d\u482e\u482f\u4830\u4831\u4832\u4833\u4834\u4835\u4836\u4837\u4838\u4839\u483a\u483b\u483c\u483d\u483e\u483f\u4840\u4841\u4842\u4843\u4844\u4845\u4846\u4847\u4848\u4849\u484a\u484b\u484c\u484d\u484e\u484f\u4850\u4851\u4852\u4853\u4854\u4855\u4856\u4857\u4858\u4859\u485a\u485b\u485c\u485d\u485e\u485f\u4860\u4861\u4862\u4863\u4864\u4865\u4866\u4867\u4868\u4869\u486a\u486b\u486c\u486d\u486e\u486f\u4870\u4871\u4872\u4873\u4874\u4875\u4876\u4877\u4878\u4879\u487a\u487b\u487c\u487d\u487e\u487f\u4880\u4881\u4882\u4883\u4884\u4885\u4886\u4887\u4888\u4889\u488a\u488b\u488c\u488d\u488e\u488f\u4890\u4891\u4892\u4893\u4894\u4895\u4896\u4897\u4898\u4899\u489a\u489b\u489c\u489d\u489e\u489f\u48a0\u48a1\u48a2\u48a3\u48a4\u48a5\u48a6\u48a7\u48a8\u48a9\u48aa\u48ab\u48ac\u48ad\u48ae\u48af\u48b0\u48b1\u48b2\u48b3\u48b4\u48b5\u48b6\u48b7\u48b8\u48b9\u48ba\u48bb\u48bc\u48bd\u48be\u48bf\u48c0\u48c1\u48c2\u48c3\u48c4\u48c5\u48c6\u48c7\u48c8\u48c9\u48ca\u48cb\u48cc\u48cd\u48ce\u48cf\u48d0\u48d1\u48d2\u48d3\u48d4\u48d5\u48d6\u48d7\u48d8\u48d9\u48da\u48db\u48dc\u48dd\u48de\u48df\u48e0\u48e1\u48e2\u48e3\u48e4\u48e5\u48e6\u48e7\u48e8\u48e9\u48ea\u48eb\u48ec\u48ed\u48ee\u48ef\u48f0\u48f1\u48f2\u48f3\u48f4\u48f5\u48f6\u48f7\u48f8\u48f9\u48fa\u48fb\u48fc\u48fd\u48fe\u48ff\u4900\u4901\u4902\u4903\u4904\u4905\u4906\u4907\u4908\u4909\u490a\u490b\u490c\u490d\u490e\u490f\u4910\u4911\u4912\u4913\u4914\u4915\u4916\u4917\u4918\u4919\u491a\u491b\u491c\u491d\u491e\u491f\u4920\u4921\u4922\u4923\u4924\u4925\u4926\u4927\u4928\u4929\u492a\u492b\u492c\u492d\u492e\u492f\u4930\u4931\u4932\u4933\u4934\u4935\u4936\u4937\u4938\u4939\u493a\u493b\u493c\u493d\u493e\u493f\u4940\u4941\u4942\u4943\u4944\u4945\u4946\u4947\u4948\u4949\u494a\u494b\u494c\u494d\u494e\u494f\u4950\u4951\u4952\u4953\u4954\u4955\u4956\u4957\u4958\u4959\u495a\u495b\u495c\u495d\u495e\u495f\u4960\u4961\u4962\u4963\u4964\u4965\u4966\u4967\u4968\u4969\u496a\u496b\u496c\u496d\u496e\u496f\u4970\u4971\u4972\u4973\u4974\u4975\u4976\u4977\u4978\u4979\u497a\u497b\u497c\u497d\u497e\u497f\u4980\u4981\u4982\u4983\u4984\u4985\u4986\u4987\u4988\u4989\u498a\u498b\u498c\u498d\u498e\u498f\u4990\u4991\u4992\u4993\u4994\u4995\u4996\u4997\u4998\u4999\u499a\u499b\u499c\u499d\u499e\u499f\u49a0\u49a1\u49a2\u49a3\u49a4\u49a5\u49a6\u49a7\u49a8\u49a9\u49aa\u49ab\u49ac\u49ad\u49ae\u49af\u49b0\u49b1\u49b2\u49b3\u49b4\u49b5\u49b6\u49b7\u49b8\u49b9\u49ba\u49bb\u49bc\u49bd\u49be\u49bf\u49c0\u49c1\u49c2\u49c3\u49c4\u49c5\u49c6\u49c7\u49c8\u49c9\u49ca\u49cb\u49cc\u49cd\u49ce\u49cf\u49d0\u49d1\u49d2\u49d3\u49d4\u49d5\u49d6\u49d7\u49d8\u49d9\u49da\u49db\u49dc\u49dd\u49de\u49df\u49e0\u49e1\u49e2\u49e3\u49e4\u49e5\u49e6\u49e7\u49e8\u49e9\u49ea\u49eb\u49ec\u49ed\u49ee\u49ef\u49f0\u49f1\u49f2\u49f3\u49f4\u49f5\u49f6\u49f7\u49f8\u49f9\u49fa\u49fb\u49fc\u49fd\u49fe\u49ff\u4a00\u4a01\u4a02\u4a03\u4a04\u4a05\u4a06\u4a07\u4a08\u4a09\u4a0a\u4a0b\u4a0c\u4a0d\u4a0e\u4a0f\u4a10\u4a11\u4a12\u4a13\u4a14\u4a15\u4a16\u4a17\u4a18\u4a19\u4a1a\u4a1b\u4a1c\u4a1d\u4a1e\u4a1f\u4a20\u4a21\u4a22\u4a23\u4a24\u4a25\u4a26\u4a27\u4a28\u4a29\u4a2a\u4a2b\u4a2c\u4a2d\u4a2e\u4a2f\u4a30\u4a31\u4a32\u4a33\u4a34\u4a35\u4a36\u4a37\u4a38\u4a39\u4a3a\u4a3b\u4a3c\u4a3d\u4a3e\u4a3f\u4a40\u4a41\u4a42\u4a43\u4a44\u4a45\u4a46\u4a47\u4a48\u4a49\u4a4a\u4a4b\u4a4c\u4a4d\u4a4e\u4a4f\u4a50\u4a51\u4a52\u4a53\u4a54\u4a55\u4a56\u4a57\u4a58\u4a59\u4a5a\u4a5b\u4a5c\u4a5d\u4a5e\u4a5f\u4a60\u4a61\u4a62\u4a63\u4a64\u4a65\u4a66\u4a67\u4a68\u4a69\u4a6a\u4a6b\u4a6c\u4a6d\u4a6e\u4a6f\u4a70\u4a71\u4a72\u4a73\u4a74\u4a75\u4a76\u4a77\u4a78\u4a79\u4a7a\u4a7b\u4a7c\u4a7d\u4a7e\u4a7f\u4a80\u4a81\u4a82\u4a83\u4a84\u4a85\u4a86\u4a87\u4a88\u4a89\u4a8a\u4a8b\u4a8c\u4a8d\u4a8e\u4a8f\u4a90\u4a91\u4a92\u4a93\u4a94\u4a95\u4a96\u4a97\u4a98\u4a99\u4a9a\u4a9b\u4a9c\u4a9d\u4a9e\u4a9f\u4aa0\u4aa1\u4aa2\u4aa3\u4aa4\u4aa5\u4aa6\u4aa7\u4aa8\u4aa9\u4aaa\u4aab\u4aac\u4aad\u4aae\u4aaf\u4ab0\u4ab1\u4ab2\u4ab3\u4ab4\u4ab5\u4ab6\u4ab7\u4ab8\u4ab9\u4aba\u4abb\u4abc\u4abd\u4abe\u4abf\u4ac0\u4ac1\u4ac2\u4ac3\u4ac4\u4ac5\u4ac6\u4ac7\u4ac8\u4ac9\u4aca\u4acb\u4acc\u4acd\u4ace\u4acf\u4ad0\u4ad1\u4ad2\u4ad3\u4ad4\u4ad5\u4ad6\u4ad7\u4ad8\u4ad9\u4ada\u4adb\u4adc\u4add\u4ade\u4adf\u4ae0\u4ae1\u4ae2\u4ae3\u4ae4\u4ae5\u4ae6\u4ae7\u4ae8\u4ae9\u4aea\u4aeb\u4aec\u4aed\u4aee\u4aef\u4af0\u4af1\u4af2\u4af3\u4af4\u4af5\u4af6\u4af7\u4af8\u4af9\u4afa\u4afb\u4afc\u4afd\u4afe\u4aff\u4b00\u4b01\u4b02\u4b03\u4b04\u4b05\u4b06\u4b07\u4b08\u4b09\u4b0a\u4b0b\u4b0c\u4b0d\u4b0e\u4b0f\u4b10\u4b11\u4b12\u4b13\u4b14\u4b15\u4b16\u4b17\u4b18\u4b19\u4b1a\u4b1b\u4b1c\u4b1d\u4b1e\u4b1f\u4b20\u4b21\u4b22\u4b23\u4b24\u4b25\u4b26\u4b27\u4b28\u4b29\u4b2a\u4b2b\u4b2c\u4b2d\u4b2e\u4b2f\u4b30\u4b31\u4b32\u4b33\u4b34\u4b35\u4b36\u4b37\u4b38\u4b39\u4b3a\u4b3b\u4b3c\u4b3d\u4b3e\u4b3f\u4b40\u4b41\u4b42\u4b43\u4b44\u4b45\u4b46\u4b47\u4b48\u4b49\u4b4a\u4b4b\u4b4c\u4b4d\u4b4e\u4b4f\u4b50\u4b51\u4b52\u4b53\u4b54\u4b55\u4b56\u4b57\u4b58\u4b59\u4b5a\u4b5b\u4b5c\u4b5d\u4b5e\u4b5f\u4b60\u4b61\u4b62\u4b63\u4b64\u4b65\u4b66\u4b67\u4b68\u4b69\u4b6a\u4b6b\u4b6c\u4b6d\u4b6e\u4b6f\u4b70\u4b71\u4b72\u4b73\u4b74\u4b75\u4b76\u4b77\u4b78\u4b79\u4b7a\u4b7b\u4b7c\u4b7d\u4b7e\u4b7f\u4b80\u4b81\u4b82\u4b83\u4b84\u4b85\u4b86\u4b87\u4b88\u4b89\u4b8a\u4b8b\u4b8c\u4b8d\u4b8e\u4b8f\u4b90\u4b91\u4b92\u4b93\u4b94\u4b95\u4b96\u4b97\u4b98\u4b99\u4b9a\u4b9b\u4b9c\u4b9d\u4b9e\u4b9f\u4ba0\u4ba1\u4ba2\u4ba3\u4ba4\u4ba5\u4ba6\u4ba7\u4ba8\u4ba9\u4baa\u4bab\u4bac\u4bad\u4bae\u4baf\u4bb0\u4bb1\u4bb2\u4bb3\u4bb4\u4bb5\u4bb6\u4bb7\u4bb8\u4bb9\u4bba\u4bbb\u4bbc\u4bbd\u4bbe\u4bbf\u4bc0\u4bc1\u4bc2\u4bc3\u4bc4\u4bc5\u4bc6\u4bc7\u4bc8\u4bc9\u4bca\u4bcb\u4bcc\u4bcd\u4bce\u4bcf\u4bd0\u4bd1\u4bd2\u4bd3\u4bd4\u4bd5\u4bd6\u4bd7\u4bd8\u4bd9\u4bda\u4bdb\u4bdc\u4bdd\u4bde\u4bdf\u4be0\u4be1\u4be2\u4be3\u4be4\u4be5\u4be6\u4be7\u4be8\u4be9\u4bea\u4beb\u4bec\u4bed\u4bee\u4bef\u4bf0\u4bf1\u4bf2\u4bf3\u4bf4\u4bf5\u4bf6\u4bf7\u4bf8\u4bf9\u4bfa\u4bfb\u4bfc\u4bfd\u4bfe\u4bff\u4c00\u4c01\u4c02\u4c03\u4c04\u4c05\u4c06\u4c07\u4c08\u4c09\u4c0a\u4c0b\u4c0c\u4c0d\u4c0e\u4c0f\u4c10\u4c11\u4c12\u4c13\u4c14\u4c15\u4c16\u4c17\u4c18\u4c19\u4c1a\u4c1b\u4c1c\u4c1d\u4c1e\u4c1f\u4c20\u4c21\u4c22\u4c23\u4c24\u4c25\u4c26\u4c27\u4c28\u4c29\u4c2a\u4c2b\u4c2c\u4c2d\u4c2e\u4c2f\u4c30\u4c31\u4c32\u4c33\u4c34\u4c35\u4c36\u4c37\u4c38\u4c39\u4c3a\u4c3b\u4c3c\u4c3d\u4c3e\u4c3f\u4c40\u4c41\u4c42\u4c43\u4c44\u4c45\u4c46\u4c47\u4c48\u4c49\u4c4a\u4c4b\u4c4c\u4c4d\u4c4e\u4c4f\u4c50\u4c51\u4c52\u4c53\u4c54\u4c55\u4c56\u4c57\u4c58\u4c59\u4c5a\u4c5b\u4c5c\u4c5d\u4c5e\u4c5f\u4c60\u4c61\u4c62\u4c63\u4c64\u4c65\u4c66\u4c67\u4c68\u4c69\u4c6a\u4c6b\u4c6c\u4c6d\u4c6e\u4c6f\u4c70\u4c71\u4c72\u4c73\u4c74\u4c75\u4c76\u4c77\u4c78\u4c79\u4c7a\u4c7b\u4c7c\u4c7d\u4c7e\u4c7f\u4c80\u4c81\u4c82\u4c83\u4c84\u4c85\u4c86\u4c87\u4c88\u4c89\u4c8a\u4c8b\u4c8c\u4c8d\u4c8e\u4c8f\u4c90\u4c91\u4c92\u4c93\u4c94\u4c95\u4c96\u4c97\u4c98\u4c99\u4c9a\u4c9b\u4c9c\u4c9d\u4c9e\u4c9f\u4ca0\u4ca1\u4ca2\u4ca3\u4ca4\u4ca5\u4ca6\u4ca7\u4ca8\u4ca9\u4caa\u4cab\u4cac\u4cad\u4cae\u4caf\u4cb0\u4cb1\u4cb2\u4cb3\u4cb4\u4cb5\u4cb6\u4cb7\u4cb8\u4cb9\u4cba\u4cbb\u4cbc\u4cbd\u4cbe\u4cbf\u4cc0\u4cc1\u4cc2\u4cc3\u4cc4\u4cc5\u4cc6\u4cc7\u4cc8\u4cc9\u4cca\u4ccb\u4ccc\u4ccd\u4cce\u4ccf\u4cd0\u4cd1\u4cd2\u4cd3\u4cd4\u4cd5\u4cd6\u4cd7\u4cd8\u4cd9\u4cda\u4cdb\u4cdc\u4cdd\u4cde\u4cdf\u4ce0\u4ce1\u4ce2\u4ce3\u4ce4\u4ce5\u4ce6\u4ce7\u4ce8\u4ce9\u4cea\u4ceb\u4cec\u4ced\u4cee\u4cef\u4cf0\u4cf1\u4cf2\u4cf3\u4cf4\u4cf5\u4cf6\u4cf7\u4cf8\u4cf9\u4cfa\u4cfb\u4cfc\u4cfd\u4cfe\u4cff\u4d00\u4d01\u4d02\u4d03\u4d04\u4d05\u4d06\u4d07\u4d08\u4d09\u4d0a\u4d0b\u4d0c\u4d0d\u4d0e\u4d0f\u4d10\u4d11\u4d12\u4d13\u4d14\u4d15\u4d16\u4d17\u4d18\u4d19\u4d1a\u4d1b\u4d1c\u4d1d\u4d1e\u4d1f\u4d20\u4d21\u4d22\u4d23\u4d24\u4d25\u4d26\u4d27\u4d28\u4d29\u4d2a\u4d2b\u4d2c\u4d2d\u4d2e\u4d2f\u4d30\u4d31\u4d32\u4d33\u4d34\u4d35\u4d36\u4d37\u4d38\u4d39\u4d3a\u4d3b\u4d3c\u4d3d\u4d3e\u4d3f\u4d40\u4d41\u4d42\u4d43\u4d44\u4d45\u4d46\u4d47\u4d48\u4d49\u4d4a\u4d4b\u4d4c\u4d4d\u4d4e\u4d4f\u4d50\u4d51\u4d52\u4d53\u4d54\u4d55\u4d56\u4d57\u4d58\u4d59\u4d5a\u4d5b\u4d5c\u4d5d\u4d5e\u4d5f\u4d60\u4d61\u4d62\u4d63\u4d64\u4d65\u4d66\u4d67\u4d68\u4d69\u4d6a\u4d6b\u4d6c\u4d6d\u4d6e\u4d6f\u4d70\u4d71\u4d72\u4d73\u4d74\u4d75\u4d76\u4d77\u4d78\u4d79\u4d7a\u4d7b\u4d7c\u4d7d\u4d7e\u4d7f\u4d80\u4d81\u4d82\u4d83\u4d84\u4d85\u4d86\u4d87\u4d88\u4d89\u4d8a\u4d8b\u4d8c\u4d8d\u4d8e\u4d8f\u4d90\u4d91\u4d92\u4d93\u4d94\u4d95\u4d96\u4d97\u4d98\u4d99\u4d9a\u4d9b\u4d9c\u4d9d\u4d9e\u4d9f\u4da0\u4da1\u4da2\u4da3\u4da4\u4da5\u4da6\u4da7\u4da8\u4da9\u4daa\u4dab\u4dac\u4dad\u4dae\u4daf\u4db0\u4db1\u4db2\u4db3\u4db4\u4db5\u4e00\u4e01\u4e02\u4e03\u4e04\u4e05\u4e06\u4e07\u4e08\u4e09\u4e0a\u4e0b\u4e0c\u4e0d\u4e0e\u4e0f\u4e10\u4e11\u4e12\u4e13\u4e14\u4e15\u4e16\u4e17\u4e18\u4e19\u4e1a\u4e1b\u4e1c\u4e1d\u4e1e\u4e1f\u4e20\u4e21\u4e22\u4e23\u4e24\u4e25\u4e26\u4e27\u4e28\u4e29\u4e2a\u4e2b\u4e2c\u4e2d\u4e2e\u4e2f\u4e30\u4e31\u4e32\u4e33\u4e34\u4e35\u4e36\u4e37\u4e38\u4e39\u4e3a\u4e3b\u4e3c\u4e3d\u4e3e\u4e3f\u4e40\u4e41\u4e42\u4e43\u4e44\u4e45\u4e46\u4e47\u4e48\u4e49\u4e4a\u4e4b\u4e4c\u4e4d\u4e4e\u4e4f\u4e50\u4e51\u4e52\u4e53\u4e54\u4e55\u4e56\u4e57\u4e58\u4e59\u4e5a\u4e5b\u4e5c\u4e5d\u4e5e\u4e5f\u4e60\u4e61\u4e62\u4e63\u4e64\u4e65\u4e66\u4e67\u4e68\u4e69\u4e6a\u4e6b\u4e6c\u4e6d\u4e6e\u4e6f\u4e70\u4e71\u4e72\u4e73\u4e74\u4e75\u4e76\u4e77\u4e78\u4e79\u4e7a\u4e7b\u4e7c\u4e7d\u4e7e\u4e7f\u4e80\u4e81\u4e82\u4e83\u4e84\u4e85\u4e86\u4e87\u4e88\u4e89\u4e8a\u4e8b\u4e8c\u4e8d\u4e8e\u4e8f\u4e90\u4e91\u4e92\u4e93\u4e94\u4e95\u4e96\u4e97\u4e98\u4e99\u4e9a\u4e9b\u4e9c\u4e9d\u4e9e\u4e9f\u4ea0\u4ea1\u4ea2\u4ea3\u4ea4\u4ea5\u4ea6\u4ea7\u4ea8\u4ea9\u4eaa\u4eab\u4eac\u4ead\u4eae\u4eaf\u4eb0\u4eb1\u4eb2\u4eb3\u4eb4\u4eb5\u4eb6\u4eb7\u4eb8\u4eb9\u4eba\u4ebb\u4ebc\u4ebd\u4ebe\u4ebf\u4ec0\u4ec1\u4ec2\u4ec3\u4ec4\u4ec5\u4ec6\u4ec7\u4ec8\u4ec9\u4eca\u4ecb\u4ecc\u4ecd\u4ece\u4ecf\u4ed0\u4ed1\u4ed2\u4ed3\u4ed4\u4ed5\u4ed6\u4ed7\u4ed8\u4ed9\u4eda\u4edb\u4edc\u4edd\u4ede\u4edf\u4ee0\u4ee1\u4ee2\u4ee3\u4ee4\u4ee5\u4ee6\u4ee7\u4ee8\u4ee9\u4eea\u4eeb\u4eec\u4eed\u4eee\u4eef\u4ef0\u4ef1\u4ef2\u4ef3\u4ef4\u4ef5\u4ef6\u4ef7\u4ef8\u4ef9\u4efa\u4efb\u4efc\u4efd\u4efe\u4eff\u4f00\u4f01\u4f02\u4f03\u4f04\u4f05\u4f06\u4f07\u4f08\u4f09\u4f0a\u4f0b\u4f0c\u4f0d\u4f0e\u4f0f\u4f10\u4f11\u4f12\u4f13\u4f14\u4f15\u4f16\u4f17\u4f18\u4f19\u4f1a\u4f1b\u4f1c\u4f1d\u4f1e\u4f1f\u4f20\u4f21\u4f22\u4f23\u4f24\u4f25\u4f26\u4f27\u4f28\u4f29\u4f2a\u4f2b\u4f2c\u4f2d\u4f2e\u4f2f\u4f30\u4f31\u4f32\u4f33\u4f34\u4f35\u4f36\u4f37\u4f38\u4f39\u4f3a\u4f3b\u4f3c\u4f3d\u4f3e\u4f3f\u4f40\u4f41\u4f42\u4f43\u4f44\u4f45\u4f46\u4f47\u4f48\u4f49\u4f4a\u4f4b\u4f4c\u4f4d\u4f4e\u4f4f\u4f50\u4f51\u4f52\u4f53\u4f54\u4f55\u4f56\u4f57\u4f58\u4f59\u4f5a\u4f5b\u4f5c\u4f5d\u4f5e\u4f5f\u4f60\u4f61\u4f62\u4f63\u4f64\u4f65\u4f66\u4f67\u4f68\u4f69\u4f6a\u4f6b\u4f6c\u4f6d\u4f6e\u4f6f\u4f70\u4f71\u4f72\u4f73\u4f74\u4f75\u4f76\u4f77\u4f78\u4f79\u4f7a\u4f7b\u4f7c\u4f7d\u4f7e\u4f7f\u4f80\u4f81\u4f82\u4f83\u4f84\u4f85\u4f86\u4f87\u4f88\u4f89\u4f8a\u4f8b\u4f8c\u4f8d\u4f8e\u4f8f\u4f90\u4f91\u4f92\u4f93\u4f94\u4f95\u4f96\u4f97\u4f98\u4f99\u4f9a\u4f9b\u4f9c\u4f9d\u4f9e\u4f9f\u4fa0\u4fa1\u4fa2\u4fa3\u4fa4\u4fa5\u4fa6\u4fa7\u4fa8\u4fa9\u4faa\u4fab\u4fac\u4fad\u4fae\u4faf\u4fb0\u4fb1\u4fb2\u4fb3\u4fb4\u4fb5\u4fb6\u4fb7\u4fb8\u4fb9\u4fba\u4fbb\u4fbc\u4fbd\u4fbe\u4fbf\u4fc0\u4fc1\u4fc2\u4fc3\u4fc4\u4fc5\u4fc6\u4fc7\u4fc8\u4fc9\u4fca\u4fcb\u4fcc\u4fcd\u4fce\u4fcf\u4fd0\u4fd1\u4fd2\u4fd3\u4fd4\u4fd5\u4fd6\u4fd7\u4fd8\u4fd9\u4fda\u4fdb\u4fdc\u4fdd\u4fde\u4fdf\u4fe0\u4fe1\u4fe2\u4fe3\u4fe4\u4fe5\u4fe6\u4fe7\u4fe8\u4fe9\u4fea\u4feb\u4fec\u4fed\u4fee\u4fef\u4ff0\u4ff1\u4ff2\u4ff3\u4ff4\u4ff5\u4ff6\u4ff7\u4ff8\u4ff9\u4ffa\u4ffb\u4ffc\u4ffd\u4ffe\u4fff\u5000\u5001\u5002\u5003\u5004\u5005\u5006\u5007\u5008\u5009\u500a\u500b\u500c\u500d\u500e\u500f\u5010\u5011\u5012\u5013\u5014\u5015\u5016\u5017\u5018\u5019\u501a\u501b\u501c\u501d\u501e\u501f\u5020\u5021\u5022\u5023\u5024\u5025\u5026\u5027\u5028\u5029\u502a\u502b\u502c\u502d\u502e\u502f\u5030\u5031\u5032\u5033\u5034\u5035\u5036\u5037\u5038\u5039\u503a\u503b\u503c\u503d\u503e\u503f\u5040\u5041\u5042\u5043\u5044\u5045\u5046\u5047\u5048\u5049\u504a\u504b\u504c\u504d\u504e\u504f\u5050\u5051\u5052\u5053\u5054\u5055\u5056\u5057\u5058\u5059\u505a\u505b\u505c\u505d\u505e\u505f\u5060\u5061\u5062\u5063\u5064\u5065\u5066\u5067\u5068\u5069\u506a\u506b\u506c\u506d\u506e\u506f\u5070\u5071\u5072\u5073\u5074\u5075\u5076\u5077\u5078\u5079\u507a\u507b\u507c\u507d\u507e\u507f\u5080\u5081\u5082\u5083\u5084\u5085\u5086\u5087\u5088\u5089\u508a\u508b\u508c\u508d\u508e\u508f\u5090\u5091\u5092\u5093\u5094\u5095\u5096\u5097\u5098\u5099\u509a\u509b\u509c\u509d\u509e\u509f\u50a0\u50a1\u50a2\u50a3\u50a4\u50a5\u50a6\u50a7\u50a8\u50a9\u50aa\u50ab\u50ac\u50ad\u50ae\u50af\u50b0\u50b1\u50b2\u50b3\u50b4\u50b5\u50b6\u50b7\u50b8\u50b9\u50ba\u50bb\u50bc\u50bd\u50be\u50bf\u50c0\u50c1\u50c2\u50c3\u50c4\u50c5\u50c6\u50c7\u50c8\u50c9\u50ca\u50cb\u50cc\u50cd\u50ce\u50cf\u50d0\u50d1\u50d2\u50d3\u50d4\u50d5\u50d6\u50d7\u50d8\u50d9\u50da\u50db\u50dc\u50dd\u50de\u50df\u50e0\u50e1\u50e2\u50e3\u50e4\u50e5\u50e6\u50e7\u50e8\u50e9\u50ea\u50eb\u50ec\u50ed\u50ee\u50ef\u50f0\u50f1\u50f2\u50f3\u50f4\u50f5\u50f6\u50f7\u50f8\u50f9\u50fa\u50fb\u50fc\u50fd\u50fe\u50ff\u5100\u5101\u5102\u5103\u5104\u5105\u5106\u5107\u5108\u5109\u510a\u510b\u510c\u510d\u510e\u510f\u5110\u5111\u5112\u5113\u5114\u5115\u5116\u5117\u5118\u5119\u511a\u511b\u511c\u511d\u511e\u511f\u5120\u5121\u5122\u5123\u5124\u5125\u5126\u5127\u5128\u5129\u512a\u512b\u512c\u512d\u512e\u512f\u5130\u5131\u5132\u5133\u5134\u5135\u5136\u5137\u5138\u5139\u513a\u513b\u513c\u513d\u513e\u513f\u5140\u5141\u5142\u5143\u5144\u5145\u5146\u5147\u5148\u5149\u514a\u514b\u514c\u514d\u514e\u514f\u5150\u5151\u5152\u5153\u5154\u5155\u5156\u5157\u5158\u5159\u515a\u515b\u515c\u515d\u515e\u515f\u5160\u5161\u5162\u5163\u5164\u5165\u5166\u5167\u5168\u5169\u516a\u516b\u516c\u516d\u516e\u516f\u5170\u5171\u5172\u5173\u5174\u5175\u5176\u5177\u5178\u5179\u517a\u517b\u517c\u517d\u517e\u517f\u5180\u5181\u5182\u5183\u5184\u5185\u5186\u5187\u5188\u5189\u518a\u518b\u518c\u518d\u518e\u518f\u5190\u5191\u5192\u5193\u5194\u5195\u5196\u5197\u5198\u5199\u519a\u519b\u519c\u519d\u519e\u519f\u51a0\u51a1\u51a2\u51a3\u51a4\u51a5\u51a6\u51a7\u51a8\u51a9\u51aa\u51ab\u51ac\u51ad\u51ae\u51af\u51b0\u51b1\u51b2\u51b3\u51b4\u51b5\u51b6\u51b7\u51b8\u51b9\u51ba\u51bb\u51bc\u51bd\u51be\u51bf\u51c0\u51c1\u51c2\u51c3\u51c4\u51c5\u51c6\u51c7\u51c8\u51c9\u51ca\u51cb\u51cc\u51cd\u51ce\u51cf\u51d0\u51d1\u51d2\u51d3\u51d4\u51d5\u51d6\u51d7\u51d8\u51d9\u51da\u51db\u51dc\u51dd\u51de\u51df\u51e0\u51e1\u51e2\u51e3\u51e4\u51e5\u51e6\u51e7\u51e8\u51e9\u51ea\u51eb\u51ec\u51ed\u51ee\u51ef\u51f0\u51f1\u51f2\u51f3\u51f4\u51f5\u51f6\u51f7\u51f8\u51f9\u51fa\u51fb\u51fc\u51fd\u51fe\u51ff\u5200\u5201\u5202\u5203\u5204\u5205\u5206\u5207\u5208\u5209\u520a\u520b\u520c\u520d\u520e\u520f\u5210\u5211\u5212\u5213\u5214\u5215\u5216\u5217\u5218\u5219\u521a\u521b\u521c\u521d\u521e\u521f\u5220\u5221\u5222\u5223\u5224\u5225\u5226\u5227\u5228\u5229\u522a\u522b\u522c\u522d\u522e\u522f\u5230\u5231\u5232\u5233\u5234\u5235\u5236\u5237\u5238\u5239\u523a\u523b\u523c\u523d\u523e\u523f\u5240\u5241\u5242\u5243\u5244\u5245\u5246\u5247\u5248\u5249\u524a\u524b\u524c\u524d\u524e\u524f\u5250\u5251\u5252\u5253\u5254\u5255\u5256\u5257\u5258\u5259\u525a\u525b\u525c\u525d\u525e\u525f\u5260\u5261\u5262\u5263\u5264\u5265\u5266\u5267\u5268\u5269\u526a\u526b\u526c\u526d\u526e\u526f\u5270\u5271\u5272\u5273\u5274\u5275\u5276\u5277\u5278\u5279\u527a\u527b\u527c\u527d\u527e\u527f\u5280\u5281\u5282\u5283\u5284\u5285\u5286\u5287\u5288\u5289\u528a\u528b\u528c\u528d\u528e\u528f\u5290\u5291\u5292\u5293\u5294\u5295\u5296\u5297\u5298\u5299\u529a\u529b\u529c\u529d\u529e\u529f\u52a0\u52a1\u52a2\u52a3\u52a4\u52a5\u52a6\u52a7\u52a8\u52a9\u52aa\u52ab\u52ac\u52ad\u52ae\u52af\u52b0\u52b1\u52b2\u52b3\u52b4\u52b5\u52b6\u52b7\u52b8\u52b9\u52ba\u52bb\u52bc\u52bd\u52be\u52bf\u52c0\u52c1\u52c2\u52c3\u52c4\u52c5\u52c6\u52c7\u52c8\u52c9\u52ca\u52cb\u52cc\u52cd\u52ce\u52cf\u52d0\u52d1\u52d2\u52d3\u52d4\u52d5\u52d6\u52d7\u52d8\u52d9\u52da\u52db\u52dc\u52dd\u52de\u52df\u52e0\u52e1\u52e2\u52e3\u52e4\u52e5\u52e6\u52e7\u52e8\u52e9\u52ea\u52eb\u52ec\u52ed\u52ee\u52ef\u52f0\u52f1\u52f2\u52f3\u52f4\u52f5\u52f6\u52f7\u52f8\u52f9\u52fa\u52fb\u52fc\u52fd\u52fe\u52ff\u5300\u5301\u5302\u5303\u5304\u5305\u5306\u5307\u5308\u5309\u530a\u530b\u530c\u530d\u530e\u530f\u5310\u5311\u5312\u5313\u5314\u5315\u5316\u5317\u5318\u5319\u531a\u531b\u531c\u531d\u531e\u531f\u5320\u5321\u5322\u5323\u5324\u5325\u5326\u5327\u5328\u5329\u532a\u532b\u532c\u532d\u532e\u532f\u5330\u5331\u5332\u5333\u5334\u5335\u5336\u5337\u5338\u5339\u533a\u533b\u533c\u533d\u533e\u533f\u5340\u5341\u5342\u5343\u5344\u5345\u5346\u5347\u5348\u5349\u534a\u534b\u534c\u534d\u534e\u534f\u5350\u5351\u5352\u5353\u5354\u5355\u5356\u5357\u5358\u5359\u535a\u535b\u535c\u535d\u535e\u535f\u5360\u5361\u5362\u5363\u5364\u5365\u5366\u5367\u5368\u5369\u536a\u536b\u536c\u536d\u536e\u536f\u5370\u5371\u5372\u5373\u5374\u5375\u5376\u5377\u5378\u5379\u537a\u537b\u537c\u537d\u537e\u537f\u5380\u5381\u5382\u5383\u5384\u5385\u5386\u5387\u5388\u5389\u538a\u538b\u538c\u538d\u538e\u538f\u5390\u5391\u5392\u5393\u5394\u5395\u5396\u5397\u5398\u5399\u539a\u539b\u539c\u539d\u539e\u539f\u53a0\u53a1\u53a2\u53a3\u53a4\u53a5\u53a6\u53a7\u53a8\u53a9\u53aa\u53ab\u53ac\u53ad\u53ae\u53af\u53b0\u53b1\u53b2\u53b3\u53b4\u53b5\u53b6\u53b7\u53b8\u53b9\u53ba\u53bb\u53bc\u53bd\u53be\u53bf\u53c0\u53c1\u53c2\u53c3\u53c4\u53c5\u53c6\u53c7\u53c8\u53c9\u53ca\u53cb\u53cc\u53cd\u53ce\u53cf\u53d0\u53d1\u53d2\u53d3\u53d4\u53d5\u53d6\u53d7\u53d8\u53d9\u53da\u53db\u53dc\u53dd\u53de\u53df\u53e0\u53e1\u53e2\u53e3\u53e4\u53e5\u53e6\u53e7\u53e8\u53e9\u53ea\u53eb\u53ec\u53ed\u53ee\u53ef\u53f0\u53f1\u53f2\u53f3\u53f4\u53f5\u53f6\u53f7\u53f8\u53f9\u53fa\u53fb\u53fc\u53fd\u53fe\u53ff\u5400\u5401\u5402\u5403\u5404\u5405\u5406\u5407\u5408\u5409\u540a\u540b\u540c\u540d\u540e\u540f\u5410\u5411\u5412\u5413\u5414\u5415\u5416\u5417\u5418\u5419\u541a\u541b\u541c\u541d\u541e\u541f\u5420\u5421\u5422\u5423\u5424\u5425\u5426\u5427\u5428\u5429\u542a\u542b\u542c\u542d\u542e\u542f\u5430\u5431\u5432\u5433\u5434\u5435\u5436\u5437\u5438\u5439\u543a\u543b\u543c\u543d\u543e\u543f\u5440\u5441\u5442\u5443\u5444\u5445\u5446\u5447\u5448\u5449\u544a\u544b\u544c\u544d\u544e\u544f\u5450\u5451\u5452\u5453\u5454\u5455\u5456\u5457\u5458\u5459\u545a\u545b\u545c\u545d\u545e\u545f\u5460\u5461\u5462\u5463\u5464\u5465\u5466\u5467\u5468\u5469\u546a\u546b\u546c\u546d\u546e\u546f\u5470\u5471\u5472\u5473\u5474\u5475\u5476\u5477\u5478\u5479\u547a\u547b\u547c\u547d\u547e\u547f\u5480\u5481\u5482\u5483\u5484\u5485\u5486\u5487\u5488\u5489\u548a\u548b\u548c\u548d\u548e\u548f\u5490\u5491\u5492\u5493\u5494\u5495\u5496\u5497\u5498\u5499\u549a\u549b\u549c\u549d\u549e\u549f\u54a0\u54a1\u54a2\u54a3\u54a4\u54a5\u54a6\u54a7\u54a8\u54a9\u54aa\u54ab\u54ac\u54ad\u54ae\u54af\u54b0\u54b1\u54b2\u54b3\u54b4\u54b5\u54b6\u54b7\u54b8\u54b9\u54ba\u54bb\u54bc\u54bd\u54be\u54bf\u54c0\u54c1\u54c2\u54c3\u54c4\u54c5\u54c6\u54c7\u54c8\u54c9\u54ca\u54cb\u54cc\u54cd\u54ce\u54cf\u54d0\u54d1\u54d2\u54d3\u54d4\u54d5\u54d6\u54d7\u54d8\u54d9\u54da\u54db\u54dc\u54dd\u54de\u54df\u54e0\u54e1\u54e2\u54e3\u54e4\u54e5\u54e6\u54e7\u54e8\u54e9\u54ea\u54eb\u54ec\u54ed\u54ee\u54ef\u54f0\u54f1\u54f2\u54f3\u54f4\u54f5\u54f6\u54f7\u54f8\u54f9\u54fa\u54fb\u54fc\u54fd\u54fe\u54ff\u5500\u5501\u5502\u5503\u5504\u5505\u5506\u5507\u5508\u5509\u550a\u550b\u550c\u550d\u550e\u550f\u5510\u5511\u5512\u5513\u5514\u5515\u5516\u5517\u5518\u5519\u551a\u551b\u551c\u551d\u551e\u551f\u5520\u5521\u5522\u5523\u5524\u5525\u5526\u5527\u5528\u5529\u552a\u552b\u552c\u552d\u552e\u552f\u5530\u5531\u5532\u5533\u5534\u5535\u5536\u5537\u5538\u5539\u553a\u553b\u553c\u553d\u553e\u553f\u5540\u5541\u5542\u5543\u5544\u5545\u5546\u5547\u5548\u5549\u554a\u554b\u554c\u554d\u554e\u554f\u5550\u5551\u5552\u5553\u5554\u5555\u5556\u5557\u5558\u5559\u555a\u555b\u555c\u555d\u555e\u555f\u5560\u5561\u5562\u5563\u5564\u5565\u5566\u5567\u5568\u5569\u556a\u556b\u556c\u556d\u556e\u556f\u5570\u5571\u5572\u5573\u5574\u5575\u5576\u5577\u5578\u5579\u557a\u557b\u557c\u557d\u557e\u557f\u5580\u5581\u5582\u5583\u5584\u5585\u5586\u5587\u5588\u5589\u558a\u558b\u558c\u558d\u558e\u558f\u5590\u5591\u5592\u5593\u5594\u5595\u5596\u5597\u5598\u5599\u559a\u559b\u559c\u559d\u559e\u559f\u55a0\u55a1\u55a2\u55a3\u55a4\u55a5\u55a6\u55a7\u55a8\u55a9\u55aa\u55ab\u55ac\u55ad\u55ae\u55af\u55b0\u55b1\u55b2\u55b3\u55b4\u55b5\u55b6\u55b7\u55b8\u55b9\u55ba\u55bb\u55bc\u55bd\u55be\u55bf\u55c0\u55c1\u55c2\u55c3\u55c4\u55c5\u55c6\u55c7\u55c8\u55c9\u55ca\u55cb\u55cc\u55cd\u55ce\u55cf\u55d0\u55d1\u55d2\u55d3\u55d4\u55d5\u55d6\u55d7\u55d8\u55d9\u55da\u55db\u55dc\u55dd\u55de\u55df\u55e0\u55e1\u55e2\u55e3\u55e4\u55e5\u55e6\u55e7\u55e8\u55e9\u55ea\u55eb\u55ec\u55ed\u55ee\u55ef\u55f0\u55f1\u55f2\u55f3\u55f4\u55f5\u55f6\u55f7\u55f8\u55f9\u55fa\u55fb\u55fc\u55fd\u55fe\u55ff\u5600\u5601\u5602\u5603\u5604\u5605\u5606\u5607\u5608\u5609\u560a\u560b\u560c\u560d\u560e\u560f\u5610\u5611\u5612\u5613\u5614\u5615\u5616\u5617\u5618\u5619\u561a\u561b\u561c\u561d\u561e\u561f\u5620\u5621\u5622\u5623\u5624\u5625\u5626\u5627\u5628\u5629\u562a\u562b\u562c\u562d\u562e\u562f\u5630\u5631\u5632\u5633\u5634\u5635\u5636\u5637\u5638\u5639\u563a\u563b\u563c\u563d\u563e\u563f\u5640\u5641\u5642\u5643\u5644\u5645\u5646\u5647\u5648\u5649\u564a\u564b\u564c\u564d\u564e\u564f\u5650\u5651\u5652\u5653\u5654\u5655\u5656\u5657\u5658\u5659\u565a\u565b\u565c\u565d\u565e\u565f\u5660\u5661\u5662\u5663\u5664\u5665\u5666\u5667\u5668\u5669\u566a\u566b\u566c\u566d\u566e\u566f\u5670\u5671\u5672\u5673\u5674\u5675\u5676\u5677\u5678\u5679\u567a\u567b\u567c\u567d\u567e\u567f\u5680\u5681\u5682\u5683\u5684\u5685\u5686\u5687\u5688\u5689\u568a\u568b\u568c\u568d\u568e\u568f\u5690\u5691\u5692\u5693\u5694\u5695\u5696\u5697\u5698\u5699\u569a\u569b\u569c\u569d\u569e\u569f\u56a0\u56a1\u56a2\u56a3\u56a4\u56a5\u56a6\u56a7\u56a8\u56a9\u56aa\u56ab\u56ac\u56ad\u56ae\u56af\u56b0\u56b1\u56b2\u56b3\u56b4\u56b5\u56b6\u56b7\u56b8\u56b9\u56ba\u56bb\u56bc\u56bd\u56be\u56bf\u56c0\u56c1\u56c2\u56c3\u56c4\u56c5\u56c6\u56c7\u56c8\u56c9\u56ca\u56cb\u56cc\u56cd\u56ce\u56cf\u56d0\u56d1\u56d2\u56d3\u56d4\u56d5\u56d6\u56d7\u56d8\u56d9\u56da\u56db\u56dc\u56dd\u56de\u56df\u56e0\u56e1\u56e2\u56e3\u56e4\u56e5\u56e6\u56e7\u56e8\u56e9\u56ea\u56eb\u56ec\u56ed\u56ee\u56ef\u56f0\u56f1\u56f2\u56f3\u56f4\u56f5\u56f6\u56f7\u56f8\u56f9\u56fa\u56fb\u56fc\u56fd\u56fe\u56ff\u5700\u5701\u5702\u5703\u5704\u5705\u5706\u5707\u5708\u5709\u570a\u570b\u570c\u570d\u570e\u570f\u5710\u5711\u5712\u5713\u5714\u5715\u5716\u5717\u5718\u5719\u571a\u571b\u571c\u571d\u571e\u571f\u5720\u5721\u5722\u5723\u5724\u5725\u5726\u5727\u5728\u5729\u572a\u572b\u572c\u572d\u572e\u572f\u5730\u5731\u5732\u5733\u5734\u5735\u5736\u5737\u5738\u5739\u573a\u573b\u573c\u573d\u573e\u573f\u5740\u5741\u5742\u5743\u5744\u5745\u5746\u5747\u5748\u5749\u574a\u574b\u574c\u574d\u574e\u574f\u5750\u5751\u5752\u5753\u5754\u5755\u5756\u5757\u5758\u5759\u575a\u575b\u575c\u575d\u575e\u575f\u5760\u5761\u5762\u5763\u5764\u5765\u5766\u5767\u5768\u5769\u576a\u576b\u576c\u576d\u576e\u576f\u5770\u5771\u5772\u5773\u5774\u5775\u5776\u5777\u5778\u5779\u577a\u577b\u577c\u577d\u577e\u577f\u5780\u5781\u5782\u5783\u5784\u5785\u5786\u5787\u5788\u5789\u578a\u578b\u578c\u578d\u578e\u578f\u5790\u5791\u5792\u5793\u5794\u5795\u5796\u5797\u5798\u5799\u579a\u579b\u579c\u579d\u579e\u579f\u57a0\u57a1\u57a2\u57a3\u57a4\u57a5\u57a6\u57a7\u57a8\u57a9\u57aa\u57ab\u57ac\u57ad\u57ae\u57af\u57b0\u57b1\u57b2\u57b3\u57b4\u57b5\u57b6\u57b7\u57b8\u57b9\u57ba\u57bb\u57bc\u57bd\u57be\u57bf\u57c0\u57c1\u57c2\u57c3\u57c4\u57c5\u57c6\u57c7\u57c8\u57c9\u57ca\u57cb\u57cc\u57cd\u57ce\u57cf\u57d0\u57d1\u57d2\u57d3\u57d4\u57d5\u57d6\u57d7\u57d8\u57d9\u57da\u57db\u57dc\u57dd\u57de\u57df\u57e0\u57e1\u57e2\u57e3\u57e4\u57e5\u57e6\u57e7\u57e8\u57e9\u57ea\u57eb\u57ec\u57ed\u57ee\u57ef\u57f0\u57f1\u57f2\u57f3\u57f4\u57f5\u57f6\u57f7\u57f8\u57f9\u57fa\u57fb\u57fc\u57fd\u57fe\u57ff\u5800\u5801\u5802\u5803\u5804\u5805\u5806\u5807\u5808\u5809\u580a\u580b\u580c\u580d\u580e\u580f\u5810\u5811\u5812\u5813\u5814\u5815\u5816\u5817\u5818\u5819\u581a\u581b\u581c\u581d\u581e\u581f\u5820\u5821\u5822\u5823\u5824\u5825\u5826\u5827\u5828\u5829\u582a\u582b\u582c\u582d\u582e\u582f\u5830\u5831\u5832\u5833\u5834\u5835\u5836\u5837\u5838\u5839\u583a\u583b\u583c\u583d\u583e\u583f\u5840\u5841\u5842\u5843\u5844\u5845\u5846\u5847\u5848\u5849\u584a\u584b\u584c\u584d\u584e\u584f\u5850\u5851\u5852\u5853\u5854\u5855\u5856\u5857\u5858\u5859\u585a\u585b\u585c\u585d\u585e\u585f\u5860\u5861\u5862\u5863\u5864\u5865\u5866\u5867\u5868\u5869\u586a\u586b\u586c\u586d\u586e\u586f\u5870\u5871\u5872\u5873\u5874\u5875\u5876\u5877\u5878\u5879\u587a\u587b\u587c\u587d\u587e\u587f\u5880\u5881\u5882\u5883\u5884\u5885\u5886\u5887\u5888\u5889\u588a\u588b\u588c\u588d\u588e\u588f\u5890\u5891\u5892\u5893\u5894\u5895\u5896\u5897\u5898\u5899\u589a\u589b\u589c\u589d\u589e\u589f\u58a0\u58a1\u58a2\u58a3\u58a4\u58a5\u58a6\u58a7\u58a8\u58a9\u58aa\u58ab\u58ac\u58ad\u58ae\u58af\u58b0\u58b1\u58b2\u58b3\u58b4\u58b5\u58b6\u58b7\u58b8\u58b9\u58ba\u58bb\u58bc\u58bd\u58be\u58bf\u58c0\u58c1\u58c2\u58c3\u58c4\u58c5\u58c6\u58c7\u58c8\u58c9\u58ca\u58cb\u58cc\u58cd\u58ce\u58cf\u58d0\u58d1\u58d2\u58d3\u58d4\u58d5\u58d6\u58d7\u58d8\u58d9\u58da\u58db\u58dc\u58dd\u58de\u58df\u58e0\u58e1\u58e2\u58e3\u58e4\u58e5\u58e6\u58e7\u58e8\u58e9\u58ea\u58eb\u58ec\u58ed\u58ee\u58ef\u58f0\u58f1\u58f2\u58f3\u58f4\u58f5\u58f6\u58f7\u58f8\u58f9\u58fa\u58fb\u58fc\u58fd\u58fe\u58ff\u5900\u5901\u5902\u5903\u5904\u5905\u5906\u5907\u5908\u5909\u590a\u590b\u590c\u590d\u590e\u590f\u5910\u5911\u5912\u5913\u5914\u5915\u5916\u5917\u5918\u5919\u591a\u591b\u591c\u591d\u591e\u591f\u5920\u5921\u5922\u5923\u5924\u5925\u5926\u5927\u5928\u5929\u592a\u592b\u592c\u592d\u592e\u592f\u5930\u5931\u5932\u5933\u5934\u5935\u5936\u5937\u5938\u5939\u593a\u593b\u593c\u593d\u593e\u593f\u5940\u5941\u5942\u5943\u5944\u5945\u5946\u5947\u5948\u5949\u594a\u594b\u594c\u594d\u594e\u594f\u5950\u5951\u5952\u5953\u5954\u5955\u5956\u5957\u5958\u5959\u595a\u595b\u595c\u595d\u595e\u595f\u5960\u5961\u5962\u5963\u5964\u5965\u5966\u5967\u5968\u5969\u596a\u596b\u596c\u596d\u596e\u596f\u5970\u5971\u5972\u5973\u5974\u5975\u5976\u5977\u5978\u5979\u597a\u597b\u597c\u597d\u597e\u597f\u5980\u5981\u5982\u5983\u5984\u5985\u5986\u5987\u5988\u5989\u598a\u598b\u598c\u598d\u598e\u598f\u5990\u5991\u5992\u5993\u5994\u5995\u5996\u5997\u5998\u5999\u599a\u599b\u599c\u599d\u599e\u599f\u59a0\u59a1\u59a2\u59a3\u59a4\u59a5\u59a6\u59a7\u59a8\u59a9\u59aa\u59ab\u59ac\u59ad\u59ae\u59af\u59b0\u59b1\u59b2\u59b3\u59b4\u59b5\u59b6\u59b7\u59b8\u59b9\u59ba\u59bb\u59bc\u59bd\u59be\u59bf\u59c0\u59c1\u59c2\u59c3\u59c4\u59c5\u59c6\u59c7\u59c8\u59c9\u59ca\u59cb\u59cc\u59cd\u59ce\u59cf\u59d0\u59d1\u59d2\u59d3\u59d4\u59d5\u59d6\u59d7\u59d8\u59d9\u59da\u59db\u59dc\u59dd\u59de\u59df\u59e0\u59e1\u59e2\u59e3\u59e4\u59e5\u59e6\u59e7\u59e8\u59e9\u59ea\u59eb\u59ec\u59ed\u59ee\u59ef\u59f0\u59f1\u59f2\u59f3\u59f4\u59f5\u59f6\u59f7\u59f8\u59f9\u59fa\u59fb\u59fc\u59fd\u59fe\u59ff\u5a00\u5a01\u5a02\u5a03\u5a04\u5a05\u5a06\u5a07\u5a08\u5a09\u5a0a\u5a0b\u5a0c\u5a0d\u5a0e\u5a0f\u5a10\u5a11\u5a12\u5a13\u5a14\u5a15\u5a16\u5a17\u5a18\u5a19\u5a1a\u5a1b\u5a1c\u5a1d\u5a1e\u5a1f\u5a20\u5a21\u5a22\u5a23\u5a24\u5a25\u5a26\u5a27\u5a28\u5a29\u5a2a\u5a2b\u5a2c\u5a2d\u5a2e\u5a2f\u5a30\u5a31\u5a32\u5a33\u5a34\u5a35\u5a36\u5a37\u5a38\u5a39\u5a3a\u5a3b\u5a3c\u5a3d\u5a3e\u5a3f\u5a40\u5a41\u5a42\u5a43\u5a44\u5a45\u5a46\u5a47\u5a48\u5a49\u5a4a\u5a4b\u5a4c\u5a4d\u5a4e\u5a4f\u5a50\u5a51\u5a52\u5a53\u5a54\u5a55\u5a56\u5a57\u5a58\u5a59\u5a5a\u5a5b\u5a5c\u5a5d\u5a5e\u5a5f\u5a60\u5a61\u5a62\u5a63\u5a64\u5a65\u5a66\u5a67\u5a68\u5a69\u5a6a\u5a6b\u5a6c\u5a6d\u5a6e\u5a6f\u5a70\u5a71\u5a72\u5a73\u5a74\u5a75\u5a76\u5a77\u5a78\u5a79\u5a7a\u5a7b\u5a7c\u5a7d\u5a7e\u5a7f\u5a80\u5a81\u5a82\u5a83\u5a84\u5a85\u5a86\u5a87\u5a88\u5a89\u5a8a\u5a8b\u5a8c\u5a8d\u5a8e\u5a8f\u5a90\u5a91\u5a92\u5a93\u5a94\u5a95\u5a96\u5a97\u5a98\u5a99\u5a9a\u5a9b\u5a9c\u5a9d\u5a9e\u5a9f\u5aa0\u5aa1\u5aa2\u5aa3\u5aa4\u5aa5\u5aa6\u5aa7\u5aa8\u5aa9\u5aaa\u5aab\u5aac\u5aad\u5aae\u5aaf\u5ab0\u5ab1\u5ab2\u5ab3\u5ab4\u5ab5\u5ab6\u5ab7\u5ab8\u5ab9\u5aba\u5abb\u5abc\u5abd\u5abe\u5abf\u5ac0\u5ac1\u5ac2\u5ac3\u5ac4\u5ac5\u5ac6\u5ac7\u5ac8\u5ac9\u5aca\u5acb\u5acc\u5acd\u5ace\u5acf\u5ad0\u5ad1\u5ad2\u5ad3\u5ad4\u5ad5\u5ad6\u5ad7\u5ad8\u5ad9\u5ada\u5adb\u5adc\u5add\u5ade\u5adf\u5ae0\u5ae1\u5ae2\u5ae3\u5ae4\u5ae5\u5ae6\u5ae7\u5ae8\u5ae9\u5aea\u5aeb\u5aec\u5aed\u5aee\u5aef\u5af0\u5af1\u5af2\u5af3\u5af4\u5af5\u5af6\u5af7\u5af8\u5af9\u5afa\u5afb\u5afc\u5afd\u5afe\u5aff\u5b00\u5b01\u5b02\u5b03\u5b04\u5b05\u5b06\u5b07\u5b08\u5b09\u5b0a\u5b0b\u5b0c\u5b0d\u5b0e\u5b0f\u5b10\u5b11\u5b12\u5b13\u5b14\u5b15\u5b16\u5b17\u5b18\u5b19\u5b1a\u5b1b\u5b1c\u5b1d\u5b1e\u5b1f\u5b20\u5b21\u5b22\u5b23\u5b24\u5b25\u5b26\u5b27\u5b28\u5b29\u5b2a\u5b2b\u5b2c\u5b2d\u5b2e\u5b2f\u5b30\u5b31\u5b32\u5b33\u5b34\u5b35\u5b36\u5b37\u5b38\u5b39\u5b3a\u5b3b\u5b3c\u5b3d\u5b3e\u5b3f\u5b40\u5b41\u5b42\u5b43\u5b44\u5b45\u5b46\u5b47\u5b48\u5b49\u5b4a\u5b4b\u5b4c\u5b4d\u5b4e\u5b4f\u5b50\u5b51\u5b52\u5b53\u5b54\u5b55\u5b56\u5b57\u5b58\u5b59\u5b5a\u5b5b\u5b5c\u5b5d\u5b5e\u5b5f\u5b60\u5b61\u5b62\u5b63\u5b64\u5b65\u5b66\u5b67\u5b68\u5b69\u5b6a\u5b6b\u5b6c\u5b6d\u5b6e\u5b6f\u5b70\u5b71\u5b72\u5b73\u5b74\u5b75\u5b76\u5b77\u5b78\u5b79\u5b7a\u5b7b\u5b7c\u5b7d\u5b7e\u5b7f\u5b80\u5b81\u5b82\u5b83\u5b84\u5b85\u5b86\u5b87\u5b88\u5b89\u5b8a\u5b8b\u5b8c\u5b8d\u5b8e\u5b8f\u5b90\u5b91\u5b92\u5b93\u5b94\u5b95\u5b96\u5b97\u5b98\u5b99\u5b9a\u5b9b\u5b9c\u5b9d\u5b9e\u5b9f\u5ba0\u5ba1\u5ba2\u5ba3\u5ba4\u5ba5\u5ba6\u5ba7\u5ba8\u5ba9\u5baa\u5bab\u5bac\u5bad\u5bae\u5baf\u5bb0\u5bb1\u5bb2\u5bb3\u5bb4\u5bb5\u5bb6\u5bb7\u5bb8\u5bb9\u5bba\u5bbb\u5bbc\u5bbd\u5bbe\u5bbf\u5bc0\u5bc1\u5bc2\u5bc3\u5bc4\u5bc5\u5bc6\u5bc7\u5bc8\u5bc9\u5bca\u5bcb\u5bcc\u5bcd\u5bce\u5bcf\u5bd0\u5bd1\u5bd2\u5bd3\u5bd4\u5bd5\u5bd6\u5bd7\u5bd8\u5bd9\u5bda\u5bdb\u5bdc\u5bdd\u5bde\u5bdf\u5be0\u5be1\u5be2\u5be3\u5be4\u5be5\u5be6\u5be7\u5be8\u5be9\u5bea\u5beb\u5bec\u5bed\u5bee\u5bef\u5bf0\u5bf1\u5bf2\u5bf3\u5bf4\u5bf5\u5bf6\u5bf7\u5bf8\u5bf9\u5bfa\u5bfb\u5bfc\u5bfd\u5bfe\u5bff\u5c00\u5c01\u5c02\u5c03\u5c04\u5c05\u5c06\u5c07\u5c08\u5c09\u5c0a\u5c0b\u5c0c\u5c0d\u5c0e\u5c0f\u5c10\u5c11\u5c12\u5c13\u5c14\u5c15\u5c16\u5c17\u5c18\u5c19\u5c1a\u5c1b\u5c1c\u5c1d\u5c1e\u5c1f\u5c20\u5c21\u5c22\u5c23\u5c24\u5c25\u5c26\u5c27\u5c28\u5c29\u5c2a\u5c2b\u5c2c\u5c2d\u5c2e\u5c2f\u5c30\u5c31\u5c32\u5c33\u5c34\u5c35\u5c36\u5c37\u5c38\u5c39\u5c3a\u5c3b\u5c3c\u5c3d\u5c3e\u5c3f\u5c40\u5c41\u5c42\u5c43\u5c44\u5c45\u5c46\u5c47\u5c48\u5c49\u5c4a\u5c4b\u5c4c\u5c4d\u5c4e\u5c4f\u5c50\u5c51\u5c52\u5c53\u5c54\u5c55\u5c56\u5c57\u5c58\u5c59\u5c5a\u5c5b\u5c5c\u5c5d\u5c5e\u5c5f\u5c60\u5c61\u5c62\u5c63\u5c64\u5c65\u5c66\u5c67\u5c68\u5c69\u5c6a\u5c6b\u5c6c\u5c6d\u5c6e\u5c6f\u5c70\u5c71\u5c72\u5c73\u5c74\u5c75\u5c76\u5c77\u5c78\u5c79\u5c7a\u5c7b\u5c7c\u5c7d\u5c7e\u5c7f\u5c80\u5c81\u5c82\u5c83\u5c84\u5c85\u5c86\u5c87\u5c88\u5c89\u5c8a\u5c8b\u5c8c\u5c8d\u5c8e\u5c8f\u5c90\u5c91\u5c92\u5c93\u5c94\u5c95\u5c96\u5c97\u5c98\u5c99\u5c9a\u5c9b\u5c9c\u5c9d\u5c9e\u5c9f\u5ca0\u5ca1\u5ca2\u5ca3\u5ca4\u5ca5\u5ca6\u5ca7\u5ca8\u5ca9\u5caa\u5cab\u5cac\u5cad\u5cae\u5caf\u5cb0\u5cb1\u5cb2\u5cb3\u5cb4\u5cb5\u5cb6\u5cb7\u5cb8\u5cb9\u5cba\u5cbb\u5cbc\u5cbd\u5cbe\u5cbf\u5cc0\u5cc1\u5cc2\u5cc3\u5cc4\u5cc5\u5cc6\u5cc7\u5cc8\u5cc9\u5cca\u5ccb\u5ccc\u5ccd\u5cce\u5ccf\u5cd0\u5cd1\u5cd2\u5cd3\u5cd4\u5cd5\u5cd6\u5cd7\u5cd8\u5cd9\u5cda\u5cdb\u5cdc\u5cdd\u5cde\u5cdf\u5ce0\u5ce1\u5ce2\u5ce3\u5ce4\u5ce5\u5ce6\u5ce7\u5ce8\u5ce9\u5cea\u5ceb\u5cec\u5ced\u5cee\u5cef\u5cf0\u5cf1\u5cf2\u5cf3\u5cf4\u5cf5\u5cf6\u5cf7\u5cf8\u5cf9\u5cfa\u5cfb\u5cfc\u5cfd\u5cfe\u5cff\u5d00\u5d01\u5d02\u5d03\u5d04\u5d05\u5d06\u5d07\u5d08\u5d09\u5d0a\u5d0b\u5d0c\u5d0d\u5d0e\u5d0f\u5d10\u5d11\u5d12\u5d13\u5d14\u5d15\u5d16\u5d17\u5d18\u5d19\u5d1a\u5d1b\u5d1c\u5d1d\u5d1e\u5d1f\u5d20\u5d21\u5d22\u5d23\u5d24\u5d25\u5d26\u5d27\u5d28\u5d29\u5d2a\u5d2b\u5d2c\u5d2d\u5d2e\u5d2f\u5d30\u5d31\u5d32\u5d33\u5d34\u5d35\u5d36\u5d37\u5d38\u5d39\u5d3a\u5d3b\u5d3c\u5d3d\u5d3e\u5d3f\u5d40\u5d41\u5d42\u5d43\u5d44\u5d45\u5d46\u5d47\u5d48\u5d49\u5d4a\u5d4b\u5d4c\u5d4d\u5d4e\u5d4f\u5d50\u5d51\u5d52\u5d53\u5d54\u5d55\u5d56\u5d57\u5d58\u5d59\u5d5a\u5d5b\u5d5c\u5d5d\u5d5e\u5d5f\u5d60\u5d61\u5d62\u5d63\u5d64\u5d65\u5d66\u5d67\u5d68\u5d69\u5d6a\u5d6b\u5d6c\u5d6d\u5d6e\u5d6f\u5d70\u5d71\u5d72\u5d73\u5d74\u5d75\u5d76\u5d77\u5d78\u5d79\u5d7a\u5d7b\u5d7c\u5d7d\u5d7e\u5d7f\u5d80\u5d81\u5d82\u5d83\u5d84\u5d85\u5d86\u5d87\u5d88\u5d89\u5d8a\u5d8b\u5d8c\u5d8d\u5d8e\u5d8f\u5d90\u5d91\u5d92\u5d93\u5d94\u5d95\u5d96\u5d97\u5d98\u5d99\u5d9a\u5d9b\u5d9c\u5d9d\u5d9e\u5d9f\u5da0\u5da1\u5da2\u5da3\u5da4\u5da5\u5da6\u5da7\u5da8\u5da9\u5daa\u5dab\u5dac\u5dad\u5dae\u5daf\u5db0\u5db1\u5db2\u5db3\u5db4\u5db5\u5db6\u5db7\u5db8\u5db9\u5dba\u5dbb\u5dbc\u5dbd\u5dbe\u5dbf\u5dc0\u5dc1\u5dc2\u5dc3\u5dc4\u5dc5\u5dc6\u5dc7\u5dc8\u5dc9\u5dca\u5dcb\u5dcc\u5dcd\u5dce\u5dcf\u5dd0\u5dd1\u5dd2\u5dd3\u5dd4\u5dd5\u5dd6\u5dd7\u5dd8\u5dd9\u5dda\u5ddb\u5ddc\u5ddd\u5dde\u5ddf\u5de0\u5de1\u5de2\u5de3\u5de4\u5de5\u5de6\u5de7\u5de8\u5de9\u5dea\u5deb\u5dec\u5ded\u5dee\u5def\u5df0\u5df1\u5df2\u5df3\u5df4\u5df5\u5df6\u5df7\u5df8\u5df9\u5dfa\u5dfb\u5dfc\u5dfd\u5dfe\u5dff\u5e00\u5e01\u5e02\u5e03\u5e04\u5e05\u5e06\u5e07\u5e08\u5e09\u5e0a\u5e0b\u5e0c\u5e0d\u5e0e\u5e0f\u5e10\u5e11\u5e12\u5e13\u5e14\u5e15\u5e16\u5e17\u5e18\u5e19\u5e1a\u5e1b\u5e1c\u5e1d\u5e1e\u5e1f\u5e20\u5e21\u5e22\u5e23\u5e24\u5e25\u5e26\u5e27\u5e28\u5e29\u5e2a\u5e2b\u5e2c\u5e2d\u5e2e\u5e2f\u5e30\u5e31\u5e32\u5e33\u5e34\u5e35\u5e36\u5e37\u5e38\u5e39\u5e3a\u5e3b\u5e3c\u5e3d\u5e3e\u5e3f\u5e40\u5e41\u5e42\u5e43\u5e44\u5e45\u5e46\u5e47\u5e48\u5e49\u5e4a\u5e4b\u5e4c\u5e4d\u5e4e\u5e4f\u5e50\u5e51\u5e52\u5e53\u5e54\u5e55\u5e56\u5e57\u5e58\u5e59\u5e5a\u5e5b\u5e5c\u5e5d\u5e5e\u5e5f\u5e60\u5e61\u5e62\u5e63\u5e64\u5e65\u5e66\u5e67\u5e68\u5e69\u5e6a\u5e6b\u5e6c\u5e6d\u5e6e\u5e6f\u5e70\u5e71\u5e72\u5e73\u5e74\u5e75\u5e76\u5e77\u5e78\u5e79\u5e7a\u5e7b\u5e7c\u5e7d\u5e7e\u5e7f\u5e80\u5e81\u5e82\u5e83\u5e84\u5e85\u5e86\u5e87\u5e88\u5e89\u5e8a\u5e8b\u5e8c\u5e8d\u5e8e\u5e8f\u5e90\u5e91\u5e92\u5e93\u5e94\u5e95\u5e96\u5e97\u5e98\u5e99\u5e9a\u5e9b\u5e9c\u5e9d\u5e9e\u5e9f\u5ea0\u5ea1\u5ea2\u5ea3\u5ea4\u5ea5\u5ea6\u5ea7\u5ea8\u5ea9\u5eaa\u5eab\u5eac\u5ead\u5eae\u5eaf\u5eb0\u5eb1\u5eb2\u5eb3\u5eb4\u5eb5\u5eb6\u5eb7\u5eb8\u5eb9\u5eba\u5ebb\u5ebc\u5ebd\u5ebe\u5ebf\u5ec0\u5ec1\u5ec2\u5ec3\u5ec4\u5ec5\u5ec6\u5ec7\u5ec8\u5ec9\u5eca\u5ecb\u5ecc\u5ecd\u5ece\u5ecf\u5ed0\u5ed1\u5ed2\u5ed3\u5ed4\u5ed5\u5ed6\u5ed7\u5ed8\u5ed9\u5eda\u5edb\u5edc\u5edd\u5ede\u5edf\u5ee0\u5ee1\u5ee2\u5ee3\u5ee4\u5ee5\u5ee6\u5ee7\u5ee8\u5ee9\u5eea\u5eeb\u5eec\u5eed\u5eee\u5eef\u5ef0\u5ef1\u5ef2\u5ef3\u5ef4\u5ef5\u5ef6\u5ef7\u5ef8\u5ef9\u5efa\u5efb\u5efc\u5efd\u5efe\u5eff\u5f00\u5f01\u5f02\u5f03\u5f04\u5f05\u5f06\u5f07\u5f08\u5f09\u5f0a\u5f0b\u5f0c\u5f0d\u5f0e\u5f0f\u5f10\u5f11\u5f12\u5f13\u5f14\u5f15\u5f16\u5f17\u5f18\u5f19\u5f1a\u5f1b\u5f1c\u5f1d\u5f1e\u5f1f\u5f20\u5f21\u5f22\u5f23\u5f24\u5f25\u5f26\u5f27\u5f28\u5f29\u5f2a\u5f2b\u5f2c\u5f2d\u5f2e\u5f2f\u5f30\u5f31\u5f32\u5f33\u5f34\u5f35\u5f36\u5f37\u5f38\u5f39\u5f3a\u5f3b\u5f3c\u5f3d\u5f3e\u5f3f\u5f40\u5f41\u5f42\u5f43\u5f44\u5f45\u5f46\u5f47\u5f48\u5f49\u5f4a\u5f4b\u5f4c\u5f4d\u5f4e\u5f4f\u5f50\u5f51\u5f52\u5f53\u5f54\u5f55\u5f56\u5f57\u5f58\u5f59\u5f5a\u5f5b\u5f5c\u5f5d\u5f5e\u5f5f\u5f60\u5f61\u5f62\u5f63\u5f64\u5f65\u5f66\u5f67\u5f68\u5f69\u5f6a\u5f6b\u5f6c\u5f6d\u5f6e\u5f6f\u5f70\u5f71\u5f72\u5f73\u5f74\u5f75\u5f76\u5f77\u5f78\u5f79\u5f7a\u5f7b\u5f7c\u5f7d\u5f7e\u5f7f\u5f80\u5f81\u5f82\u5f83\u5f84\u5f85\u5f86\u5f87\u5f88\u5f89\u5f8a\u5f8b\u5f8c\u5f8d\u5f8e\u5f8f\u5f90\u5f91\u5f92\u5f93\u5f94\u5f95\u5f96\u5f97\u5f98\u5f99\u5f9a\u5f9b\u5f9c\u5f9d\u5f9e\u5f9f\u5fa0\u5fa1\u5fa2\u5fa3\u5fa4\u5fa5\u5fa6\u5fa7\u5fa8\u5fa9\u5faa\u5fab\u5fac\u5fad\u5fae\u5faf\u5fb0\u5fb1\u5fb2\u5fb3\u5fb4\u5fb5\u5fb6\u5fb7\u5fb8\u5fb9\u5fba\u5fbb\u5fbc\u5fbd\u5fbe\u5fbf\u5fc0\u5fc1\u5fc2\u5fc3\u5fc4\u5fc5\u5fc6\u5fc7\u5fc8\u5fc9\u5fca\u5fcb\u5fcc\u5fcd\u5fce\u5fcf\u5fd0\u5fd1\u5fd2\u5fd3\u5fd4\u5fd5\u5fd6\u5fd7\u5fd8\u5fd9\u5fda\u5fdb\u5fdc\u5fdd\u5fde\u5fdf\u5fe0\u5fe1\u5fe2\u5fe3\u5fe4\u5fe5\u5fe6\u5fe7\u5fe8\u5fe9\u5fea\u5feb\u5fec\u5fed\u5fee\u5fef\u5ff0\u5ff1\u5ff2\u5ff3\u5ff4\u5ff5\u5ff6\u5ff7\u5ff8\u5ff9\u5ffa\u5ffb\u5ffc\u5ffd\u5ffe\u5fff\u6000\u6001\u6002\u6003\u6004\u6005\u6006\u6007\u6008\u6009\u600a\u600b\u600c\u600d\u600e\u600f\u6010\u6011\u6012\u6013\u6014\u6015\u6016\u6017\u6018\u6019\u601a\u601b\u601c\u601d\u601e\u601f\u6020\u6021\u6022\u6023\u6024\u6025\u6026\u6027\u6028\u6029\u602a\u602b\u602c\u602d\u602e\u602f\u6030\u6031\u6032\u6033\u6034\u6035\u6036\u6037\u6038\u6039\u603a\u603b\u603c\u603d\u603e\u603f\u6040\u6041\u6042\u6043\u6044\u6045\u6046\u6047\u6048\u6049\u604a\u604b\u604c\u604d\u604e\u604f\u6050\u6051\u6052\u6053\u6054\u6055\u6056\u6057\u6058\u6059\u605a\u605b\u605c\u605d\u605e\u605f\u6060\u6061\u6062\u6063\u6064\u6065\u6066\u6067\u6068\u6069\u606a\u606b\u606c\u606d\u606e\u606f\u6070\u6071\u6072\u6073\u6074\u6075\u6076\u6077\u6078\u6079\u607a\u607b\u607c\u607d\u607e\u607f\u6080\u6081\u6082\u6083\u6084\u6085\u6086\u6087\u6088\u6089\u608a\u608b\u608c\u608d\u608e\u608f\u6090\u6091\u6092\u6093\u6094\u6095\u6096\u6097\u6098\u6099\u609a\u609b\u609c\u609d\u609e\u609f\u60a0\u60a1\u60a2\u60a3\u60a4\u60a5\u60a6\u60a7\u60a8\u60a9\u60aa\u60ab\u60ac\u60ad\u60ae\u60af\u60b0\u60b1\u60b2\u60b3\u60b4\u60b5\u60b6\u60b7\u60b8\u60b9\u60ba\u60bb\u60bc\u60bd\u60be\u60bf\u60c0\u60c1\u60c2\u60c3\u60c4\u60c5\u60c6\u60c7\u60c8\u60c9\u60ca\u60cb\u60cc\u60cd\u60ce\u60cf\u60d0\u60d1\u60d2\u60d3\u60d4\u60d5\u60d6\u60d7\u60d8\u60d9\u60da\u60db\u60dc\u60dd\u60de\u60df\u60e0\u60e1\u60e2\u60e3\u60e4\u60e5\u60e6\u60e7\u60e8\u60e9\u60ea\u60eb\u60ec\u60ed\u60ee\u60ef\u60f0\u60f1\u60f2\u60f3\u60f4\u60f5\u60f6\u60f7\u60f8\u60f9\u60fa\u60fb\u60fc\u60fd\u60fe\u60ff\u6100\u6101\u6102\u6103\u6104\u6105\u6106\u6107\u6108\u6109\u610a\u610b\u610c\u610d\u610e\u610f\u6110\u6111\u6112\u6113\u6114\u6115\u6116\u6117\u6118\u6119\u611a\u611b\u611c\u611d\u611e\u611f\u6120\u6121\u6122\u6123\u6124\u6125\u6126\u6127\u6128\u6129\u612a\u612b\u612c\u612d\u612e\u612f\u6130\u6131\u6132\u6133\u6134\u6135\u6136\u6137\u6138\u6139\u613a\u613b\u613c\u613d\u613e\u613f\u6140\u6141\u6142\u6143\u6144\u6145\u6146\u6147\u6148\u6149\u614a\u614b\u614c\u614d\u614e\u614f\u6150\u6151\u6152\u6153\u6154\u6155\u6156\u6157\u6158\u6159\u615a\u615b\u615c\u615d\u615e\u615f\u6160\u6161\u6162\u6163\u6164\u6165\u6166\u6167\u6168\u6169\u616a\u616b\u616c\u616d\u616e\u616f\u6170\u6171\u6172\u6173\u6174\u6175\u6176\u6177\u6178\u6179\u617a\u617b\u617c\u617d\u617e\u617f\u6180\u6181\u6182\u6183\u6184\u6185\u6186\u6187\u6188\u6189\u618a\u618b\u618c\u618d\u618e\u618f\u6190\u6191\u6192\u6193\u6194\u6195\u6196\u6197\u6198\u6199\u619a\u619b\u619c\u619d\u619e\u619f\u61a0\u61a1\u61a2\u61a3\u61a4\u61a5\u61a6\u61a7\u61a8\u61a9\u61aa\u61ab\u61ac\u61ad\u61ae\u61af\u61b0\u61b1\u61b2\u61b3\u61b4\u61b5\u61b6\u61b7\u61b8\u61b9\u61ba\u61bb\u61bc\u61bd\u61be\u61bf\u61c0\u61c1\u61c2\u61c3\u61c4\u61c5\u61c6\u61c7\u61c8\u61c9\u61ca\u61cb\u61cc\u61cd\u61ce\u61cf\u61d0\u61d1\u61d2\u61d3\u61d4\u61d5\u61d6\u61d7\u61d8\u61d9\u61da\u61db\u61dc\u61dd\u61de\u61df\u61e0\u61e1\u61e2\u61e3\u61e4\u61e5\u61e6\u61e7\u61e8\u61e9\u61ea\u61eb\u61ec\u61ed\u61ee\u61ef\u61f0\u61f1\u61f2\u61f3\u61f4\u61f5\u61f6\u61f7\u61f8\u61f9\u61fa\u61fb\u61fc\u61fd\u61fe\u61ff\u6200\u6201\u6202\u6203\u6204\u6205\u6206\u6207\u6208\u6209\u620a\u620b\u620c\u620d\u620e\u620f\u6210\u6211\u6212\u6213\u6214\u6215\u6216\u6217\u6218\u6219\u621a\u621b\u621c\u621d\u621e\u621f\u6220\u6221\u6222\u6223\u6224\u6225\u6226\u6227\u6228\u6229\u622a\u622b\u622c\u622d\u622e\u622f\u6230\u6231\u6232\u6233\u6234\u6235\u6236\u6237\u6238\u6239\u623a\u623b\u623c\u623d\u623e\u623f\u6240\u6241\u6242\u6243\u6244\u6245\u6246\u6247\u6248\u6249\u624a\u624b\u624c\u624d\u624e\u624f\u6250\u6251\u6252\u6253\u6254\u6255\u6256\u6257\u6258\u6259\u625a\u625b\u625c\u625d\u625e\u625f\u6260\u6261\u6262\u6263\u6264\u6265\u6266\u6267\u6268\u6269\u626a\u626b\u626c\u626d\u626e\u626f\u6270\u6271\u6272\u6273\u6274\u6275\u6276\u6277\u6278\u6279\u627a\u627b\u627c\u627d\u627e\u627f\u6280\u6281\u6282\u6283\u6284\u6285\u6286\u6287\u6288\u6289\u628a\u628b\u628c\u628d\u628e\u628f\u6290\u6291\u6292\u6293\u6294\u6295\u6296\u6297\u6298\u6299\u629a\u629b\u629c\u629d\u629e\u629f\u62a0\u62a1\u62a2\u62a3\u62a4\u62a5\u62a6\u62a7\u62a8\u62a9\u62aa\u62ab\u62ac\u62ad\u62ae\u62af\u62b0\u62b1\u62b2\u62b3\u62b4\u62b5\u62b6\u62b7\u62b8\u62b9\u62ba\u62bb\u62bc\u62bd\u62be\u62bf\u62c0\u62c1\u62c2\u62c3\u62c4\u62c5\u62c6\u62c7\u62c8\u62c9\u62ca\u62cb\u62cc\u62cd\u62ce\u62cf\u62d0\u62d1\u62d2\u62d3\u62d4\u62d5\u62d6\u62d7\u62d8\u62d9\u62da\u62db\u62dc\u62dd\u62de\u62df\u62e0\u62e1\u62e2\u62e3\u62e4\u62e5\u62e6\u62e7\u62e8\u62e9\u62ea\u62eb\u62ec\u62ed\u62ee\u62ef\u62f0\u62f1\u62f2\u62f3\u62f4\u62f5\u62f6\u62f7\u62f8\u62f9\u62fa\u62fb\u62fc\u62fd\u62fe\u62ff\u6300\u6301\u6302\u6303\u6304\u6305\u6306\u6307\u6308\u6309\u630a\u630b\u630c\u630d\u630e\u630f\u6310\u6311\u6312\u6313\u6314\u6315\u6316\u6317\u6318\u6319\u631a\u631b\u631c\u631d\u631e\u631f\u6320\u6321\u6322\u6323\u6324\u6325\u6326\u6327\u6328\u6329\u632a\u632b\u632c\u632d\u632e\u632f\u6330\u6331\u6332\u6333\u6334\u6335\u6336\u6337\u6338\u6339\u633a\u633b\u633c\u633d\u633e\u633f\u6340\u6341\u6342\u6343\u6344\u6345\u6346\u6347\u6348\u6349\u634a\u634b\u634c\u634d\u634e\u634f\u6350\u6351\u6352\u6353\u6354\u6355\u6356\u6357\u6358\u6359\u635a\u635b\u635c\u635d\u635e\u635f\u6360\u6361\u6362\u6363\u6364\u6365\u6366\u6367\u6368\u6369\u636a\u636b\u636c\u636d\u636e\u636f\u6370\u6371\u6372\u6373\u6374\u6375\u6376\u6377\u6378\u6379\u637a\u637b\u637c\u637d\u637e\u637f\u6380\u6381\u6382\u6383\u6384\u6385\u6386\u6387\u6388\u6389\u638a\u638b\u638c\u638d\u638e\u638f\u6390\u6391\u6392\u6393\u6394\u6395\u6396\u6397\u6398\u6399\u639a\u639b\u639c\u639d\u639e\u639f\u63a0\u63a1\u63a2\u63a3\u63a4\u63a5\u63a6\u63a7\u63a8\u63a9\u63aa\u63ab\u63ac\u63ad\u63ae\u63af\u63b0\u63b1\u63b2\u63b3\u63b4\u63b5\u63b6\u63b7\u63b8\u63b9\u63ba\u63bb\u63bc\u63bd\u63be\u63bf\u63c0\u63c1\u63c2\u63c3\u63c4\u63c5\u63c6\u63c7\u63c8\u63c9\u63ca\u63cb\u63cc\u63cd\u63ce\u63cf\u63d0\u63d1\u63d2\u63d3\u63d4\u63d5\u63d6\u63d7\u63d8\u63d9\u63da\u63db\u63dc\u63dd\u63de\u63df\u63e0\u63e1\u63e2\u63e3\u63e4\u63e5\u63e6\u63e7\u63e8\u63e9\u63ea\u63eb\u63ec\u63ed\u63ee\u63ef\u63f0\u63f1\u63f2\u63f3\u63f4\u63f5\u63f6\u63f7\u63f8\u63f9\u63fa\u63fb\u63fc\u63fd\u63fe\u63ff\u6400\u6401\u6402\u6403\u6404\u6405\u6406\u6407\u6408\u6409\u640a\u640b\u640c\u640d\u640e\u640f\u6410\u6411\u6412\u6413\u6414\u6415\u6416\u6417\u6418\u6419\u641a\u641b\u641c\u641d\u641e\u641f\u6420\u6421\u6422\u6423\u6424\u6425\u6426\u6427\u6428\u6429\u642a\u642b\u642c\u642d\u642e\u642f\u6430\u6431\u6432\u6433\u6434\u6435\u6436\u6437\u6438\u6439\u643a\u643b\u643c\u643d\u643e\u643f\u6440\u6441\u6442\u6443\u6444\u6445\u6446\u6447\u6448\u6449\u644a\u644b\u644c\u644d\u644e\u644f\u6450\u6451\u6452\u6453\u6454\u6455\u6456\u6457\u6458\u6459\u645a\u645b\u645c\u645d\u645e\u645f\u6460\u6461\u6462\u6463\u6464\u6465\u6466\u6467\u6468\u6469\u646a\u646b\u646c\u646d\u646e\u646f\u6470\u6471\u6472\u6473\u6474\u6475\u6476\u6477\u6478\u6479\u647a\u647b\u647c\u647d\u647e\u647f\u6480\u6481\u6482\u6483\u6484\u6485\u6486\u6487\u6488\u6489\u648a\u648b\u648c\u648d\u648e\u648f\u6490\u6491\u6492\u6493\u6494\u6495\u6496\u6497\u6498\u6499\u649a\u649b\u649c\u649d\u649e\u649f\u64a0\u64a1\u64a2\u64a3\u64a4\u64a5\u64a6\u64a7\u64a8\u64a9\u64aa\u64ab\u64ac\u64ad\u64ae\u64af\u64b0\u64b1\u64b2\u64b3\u64b4\u64b5\u64b6\u64b7\u64b8\u64b9\u64ba\u64bb\u64bc\u64bd\u64be\u64bf\u64c0\u64c1\u64c2\u64c3\u64c4\u64c5\u64c6\u64c7\u64c8\u64c9\u64ca\u64cb\u64cc\u64cd\u64ce\u64cf\u64d0\u64d1\u64d2\u64d3\u64d4\u64d5\u64d6\u64d7\u64d8\u64d9\u64da\u64db\u64dc\u64dd\u64de\u64df\u64e0\u64e1\u64e2\u64e3\u64e4\u64e5\u64e6\u64e7\u64e8\u64e9\u64ea\u64eb\u64ec\u64ed\u64ee\u64ef\u64f0\u64f1\u64f2\u64f3\u64f4\u64f5\u64f6\u64f7\u64f8\u64f9\u64fa\u64fb\u64fc\u64fd\u64fe\u64ff\u6500\u6501\u6502\u6503\u6504\u6505\u6506\u6507\u6508\u6509\u650a\u650b\u650c\u650d\u650e\u650f\u6510\u6511\u6512\u6513\u6514\u6515\u6516\u6517\u6518\u6519\u651a\u651b\u651c\u651d\u651e\u651f\u6520\u6521\u6522\u6523\u6524\u6525\u6526\u6527\u6528\u6529\u652a\u652b\u652c\u652d\u652e\u652f\u6530\u6531\u6532\u6533\u6534\u6535\u6536\u6537\u6538\u6539\u653a\u653b\u653c\u653d\u653e\u653f\u6540\u6541\u6542\u6543\u6544\u6545\u6546\u6547\u6548\u6549\u654a\u654b\u654c\u654d\u654e\u654f\u6550\u6551\u6552\u6553\u6554\u6555\u6556\u6557\u6558\u6559\u655a\u655b\u655c\u655d\u655e\u655f\u6560\u6561\u6562\u6563\u6564\u6565\u6566\u6567\u6568\u6569\u656a\u656b\u656c\u656d\u656e\u656f\u6570\u6571\u6572\u6573\u6574\u6575\u6576\u6577\u6578\u6579\u657a\u657b\u657c\u657d\u657e\u657f\u6580\u6581\u6582\u6583\u6584\u6585\u6586\u6587\u6588\u6589\u658a\u658b\u658c\u658d\u658e\u658f\u6590\u6591\u6592\u6593\u6594\u6595\u6596\u6597\u6598\u6599\u659a\u659b\u659c\u659d\u659e\u659f\u65a0\u65a1\u65a2\u65a3\u65a4\u65a5\u65a6\u65a7\u65a8\u65a9\u65aa\u65ab\u65ac\u65ad\u65ae\u65af\u65b0\u65b1\u65b2\u65b3\u65b4\u65b5\u65b6\u65b7\u65b8\u65b9\u65ba\u65bb\u65bc\u65bd\u65be\u65bf\u65c0\u65c1\u65c2\u65c3\u65c4\u65c5\u65c6\u65c7\u65c8\u65c9\u65ca\u65cb\u65cc\u65cd\u65ce\u65cf\u65d0\u65d1\u65d2\u65d3\u65d4\u65d5\u65d6\u65d7\u65d8\u65d9\u65da\u65db\u65dc\u65dd\u65de\u65df\u65e0\u65e1\u65e2\u65e3\u65e4\u65e5\u65e6\u65e7\u65e8\u65e9\u65ea\u65eb\u65ec\u65ed\u65ee\u65ef\u65f0\u65f1\u65f2\u65f3\u65f4\u65f5\u65f6\u65f7\u65f8\u65f9\u65fa\u65fb\u65fc\u65fd\u65fe\u65ff\u6600\u6601\u6602\u6603\u6604\u6605\u6606\u6607\u6608\u6609\u660a\u660b\u660c\u660d\u660e\u660f\u6610\u6611\u6612\u6613\u6614\u6615\u6616\u6617\u6618\u6619\u661a\u661b\u661c\u661d\u661e\u661f\u6620\u6621\u6622\u6623\u6624\u6625\u6626\u6627\u6628\u6629\u662a\u662b\u662c\u662d\u662e\u662f\u6630\u6631\u6632\u6633\u6634\u6635\u6636\u6637\u6638\u6639\u663a\u663b\u663c\u663d\u663e\u663f\u6640\u6641\u6642\u6643\u6644\u6645\u6646\u6647\u6648\u6649\u664a\u664b\u664c\u664d\u664e\u664f\u6650\u6651\u6652\u6653\u6654\u6655\u6656\u6657\u6658\u6659\u665a\u665b\u665c\u665d\u665e\u665f\u6660\u6661\u6662\u6663\u6664\u6665\u6666\u6667\u6668\u6669\u666a\u666b\u666c\u666d\u666e\u666f\u6670\u6671\u6672\u6673\u6674\u6675\u6676\u6677\u6678\u6679\u667a\u667b\u667c\u667d\u667e\u667f\u6680\u6681\u6682\u6683\u6684\u6685\u6686\u6687\u6688\u6689\u668a\u668b\u668c\u668d\u668e\u668f\u6690\u6691\u6692\u6693\u6694\u6695\u6696\u6697\u6698\u6699\u669a\u669b\u669c\u669d\u669e\u669f\u66a0\u66a1\u66a2\u66a3\u66a4\u66a5\u66a6\u66a7\u66a8\u66a9\u66aa\u66ab\u66ac\u66ad\u66ae\u66af\u66b0\u66b1\u66b2\u66b3\u66b4\u66b5\u66b6\u66b7\u66b8\u66b9\u66ba\u66bb\u66bc\u66bd\u66be\u66bf\u66c0\u66c1\u66c2\u66c3\u66c4\u66c5\u66c6\u66c7\u66c8\u66c9\u66ca\u66cb\u66cc\u66cd\u66ce\u66cf\u66d0\u66d1\u66d2\u66d3\u66d4\u66d5\u66d6\u66d7\u66d8\u66d9\u66da\u66db\u66dc\u66dd\u66de\u66df\u66e0\u66e1\u66e2\u66e3\u66e4\u66e5\u66e6\u66e7\u66e8\u66e9\u66ea\u66eb\u66ec\u66ed\u66ee\u66ef\u66f0\u66f1\u66f2\u66f3\u66f4\u66f5\u66f6\u66f7\u66f8\u66f9\u66fa\u66fb\u66fc\u66fd\u66fe\u66ff\u6700\u6701\u6702\u6703\u6704\u6705\u6706\u6707\u6708\u6709\u670a\u670b\u670c\u670d\u670e\u670f\u6710\u6711\u6712\u6713\u6714\u6715\u6716\u6717\u6718\u6719\u671a\u671b\u671c\u671d\u671e\u671f\u6720\u6721\u6722\u6723\u6724\u6725\u6726\u6727\u6728\u6729\u672a\u672b\u672c\u672d\u672e\u672f\u6730\u6731\u6732\u6733\u6734\u6735\u6736\u6737\u6738\u6739\u673a\u673b\u673c\u673d\u673e\u673f\u6740\u6741\u6742\u6743\u6744\u6745\u6746\u6747\u6748\u6749\u674a\u674b\u674c\u674d\u674e\u674f\u6750\u6751\u6752\u6753\u6754\u6755\u6756\u6757\u6758\u6759\u675a\u675b\u675c\u675d\u675e\u675f\u6760\u6761\u6762\u6763\u6764\u6765\u6766\u6767\u6768\u6769\u676a\u676b\u676c\u676d\u676e\u676f\u6770\u6771\u6772\u6773\u6774\u6775\u6776\u6777\u6778\u6779\u677a\u677b\u677c\u677d\u677e\u677f\u6780\u6781\u6782\u6783\u6784\u6785\u6786\u6787\u6788\u6789\u678a\u678b\u678c\u678d\u678e\u678f\u6790\u6791\u6792\u6793\u6794\u6795\u6796\u6797\u6798\u6799\u679a\u679b\u679c\u679d\u679e\u679f\u67a0\u67a1\u67a2\u67a3\u67a4\u67a5\u67a6\u67a7\u67a8\u67a9\u67aa\u67ab\u67ac\u67ad\u67ae\u67af\u67b0\u67b1\u67b2\u67b3\u67b4\u67b5\u67b6\u67b7\u67b8\u67b9\u67ba\u67bb\u67bc\u67bd\u67be\u67bf\u67c0\u67c1\u67c2\u67c3\u67c4\u67c5\u67c6\u67c7\u67c8\u67c9\u67ca\u67cb\u67cc\u67cd\u67ce\u67cf\u67d0\u67d1\u67d2\u67d3\u67d4\u67d5\u67d6\u67d7\u67d8\u67d9\u67da\u67db\u67dc\u67dd\u67de\u67df\u67e0\u67e1\u67e2\u67e3\u67e4\u67e5\u67e6\u67e7\u67e8\u67e9\u67ea\u67eb\u67ec\u67ed\u67ee\u67ef\u67f0\u67f1\u67f2\u67f3\u67f4\u67f5\u67f6\u67f7\u67f8\u67f9\u67fa\u67fb\u67fc\u67fd\u67fe\u67ff\u6800\u6801\u6802\u6803\u6804\u6805\u6806\u6807\u6808\u6809\u680a\u680b\u680c\u680d\u680e\u680f\u6810\u6811\u6812\u6813\u6814\u6815\u6816\u6817\u6818\u6819\u681a\u681b\u681c\u681d\u681e\u681f\u6820\u6821\u6822\u6823\u6824\u6825\u6826\u6827\u6828\u6829\u682a\u682b\u682c\u682d\u682e\u682f\u6830\u6831\u6832\u6833\u6834\u6835\u6836\u6837\u6838\u6839\u683a\u683b\u683c\u683d\u683e\u683f\u6840\u6841\u6842\u6843\u6844\u6845\u6846\u6847\u6848\u6849\u684a\u684b\u684c\u684d\u684e\u684f\u6850\u6851\u6852\u6853\u6854\u6855\u6856\u6857\u6858\u6859\u685a\u685b\u685c\u685d\u685e\u685f\u6860\u6861\u6862\u6863\u6864\u6865\u6866\u6867\u6868\u6869\u686a\u686b\u686c\u686d\u686e\u686f\u6870\u6871\u6872\u6873\u6874\u6875\u6876\u6877\u6878\u6879\u687a\u687b\u687c\u687d\u687e\u687f\u6880\u6881\u6882\u6883\u6884\u6885\u6886\u6887\u6888\u6889\u688a\u688b\u688c\u688d\u688e\u688f\u6890\u6891\u6892\u6893\u6894\u6895\u6896\u6897\u6898\u6899\u689a\u689b\u689c\u689d\u689e\u689f\u68a0\u68a1\u68a2\u68a3\u68a4\u68a5\u68a6\u68a7\u68a8\u68a9\u68aa\u68ab\u68ac\u68ad\u68ae\u68af\u68b0\u68b1\u68b2\u68b3\u68b4\u68b5\u68b6\u68b7\u68b8\u68b9\u68ba\u68bb\u68bc\u68bd\u68be\u68bf\u68c0\u68c1\u68c2\u68c3\u68c4\u68c5\u68c6\u68c7\u68c8\u68c9\u68ca\u68cb\u68cc\u68cd\u68ce\u68cf\u68d0\u68d1\u68d2\u68d3\u68d4\u68d5\u68d6\u68d7\u68d8\u68d9\u68da\u68db\u68dc\u68dd\u68de\u68df\u68e0\u68e1\u68e2\u68e3\u68e4\u68e5\u68e6\u68e7\u68e8\u68e9\u68ea\u68eb\u68ec\u68ed\u68ee\u68ef\u68f0\u68f1\u68f2\u68f3\u68f4\u68f5\u68f6\u68f7\u68f8\u68f9\u68fa\u68fb\u68fc\u68fd\u68fe\u68ff\u6900\u6901\u6902\u6903\u6904\u6905\u6906\u6907\u6908\u6909\u690a\u690b\u690c\u690d\u690e\u690f\u6910\u6911\u6912\u6913\u6914\u6915\u6916\u6917\u6918\u6919\u691a\u691b\u691c\u691d\u691e\u691f\u6920\u6921\u6922\u6923\u6924\u6925\u6926\u6927\u6928\u6929\u692a\u692b\u692c\u692d\u692e\u692f\u6930\u6931\u6932\u6933\u6934\u6935\u6936\u6937\u6938\u6939\u693a\u693b\u693c\u693d\u693e\u693f\u6940\u6941\u6942\u6943\u6944\u6945\u6946\u6947\u6948\u6949\u694a\u694b\u694c\u694d\u694e\u694f\u6950\u6951\u6952\u6953\u6954\u6955\u6956\u6957\u6958\u6959\u695a\u695b\u695c\u695d\u695e\u695f\u6960\u6961\u6962\u6963\u6964\u6965\u6966\u6967\u6968\u6969\u696a\u696b\u696c\u696d\u696e\u696f\u6970\u6971\u6972\u6973\u6974\u6975\u6976\u6977\u6978\u6979\u697a\u697b\u697c\u697d\u697e\u697f\u6980\u6981\u6982\u6983\u6984\u6985\u6986\u6987\u6988\u6989\u698a\u698b\u698c\u698d\u698e\u698f\u6990\u6991\u6992\u6993\u6994\u6995\u6996\u6997\u6998\u6999\u699a\u699b\u699c\u699d\u699e\u699f\u69a0\u69a1\u69a2\u69a3\u69a4\u69a5\u69a6\u69a7\u69a8\u69a9\u69aa\u69ab\u69ac\u69ad\u69ae\u69af\u69b0\u69b1\u69b2\u69b3\u69b4\u69b5\u69b6\u69b7\u69b8\u69b9\u69ba\u69bb\u69bc\u69bd\u69be\u69bf\u69c0\u69c1\u69c2\u69c3\u69c4\u69c5\u69c6\u69c7\u69c8\u69c9\u69ca\u69cb\u69cc\u69cd\u69ce\u69cf\u69d0\u69d1\u69d2\u69d3\u69d4\u69d5\u69d6\u69d7\u69d8\u69d9\u69da\u69db\u69dc\u69dd\u69de\u69df\u69e0\u69e1\u69e2\u69e3\u69e4\u69e5\u69e6\u69e7\u69e8\u69e9\u69ea\u69eb\u69ec\u69ed\u69ee\u69ef\u69f0\u69f1\u69f2\u69f3\u69f4\u69f5\u69f6\u69f7\u69f8\u69f9\u69fa\u69fb\u69fc\u69fd\u69fe\u69ff\u6a00\u6a01\u6a02\u6a03\u6a04\u6a05\u6a06\u6a07\u6a08\u6a09\u6a0a\u6a0b\u6a0c\u6a0d\u6a0e\u6a0f\u6a10\u6a11\u6a12\u6a13\u6a14\u6a15\u6a16\u6a17\u6a18\u6a19\u6a1a\u6a1b\u6a1c\u6a1d\u6a1e\u6a1f\u6a20\u6a21\u6a22\u6a23\u6a24\u6a25\u6a26\u6a27\u6a28\u6a29\u6a2a\u6a2b\u6a2c\u6a2d\u6a2e\u6a2f\u6a30\u6a31\u6a32\u6a33\u6a34\u6a35\u6a36\u6a37\u6a38\u6a39\u6a3a\u6a3b\u6a3c\u6a3d\u6a3e\u6a3f\u6a40\u6a41\u6a42\u6a43\u6a44\u6a45\u6a46\u6a47\u6a48\u6a49\u6a4a\u6a4b\u6a4c\u6a4d\u6a4e\u6a4f\u6a50\u6a51\u6a52\u6a53\u6a54\u6a55\u6a56\u6a57\u6a58\u6a59\u6a5a\u6a5b\u6a5c\u6a5d\u6a5e\u6a5f\u6a60\u6a61\u6a62\u6a63\u6a64\u6a65\u6a66\u6a67\u6a68\u6a69\u6a6a\u6a6b\u6a6c\u6a6d\u6a6e\u6a6f\u6a70\u6a71\u6a72\u6a73\u6a74\u6a75\u6a76\u6a77\u6a78\u6a79\u6a7a\u6a7b\u6a7c\u6a7d\u6a7e\u6a7f\u6a80\u6a81\u6a82\u6a83\u6a84\u6a85\u6a86\u6a87\u6a88\u6a89\u6a8a\u6a8b\u6a8c\u6a8d\u6a8e\u6a8f\u6a90\u6a91\u6a92\u6a93\u6a94\u6a95\u6a96\u6a97\u6a98\u6a99\u6a9a\u6a9b\u6a9c\u6a9d\u6a9e\u6a9f\u6aa0\u6aa1\u6aa2\u6aa3\u6aa4\u6aa5\u6aa6\u6aa7\u6aa8\u6aa9\u6aaa\u6aab\u6aac\u6aad\u6aae\u6aaf\u6ab0\u6ab1\u6ab2\u6ab3\u6ab4\u6ab5\u6ab6\u6ab7\u6ab8\u6ab9\u6aba\u6abb\u6abc\u6abd\u6abe\u6abf\u6ac0\u6ac1\u6ac2\u6ac3\u6ac4\u6ac5\u6ac6\u6ac7\u6ac8\u6ac9\u6aca\u6acb\u6acc\u6acd\u6ace\u6acf\u6ad0\u6ad1\u6ad2\u6ad3\u6ad4\u6ad5\u6ad6\u6ad7\u6ad8\u6ad9\u6ada\u6adb\u6adc\u6add\u6ade\u6adf\u6ae0\u6ae1\u6ae2\u6ae3\u6ae4\u6ae5\u6ae6\u6ae7\u6ae8\u6ae9\u6aea\u6aeb\u6aec\u6aed\u6aee\u6aef\u6af0\u6af1\u6af2\u6af3\u6af4\u6af5\u6af6\u6af7\u6af8\u6af9\u6afa\u6afb\u6afc\u6afd\u6afe\u6aff\u6b00\u6b01\u6b02\u6b03\u6b04\u6b05\u6b06\u6b07\u6b08\u6b09\u6b0a\u6b0b\u6b0c\u6b0d\u6b0e\u6b0f\u6b10\u6b11\u6b12\u6b13\u6b14\u6b15\u6b16\u6b17\u6b18\u6b19\u6b1a\u6b1b\u6b1c\u6b1d\u6b1e\u6b1f\u6b20\u6b21\u6b22\u6b23\u6b24\u6b25\u6b26\u6b27\u6b28\u6b29\u6b2a\u6b2b\u6b2c\u6b2d\u6b2e\u6b2f\u6b30\u6b31\u6b32\u6b33\u6b34\u6b35\u6b36\u6b37\u6b38\u6b39\u6b3a\u6b3b\u6b3c\u6b3d\u6b3e\u6b3f\u6b40\u6b41\u6b42\u6b43\u6b44\u6b45\u6b46\u6b47\u6b48\u6b49\u6b4a\u6b4b\u6b4c\u6b4d\u6b4e\u6b4f\u6b50\u6b51\u6b52\u6b53\u6b54\u6b55\u6b56\u6b57\u6b58\u6b59\u6b5a\u6b5b\u6b5c\u6b5d\u6b5e\u6b5f\u6b60\u6b61\u6b62\u6b63\u6b64\u6b65\u6b66\u6b67\u6b68\u6b69\u6b6a\u6b6b\u6b6c\u6b6d\u6b6e\u6b6f\u6b70\u6b71\u6b72\u6b73\u6b74\u6b75\u6b76\u6b77\u6b78\u6b79\u6b7a\u6b7b\u6b7c\u6b7d\u6b7e\u6b7f\u6b80\u6b81\u6b82\u6b83\u6b84\u6b85\u6b86\u6b87\u6b88\u6b89\u6b8a\u6b8b\u6b8c\u6b8d\u6b8e\u6b8f\u6b90\u6b91\u6b92\u6b93\u6b94\u6b95\u6b96\u6b97\u6b98\u6b99\u6b9a\u6b9b\u6b9c\u6b9d\u6b9e\u6b9f\u6ba0\u6ba1\u6ba2\u6ba3\u6ba4\u6ba5\u6ba6\u6ba7\u6ba8\u6ba9\u6baa\u6bab\u6bac\u6bad\u6bae\u6baf\u6bb0\u6bb1\u6bb2\u6bb3\u6bb4\u6bb5\u6bb6\u6bb7\u6bb8\u6bb9\u6bba\u6bbb\u6bbc\u6bbd\u6bbe\u6bbf\u6bc0\u6bc1\u6bc2\u6bc3\u6bc4\u6bc5\u6bc6\u6bc7\u6bc8\u6bc9\u6bca\u6bcb\u6bcc\u6bcd\u6bce\u6bcf\u6bd0\u6bd1\u6bd2\u6bd3\u6bd4\u6bd5\u6bd6\u6bd7\u6bd8\u6bd9\u6bda\u6bdb\u6bdc\u6bdd\u6bde\u6bdf\u6be0\u6be1\u6be2\u6be3\u6be4\u6be5\u6be6\u6be7\u6be8\u6be9\u6bea\u6beb\u6bec\u6bed\u6bee\u6bef\u6bf0\u6bf1\u6bf2\u6bf3\u6bf4\u6bf5\u6bf6\u6bf7\u6bf8\u6bf9\u6bfa\u6bfb\u6bfc\u6bfd\u6bfe\u6bff\u6c00\u6c01\u6c02\u6c03\u6c04\u6c05\u6c06\u6c07\u6c08\u6c09\u6c0a\u6c0b\u6c0c\u6c0d\u6c0e\u6c0f\u6c10\u6c11\u6c12\u6c13\u6c14\u6c15\u6c16\u6c17\u6c18\u6c19\u6c1a\u6c1b\u6c1c\u6c1d\u6c1e\u6c1f\u6c20\u6c21\u6c22\u6c23\u6c24\u6c25\u6c26\u6c27\u6c28\u6c29\u6c2a\u6c2b\u6c2c\u6c2d\u6c2e\u6c2f\u6c30\u6c31\u6c32\u6c33\u6c34\u6c35\u6c36\u6c37\u6c38\u6c39\u6c3a\u6c3b\u6c3c\u6c3d\u6c3e\u6c3f\u6c40\u6c41\u6c42\u6c43\u6c44\u6c45\u6c46\u6c47\u6c48\u6c49\u6c4a\u6c4b\u6c4c\u6c4d\u6c4e\u6c4f\u6c50\u6c51\u6c52\u6c53\u6c54\u6c55\u6c56\u6c57\u6c58\u6c59\u6c5a\u6c5b\u6c5c\u6c5d\u6c5e\u6c5f\u6c60\u6c61\u6c62\u6c63\u6c64\u6c65\u6c66\u6c67\u6c68\u6c69\u6c6a\u6c6b\u6c6c\u6c6d\u6c6e\u6c6f\u6c70\u6c71\u6c72\u6c73\u6c74\u6c75\u6c76\u6c77\u6c78\u6c79\u6c7a\u6c7b\u6c7c\u6c7d\u6c7e\u6c7f\u6c80\u6c81\u6c82\u6c83\u6c84\u6c85\u6c86\u6c87\u6c88\u6c89\u6c8a\u6c8b\u6c8c\u6c8d\u6c8e\u6c8f\u6c90\u6c91\u6c92\u6c93\u6c94\u6c95\u6c96\u6c97\u6c98\u6c99\u6c9a\u6c9b\u6c9c\u6c9d\u6c9e\u6c9f\u6ca0\u6ca1\u6ca2\u6ca3\u6ca4\u6ca5\u6ca6\u6ca7\u6ca8\u6ca9\u6caa\u6cab\u6cac\u6cad\u6cae\u6caf\u6cb0\u6cb1\u6cb2\u6cb3\u6cb4\u6cb5\u6cb6\u6cb7\u6cb8\u6cb9\u6cba\u6cbb\u6cbc\u6cbd\u6cbe\u6cbf\u6cc0\u6cc1\u6cc2\u6cc3\u6cc4\u6cc5\u6cc6\u6cc7\u6cc8\u6cc9\u6cca\u6ccb\u6ccc\u6ccd\u6cce\u6ccf\u6cd0\u6cd1\u6cd2\u6cd3\u6cd4\u6cd5\u6cd6\u6cd7\u6cd8\u6cd9\u6cda\u6cdb\u6cdc\u6cdd\u6cde\u6cdf\u6ce0\u6ce1\u6ce2\u6ce3\u6ce4\u6ce5\u6ce6\u6ce7\u6ce8\u6ce9\u6cea\u6ceb\u6cec\u6ced\u6cee\u6cef\u6cf0\u6cf1\u6cf2\u6cf3\u6cf4\u6cf5\u6cf6\u6cf7\u6cf8\u6cf9\u6cfa\u6cfb\u6cfc\u6cfd\u6cfe\u6cff\u6d00\u6d01\u6d02\u6d03\u6d04\u6d05\u6d06\u6d07\u6d08\u6d09\u6d0a\u6d0b\u6d0c\u6d0d\u6d0e\u6d0f\u6d10\u6d11\u6d12\u6d13\u6d14\u6d15\u6d16\u6d17\u6d18\u6d19\u6d1a\u6d1b\u6d1c\u6d1d\u6d1e\u6d1f\u6d20\u6d21\u6d22\u6d23\u6d24\u6d25\u6d26\u6d27\u6d28\u6d29\u6d2a\u6d2b\u6d2c\u6d2d\u6d2e\u6d2f\u6d30\u6d31\u6d32\u6d33\u6d34\u6d35\u6d36\u6d37\u6d38\u6d39\u6d3a\u6d3b\u6d3c\u6d3d\u6d3e\u6d3f\u6d40\u6d41\u6d42\u6d43\u6d44\u6d45\u6d46\u6d47\u6d48\u6d49\u6d4a\u6d4b\u6d4c\u6d4d\u6d4e\u6d4f\u6d50\u6d51\u6d52\u6d53\u6d54\u6d55\u6d56\u6d57\u6d58\u6d59\u6d5a\u6d5b\u6d5c\u6d5d\u6d5e\u6d5f\u6d60\u6d61\u6d62\u6d63\u6d64\u6d65\u6d66\u6d67\u6d68\u6d69\u6d6a\u6d6b\u6d6c\u6d6d\u6d6e\u6d6f\u6d70\u6d71\u6d72\u6d73\u6d74\u6d75\u6d76\u6d77\u6d78\u6d79\u6d7a\u6d7b\u6d7c\u6d7d\u6d7e\u6d7f\u6d80\u6d81\u6d82\u6d83\u6d84\u6d85\u6d86\u6d87\u6d88\u6d89\u6d8a\u6d8b\u6d8c\u6d8d\u6d8e\u6d8f\u6d90\u6d91\u6d92\u6d93\u6d94\u6d95\u6d96\u6d97\u6d98\u6d99\u6d9a\u6d9b\u6d9c\u6d9d\u6d9e\u6d9f\u6da0\u6da1\u6da2\u6da3\u6da4\u6da5\u6da6\u6da7\u6da8\u6da9\u6daa\u6dab\u6dac\u6dad\u6dae\u6daf\u6db0\u6db1\u6db2\u6db3\u6db4\u6db5\u6db6\u6db7\u6db8\u6db9\u6dba\u6dbb\u6dbc\u6dbd\u6dbe\u6dbf\u6dc0\u6dc1\u6dc2\u6dc3\u6dc4\u6dc5\u6dc6\u6dc7\u6dc8\u6dc9\u6dca\u6dcb\u6dcc\u6dcd\u6dce\u6dcf\u6dd0\u6dd1\u6dd2\u6dd3\u6dd4\u6dd5\u6dd6\u6dd7\u6dd8\u6dd9\u6dda\u6ddb\u6ddc\u6ddd\u6dde\u6ddf\u6de0\u6de1\u6de2\u6de3\u6de4\u6de5\u6de6\u6de7\u6de8\u6de9\u6dea\u6deb\u6dec\u6ded\u6dee\u6def\u6df0\u6df1\u6df2\u6df3\u6df4\u6df5\u6df6\u6df7\u6df8\u6df9\u6dfa\u6dfb\u6dfc\u6dfd\u6dfe\u6dff\u6e00\u6e01\u6e02\u6e03\u6e04\u6e05\u6e06\u6e07\u6e08\u6e09\u6e0a\u6e0b\u6e0c\u6e0d\u6e0e\u6e0f\u6e10\u6e11\u6e12\u6e13\u6e14\u6e15\u6e16\u6e17\u6e18\u6e19\u6e1a\u6e1b\u6e1c\u6e1d\u6e1e\u6e1f\u6e20\u6e21\u6e22\u6e23\u6e24\u6e25\u6e26\u6e27\u6e28\u6e29\u6e2a\u6e2b\u6e2c\u6e2d\u6e2e\u6e2f\u6e30\u6e31\u6e32\u6e33\u6e34\u6e35\u6e36\u6e37\u6e38\u6e39\u6e3a\u6e3b\u6e3c\u6e3d\u6e3e\u6e3f\u6e40\u6e41\u6e42\u6e43\u6e44\u6e45\u6e46\u6e47\u6e48\u6e49\u6e4a\u6e4b\u6e4c\u6e4d\u6e4e\u6e4f\u6e50\u6e51\u6e52\u6e53\u6e54\u6e55\u6e56\u6e57\u6e58\u6e59\u6e5a\u6e5b\u6e5c\u6e5d\u6e5e\u6e5f\u6e60\u6e61\u6e62\u6e63\u6e64\u6e65\u6e66\u6e67\u6e68\u6e69\u6e6a\u6e6b\u6e6c\u6e6d\u6e6e\u6e6f\u6e70\u6e71\u6e72\u6e73\u6e74\u6e75\u6e76\u6e77\u6e78\u6e79\u6e7a\u6e7b\u6e7c\u6e7d\u6e7e\u6e7f\u6e80\u6e81\u6e82\u6e83\u6e84\u6e85\u6e86\u6e87\u6e88\u6e89\u6e8a\u6e8b\u6e8c\u6e8d\u6e8e\u6e8f\u6e90\u6e91\u6e92\u6e93\u6e94\u6e95\u6e96\u6e97\u6e98\u6e99\u6e9a\u6e9b\u6e9c\u6e9d\u6e9e\u6e9f\u6ea0\u6ea1\u6ea2\u6ea3\u6ea4\u6ea5\u6ea6\u6ea7\u6ea8\u6ea9\u6eaa\u6eab\u6eac\u6ead\u6eae\u6eaf\u6eb0\u6eb1\u6eb2\u6eb3\u6eb4\u6eb5\u6eb6\u6eb7\u6eb8\u6eb9\u6eba\u6ebb\u6ebc\u6ebd\u6ebe\u6ebf\u6ec0\u6ec1\u6ec2\u6ec3\u6ec4\u6ec5\u6ec6\u6ec7\u6ec8\u6ec9\u6eca\u6ecb\u6ecc\u6ecd\u6ece\u6ecf\u6ed0\u6ed1\u6ed2\u6ed3\u6ed4\u6ed5\u6ed6\u6ed7\u6ed8\u6ed9\u6eda\u6edb\u6edc\u6edd\u6ede\u6edf\u6ee0\u6ee1\u6ee2\u6ee3\u6ee4\u6ee5\u6ee6\u6ee7\u6ee8\u6ee9\u6eea\u6eeb\u6eec\u6eed\u6eee\u6eef\u6ef0\u6ef1\u6ef2\u6ef3\u6ef4\u6ef5\u6ef6\u6ef7\u6ef8\u6ef9\u6efa\u6efb\u6efc\u6efd\u6efe\u6eff\u6f00\u6f01\u6f02\u6f03\u6f04\u6f05\u6f06\u6f07\u6f08\u6f09\u6f0a\u6f0b\u6f0c\u6f0d\u6f0e\u6f0f\u6f10\u6f11\u6f12\u6f13\u6f14\u6f15\u6f16\u6f17\u6f18\u6f19\u6f1a\u6f1b\u6f1c\u6f1d\u6f1e\u6f1f\u6f20\u6f21\u6f22\u6f23\u6f24\u6f25\u6f26\u6f27\u6f28\u6f29\u6f2a\u6f2b\u6f2c\u6f2d\u6f2e\u6f2f\u6f30\u6f31\u6f32\u6f33\u6f34\u6f35\u6f36\u6f37\u6f38\u6f39\u6f3a\u6f3b\u6f3c\u6f3d\u6f3e\u6f3f\u6f40\u6f41\u6f42\u6f43\u6f44\u6f45\u6f46\u6f47\u6f48\u6f49\u6f4a\u6f4b\u6f4c\u6f4d\u6f4e\u6f4f\u6f50\u6f51\u6f52\u6f53\u6f54\u6f55\u6f56\u6f57\u6f58\u6f59\u6f5a\u6f5b\u6f5c\u6f5d\u6f5e\u6f5f\u6f60\u6f61\u6f62\u6f63\u6f64\u6f65\u6f66\u6f67\u6f68\u6f69\u6f6a\u6f6b\u6f6c\u6f6d\u6f6e\u6f6f\u6f70\u6f71\u6f72\u6f73\u6f74\u6f75\u6f76\u6f77\u6f78\u6f79\u6f7a\u6f7b\u6f7c\u6f7d\u6f7e\u6f7f\u6f80\u6f81\u6f82\u6f83\u6f84\u6f85\u6f86\u6f87\u6f88\u6f89\u6f8a\u6f8b\u6f8c\u6f8d\u6f8e\u6f8f\u6f90\u6f91\u6f92\u6f93\u6f94\u6f95\u6f96\u6f97\u6f98\u6f99\u6f9a\u6f9b\u6f9c\u6f9d\u6f9e\u6f9f\u6fa0\u6fa1\u6fa2\u6fa3\u6fa4\u6fa5\u6fa6\u6fa7\u6fa8\u6fa9\u6faa\u6fab\u6fac\u6fad\u6fae\u6faf\u6fb0\u6fb1\u6fb2\u6fb3\u6fb4\u6fb5\u6fb6\u6fb7\u6fb8\u6fb9\u6fba\u6fbb\u6fbc\u6fbd\u6fbe\u6fbf\u6fc0\u6fc1\u6fc2\u6fc3\u6fc4\u6fc5\u6fc6\u6fc7\u6fc8\u6fc9\u6fca\u6fcb\u6fcc\u6fcd\u6fce\u6fcf\u6fd0\u6fd1\u6fd2\u6fd3\u6fd4\u6fd5\u6fd6\u6fd7\u6fd8\u6fd9\u6fda\u6fdb\u6fdc\u6fdd\u6fde\u6fdf\u6fe0\u6fe1\u6fe2\u6fe3\u6fe4\u6fe5\u6fe6\u6fe7\u6fe8\u6fe9\u6fea\u6feb\u6fec\u6fed\u6fee\u6fef\u6ff0\u6ff1\u6ff2\u6ff3\u6ff4\u6ff5\u6ff6\u6ff7\u6ff8\u6ff9\u6ffa\u6ffb\u6ffc\u6ffd\u6ffe\u6fff\u7000\u7001\u7002\u7003\u7004\u7005\u7006\u7007\u7008\u7009\u700a\u700b\u700c\u700d\u700e\u700f\u7010\u7011\u7012\u7013\u7014\u7015\u7016\u7017\u7018\u7019\u701a\u701b\u701c\u701d\u701e\u701f\u7020\u7021\u7022\u7023\u7024\u7025\u7026\u7027\u7028\u7029\u702a\u702b\u702c\u702d\u702e\u702f\u7030\u7031\u7032\u7033\u7034\u7035\u7036\u7037\u7038\u7039\u703a\u703b\u703c\u703d\u703e\u703f\u7040\u7041\u7042\u7043\u7044\u7045\u7046\u7047\u7048\u7049\u704a\u704b\u704c\u704d\u704e\u704f\u7050\u7051\u7052\u7053\u7054\u7055\u7056\u7057\u7058\u7059\u705a\u705b\u705c\u705d\u705e\u705f\u7060\u7061\u7062\u7063\u7064\u7065\u7066\u7067\u7068\u7069\u706a\u706b\u706c\u706d\u706e\u706f\u7070\u7071\u7072\u7073\u7074\u7075\u7076\u7077\u7078\u7079\u707a\u707b\u707c\u707d\u707e\u707f\u7080\u7081\u7082\u7083\u7084\u7085\u7086\u7087\u7088\u7089\u708a\u708b\u708c\u708d\u708e\u708f\u7090\u7091\u7092\u7093\u7094\u7095\u7096\u7097\u7098\u7099\u709a\u709b\u709c\u709d\u709e\u709f\u70a0\u70a1\u70a2\u70a3\u70a4\u70a5\u70a6\u70a7\u70a8\u70a9\u70aa\u70ab\u70ac\u70ad\u70ae\u70af\u70b0\u70b1\u70b2\u70b3\u70b4\u70b5\u70b6\u70b7\u70b8\u70b9\u70ba\u70bb\u70bc\u70bd\u70be\u70bf\u70c0\u70c1\u70c2\u70c3\u70c4\u70c5\u70c6\u70c7\u70c8\u70c9\u70ca\u70cb\u70cc\u70cd\u70ce\u70cf\u70d0\u70d1\u70d2\u70d3\u70d4\u70d5\u70d6\u70d7\u70d8\u70d9\u70da\u70db\u70dc\u70dd\u70de\u70df\u70e0\u70e1\u70e2\u70e3\u70e4\u70e5\u70e6\u70e7\u70e8\u70e9\u70ea\u70eb\u70ec\u70ed\u70ee\u70ef\u70f0\u70f1\u70f2\u70f3\u70f4\u70f5\u70f6\u70f7\u70f8\u70f9\u70fa\u70fb\u70fc\u70fd\u70fe\u70ff\u7100\u7101\u7102\u7103\u7104\u7105\u7106\u7107\u7108\u7109\u710a\u710b\u710c\u710d\u710e\u710f\u7110\u7111\u7112\u7113\u7114\u7115\u7116\u7117\u7118\u7119\u711a\u711b\u711c\u711d\u711e\u711f\u7120\u7121\u7122\u7123\u7124\u7125\u7126\u7127\u7128\u7129\u712a\u712b\u712c\u712d\u712e\u712f\u7130\u7131\u7132\u7133\u7134\u7135\u7136\u7137\u7138\u7139\u713a\u713b\u713c\u713d\u713e\u713f\u7140\u7141\u7142\u7143\u7144\u7145\u7146\u7147\u7148\u7149\u714a\u714b\u714c\u714d\u714e\u714f\u7150\u7151\u7152\u7153\u7154\u7155\u7156\u7157\u7158\u7159\u715a\u715b\u715c\u715d\u715e\u715f\u7160\u7161\u7162\u7163\u7164\u7165\u7166\u7167\u7168\u7169\u716a\u716b\u716c\u716d\u716e\u716f\u7170\u7171\u7172\u7173\u7174\u7175\u7176\u7177\u7178\u7179\u717a\u717b\u717c\u717d\u717e\u717f\u7180\u7181\u7182\u7183\u7184\u7185\u7186\u7187\u7188\u7189\u718a\u718b\u718c\u718d\u718e\u718f\u7190\u7191\u7192\u7193\u7194\u7195\u7196\u7197\u7198\u7199\u719a\u719b\u719c\u719d\u719e\u719f\u71a0\u71a1\u71a2\u71a3\u71a4\u71a5\u71a6\u71a7\u71a8\u71a9\u71aa\u71ab\u71ac\u71ad\u71ae\u71af\u71b0\u71b1\u71b2\u71b3\u71b4\u71b5\u71b6\u71b7\u71b8\u71b9\u71ba\u71bb\u71bc\u71bd\u71be\u71bf\u71c0\u71c1\u71c2\u71c3\u71c4\u71c5\u71c6\u71c7\u71c8\u71c9\u71ca\u71cb\u71cc\u71cd\u71ce\u71cf\u71d0\u71d1\u71d2\u71d3\u71d4\u71d5\u71d6\u71d7\u71d8\u71d9\u71da\u71db\u71dc\u71dd\u71de\u71df\u71e0\u71e1\u71e2\u71e3\u71e4\u71e5\u71e6\u71e7\u71e8\u71e9\u71ea\u71eb\u71ec\u71ed\u71ee\u71ef\u71f0\u71f1\u71f2\u71f3\u71f4\u71f5\u71f6\u71f7\u71f8\u71f9\u71fa\u71fb\u71fc\u71fd\u71fe\u71ff\u7200\u7201\u7202\u7203\u7204\u7205\u7206\u7207\u7208\u7209\u720a\u720b\u720c\u720d\u720e\u720f\u7210\u7211\u7212\u7213\u7214\u7215\u7216\u7217\u7218\u7219\u721a\u721b\u721c\u721d\u721e\u721f\u7220\u7221\u7222\u7223\u7224\u7225\u7226\u7227\u7228\u7229\u722a\u722b\u722c\u722d\u722e\u722f\u7230\u7231\u7232\u7233\u7234\u7235\u7236\u7237\u7238\u7239\u723a\u723b\u723c\u723d\u723e\u723f\u7240\u7241\u7242\u7243\u7244\u7245\u7246\u7247\u7248\u7249\u724a\u724b\u724c\u724d\u724e\u724f\u7250\u7251\u7252\u7253\u7254\u7255\u7256\u7257\u7258\u7259\u725a\u725b\u725c\u725d\u725e\u725f\u7260\u7261\u7262\u7263\u7264\u7265\u7266\u7267\u7268\u7269\u726a\u726b\u726c\u726d\u726e\u726f\u7270\u7271\u7272\u7273\u7274\u7275\u7276\u7277\u7278\u7279\u727a\u727b\u727c\u727d\u727e\u727f\u7280\u7281\u7282\u7283\u7284\u7285\u7286\u7287\u7288\u7289\u728a\u728b\u728c\u728d\u728e\u728f\u7290\u7291\u7292\u7293\u7294\u7295\u7296\u7297\u7298\u7299\u729a\u729b\u729c\u729d\u729e\u729f\u72a0\u72a1\u72a2\u72a3\u72a4\u72a5\u72a6\u72a7\u72a8\u72a9\u72aa\u72ab\u72ac\u72ad\u72ae\u72af\u72b0\u72b1\u72b2\u72b3\u72b4\u72b5\u72b6\u72b7\u72b8\u72b9\u72ba\u72bb\u72bc\u72bd\u72be\u72bf\u72c0\u72c1\u72c2\u72c3\u72c4\u72c5\u72c6\u72c7\u72c8\u72c9\u72ca\u72cb\u72cc\u72cd\u72ce\u72cf\u72d0\u72d1\u72d2\u72d3\u72d4\u72d5\u72d6\u72d7\u72d8\u72d9\u72da\u72db\u72dc\u72dd\u72de\u72df\u72e0\u72e1\u72e2\u72e3\u72e4\u72e5\u72e6\u72e7\u72e8\u72e9\u72ea\u72eb\u72ec\u72ed\u72ee\u72ef\u72f0\u72f1\u72f2\u72f3\u72f4\u72f5\u72f6\u72f7\u72f8\u72f9\u72fa\u72fb\u72fc\u72fd\u72fe\u72ff\u7300\u7301\u7302\u7303\u7304\u7305\u7306\u7307\u7308\u7309\u730a\u730b\u730c\u730d\u730e\u730f\u7310\u7311\u7312\u7313\u7314\u7315\u7316\u7317\u7318\u7319\u731a\u731b\u731c\u731d\u731e\u731f\u7320\u7321\u7322\u7323\u7324\u7325\u7326\u7327\u7328\u7329\u732a\u732b\u732c\u732d\u732e\u732f\u7330\u7331\u7332\u7333\u7334\u7335\u7336\u7337\u7338\u7339\u733a\u733b\u733c\u733d\u733e\u733f\u7340\u7341\u7342\u7343\u7344\u7345\u7346\u7347\u7348\u7349\u734a\u734b\u734c\u734d\u734e\u734f\u7350\u7351\u7352\u7353\u7354\u7355\u7356\u7357\u7358\u7359\u735a\u735b\u735c\u735d\u735e\u735f\u7360\u7361\u7362\u7363\u7364\u7365\u7366\u7367\u7368\u7369\u736a\u736b\u736c\u736d\u736e\u736f\u7370\u7371\u7372\u7373\u7374\u7375\u7376\u7377\u7378\u7379\u737a\u737b\u737c\u737d\u737e\u737f\u7380\u7381\u7382\u7383\u7384\u7385\u7386\u7387\u7388\u7389\u738a\u738b\u738c\u738d\u738e\u738f\u7390\u7391\u7392\u7393\u7394\u7395\u7396\u7397\u7398\u7399\u739a\u739b\u739c\u739d\u739e\u739f\u73a0\u73a1\u73a2\u73a3\u73a4\u73a5\u73a6\u73a7\u73a8\u73a9\u73aa\u73ab\u73ac\u73ad\u73ae\u73af\u73b0\u73b1\u73b2\u73b3\u73b4\u73b5\u73b6\u73b7\u73b8\u73b9\u73ba\u73bb\u73bc\u73bd\u73be\u73bf\u73c0\u73c1\u73c2\u73c3\u73c4\u73c5\u73c6\u73c7\u73c8\u73c9\u73ca\u73cb\u73cc\u73cd\u73ce\u73cf\u73d0\u73d1\u73d2\u73d3\u73d4\u73d5\u73d6\u73d7\u73d8\u73d9\u73da\u73db\u73dc\u73dd\u73de\u73df\u73e0\u73e1\u73e2\u73e3\u73e4\u73e5\u73e6\u73e7\u73e8\u73e9\u73ea\u73eb\u73ec\u73ed\u73ee\u73ef\u73f0\u73f1\u73f2\u73f3\u73f4\u73f5\u73f6\u73f7\u73f8\u73f9\u73fa\u73fb\u73fc\u73fd\u73fe\u73ff\u7400\u7401\u7402\u7403\u7404\u7405\u7406\u7407\u7408\u7409\u740a\u740b\u740c\u740d\u740e\u740f\u7410\u7411\u7412\u7413\u7414\u7415\u7416\u7417\u7418\u7419\u741a\u741b\u741c\u741d\u741e\u741f\u7420\u7421\u7422\u7423\u7424\u7425\u7426\u7427\u7428\u7429\u742a\u742b\u742c\u742d\u742e\u742f\u7430\u7431\u7432\u7433\u7434\u7435\u7436\u7437\u7438\u7439\u743a\u743b\u743c\u743d\u743e\u743f\u7440\u7441\u7442\u7443\u7444\u7445\u7446\u7447\u7448\u7449\u744a\u744b\u744c\u744d\u744e\u744f\u7450\u7451\u7452\u7453\u7454\u7455\u7456\u7457\u7458\u7459\u745a\u745b\u745c\u745d\u745e\u745f\u7460\u7461\u7462\u7463\u7464\u7465\u7466\u7467\u7468\u7469\u746a\u746b\u746c\u746d\u746e\u746f\u7470\u7471\u7472\u7473\u7474\u7475\u7476\u7477\u7478\u7479\u747a\u747b\u747c\u747d\u747e\u747f\u7480\u7481\u7482\u7483\u7484\u7485\u7486\u7487\u7488\u7489\u748a\u748b\u748c\u748d\u748e\u748f\u7490\u7491\u7492\u7493\u7494\u7495\u7496\u7497\u7498\u7499\u749a\u749b\u749c\u749d\u749e\u749f\u74a0\u74a1\u74a2\u74a3\u74a4\u74a5\u74a6\u74a7\u74a8\u74a9\u74aa\u74ab\u74ac\u74ad\u74ae\u74af\u74b0\u74b1\u74b2\u74b3\u74b4\u74b5\u74b6\u74b7\u74b8\u74b9\u74ba\u74bb\u74bc\u74bd\u74be\u74bf\u74c0\u74c1\u74c2\u74c3\u74c4\u74c5\u74c6\u74c7\u74c8\u74c9\u74ca\u74cb\u74cc\u74cd\u74ce\u74cf\u74d0\u74d1\u74d2\u74d3\u74d4\u74d5\u74d6\u74d7\u74d8\u74d9\u74da\u74db\u74dc\u74dd\u74de\u74df\u74e0\u74e1\u74e2\u74e3\u74e4\u74e5\u74e6\u74e7\u74e8\u74e9\u74ea\u74eb\u74ec\u74ed\u74ee\u74ef\u74f0\u74f1\u74f2\u74f3\u74f4\u74f5\u74f6\u74f7\u74f8\u74f9\u74fa\u74fb\u74fc\u74fd\u74fe\u74ff\u7500\u7501\u7502\u7503\u7504\u7505\u7506\u7507\u7508\u7509\u750a\u750b\u750c\u750d\u750e\u750f\u7510\u7511\u7512\u7513\u7514\u7515\u7516\u7517\u7518\u7519\u751a\u751b\u751c\u751d\u751e\u751f\u7520\u7521\u7522\u7523\u7524\u7525\u7526\u7527\u7528\u7529\u752a\u752b\u752c\u752d\u752e\u752f\u7530\u7531\u7532\u7533\u7534\u7535\u7536\u7537\u7538\u7539\u753a\u753b\u753c\u753d\u753e\u753f\u7540\u7541\u7542\u7543\u7544\u7545\u7546\u7547\u7548\u7549\u754a\u754b\u754c\u754d\u754e\u754f\u7550\u7551\u7552\u7553\u7554\u7555\u7556\u7557\u7558\u7559\u755a\u755b\u755c\u755d\u755e\u755f\u7560\u7561\u7562\u7563\u7564\u7565\u7566\u7567\u7568\u7569\u756a\u756b\u756c\u756d\u756e\u756f\u7570\u7571\u7572\u7573\u7574\u7575\u7576\u7577\u7578\u7579\u757a\u757b\u757c\u757d\u757e\u757f\u7580\u7581\u7582\u7583\u7584\u7585\u7586\u7587\u7588\u7589\u758a\u758b\u758c\u758d\u758e\u758f\u7590\u7591\u7592\u7593\u7594\u7595\u7596\u7597\u7598\u7599\u759a\u759b\u759c\u759d\u759e\u759f\u75a0\u75a1\u75a2\u75a3\u75a4\u75a5\u75a6\u75a7\u75a8\u75a9\u75aa\u75ab\u75ac\u75ad\u75ae\u75af\u75b0\u75b1\u75b2\u75b3\u75b4\u75b5\u75b6\u75b7\u75b8\u75b9\u75ba\u75bb\u75bc\u75bd\u75be\u75bf\u75c0\u75c1\u75c2\u75c3\u75c4\u75c5\u75c6\u75c7\u75c8\u75c9\u75ca\u75cb\u75cc\u75cd\u75ce\u75cf\u75d0\u75d1\u75d2\u75d3\u75d4\u75d5\u75d6\u75d7\u75d8\u75d9\u75da\u75db\u75dc\u75dd\u75de\u75df\u75e0\u75e1\u75e2\u75e3\u75e4\u75e5\u75e6\u75e7\u75e8\u75e9\u75ea\u75eb\u75ec\u75ed\u75ee\u75ef\u75f0\u75f1\u75f2\u75f3\u75f4\u75f5\u75f6\u75f7\u75f8\u75f9\u75fa\u75fb\u75fc\u75fd\u75fe\u75ff\u7600\u7601\u7602\u7603\u7604\u7605\u7606\u7607\u7608\u7609\u760a\u760b\u760c\u760d\u760e\u760f\u7610\u7611\u7612\u7613\u7614\u7615\u7616\u7617\u7618\u7619\u761a\u761b\u761c\u761d\u761e\u761f\u7620\u7621\u7622\u7623\u7624\u7625\u7626\u7627\u7628\u7629\u762a\u762b\u762c\u762d\u762e\u762f\u7630\u7631\u7632\u7633\u7634\u7635\u7636\u7637\u7638\u7639\u763a\u763b\u763c\u763d\u763e\u763f\u7640\u7641\u7642\u7643\u7644\u7645\u7646\u7647\u7648\u7649\u764a\u764b\u764c\u764d\u764e\u764f\u7650\u7651\u7652\u7653\u7654\u7655\u7656\u7657\u7658\u7659\u765a\u765b\u765c\u765d\u765e\u765f\u7660\u7661\u7662\u7663\u7664\u7665\u7666\u7667\u7668\u7669\u766a\u766b\u766c\u766d\u766e\u766f\u7670\u7671\u7672\u7673\u7674\u7675\u7676\u7677\u7678\u7679\u767a\u767b\u767c\u767d\u767e\u767f\u7680\u7681\u7682\u7683\u7684\u7685\u7686\u7687\u7688\u7689\u768a\u768b\u768c\u768d\u768e\u768f\u7690\u7691\u7692\u7693\u7694\u7695\u7696\u7697\u7698\u7699\u769a\u769b\u769c\u769d\u769e\u769f\u76a0\u76a1\u76a2\u76a3\u76a4\u76a5\u76a6\u76a7\u76a8\u76a9\u76aa\u76ab\u76ac\u76ad\u76ae\u76af\u76b0\u76b1\u76b2\u76b3\u76b4\u76b5\u76b6\u76b7\u76b8\u76b9\u76ba\u76bb\u76bc\u76bd\u76be\u76bf\u76c0\u76c1\u76c2\u76c3\u76c4\u76c5\u76c6\u76c7\u76c8\u76c9\u76ca\u76cb\u76cc\u76cd\u76ce\u76cf\u76d0\u76d1\u76d2\u76d3\u76d4\u76d5\u76d6\u76d7\u76d8\u76d9\u76da\u76db\u76dc\u76dd\u76de\u76df\u76e0\u76e1\u76e2\u76e3\u76e4\u76e5\u76e6\u76e7\u76e8\u76e9\u76ea\u76eb\u76ec\u76ed\u76ee\u76ef\u76f0\u76f1\u76f2\u76f3\u76f4\u76f5\u76f6\u76f7\u76f8\u76f9\u76fa\u76fb\u76fc\u76fd\u76fe\u76ff\u7700\u7701\u7702\u7703\u7704\u7705\u7706\u7707\u7708\u7709\u770a\u770b\u770c\u770d\u770e\u770f\u7710\u7711\u7712\u7713\u7714\u7715\u7716\u7717\u7718\u7719\u771a\u771b\u771c\u771d\u771e\u771f\u7720\u7721\u7722\u7723\u7724\u7725\u7726\u7727\u7728\u7729\u772a\u772b\u772c\u772d\u772e\u772f\u7730\u7731\u7732\u7733\u7734\u7735\u7736\u7737\u7738\u7739\u773a\u773b\u773c\u773d\u773e\u773f\u7740\u7741\u7742\u7743\u7744\u7745\u7746\u7747\u7748\u7749\u774a\u774b\u774c\u774d\u774e\u774f\u7750\u7751\u7752\u7753\u7754\u7755\u7756\u7757\u7758\u7759\u775a\u775b\u775c\u775d\u775e\u775f\u7760\u7761\u7762\u7763\u7764\u7765\u7766\u7767\u7768\u7769\u776a\u776b\u776c\u776d\u776e\u776f\u7770\u7771\u7772\u7773\u7774\u7775\u7776\u7777\u7778\u7779\u777a\u777b\u777c\u777d\u777e\u777f\u7780\u7781\u7782\u7783\u7784\u7785\u7786\u7787\u7788\u7789\u778a\u778b\u778c\u778d\u778e\u778f\u7790\u7791\u7792\u7793\u7794\u7795\u7796\u7797\u7798\u7799\u779a\u779b\u779c\u779d\u779e\u779f\u77a0\u77a1\u77a2\u77a3\u77a4\u77a5\u77a6\u77a7\u77a8\u77a9\u77aa\u77ab\u77ac\u77ad\u77ae\u77af\u77b0\u77b1\u77b2\u77b3\u77b4\u77b5\u77b6\u77b7\u77b8\u77b9\u77ba\u77bb\u77bc\u77bd\u77be\u77bf\u77c0\u77c1\u77c2\u77c3\u77c4\u77c5\u77c6\u77c7\u77c8\u77c9\u77ca\u77cb\u77cc\u77cd\u77ce\u77cf\u77d0\u77d1\u77d2\u77d3\u77d4\u77d5\u77d6\u77d7\u77d8\u77d9\u77da\u77db\u77dc\u77dd\u77de\u77df\u77e0\u77e1\u77e2\u77e3\u77e4\u77e5\u77e6\u77e7\u77e8\u77e9\u77ea\u77eb\u77ec\u77ed\u77ee\u77ef\u77f0\u77f1\u77f2\u77f3\u77f4\u77f5\u77f6\u77f7\u77f8\u77f9\u77fa\u77fb\u77fc\u77fd\u77fe\u77ff\u7800\u7801\u7802\u7803\u7804\u7805\u7806\u7807\u7808\u7809\u780a\u780b\u780c\u780d\u780e\u780f\u7810\u7811\u7812\u7813\u7814\u7815\u7816\u7817\u7818\u7819\u781a\u781b\u781c\u781d\u781e\u781f\u7820\u7821\u7822\u7823\u7824\u7825\u7826\u7827\u7828\u7829\u782a\u782b\u782c\u782d\u782e\u782f\u7830\u7831\u7832\u7833\u7834\u7835\u7836\u7837\u7838\u7839\u783a\u783b\u783c\u783d\u783e\u783f\u7840\u7841\u7842\u7843\u7844\u7845\u7846\u7847\u7848\u7849\u784a\u784b\u784c\u784d\u784e\u784f\u7850\u7851\u7852\u7853\u7854\u7855\u7856\u7857\u7858\u7859\u785a\u785b\u785c\u785d\u785e\u785f\u7860\u7861\u7862\u7863\u7864\u7865\u7866\u7867\u7868\u7869\u786a\u786b\u786c\u786d\u786e\u786f\u7870\u7871\u7872\u7873\u7874\u7875\u7876\u7877\u7878\u7879\u787a\u787b\u787c\u787d\u787e\u787f\u7880\u7881\u7882\u7883\u7884\u7885\u7886\u7887\u7888\u7889\u788a\u788b\u788c\u788d\u788e\u788f\u7890\u7891\u7892\u7893\u7894\u7895\u7896\u7897\u7898\u7899\u789a\u789b\u789c\u789d\u789e\u789f\u78a0\u78a1\u78a2\u78a3\u78a4\u78a5\u78a6\u78a7\u78a8\u78a9\u78aa\u78ab\u78ac\u78ad\u78ae\u78af\u78b0\u78b1\u78b2\u78b3\u78b4\u78b5\u78b6\u78b7\u78b8\u78b9\u78ba\u78bb\u78bc\u78bd\u78be\u78bf\u78c0\u78c1\u78c2\u78c3\u78c4\u78c5\u78c6\u78c7\u78c8\u78c9\u78ca\u78cb\u78cc\u78cd\u78ce\u78cf\u78d0\u78d1\u78d2\u78d3\u78d4\u78d5\u78d6\u78d7\u78d8\u78d9\u78da\u78db\u78dc\u78dd\u78de\u78df\u78e0\u78e1\u78e2\u78e3\u78e4\u78e5\u78e6\u78e7\u78e8\u78e9\u78ea\u78eb\u78ec\u78ed\u78ee\u78ef\u78f0\u78f1\u78f2\u78f3\u78f4\u78f5\u78f6\u78f7\u78f8\u78f9\u78fa\u78fb\u78fc\u78fd\u78fe\u78ff\u7900\u7901\u7902\u7903\u7904\u7905\u7906\u7907\u7908\u7909\u790a\u790b\u790c\u790d\u790e\u790f\u7910\u7911\u7912\u7913\u7914\u7915\u7916\u7917\u7918\u7919\u791a\u791b\u791c\u791d\u791e\u791f\u7920\u7921\u7922\u7923\u7924\u7925\u7926\u7927\u7928\u7929\u792a\u792b\u792c\u792d\u792e\u792f\u7930\u7931\u7932\u7933\u7934\u7935\u7936\u7937\u7938\u7939\u793a\u793b\u793c\u793d\u793e\u793f\u7940\u7941\u7942\u7943\u7944\u7945\u7946\u7947\u7948\u7949\u794a\u794b\u794c\u794d\u794e\u794f\u7950\u7951\u7952\u7953\u7954\u7955\u7956\u7957\u7958\u7959\u795a\u795b\u795c\u795d\u795e\u795f\u7960\u7961\u7962\u7963\u7964\u7965\u7966\u7967\u7968\u7969\u796a\u796b\u796c\u796d\u796e\u796f\u7970\u7971\u7972\u7973\u7974\u7975\u7976\u7977\u7978\u7979\u797a\u797b\u797c\u797d\u797e\u797f\u7980\u7981\u7982\u7983\u7984\u7985\u7986\u7987\u7988\u7989\u798a\u798b\u798c\u798d\u798e\u798f\u7990\u7991\u7992\u7993\u7994\u7995\u7996\u7997\u7998\u7999\u799a\u799b\u799c\u799d\u799e\u799f\u79a0\u79a1\u79a2\u79a3\u79a4\u79a5\u79a6\u79a7\u79a8\u79a9\u79aa\u79ab\u79ac\u79ad\u79ae\u79af\u79b0\u79b1\u79b2\u79b3\u79b4\u79b5\u79b6\u79b7\u79b8\u79b9\u79ba\u79bb\u79bc\u79bd\u79be\u79bf\u79c0\u79c1\u79c2\u79c3\u79c4\u79c5\u79c6\u79c7\u79c8\u79c9\u79ca\u79cb\u79cc\u79cd\u79ce\u79cf\u79d0\u79d1\u79d2\u79d3\u79d4\u79d5\u79d6\u79d7\u79d8\u79d9\u79da\u79db\u79dc\u79dd\u79de\u79df\u79e0\u79e1\u79e2\u79e3\u79e4\u79e5\u79e6\u79e7\u79e8\u79e9\u79ea\u79eb\u79ec\u79ed\u79ee\u79ef\u79f0\u79f1\u79f2\u79f3\u79f4\u79f5\u79f6\u79f7\u79f8\u79f9\u79fa\u79fb\u79fc\u79fd\u79fe\u79ff\u7a00\u7a01\u7a02\u7a03\u7a04\u7a05\u7a06\u7a07\u7a08\u7a09\u7a0a\u7a0b\u7a0c\u7a0d\u7a0e\u7a0f\u7a10\u7a11\u7a12\u7a13\u7a14\u7a15\u7a16\u7a17\u7a18\u7a19\u7a1a\u7a1b\u7a1c\u7a1d\u7a1e\u7a1f\u7a20\u7a21\u7a22\u7a23\u7a24\u7a25\u7a26\u7a27\u7a28\u7a29\u7a2a\u7a2b\u7a2c\u7a2d\u7a2e\u7a2f\u7a30\u7a31\u7a32\u7a33\u7a34\u7a35\u7a36\u7a37\u7a38\u7a39\u7a3a\u7a3b\u7a3c\u7a3d\u7a3e\u7a3f\u7a40\u7a41\u7a42\u7a43\u7a44\u7a45\u7a46\u7a47\u7a48\u7a49\u7a4a\u7a4b\u7a4c\u7a4d\u7a4e\u7a4f\u7a50\u7a51\u7a52\u7a53\u7a54\u7a55\u7a56\u7a57\u7a58\u7a59\u7a5a\u7a5b\u7a5c\u7a5d\u7a5e\u7a5f\u7a60\u7a61\u7a62\u7a63\u7a64\u7a65\u7a66\u7a67\u7a68\u7a69\u7a6a\u7a6b\u7a6c\u7a6d\u7a6e\u7a6f\u7a70\u7a71\u7a72\u7a73\u7a74\u7a75\u7a76\u7a77\u7a78\u7a79\u7a7a\u7a7b\u7a7c\u7a7d\u7a7e\u7a7f\u7a80\u7a81\u7a82\u7a83\u7a84\u7a85\u7a86\u7a87\u7a88\u7a89\u7a8a\u7a8b\u7a8c\u7a8d\u7a8e\u7a8f\u7a90\u7a91\u7a92\u7a93\u7a94\u7a95\u7a96\u7a97\u7a98\u7a99\u7a9a\u7a9b\u7a9c\u7a9d\u7a9e\u7a9f\u7aa0\u7aa1\u7aa2\u7aa3\u7aa4\u7aa5\u7aa6\u7aa7\u7aa8\u7aa9\u7aaa\u7aab\u7aac\u7aad\u7aae\u7aaf\u7ab0\u7ab1\u7ab2\u7ab3\u7ab4\u7ab5\u7ab6\u7ab7\u7ab8\u7ab9\u7aba\u7abb\u7abc\u7abd\u7abe\u7abf\u7ac0\u7ac1\u7ac2\u7ac3\u7ac4\u7ac5\u7ac6\u7ac7\u7ac8\u7ac9\u7aca\u7acb\u7acc\u7acd\u7ace\u7acf\u7ad0\u7ad1\u7ad2\u7ad3\u7ad4\u7ad5\u7ad6\u7ad7\u7ad8\u7ad9\u7ada\u7adb\u7adc\u7add\u7ade\u7adf\u7ae0\u7ae1\u7ae2\u7ae3\u7ae4\u7ae5\u7ae6\u7ae7\u7ae8\u7ae9\u7aea\u7aeb\u7aec\u7aed\u7aee\u7aef\u7af0\u7af1\u7af2\u7af3\u7af4\u7af5\u7af6\u7af7\u7af8\u7af9\u7afa\u7afb\u7afc\u7afd\u7afe\u7aff\u7b00\u7b01\u7b02\u7b03\u7b04\u7b05\u7b06\u7b07\u7b08\u7b09\u7b0a\u7b0b\u7b0c\u7b0d\u7b0e\u7b0f\u7b10\u7b11\u7b12\u7b13\u7b14\u7b15\u7b16\u7b17\u7b18\u7b19\u7b1a\u7b1b\u7b1c\u7b1d\u7b1e\u7b1f\u7b20\u7b21\u7b22\u7b23\u7b24\u7b25\u7b26\u7b27\u7b28\u7b29\u7b2a\u7b2b\u7b2c\u7b2d\u7b2e\u7b2f\u7b30\u7b31\u7b32\u7b33\u7b34\u7b35\u7b36\u7b37\u7b38\u7b39\u7b3a\u7b3b\u7b3c\u7b3d\u7b3e\u7b3f\u7b40\u7b41\u7b42\u7b43\u7b44\u7b45\u7b46\u7b47\u7b48\u7b49\u7b4a\u7b4b\u7b4c\u7b4d\u7b4e\u7b4f\u7b50\u7b51\u7b52\u7b53\u7b54\u7b55\u7b56\u7b57\u7b58\u7b59\u7b5a\u7b5b\u7b5c\u7b5d\u7b5e\u7b5f\u7b60\u7b61\u7b62\u7b63\u7b64\u7b65\u7b66\u7b67\u7b68\u7b69\u7b6a\u7b6b\u7b6c\u7b6d\u7b6e\u7b6f\u7b70\u7b71\u7b72\u7b73\u7b74\u7b75\u7b76\u7b77\u7b78\u7b79\u7b7a\u7b7b\u7b7c\u7b7d\u7b7e\u7b7f\u7b80\u7b81\u7b82\u7b83\u7b84\u7b85\u7b86\u7b87\u7b88\u7b89\u7b8a\u7b8b\u7b8c\u7b8d\u7b8e\u7b8f\u7b90\u7b91\u7b92\u7b93\u7b94\u7b95\u7b96\u7b97\u7b98\u7b99\u7b9a\u7b9b\u7b9c\u7b9d\u7b9e\u7b9f\u7ba0\u7ba1\u7ba2\u7ba3\u7ba4\u7ba5\u7ba6\u7ba7\u7ba8\u7ba9\u7baa\u7bab\u7bac\u7bad\u7bae\u7baf\u7bb0\u7bb1\u7bb2\u7bb3\u7bb4\u7bb5\u7bb6\u7bb7\u7bb8\u7bb9\u7bba\u7bbb\u7bbc\u7bbd\u7bbe\u7bbf\u7bc0\u7bc1\u7bc2\u7bc3\u7bc4\u7bc5\u7bc6\u7bc7\u7bc8\u7bc9\u7bca\u7bcb\u7bcc\u7bcd\u7bce\u7bcf\u7bd0\u7bd1\u7bd2\u7bd3\u7bd4\u7bd5\u7bd6\u7bd7\u7bd8\u7bd9\u7bda\u7bdb\u7bdc\u7bdd\u7bde\u7bdf\u7be0\u7be1\u7be2\u7be3\u7be4\u7be5\u7be6\u7be7\u7be8\u7be9\u7bea\u7beb\u7bec\u7bed\u7bee\u7bef\u7bf0\u7bf1\u7bf2\u7bf3\u7bf4\u7bf5\u7bf6\u7bf7\u7bf8\u7bf9\u7bfa\u7bfb\u7bfc\u7bfd\u7bfe\u7bff\u7c00\u7c01\u7c02\u7c03\u7c04\u7c05\u7c06\u7c07\u7c08\u7c09\u7c0a\u7c0b\u7c0c\u7c0d\u7c0e\u7c0f\u7c10\u7c11\u7c12\u7c13\u7c14\u7c15\u7c16\u7c17\u7c18\u7c19\u7c1a\u7c1b\u7c1c\u7c1d\u7c1e\u7c1f\u7c20\u7c21\u7c22\u7c23\u7c24\u7c25\u7c26\u7c27\u7c28\u7c29\u7c2a\u7c2b\u7c2c\u7c2d\u7c2e\u7c2f\u7c30\u7c31\u7c32\u7c33\u7c34\u7c35\u7c36\u7c37\u7c38\u7c39\u7c3a\u7c3b\u7c3c\u7c3d\u7c3e\u7c3f\u7c40\u7c41\u7c42\u7c43\u7c44\u7c45\u7c46\u7c47\u7c48\u7c49\u7c4a\u7c4b\u7c4c\u7c4d\u7c4e\u7c4f\u7c50\u7c51\u7c52\u7c53\u7c54\u7c55\u7c56\u7c57\u7c58\u7c59\u7c5a\u7c5b\u7c5c\u7c5d\u7c5e\u7c5f\u7c60\u7c61\u7c62\u7c63\u7c64\u7c65\u7c66\u7c67\u7c68\u7c69\u7c6a\u7c6b\u7c6c\u7c6d\u7c6e\u7c6f\u7c70\u7c71\u7c72\u7c73\u7c74\u7c75\u7c76\u7c77\u7c78\u7c79\u7c7a\u7c7b\u7c7c\u7c7d\u7c7e\u7c7f\u7c80\u7c81\u7c82\u7c83\u7c84\u7c85\u7c86\u7c87\u7c88\u7c89\u7c8a\u7c8b\u7c8c\u7c8d\u7c8e\u7c8f\u7c90\u7c91\u7c92\u7c93\u7c94\u7c95\u7c96\u7c97\u7c98\u7c99\u7c9a\u7c9b\u7c9c\u7c9d\u7c9e\u7c9f\u7ca0\u7ca1\u7ca2\u7ca3\u7ca4\u7ca5\u7ca6\u7ca7\u7ca8\u7ca9\u7caa\u7cab\u7cac\u7cad\u7cae\u7caf\u7cb0\u7cb1\u7cb2\u7cb3\u7cb4\u7cb5\u7cb6\u7cb7\u7cb8\u7cb9\u7cba\u7cbb\u7cbc\u7cbd\u7cbe\u7cbf\u7cc0\u7cc1\u7cc2\u7cc3\u7cc4\u7cc5\u7cc6\u7cc7\u7cc8\u7cc9\u7cca\u7ccb\u7ccc\u7ccd\u7cce\u7ccf\u7cd0\u7cd1\u7cd2\u7cd3\u7cd4\u7cd5\u7cd6\u7cd7\u7cd8\u7cd9\u7cda\u7cdb\u7cdc\u7cdd\u7cde\u7cdf\u7ce0\u7ce1\u7ce2\u7ce3\u7ce4\u7ce5\u7ce6\u7ce7\u7ce8\u7ce9\u7cea\u7ceb\u7cec\u7ced\u7cee\u7cef\u7cf0\u7cf1\u7cf2\u7cf3\u7cf4\u7cf5\u7cf6\u7cf7\u7cf8\u7cf9\u7cfa\u7cfb\u7cfc\u7cfd\u7cfe\u7cff\u7d00\u7d01\u7d02\u7d03\u7d04\u7d05\u7d06\u7d07\u7d08\u7d09\u7d0a\u7d0b\u7d0c\u7d0d\u7d0e\u7d0f\u7d10\u7d11\u7d12\u7d13\u7d14\u7d15\u7d16\u7d17\u7d18\u7d19\u7d1a\u7d1b\u7d1c\u7d1d\u7d1e\u7d1f\u7d20\u7d21\u7d22\u7d23\u7d24\u7d25\u7d26\u7d27\u7d28\u7d29\u7d2a\u7d2b\u7d2c\u7d2d\u7d2e\u7d2f\u7d30\u7d31\u7d32\u7d33\u7d34\u7d35\u7d36\u7d37\u7d38\u7d39\u7d3a\u7d3b\u7d3c\u7d3d\u7d3e\u7d3f\u7d40\u7d41\u7d42\u7d43\u7d44\u7d45\u7d46\u7d47\u7d48\u7d49\u7d4a\u7d4b\u7d4c\u7d4d\u7d4e\u7d4f\u7d50\u7d51\u7d52\u7d53\u7d54\u7d55\u7d56\u7d57\u7d58\u7d59\u7d5a\u7d5b\u7d5c\u7d5d\u7d5e\u7d5f\u7d60\u7d61\u7d62\u7d63\u7d64\u7d65\u7d66\u7d67\u7d68\u7d69\u7d6a\u7d6b\u7d6c\u7d6d\u7d6e\u7d6f\u7d70\u7d71\u7d72\u7d73\u7d74\u7d75\u7d76\u7d77\u7d78\u7d79\u7d7a\u7d7b\u7d7c\u7d7d\u7d7e\u7d7f\u7d80\u7d81\u7d82\u7d83\u7d84\u7d85\u7d86\u7d87\u7d88\u7d89\u7d8a\u7d8b\u7d8c\u7d8d\u7d8e\u7d8f\u7d90\u7d91\u7d92\u7d93\u7d94\u7d95\u7d96\u7d97\u7d98\u7d99\u7d9a\u7d9b\u7d9c\u7d9d\u7d9e\u7d9f\u7da0\u7da1\u7da2\u7da3\u7da4\u7da5\u7da6\u7da7\u7da8\u7da9\u7daa\u7dab\u7dac\u7dad\u7dae\u7daf\u7db0\u7db1\u7db2\u7db3\u7db4\u7db5\u7db6\u7db7\u7db8\u7db9\u7dba\u7dbb\u7dbc\u7dbd\u7dbe\u7dbf\u7dc0\u7dc1\u7dc2\u7dc3\u7dc4\u7dc5\u7dc6\u7dc7\u7dc8\u7dc9\u7dca\u7dcb\u7dcc\u7dcd\u7dce\u7dcf\u7dd0\u7dd1\u7dd2\u7dd3\u7dd4\u7dd5\u7dd6\u7dd7\u7dd8\u7dd9\u7dda\u7ddb\u7ddc\u7ddd\u7dde\u7ddf\u7de0\u7de1\u7de2\u7de3\u7de4\u7de5\u7de6\u7de7\u7de8\u7de9\u7dea\u7deb\u7dec\u7ded\u7dee\u7def\u7df0\u7df1\u7df2\u7df3\u7df4\u7df5\u7df6\u7df7\u7df8\u7df9\u7dfa\u7dfb\u7dfc\u7dfd\u7dfe\u7dff\u7e00\u7e01\u7e02\u7e03\u7e04\u7e05\u7e06\u7e07\u7e08\u7e09\u7e0a\u7e0b\u7e0c\u7e0d\u7e0e\u7e0f\u7e10\u7e11\u7e12\u7e13\u7e14\u7e15\u7e16\u7e17\u7e18\u7e19\u7e1a\u7e1b\u7e1c\u7e1d\u7e1e\u7e1f\u7e20\u7e21\u7e22\u7e23\u7e24\u7e25\u7e26\u7e27\u7e28\u7e29\u7e2a\u7e2b\u7e2c\u7e2d\u7e2e\u7e2f\u7e30\u7e31\u7e32\u7e33\u7e34\u7e35\u7e36\u7e37\u7e38\u7e39\u7e3a\u7e3b\u7e3c\u7e3d\u7e3e\u7e3f\u7e40\u7e41\u7e42\u7e43\u7e44\u7e45\u7e46\u7e47\u7e48\u7e49\u7e4a\u7e4b\u7e4c\u7e4d\u7e4e\u7e4f\u7e50\u7e51\u7e52\u7e53\u7e54\u7e55\u7e56\u7e57\u7e58\u7e59\u7e5a\u7e5b\u7e5c\u7e5d\u7e5e\u7e5f\u7e60\u7e61\u7e62\u7e63\u7e64\u7e65\u7e66\u7e67\u7e68\u7e69\u7e6a\u7e6b\u7e6c\u7e6d\u7e6e\u7e6f\u7e70\u7e71\u7e72\u7e73\u7e74\u7e75\u7e76\u7e77\u7e78\u7e79\u7e7a\u7e7b\u7e7c\u7e7d\u7e7e\u7e7f\u7e80\u7e81\u7e82\u7e83\u7e84\u7e85\u7e86\u7e87\u7e88\u7e89\u7e8a\u7e8b\u7e8c\u7e8d\u7e8e\u7e8f\u7e90\u7e91\u7e92\u7e93\u7e94\u7e95\u7e96\u7e97\u7e98\u7e99\u7e9a\u7e9b\u7e9c\u7e9d\u7e9e\u7e9f\u7ea0\u7ea1\u7ea2\u7ea3\u7ea4\u7ea5\u7ea6\u7ea7\u7ea8\u7ea9\u7eaa\u7eab\u7eac\u7ead\u7eae\u7eaf\u7eb0\u7eb1\u7eb2\u7eb3\u7eb4\u7eb5\u7eb6\u7eb7\u7eb8\u7eb9\u7eba\u7ebb\u7ebc\u7ebd\u7ebe\u7ebf\u7ec0\u7ec1\u7ec2\u7ec3\u7ec4\u7ec5\u7ec6\u7ec7\u7ec8\u7ec9\u7eca\u7ecb\u7ecc\u7ecd\u7ece\u7ecf\u7ed0\u7ed1\u7ed2\u7ed3\u7ed4\u7ed5\u7ed6\u7ed7\u7ed8\u7ed9\u7eda\u7edb\u7edc\u7edd\u7ede\u7edf\u7ee0\u7ee1\u7ee2\u7ee3\u7ee4\u7ee5\u7ee6\u7ee7\u7ee8\u7ee9\u7eea\u7eeb\u7eec\u7eed\u7eee\u7eef\u7ef0\u7ef1\u7ef2\u7ef3\u7ef4\u7ef5\u7ef6\u7ef7\u7ef8\u7ef9\u7efa\u7efb\u7efc\u7efd\u7efe\u7eff\u7f00\u7f01\u7f02\u7f03\u7f04\u7f05\u7f06\u7f07\u7f08\u7f09\u7f0a\u7f0b\u7f0c\u7f0d\u7f0e\u7f0f\u7f10\u7f11\u7f12\u7f13\u7f14\u7f15\u7f16\u7f17\u7f18\u7f19\u7f1a\u7f1b\u7f1c\u7f1d\u7f1e\u7f1f\u7f20\u7f21\u7f22\u7f23\u7f24\u7f25\u7f26\u7f27\u7f28\u7f29\u7f2a\u7f2b\u7f2c\u7f2d\u7f2e\u7f2f\u7f30\u7f31\u7f32\u7f33\u7f34\u7f35\u7f36\u7f37\u7f38\u7f39\u7f3a\u7f3b\u7f3c\u7f3d\u7f3e\u7f3f\u7f40\u7f41\u7f42\u7f43\u7f44\u7f45\u7f46\u7f47\u7f48\u7f49\u7f4a\u7f4b\u7f4c\u7f4d\u7f4e\u7f4f\u7f50\u7f51\u7f52\u7f53\u7f54\u7f55\u7f56\u7f57\u7f58\u7f59\u7f5a\u7f5b\u7f5c\u7f5d\u7f5e\u7f5f\u7f60\u7f61\u7f62\u7f63\u7f64\u7f65\u7f66\u7f67\u7f68\u7f69\u7f6a\u7f6b\u7f6c\u7f6d\u7f6e\u7f6f\u7f70\u7f71\u7f72\u7f73\u7f74\u7f75\u7f76\u7f77\u7f78\u7f79\u7f7a\u7f7b\u7f7c\u7f7d\u7f7e\u7f7f\u7f80\u7f81\u7f82\u7f83\u7f84\u7f85\u7f86\u7f87\u7f88\u7f89\u7f8a\u7f8b\u7f8c\u7f8d\u7f8e\u7f8f\u7f90\u7f91\u7f92\u7f93\u7f94\u7f95\u7f96\u7f97\u7f98\u7f99\u7f9a\u7f9b\u7f9c\u7f9d\u7f9e\u7f9f\u7fa0\u7fa1\u7fa2\u7fa3\u7fa4\u7fa5\u7fa6\u7fa7\u7fa8\u7fa9\u7faa\u7fab\u7fac\u7fad\u7fae\u7faf\u7fb0\u7fb1\u7fb2\u7fb3\u7fb4\u7fb5\u7fb6\u7fb7\u7fb8\u7fb9\u7fba\u7fbb\u7fbc\u7fbd\u7fbe\u7fbf\u7fc0\u7fc1\u7fc2\u7fc3\u7fc4\u7fc5\u7fc6\u7fc7\u7fc8\u7fc9\u7fca\u7fcb\u7fcc\u7fcd\u7fce\u7fcf\u7fd0\u7fd1\u7fd2\u7fd3\u7fd4\u7fd5\u7fd6\u7fd7\u7fd8\u7fd9\u7fda\u7fdb\u7fdc\u7fdd\u7fde\u7fdf\u7fe0\u7fe1\u7fe2\u7fe3\u7fe4\u7fe5\u7fe6\u7fe7\u7fe8\u7fe9\u7fea\u7feb\u7fec\u7fed\u7fee\u7fef\u7ff0\u7ff1\u7ff2\u7ff3\u7ff4\u7ff5\u7ff6\u7ff7\u7ff8\u7ff9\u7ffa\u7ffb\u7ffc\u7ffd\u7ffe\u7fff\u8000\u8001\u8002\u8003\u8004\u8005\u8006\u8007\u8008\u8009\u800a\u800b\u800c\u800d\u800e\u800f\u8010\u8011\u8012\u8013\u8014\u8015\u8016\u8017\u8018\u8019\u801a\u801b\u801c\u801d\u801e\u801f\u8020\u8021\u8022\u8023\u8024\u8025\u8026\u8027\u8028\u8029\u802a\u802b\u802c\u802d\u802e\u802f\u8030\u8031\u8032\u8033\u8034\u8035\u8036\u8037\u8038\u8039\u803a\u803b\u803c\u803d\u803e\u803f\u8040\u8041\u8042\u8043\u8044\u8045\u8046\u8047\u8048\u8049\u804a\u804b\u804c\u804d\u804e\u804f\u8050\u8051\u8052\u8053\u8054\u8055\u8056\u8057\u8058\u8059\u805a\u805b\u805c\u805d\u805e\u805f\u8060\u8061\u8062\u8063\u8064\u8065\u8066\u8067\u8068\u8069\u806a\u806b\u806c\u806d\u806e\u806f\u8070\u8071\u8072\u8073\u8074\u8075\u8076\u8077\u8078\u8079\u807a\u807b\u807c\u807d\u807e\u807f\u8080\u8081\u8082\u8083\u8084\u8085\u8086\u8087\u8088\u8089\u808a\u808b\u808c\u808d\u808e\u808f\u8090\u8091\u8092\u8093\u8094\u8095\u8096\u8097\u8098\u8099\u809a\u809b\u809c\u809d\u809e\u809f\u80a0\u80a1\u80a2\u80a3\u80a4\u80a5\u80a6\u80a7\u80a8\u80a9\u80aa\u80ab\u80ac\u80ad\u80ae\u80af\u80b0\u80b1\u80b2\u80b3\u80b4\u80b5\u80b6\u80b7\u80b8\u80b9\u80ba\u80bb\u80bc\u80bd\u80be\u80bf\u80c0\u80c1\u80c2\u80c3\u80c4\u80c5\u80c6\u80c7\u80c8\u80c9\u80ca\u80cb\u80cc\u80cd\u80ce\u80cf\u80d0\u80d1\u80d2\u80d3\u80d4\u80d5\u80d6\u80d7\u80d8\u80d9\u80da\u80db\u80dc\u80dd\u80de\u80df\u80e0\u80e1\u80e2\u80e3\u80e4\u80e5\u80e6\u80e7\u80e8\u80e9\u80ea\u80eb\u80ec\u80ed\u80ee\u80ef\u80f0\u80f1\u80f2\u80f3\u80f4\u80f5\u80f6\u80f7\u80f8\u80f9\u80fa\u80fb\u80fc\u80fd\u80fe\u80ff\u8100\u8101\u8102\u8103\u8104\u8105\u8106\u8107\u8108\u8109\u810a\u810b\u810c\u810d\u810e\u810f\u8110\u8111\u8112\u8113\u8114\u8115\u8116\u8117\u8118\u8119\u811a\u811b\u811c\u811d\u811e\u811f\u8120\u8121\u8122\u8123\u8124\u8125\u8126\u8127\u8128\u8129\u812a\u812b\u812c\u812d\u812e\u812f\u8130\u8131\u8132\u8133\u8134\u8135\u8136\u8137\u8138\u8139\u813a\u813b\u813c\u813d\u813e\u813f\u8140\u8141\u8142\u8143\u8144\u8145\u8146\u8147\u8148\u8149\u814a\u814b\u814c\u814d\u814e\u814f\u8150\u8151\u8152\u8153\u8154\u8155\u8156\u8157\u8158\u8159\u815a\u815b\u815c\u815d\u815e\u815f\u8160\u8161\u8162\u8163\u8164\u8165\u8166\u8167\u8168\u8169\u816a\u816b\u816c\u816d\u816e\u816f\u8170\u8171\u8172\u8173\u8174\u8175\u8176\u8177\u8178\u8179\u817a\u817b\u817c\u817d\u817e\u817f\u8180\u8181\u8182\u8183\u8184\u8185\u8186\u8187\u8188\u8189\u818a\u818b\u818c\u818d\u818e\u818f\u8190\u8191\u8192\u8193\u8194\u8195\u8196\u8197\u8198\u8199\u819a\u819b\u819c\u819d\u819e\u819f\u81a0\u81a1\u81a2\u81a3\u81a4\u81a5\u81a6\u81a7\u81a8\u81a9\u81aa\u81ab\u81ac\u81ad\u81ae\u81af\u81b0\u81b1\u81b2\u81b3\u81b4\u81b5\u81b6\u81b7\u81b8\u81b9\u81ba\u81bb\u81bc\u81bd\u81be\u81bf\u81c0\u81c1\u81c2\u81c3\u81c4\u81c5\u81c6\u81c7\u81c8\u81c9\u81ca\u81cb\u81cc\u81cd\u81ce\u81cf\u81d0\u81d1\u81d2\u81d3\u81d4\u81d5\u81d6\u81d7\u81d8\u81d9\u81da\u81db\u81dc\u81dd\u81de\u81df\u81e0\u81e1\u81e2\u81e3\u81e4\u81e5\u81e6\u81e7\u81e8\u81e9\u81ea\u81eb\u81ec\u81ed\u81ee\u81ef\u81f0\u81f1\u81f2\u81f3\u81f4\u81f5\u81f6\u81f7\u81f8\u81f9\u81fa\u81fb\u81fc\u81fd\u81fe\u81ff\u8200\u8201\u8202\u8203\u8204\u8205\u8206\u8207\u8208\u8209\u820a\u820b\u820c\u820d\u820e\u820f\u8210\u8211\u8212\u8213\u8214\u8215\u8216\u8217\u8218\u8219\u821a\u821b\u821c\u821d\u821e\u821f\u8220\u8221\u8222\u8223\u8224\u8225\u8226\u8227\u8228\u8229\u822a\u822b\u822c\u822d\u822e\u822f\u8230\u8231\u8232\u8233\u8234\u8235\u8236\u8237\u8238\u8239\u823a\u823b\u823c\u823d\u823e\u823f\u8240\u8241\u8242\u8243\u8244\u8245\u8246\u8247\u8248\u8249\u824a\u824b\u824c\u824d\u824e\u824f\u8250\u8251\u8252\u8253\u8254\u8255\u8256\u8257\u8258\u8259\u825a\u825b\u825c\u825d\u825e\u825f\u8260\u8261\u8262\u8263\u8264\u8265\u8266\u8267\u8268\u8269\u826a\u826b\u826c\u826d\u826e\u826f\u8270\u8271\u8272\u8273\u8274\u8275\u8276\u8277\u8278\u8279\u827a\u827b\u827c\u827d\u827e\u827f\u8280\u8281\u8282\u8283\u8284\u8285\u8286\u8287\u8288\u8289\u828a\u828b\u828c\u828d\u828e\u828f\u8290\u8291\u8292\u8293\u8294\u8295\u8296\u8297\u8298\u8299\u829a\u829b\u829c\u829d\u829e\u829f\u82a0\u82a1\u82a2\u82a3\u82a4\u82a5\u82a6\u82a7\u82a8\u82a9\u82aa\u82ab\u82ac\u82ad\u82ae\u82af\u82b0\u82b1\u82b2\u82b3\u82b4\u82b5\u82b6\u82b7\u82b8\u82b9\u82ba\u82bb\u82bc\u82bd\u82be\u82bf\u82c0\u82c1\u82c2\u82c3\u82c4\u82c5\u82c6\u82c7\u82c8\u82c9\u82ca\u82cb\u82cc\u82cd\u82ce\u82cf\u82d0\u82d1\u82d2\u82d3\u82d4\u82d5\u82d6\u82d7\u82d8\u82d9\u82da\u82db\u82dc\u82dd\u82de\u82df\u82e0\u82e1\u82e2\u82e3\u82e4\u82e5\u82e6\u82e7\u82e8\u82e9\u82ea\u82eb\u82ec\u82ed\u82ee\u82ef\u82f0\u82f1\u82f2\u82f3\u82f4\u82f5\u82f6\u82f7\u82f8\u82f9\u82fa\u82fb\u82fc\u82fd\u82fe\u82ff\u8300\u8301\u8302\u8303\u8304\u8305\u8306\u8307\u8308\u8309\u830a\u830b\u830c\u830d\u830e\u830f\u8310\u8311\u8312\u8313\u8314\u8315\u8316\u8317\u8318\u8319\u831a\u831b\u831c\u831d\u831e\u831f\u8320\u8321\u8322\u8323\u8324\u8325\u8326\u8327\u8328\u8329\u832a\u832b\u832c\u832d\u832e\u832f\u8330\u8331\u8332\u8333\u8334\u8335\u8336\u8337\u8338\u8339\u833a\u833b\u833c\u833d\u833e\u833f\u8340\u8341\u8342\u8343\u8344\u8345\u8346\u8347\u8348\u8349\u834a\u834b\u834c\u834d\u834e\u834f\u8350\u8351\u8352\u8353\u8354\u8355\u8356\u8357\u8358\u8359\u835a\u835b\u835c\u835d\u835e\u835f\u8360\u8361\u8362\u8363\u8364\u8365\u8366\u8367\u8368\u8369\u836a\u836b\u836c\u836d\u836e\u836f\u8370\u8371\u8372\u8373\u8374\u8375\u8376\u8377\u8378\u8379\u837a\u837b\u837c\u837d\u837e\u837f\u8380\u8381\u8382\u8383\u8384\u8385\u8386\u8387\u8388\u8389\u838a\u838b\u838c\u838d\u838e\u838f\u8390\u8391\u8392\u8393\u8394\u8395\u8396\u8397\u8398\u8399\u839a\u839b\u839c\u839d\u839e\u839f\u83a0\u83a1\u83a2\u83a3\u83a4\u83a5\u83a6\u83a7\u83a8\u83a9\u83aa\u83ab\u83ac\u83ad\u83ae\u83af\u83b0\u83b1\u83b2\u83b3\u83b4\u83b5\u83b6\u83b7\u83b8\u83b9\u83ba\u83bb\u83bc\u83bd\u83be\u83bf\u83c0\u83c1\u83c2\u83c3\u83c4\u83c5\u83c6\u83c7\u83c8\u83c9\u83ca\u83cb\u83cc\u83cd\u83ce\u83cf\u83d0\u83d1\u83d2\u83d3\u83d4\u83d5\u83d6\u83d7\u83d8\u83d9\u83da\u83db\u83dc\u83dd\u83de\u83df\u83e0\u83e1\u83e2\u83e3\u83e4\u83e5\u83e6\u83e7\u83e8\u83e9\u83ea\u83eb\u83ec\u83ed\u83ee\u83ef\u83f0\u83f1\u83f2\u83f3\u83f4\u83f5\u83f6\u83f7\u83f8\u83f9\u83fa\u83fb\u83fc\u83fd\u83fe\u83ff\u8400\u8401\u8402\u8403\u8404\u8405\u8406\u8407\u8408\u8409\u840a\u840b\u840c\u840d\u840e\u840f\u8410\u8411\u8412\u8413\u8414\u8415\u8416\u8417\u8418\u8419\u841a\u841b\u841c\u841d\u841e\u841f\u8420\u8421\u8422\u8423\u8424\u8425\u8426\u8427\u8428\u8429\u842a\u842b\u842c\u842d\u842e\u842f\u8430\u8431\u8432\u8433\u8434\u8435\u8436\u8437\u8438\u8439\u843a\u843b\u843c\u843d\u843e\u843f\u8440\u8441\u8442\u8443\u8444\u8445\u8446\u8447\u8448\u8449\u844a\u844b\u844c\u844d\u844e\u844f\u8450\u8451\u8452\u8453\u8454\u8455\u8456\u8457\u8458\u8459\u845a\u845b\u845c\u845d\u845e\u845f\u8460\u8461\u8462\u8463\u8464\u8465\u8466\u8467\u8468\u8469\u846a\u846b\u846c\u846d\u846e\u846f\u8470\u8471\u8472\u8473\u8474\u8475\u8476\u8477\u8478\u8479\u847a\u847b\u847c\u847d\u847e\u847f\u8480\u8481\u8482\u8483\u8484\u8485\u8486\u8487\u8488\u8489\u848a\u848b\u848c\u848d\u848e\u848f\u8490\u8491\u8492\u8493\u8494\u8495\u8496\u8497\u8498\u8499\u849a\u849b\u849c\u849d\u849e\u849f\u84a0\u84a1\u84a2\u84a3\u84a4\u84a5\u84a6\u84a7\u84a8\u84a9\u84aa\u84ab\u84ac\u84ad\u84ae\u84af\u84b0\u84b1\u84b2\u84b3\u84b4\u84b5\u84b6\u84b7\u84b8\u84b9\u84ba\u84bb\u84bc\u84bd\u84be\u84bf\u84c0\u84c1\u84c2\u84c3\u84c4\u84c5\u84c6\u84c7\u84c8\u84c9\u84ca\u84cb\u84cc\u84cd\u84ce\u84cf\u84d0\u84d1\u84d2\u84d3\u84d4\u84d5\u84d6\u84d7\u84d8\u84d9\u84da\u84db\u84dc\u84dd\u84de\u84df\u84e0\u84e1\u84e2\u84e3\u84e4\u84e5\u84e6\u84e7\u84e8\u84e9\u84ea\u84eb\u84ec\u84ed\u84ee\u84ef\u84f0\u84f1\u84f2\u84f3\u84f4\u84f5\u84f6\u84f7\u84f8\u84f9\u84fa\u84fb\u84fc\u84fd\u84fe\u84ff\u8500\u8501\u8502\u8503\u8504\u8505\u8506\u8507\u8508\u8509\u850a\u850b\u850c\u850d\u850e\u850f\u8510\u8511\u8512\u8513\u8514\u8515\u8516\u8517\u8518\u8519\u851a\u851b\u851c\u851d\u851e\u851f\u8520\u8521\u8522\u8523\u8524\u8525\u8526\u8527\u8528\u8529\u852a\u852b\u852c\u852d\u852e\u852f\u8530\u8531\u8532\u8533\u8534\u8535\u8536\u8537\u8538\u8539\u853a\u853b\u853c\u853d\u853e\u853f\u8540\u8541\u8542\u8543\u8544\u8545\u8546\u8547\u8548\u8549\u854a\u854b\u854c\u854d\u854e\u854f\u8550\u8551\u8552\u8553\u8554\u8555\u8556\u8557\u8558\u8559\u855a\u855b\u855c\u855d\u855e\u855f\u8560\u8561\u8562\u8563\u8564\u8565\u8566\u8567\u8568\u8569\u856a\u856b\u856c\u856d\u856e\u856f\u8570\u8571\u8572\u8573\u8574\u8575\u8576\u8577\u8578\u8579\u857a\u857b\u857c\u857d\u857e\u857f\u8580\u8581\u8582\u8583\u8584\u8585\u8586\u8587\u8588\u8589\u858a\u858b\u858c\u858d\u858e\u858f\u8590\u8591\u8592\u8593\u8594\u8595\u8596\u8597\u8598\u8599\u859a\u859b\u859c\u859d\u859e\u859f\u85a0\u85a1\u85a2\u85a3\u85a4\u85a5\u85a6\u85a7\u85a8\u85a9\u85aa\u85ab\u85ac\u85ad\u85ae\u85af\u85b0\u85b1\u85b2\u85b3\u85b4\u85b5\u85b6\u85b7\u85b8\u85b9\u85ba\u85bb\u85bc\u85bd\u85be\u85bf\u85c0\u85c1\u85c2\u85c3\u85c4\u85c5\u85c6\u85c7\u85c8\u85c9\u85ca\u85cb\u85cc\u85cd\u85ce\u85cf\u85d0\u85d1\u85d2\u85d3\u85d4\u85d5\u85d6\u85d7\u85d8\u85d9\u85da\u85db\u85dc\u85dd\u85de\u85df\u85e0\u85e1\u85e2\u85e3\u85e4\u85e5\u85e6\u85e7\u85e8\u85e9\u85ea\u85eb\u85ec\u85ed\u85ee\u85ef\u85f0\u85f1\u85f2\u85f3\u85f4\u85f5\u85f6\u85f7\u85f8\u85f9\u85fa\u85fb\u85fc\u85fd\u85fe\u85ff\u8600\u8601\u8602\u8603\u8604\u8605\u8606\u8607\u8608\u8609\u860a\u860b\u860c\u860d\u860e\u860f\u8610\u8611\u8612\u8613\u8614\u8615\u8616\u8617\u8618\u8619\u861a\u861b\u861c\u861d\u861e\u861f\u8620\u8621\u8622\u8623\u8624\u8625\u8626\u8627\u8628\u8629\u862a\u862b\u862c\u862d\u862e\u862f\u8630\u8631\u8632\u8633\u8634\u8635\u8636\u8637\u8638\u8639\u863a\u863b\u863c\u863d\u863e\u863f\u8640\u8641\u8642\u8643\u8644\u8645\u8646\u8647\u8648\u8649\u864a\u864b\u864c\u864d\u864e\u864f\u8650\u8651\u8652\u8653\u8654\u8655\u8656\u8657\u8658\u8659\u865a\u865b\u865c\u865d\u865e\u865f\u8660\u8661\u8662\u8663\u8664\u8665\u8666\u8667\u8668\u8669\u866a\u866b\u866c\u866d\u866e\u866f\u8670\u8671\u8672\u8673\u8674\u8675\u8676\u8677\u8678\u8679\u867a\u867b\u867c\u867d\u867e\u867f\u8680\u8681\u8682\u8683\u8684\u8685\u8686\u8687\u8688\u8689\u868a\u868b\u868c\u868d\u868e\u868f\u8690\u8691\u8692\u8693\u8694\u8695\u8696\u8697\u8698\u8699\u869a\u869b\u869c\u869d\u869e\u869f\u86a0\u86a1\u86a2\u86a3\u86a4\u86a5\u86a6\u86a7\u86a8\u86a9\u86aa\u86ab\u86ac\u86ad\u86ae\u86af\u86b0\u86b1\u86b2\u86b3\u86b4\u86b5\u86b6\u86b7\u86b8\u86b9\u86ba\u86bb\u86bc\u86bd\u86be\u86bf\u86c0\u86c1\u86c2\u86c3\u86c4\u86c5\u86c6\u86c7\u86c8\u86c9\u86ca\u86cb\u86cc\u86cd\u86ce\u86cf\u86d0\u86d1\u86d2\u86d3\u86d4\u86d5\u86d6\u86d7\u86d8\u86d9\u86da\u86db\u86dc\u86dd\u86de\u86df\u86e0\u86e1\u86e2\u86e3\u86e4\u86e5\u86e6\u86e7\u86e8\u86e9\u86ea\u86eb\u86ec\u86ed\u86ee\u86ef\u86f0\u86f1\u86f2\u86f3\u86f4\u86f5\u86f6\u86f7\u86f8\u86f9\u86fa\u86fb\u86fc\u86fd\u86fe\u86ff\u8700\u8701\u8702\u8703\u8704\u8705\u8706\u8707\u8708\u8709\u870a\u870b\u870c\u870d\u870e\u870f\u8710\u8711\u8712\u8713\u8714\u8715\u8716\u8717\u8718\u8719\u871a\u871b\u871c\u871d\u871e\u871f\u8720\u8721\u8722\u8723\u8724\u8725\u8726\u8727\u8728\u8729\u872a\u872b\u872c\u872d\u872e\u872f\u8730\u8731\u8732\u8733\u8734\u8735\u8736\u8737\u8738\u8739\u873a\u873b\u873c\u873d\u873e\u873f\u8740\u8741\u8742\u8743\u8744\u8745\u8746\u8747\u8748\u8749\u874a\u874b\u874c\u874d\u874e\u874f\u8750\u8751\u8752\u8753\u8754\u8755\u8756\u8757\u8758\u8759\u875a\u875b\u875c\u875d\u875e\u875f\u8760\u8761\u8762\u8763\u8764\u8765\u8766\u8767\u8768\u8769\u876a\u876b\u876c\u876d\u876e\u876f\u8770\u8771\u8772\u8773\u8774\u8775\u8776\u8777\u8778\u8779\u877a\u877b\u877c\u877d\u877e\u877f\u8780\u8781\u8782\u8783\u8784\u8785\u8786\u8787\u8788\u8789\u878a\u878b\u878c\u878d\u878e\u878f\u8790\u8791\u8792\u8793\u8794\u8795\u8796\u8797\u8798\u8799\u879a\u879b\u879c\u879d\u879e\u879f\u87a0\u87a1\u87a2\u87a3\u87a4\u87a5\u87a6\u87a7\u87a8\u87a9\u87aa\u87ab\u87ac\u87ad\u87ae\u87af\u87b0\u87b1\u87b2\u87b3\u87b4\u87b5\u87b6\u87b7\u87b8\u87b9\u87ba\u87bb\u87bc\u87bd\u87be\u87bf\u87c0\u87c1\u87c2\u87c3\u87c4\u87c5\u87c6\u87c7\u87c8\u87c9\u87ca\u87cb\u87cc\u87cd\u87ce\u87cf\u87d0\u87d1\u87d2\u87d3\u87d4\u87d5\u87d6\u87d7\u87d8\u87d9\u87da\u87db\u87dc\u87dd\u87de\u87df\u87e0\u87e1\u87e2\u87e3\u87e4\u87e5\u87e6\u87e7\u87e8\u87e9\u87ea\u87eb\u87ec\u87ed\u87ee\u87ef\u87f0\u87f1\u87f2\u87f3\u87f4\u87f5\u87f6\u87f7\u87f8\u87f9\u87fa\u87fb\u87fc\u87fd\u87fe\u87ff\u8800\u8801\u8802\u8803\u8804\u8805\u8806\u8807\u8808\u8809\u880a\u880b\u880c\u880d\u880e\u880f\u8810\u8811\u8812\u8813\u8814\u8815\u8816\u8817\u8818\u8819\u881a\u881b\u881c\u881d\u881e\u881f\u8820\u8821\u8822\u8823\u8824\u8825\u8826\u8827\u8828\u8829\u882a\u882b\u882c\u882d\u882e\u882f\u8830\u8831\u8832\u8833\u8834\u8835\u8836\u8837\u8838\u8839\u883a\u883b\u883c\u883d\u883e\u883f\u8840\u8841\u8842\u8843\u8844\u8845\u8846\u8847\u8848\u8849\u884a\u884b\u884c\u884d\u884e\u884f\u8850\u8851\u8852\u8853\u8854\u8855\u8856\u8857\u8858\u8859\u885a\u885b\u885c\u885d\u885e\u885f\u8860\u8861\u8862\u8863\u8864\u8865\u8866\u8867\u8868\u8869\u886a\u886b\u886c\u886d\u886e\u886f\u8870\u8871\u8872\u8873\u8874\u8875\u8876\u8877\u8878\u8879\u887a\u887b\u887c\u887d\u887e\u887f\u8880\u8881\u8882\u8883\u8884\u8885\u8886\u8887\u8888\u8889\u888a\u888b\u888c\u888d\u888e\u888f\u8890\u8891\u8892\u8893\u8894\u8895\u8896\u8897\u8898\u8899\u889a\u889b\u889c\u889d\u889e\u889f\u88a0\u88a1\u88a2\u88a3\u88a4\u88a5\u88a6\u88a7\u88a8\u88a9\u88aa\u88ab\u88ac\u88ad\u88ae\u88af\u88b0\u88b1\u88b2\u88b3\u88b4\u88b5\u88b6\u88b7\u88b8\u88b9\u88ba\u88bb\u88bc\u88bd\u88be\u88bf\u88c0\u88c1\u88c2\u88c3\u88c4\u88c5\u88c6\u88c7\u88c8\u88c9\u88ca\u88cb\u88cc\u88cd\u88ce\u88cf\u88d0\u88d1\u88d2\u88d3\u88d4\u88d5\u88d6\u88d7\u88d8\u88d9\u88da\u88db\u88dc\u88dd\u88de\u88df\u88e0\u88e1\u88e2\u88e3\u88e4\u88e5\u88e6\u88e7\u88e8\u88e9\u88ea\u88eb\u88ec\u88ed\u88ee\u88ef\u88f0\u88f1\u88f2\u88f3\u88f4\u88f5\u88f6\u88f7\u88f8\u88f9\u88fa\u88fb\u88fc\u88fd\u88fe\u88ff\u8900\u8901\u8902\u8903\u8904\u8905\u8906\u8907\u8908\u8909\u890a\u890b\u890c\u890d\u890e\u890f\u8910\u8911\u8912\u8913\u8914\u8915\u8916\u8917\u8918\u8919\u891a\u891b\u891c\u891d\u891e\u891f\u8920\u8921\u8922\u8923\u8924\u8925\u8926\u8927\u8928\u8929\u892a\u892b\u892c\u892d\u892e\u892f\u8930\u8931\u8932\u8933\u8934\u8935\u8936\u8937\u8938\u8939\u893a\u893b\u893c\u893d\u893e\u893f\u8940\u8941\u8942\u8943\u8944\u8945\u8946\u8947\u8948\u8949\u894a\u894b\u894c\u894d\u894e\u894f\u8950\u8951\u8952\u8953\u8954\u8955\u8956\u8957\u8958\u8959\u895a\u895b\u895c\u895d\u895e\u895f\u8960\u8961\u8962\u8963\u8964\u8965\u8966\u8967\u8968\u8969\u896a\u896b\u896c\u896d\u896e\u896f\u8970\u8971\u8972\u8973\u8974\u8975\u8976\u8977\u8978\u8979\u897a\u897b\u897c\u897d\u897e\u897f\u8980\u8981\u8982\u8983\u8984\u8985\u8986\u8987\u8988\u8989\u898a\u898b\u898c\u898d\u898e\u898f\u8990\u8991\u8992\u8993\u8994\u8995\u8996\u8997\u8998\u8999\u899a\u899b\u899c\u899d\u899e\u899f\u89a0\u89a1\u89a2\u89a3\u89a4\u89a5\u89a6\u89a7\u89a8\u89a9\u89aa\u89ab\u89ac\u89ad\u89ae\u89af\u89b0\u89b1\u89b2\u89b3\u89b4\u89b5\u89b6\u89b7\u89b8\u89b9\u89ba\u89bb\u89bc\u89bd\u89be\u89bf\u89c0\u89c1\u89c2\u89c3\u89c4\u89c5\u89c6\u89c7\u89c8\u89c9\u89ca\u89cb\u89cc\u89cd\u89ce\u89cf\u89d0\u89d1\u89d2\u89d3\u89d4\u89d5\u89d6\u89d7\u89d8\u89d9\u89da\u89db\u89dc\u89dd\u89de\u89df\u89e0\u89e1\u89e2\u89e3\u89e4\u89e5\u89e6\u89e7\u89e8\u89e9\u89ea\u89eb\u89ec\u89ed\u89ee\u89ef\u89f0\u89f1\u89f2\u89f3\u89f4\u89f5\u89f6\u89f7\u89f8\u89f9\u89fa\u89fb\u89fc\u89fd\u89fe\u89ff\u8a00\u8a01\u8a02\u8a03\u8a04\u8a05\u8a06\u8a07\u8a08\u8a09\u8a0a\u8a0b\u8a0c\u8a0d\u8a0e\u8a0f\u8a10\u8a11\u8a12\u8a13\u8a14\u8a15\u8a16\u8a17\u8a18\u8a19\u8a1a\u8a1b\u8a1c\u8a1d\u8a1e\u8a1f\u8a20\u8a21\u8a22\u8a23\u8a24\u8a25\u8a26\u8a27\u8a28\u8a29\u8a2a\u8a2b\u8a2c\u8a2d\u8a2e\u8a2f\u8a30\u8a31\u8a32\u8a33\u8a34\u8a35\u8a36\u8a37\u8a38\u8a39\u8a3a\u8a3b\u8a3c\u8a3d\u8a3e\u8a3f\u8a40\u8a41\u8a42\u8a43\u8a44\u8a45\u8a46\u8a47\u8a48\u8a49\u8a4a\u8a4b\u8a4c\u8a4d\u8a4e\u8a4f\u8a50\u8a51\u8a52\u8a53\u8a54\u8a55\u8a56\u8a57\u8a58\u8a59\u8a5a\u8a5b\u8a5c\u8a5d\u8a5e\u8a5f\u8a60\u8a61\u8a62\u8a63\u8a64\u8a65\u8a66\u8a67\u8a68\u8a69\u8a6a\u8a6b\u8a6c\u8a6d\u8a6e\u8a6f\u8a70\u8a71\u8a72\u8a73\u8a74\u8a75\u8a76\u8a77\u8a78\u8a79\u8a7a\u8a7b\u8a7c\u8a7d\u8a7e\u8a7f\u8a80\u8a81\u8a82\u8a83\u8a84\u8a85\u8a86\u8a87\u8a88\u8a89\u8a8a\u8a8b\u8a8c\u8a8d\u8a8e\u8a8f\u8a90\u8a91\u8a92\u8a93\u8a94\u8a95\u8a96\u8a97\u8a98\u8a99\u8a9a\u8a9b\u8a9c\u8a9d\u8a9e\u8a9f\u8aa0\u8aa1\u8aa2\u8aa3\u8aa4\u8aa5\u8aa6\u8aa7\u8aa8\u8aa9\u8aaa\u8aab\u8aac\u8aad\u8aae\u8aaf\u8ab0\u8ab1\u8ab2\u8ab3\u8ab4\u8ab5\u8ab6\u8ab7\u8ab8\u8ab9\u8aba\u8abb\u8abc\u8abd\u8abe\u8abf\u8ac0\u8ac1\u8ac2\u8ac3\u8ac4\u8ac5\u8ac6\u8ac7\u8ac8\u8ac9\u8aca\u8acb\u8acc\u8acd\u8ace\u8acf\u8ad0\u8ad1\u8ad2\u8ad3\u8ad4\u8ad5\u8ad6\u8ad7\u8ad8\u8ad9\u8ada\u8adb\u8adc\u8add\u8ade\u8adf\u8ae0\u8ae1\u8ae2\u8ae3\u8ae4\u8ae5\u8ae6\u8ae7\u8ae8\u8ae9\u8aea\u8aeb\u8aec\u8aed\u8aee\u8aef\u8af0\u8af1\u8af2\u8af3\u8af4\u8af5\u8af6\u8af7\u8af8\u8af9\u8afa\u8afb\u8afc\u8afd\u8afe\u8aff\u8b00\u8b01\u8b02\u8b03\u8b04\u8b05\u8b06\u8b07\u8b08\u8b09\u8b0a\u8b0b\u8b0c\u8b0d\u8b0e\u8b0f\u8b10\u8b11\u8b12\u8b13\u8b14\u8b15\u8b16\u8b17\u8b18\u8b19\u8b1a\u8b1b\u8b1c\u8b1d\u8b1e\u8b1f\u8b20\u8b21\u8b22\u8b23\u8b24\u8b25\u8b26\u8b27\u8b28\u8b29\u8b2a\u8b2b\u8b2c\u8b2d\u8b2e\u8b2f\u8b30\u8b31\u8b32\u8b33\u8b34\u8b35\u8b36\u8b37\u8b38\u8b39\u8b3a\u8b3b\u8b3c\u8b3d\u8b3e\u8b3f\u8b40\u8b41\u8b42\u8b43\u8b44\u8b45\u8b46\u8b47\u8b48\u8b49\u8b4a\u8b4b\u8b4c\u8b4d\u8b4e\u8b4f\u8b50\u8b51\u8b52\u8b53\u8b54\u8b55\u8b56\u8b57\u8b58\u8b59\u8b5a\u8b5b\u8b5c\u8b5d\u8b5e\u8b5f\u8b60\u8b61\u8b62\u8b63\u8b64\u8b65\u8b66\u8b67\u8b68\u8b69\u8b6a\u8b6b\u8b6c\u8b6d\u8b6e\u8b6f\u8b70\u8b71\u8b72\u8b73\u8b74\u8b75\u8b76\u8b77\u8b78\u8b79\u8b7a\u8b7b\u8b7c\u8b7d\u8b7e\u8b7f\u8b80\u8b81\u8b82\u8b83\u8b84\u8b85\u8b86\u8b87\u8b88\u8b89\u8b8a\u8b8b\u8b8c\u8b8d\u8b8e\u8b8f\u8b90\u8b91\u8b92\u8b93\u8b94\u8b95\u8b96\u8b97\u8b98\u8b99\u8b9a\u8b9b\u8b9c\u8b9d\u8b9e\u8b9f\u8ba0\u8ba1\u8ba2\u8ba3\u8ba4\u8ba5\u8ba6\u8ba7\u8ba8\u8ba9\u8baa\u8bab\u8bac\u8bad\u8bae\u8baf\u8bb0\u8bb1\u8bb2\u8bb3\u8bb4\u8bb5\u8bb6\u8bb7\u8bb8\u8bb9\u8bba\u8bbb\u8bbc\u8bbd\u8bbe\u8bbf\u8bc0\u8bc1\u8bc2\u8bc3\u8bc4\u8bc5\u8bc6\u8bc7\u8bc8\u8bc9\u8bca\u8bcb\u8bcc\u8bcd\u8bce\u8bcf\u8bd0\u8bd1\u8bd2\u8bd3\u8bd4\u8bd5\u8bd6\u8bd7\u8bd8\u8bd9\u8bda\u8bdb\u8bdc\u8bdd\u8bde\u8bdf\u8be0\u8be1\u8be2\u8be3\u8be4\u8be5\u8be6\u8be7\u8be8\u8be9\u8bea\u8beb\u8bec\u8bed\u8bee\u8bef\u8bf0\u8bf1\u8bf2\u8bf3\u8bf4\u8bf5\u8bf6\u8bf7\u8bf8\u8bf9\u8bfa\u8bfb\u8bfc\u8bfd\u8bfe\u8bff\u8c00\u8c01\u8c02\u8c03\u8c04\u8c05\u8c06\u8c07\u8c08\u8c09\u8c0a\u8c0b\u8c0c\u8c0d\u8c0e\u8c0f\u8c10\u8c11\u8c12\u8c13\u8c14\u8c15\u8c16\u8c17\u8c18\u8c19\u8c1a\u8c1b\u8c1c\u8c1d\u8c1e\u8c1f\u8c20\u8c21\u8c22\u8c23\u8c24\u8c25\u8c26\u8c27\u8c28\u8c29\u8c2a\u8c2b\u8c2c\u8c2d\u8c2e\u8c2f\u8c30\u8c31\u8c32\u8c33\u8c34\u8c35\u8c36\u8c37\u8c38\u8c39\u8c3a\u8c3b\u8c3c\u8c3d\u8c3e\u8c3f\u8c40\u8c41\u8c42\u8c43\u8c44\u8c45\u8c46\u8c47\u8c48\u8c49\u8c4a\u8c4b\u8c4c\u8c4d\u8c4e\u8c4f\u8c50\u8c51\u8c52\u8c53\u8c54\u8c55\u8c56\u8c57\u8c58\u8c59\u8c5a\u8c5b\u8c5c\u8c5d\u8c5e\u8c5f\u8c60\u8c61\u8c62\u8c63\u8c64\u8c65\u8c66\u8c67\u8c68\u8c69\u8c6a\u8c6b\u8c6c\u8c6d\u8c6e\u8c6f\u8c70\u8c71\u8c72\u8c73\u8c74\u8c75\u8c76\u8c77\u8c78\u8c79\u8c7a\u8c7b\u8c7c\u8c7d\u8c7e\u8c7f\u8c80\u8c81\u8c82\u8c83\u8c84\u8c85\u8c86\u8c87\u8c88\u8c89\u8c8a\u8c8b\u8c8c\u8c8d\u8c8e\u8c8f\u8c90\u8c91\u8c92\u8c93\u8c94\u8c95\u8c96\u8c97\u8c98\u8c99\u8c9a\u8c9b\u8c9c\u8c9d\u8c9e\u8c9f\u8ca0\u8ca1\u8ca2\u8ca3\u8ca4\u8ca5\u8ca6\u8ca7\u8ca8\u8ca9\u8caa\u8cab\u8cac\u8cad\u8cae\u8caf\u8cb0\u8cb1\u8cb2\u8cb3\u8cb4\u8cb5\u8cb6\u8cb7\u8cb8\u8cb9\u8cba\u8cbb\u8cbc\u8cbd\u8cbe\u8cbf\u8cc0\u8cc1\u8cc2\u8cc3\u8cc4\u8cc5\u8cc6\u8cc7\u8cc8\u8cc9\u8cca\u8ccb\u8ccc\u8ccd\u8cce\u8ccf\u8cd0\u8cd1\u8cd2\u8cd3\u8cd4\u8cd5\u8cd6\u8cd7\u8cd8\u8cd9\u8cda\u8cdb\u8cdc\u8cdd\u8cde\u8cdf\u8ce0\u8ce1\u8ce2\u8ce3\u8ce4\u8ce5\u8ce6\u8ce7\u8ce8\u8ce9\u8cea\u8ceb\u8cec\u8ced\u8cee\u8cef\u8cf0\u8cf1\u8cf2\u8cf3\u8cf4\u8cf5\u8cf6\u8cf7\u8cf8\u8cf9\u8cfa\u8cfb\u8cfc\u8cfd\u8cfe\u8cff\u8d00\u8d01\u8d02\u8d03\u8d04\u8d05\u8d06\u8d07\u8d08\u8d09\u8d0a\u8d0b\u8d0c\u8d0d\u8d0e\u8d0f\u8d10\u8d11\u8d12\u8d13\u8d14\u8d15\u8d16\u8d17\u8d18\u8d19\u8d1a\u8d1b\u8d1c\u8d1d\u8d1e\u8d1f\u8d20\u8d21\u8d22\u8d23\u8d24\u8d25\u8d26\u8d27\u8d28\u8d29\u8d2a\u8d2b\u8d2c\u8d2d\u8d2e\u8d2f\u8d30\u8d31\u8d32\u8d33\u8d34\u8d35\u8d36\u8d37\u8d38\u8d39\u8d3a\u8d3b\u8d3c\u8d3d\u8d3e\u8d3f\u8d40\u8d41\u8d42\u8d43\u8d44\u8d45\u8d46\u8d47\u8d48\u8d49\u8d4a\u8d4b\u8d4c\u8d4d\u8d4e\u8d4f\u8d50\u8d51\u8d52\u8d53\u8d54\u8d55\u8d56\u8d57\u8d58\u8d59\u8d5a\u8d5b\u8d5c\u8d5d\u8d5e\u8d5f\u8d60\u8d61\u8d62\u8d63\u8d64\u8d65\u8d66\u8d67\u8d68\u8d69\u8d6a\u8d6b\u8d6c\u8d6d\u8d6e\u8d6f\u8d70\u8d71\u8d72\u8d73\u8d74\u8d75\u8d76\u8d77\u8d78\u8d79\u8d7a\u8d7b\u8d7c\u8d7d\u8d7e\u8d7f\u8d80\u8d81\u8d82\u8d83\u8d84\u8d85\u8d86\u8d87\u8d88\u8d89\u8d8a\u8d8b\u8d8c\u8d8d\u8d8e\u8d8f\u8d90\u8d91\u8d92\u8d93\u8d94\u8d95\u8d96\u8d97\u8d98\u8d99\u8d9a\u8d9b\u8d9c\u8d9d\u8d9e\u8d9f\u8da0\u8da1\u8da2\u8da3\u8da4\u8da5\u8da6\u8da7\u8da8\u8da9\u8daa\u8dab\u8dac\u8dad\u8dae\u8daf\u8db0\u8db1\u8db2\u8db3\u8db4\u8db5\u8db6\u8db7\u8db8\u8db9\u8dba\u8dbb\u8dbc\u8dbd\u8dbe\u8dbf\u8dc0\u8dc1\u8dc2\u8dc3\u8dc4\u8dc5\u8dc6\u8dc7\u8dc8\u8dc9\u8dca\u8dcb\u8dcc\u8dcd\u8dce\u8dcf\u8dd0\u8dd1\u8dd2\u8dd3\u8dd4\u8dd5\u8dd6\u8dd7\u8dd8\u8dd9\u8dda\u8ddb\u8ddc\u8ddd\u8dde\u8ddf\u8de0\u8de1\u8de2\u8de3\u8de4\u8de5\u8de6\u8de7\u8de8\u8de9\u8dea\u8deb\u8dec\u8ded\u8dee\u8def\u8df0\u8df1\u8df2\u8df3\u8df4\u8df5\u8df6\u8df7\u8df8\u8df9\u8dfa\u8dfb\u8dfc\u8dfd\u8dfe\u8dff\u8e00\u8e01\u8e02\u8e03\u8e04\u8e05\u8e06\u8e07\u8e08\u8e09\u8e0a\u8e0b\u8e0c\u8e0d\u8e0e\u8e0f\u8e10\u8e11\u8e12\u8e13\u8e14\u8e15\u8e16\u8e17\u8e18\u8e19\u8e1a\u8e1b\u8e1c\u8e1d\u8e1e\u8e1f\u8e20\u8e21\u8e22\u8e23\u8e24\u8e25\u8e26\u8e27\u8e28\u8e29\u8e2a\u8e2b\u8e2c\u8e2d\u8e2e\u8e2f\u8e30\u8e31\u8e32\u8e33\u8e34\u8e35\u8e36\u8e37\u8e38\u8e39\u8e3a\u8e3b\u8e3c\u8e3d\u8e3e\u8e3f\u8e40\u8e41\u8e42\u8e43\u8e44\u8e45\u8e46\u8e47\u8e48\u8e49\u8e4a\u8e4b\u8e4c\u8e4d\u8e4e\u8e4f\u8e50\u8e51\u8e52\u8e53\u8e54\u8e55\u8e56\u8e57\u8e58\u8e59\u8e5a\u8e5b\u8e5c\u8e5d\u8e5e\u8e5f\u8e60\u8e61\u8e62\u8e63\u8e64\u8e65\u8e66\u8e67\u8e68\u8e69\u8e6a\u8e6b\u8e6c\u8e6d\u8e6e\u8e6f\u8e70\u8e71\u8e72\u8e73\u8e74\u8e75\u8e76\u8e77\u8e78\u8e79\u8e7a\u8e7b\u8e7c\u8e7d\u8e7e\u8e7f\u8e80\u8e81\u8e82\u8e83\u8e84\u8e85\u8e86\u8e87\u8e88\u8e89\u8e8a\u8e8b\u8e8c\u8e8d\u8e8e\u8e8f\u8e90\u8e91\u8e92\u8e93\u8e94\u8e95\u8e96\u8e97\u8e98\u8e99\u8e9a\u8e9b\u8e9c\u8e9d\u8e9e\u8e9f\u8ea0\u8ea1\u8ea2\u8ea3\u8ea4\u8ea5\u8ea6\u8ea7\u8ea8\u8ea9\u8eaa\u8eab\u8eac\u8ead\u8eae\u8eaf\u8eb0\u8eb1\u8eb2\u8eb3\u8eb4\u8eb5\u8eb6\u8eb7\u8eb8\u8eb9\u8eba\u8ebb\u8ebc\u8ebd\u8ebe\u8ebf\u8ec0\u8ec1\u8ec2\u8ec3\u8ec4\u8ec5\u8ec6\u8ec7\u8ec8\u8ec9\u8eca\u8ecb\u8ecc\u8ecd\u8ece\u8ecf\u8ed0\u8ed1\u8ed2\u8ed3\u8ed4\u8ed5\u8ed6\u8ed7\u8ed8\u8ed9\u8eda\u8edb\u8edc\u8edd\u8ede\u8edf\u8ee0\u8ee1\u8ee2\u8ee3\u8ee4\u8ee5\u8ee6\u8ee7\u8ee8\u8ee9\u8eea\u8eeb\u8eec\u8eed\u8eee\u8eef\u8ef0\u8ef1\u8ef2\u8ef3\u8ef4\u8ef5\u8ef6\u8ef7\u8ef8\u8ef9\u8efa\u8efb\u8efc\u8efd\u8efe\u8eff\u8f00\u8f01\u8f02\u8f03\u8f04\u8f05\u8f06\u8f07\u8f08\u8f09\u8f0a\u8f0b\u8f0c\u8f0d\u8f0e\u8f0f\u8f10\u8f11\u8f12\u8f13\u8f14\u8f15\u8f16\u8f17\u8f18\u8f19\u8f1a\u8f1b\u8f1c\u8f1d\u8f1e\u8f1f\u8f20\u8f21\u8f22\u8f23\u8f24\u8f25\u8f26\u8f27\u8f28\u8f29\u8f2a\u8f2b\u8f2c\u8f2d\u8f2e\u8f2f\u8f30\u8f31\u8f32\u8f33\u8f34\u8f35\u8f36\u8f37\u8f38\u8f39\u8f3a\u8f3b\u8f3c\u8f3d\u8f3e\u8f3f\u8f40\u8f41\u8f42\u8f43\u8f44\u8f45\u8f46\u8f47\u8f48\u8f49\u8f4a\u8f4b\u8f4c\u8f4d\u8f4e\u8f4f\u8f50\u8f51\u8f52\u8f53\u8f54\u8f55\u8f56\u8f57\u8f58\u8f59\u8f5a\u8f5b\u8f5c\u8f5d\u8f5e\u8f5f\u8f60\u8f61\u8f62\u8f63\u8f64\u8f65\u8f66\u8f67\u8f68\u8f69\u8f6a\u8f6b\u8f6c\u8f6d\u8f6e\u8f6f\u8f70\u8f71\u8f72\u8f73\u8f74\u8f75\u8f76\u8f77\u8f78\u8f79\u8f7a\u8f7b\u8f7c\u8f7d\u8f7e\u8f7f\u8f80\u8f81\u8f82\u8f83\u8f84\u8f85\u8f86\u8f87\u8f88\u8f89\u8f8a\u8f8b\u8f8c\u8f8d\u8f8e\u8f8f\u8f90\u8f91\u8f92\u8f93\u8f94\u8f95\u8f96\u8f97\u8f98\u8f99\u8f9a\u8f9b\u8f9c\u8f9d\u8f9e\u8f9f\u8fa0\u8fa1\u8fa2\u8fa3\u8fa4\u8fa5\u8fa6\u8fa7\u8fa8\u8fa9\u8faa\u8fab\u8fac\u8fad\u8fae\u8faf\u8fb0\u8fb1\u8fb2\u8fb3\u8fb4\u8fb5\u8fb6\u8fb7\u8fb8\u8fb9\u8fba\u8fbb\u8fbc\u8fbd\u8fbe\u8fbf\u8fc0\u8fc1\u8fc2\u8fc3\u8fc4\u8fc5\u8fc6\u8fc7\u8fc8\u8fc9\u8fca\u8fcb\u8fcc\u8fcd\u8fce\u8fcf\u8fd0\u8fd1\u8fd2\u8fd3\u8fd4\u8fd5\u8fd6\u8fd7\u8fd8\u8fd9\u8fda\u8fdb\u8fdc\u8fdd\u8fde\u8fdf\u8fe0\u8fe1\u8fe2\u8fe3\u8fe4\u8fe5\u8fe6\u8fe7\u8fe8\u8fe9\u8fea\u8feb\u8fec\u8fed\u8fee\u8fef\u8ff0\u8ff1\u8ff2\u8ff3\u8ff4\u8ff5\u8ff6\u8ff7\u8ff8\u8ff9\u8ffa\u8ffb\u8ffc\u8ffd\u8ffe\u8fff\u9000\u9001\u9002\u9003\u9004\u9005\u9006\u9007\u9008\u9009\u900a\u900b\u900c\u900d\u900e\u900f\u9010\u9011\u9012\u9013\u9014\u9015\u9016\u9017\u9018\u9019\u901a\u901b\u901c\u901d\u901e\u901f\u9020\u9021\u9022\u9023\u9024\u9025\u9026\u9027\u9028\u9029\u902a\u902b\u902c\u902d\u902e\u902f\u9030\u9031\u9032\u9033\u9034\u9035\u9036\u9037\u9038\u9039\u903a\u903b\u903c\u903d\u903e\u903f\u9040\u9041\u9042\u9043\u9044\u9045\u9046\u9047\u9048\u9049\u904a\u904b\u904c\u904d\u904e\u904f\u9050\u9051\u9052\u9053\u9054\u9055\u9056\u9057\u9058\u9059\u905a\u905b\u905c\u905d\u905e\u905f\u9060\u9061\u9062\u9063\u9064\u9065\u9066\u9067\u9068\u9069\u906a\u906b\u906c\u906d\u906e\u906f\u9070\u9071\u9072\u9073\u9074\u9075\u9076\u9077\u9078\u9079\u907a\u907b\u907c\u907d\u907e\u907f\u9080\u9081\u9082\u9083\u9084\u9085\u9086\u9087\u9088\u9089\u908a\u908b\u908c\u908d\u908e\u908f\u9090\u9091\u9092\u9093\u9094\u9095\u9096\u9097\u9098\u9099\u909a\u909b\u909c\u909d\u909e\u909f\u90a0\u90a1\u90a2\u90a3\u90a4\u90a5\u90a6\u90a7\u90a8\u90a9\u90aa\u90ab\u90ac\u90ad\u90ae\u90af\u90b0\u90b1\u90b2\u90b3\u90b4\u90b5\u90b6\u90b7\u90b8\u90b9\u90ba\u90bb\u90bc\u90bd\u90be\u90bf\u90c0\u90c1\u90c2\u90c3\u90c4\u90c5\u90c6\u90c7\u90c8\u90c9\u90ca\u90cb\u90cc\u90cd\u90ce\u90cf\u90d0\u90d1\u90d2\u90d3\u90d4\u90d5\u90d6\u90d7\u90d8\u90d9\u90da\u90db\u90dc\u90dd\u90de\u90df\u90e0\u90e1\u90e2\u90e3\u90e4\u90e5\u90e6\u90e7\u90e8\u90e9\u90ea\u90eb\u90ec\u90ed\u90ee\u90ef\u90f0\u90f1\u90f2\u90f3\u90f4\u90f5\u90f6\u90f7\u90f8\u90f9\u90fa\u90fb\u90fc\u90fd\u90fe\u90ff\u9100\u9101\u9102\u9103\u9104\u9105\u9106\u9107\u9108\u9109\u910a\u910b\u910c\u910d\u910e\u910f\u9110\u9111\u9112\u9113\u9114\u9115\u9116\u9117\u9118\u9119\u911a\u911b\u911c\u911d\u911e\u911f\u9120\u9121\u9122\u9123\u9124\u9125\u9126\u9127\u9128\u9129\u912a\u912b\u912c\u912d\u912e\u912f\u9130\u9131\u9132\u9133\u9134\u9135\u9136\u9137\u9138\u9139\u913a\u913b\u913c\u913d\u913e\u913f\u9140\u9141\u9142\u9143\u9144\u9145\u9146\u9147\u9148\u9149\u914a\u914b\u914c\u914d\u914e\u914f\u9150\u9151\u9152\u9153\u9154\u9155\u9156\u9157\u9158\u9159\u915a\u915b\u915c\u915d\u915e\u915f\u9160\u9161\u9162\u9163\u9164\u9165\u9166\u9167\u9168\u9169\u916a\u916b\u916c\u916d\u916e\u916f\u9170\u9171\u9172\u9173\u9174\u9175\u9176\u9177\u9178\u9179\u917a\u917b\u917c\u917d\u917e\u917f\u9180\u9181\u9182\u9183\u9184\u9185\u9186\u9187\u9188\u9189\u918a\u918b\u918c\u918d\u918e\u918f\u9190\u9191\u9192\u9193\u9194\u9195\u9196\u9197\u9198\u9199\u919a\u919b\u919c\u919d\u919e\u919f\u91a0\u91a1\u91a2\u91a3\u91a4\u91a5\u91a6\u91a7\u91a8\u91a9\u91aa\u91ab\u91ac\u91ad\u91ae\u91af\u91b0\u91b1\u91b2\u91b3\u91b4\u91b5\u91b6\u91b7\u91b8\u91b9\u91ba\u91bb\u91bc\u91bd\u91be\u91bf\u91c0\u91c1\u91c2\u91c3\u91c4\u91c5\u91c6\u91c7\u91c8\u91c9\u91ca\u91cb\u91cc\u91cd\u91ce\u91cf\u91d0\u91d1\u91d2\u91d3\u91d4\u91d5\u91d6\u91d7\u91d8\u91d9\u91da\u91db\u91dc\u91dd\u91de\u91df\u91e0\u91e1\u91e2\u91e3\u91e4\u91e5\u91e6\u91e7\u91e8\u91e9\u91ea\u91eb\u91ec\u91ed\u91ee\u91ef\u91f0\u91f1\u91f2\u91f3\u91f4\u91f5\u91f6\u91f7\u91f8\u91f9\u91fa\u91fb\u91fc\u91fd\u91fe\u91ff\u9200\u9201\u9202\u9203\u9204\u9205\u9206\u9207\u9208\u9209\u920a\u920b\u920c\u920d\u920e\u920f\u9210\u9211\u9212\u9213\u9214\u9215\u9216\u9217\u9218\u9219\u921a\u921b\u921c\u921d\u921e\u921f\u9220\u9221\u9222\u9223\u9224\u9225\u9226\u9227\u9228\u9229\u922a\u922b\u922c\u922d\u922e\u922f\u9230\u9231\u9232\u9233\u9234\u9235\u9236\u9237\u9238\u9239\u923a\u923b\u923c\u923d\u923e\u923f\u9240\u9241\u9242\u9243\u9244\u9245\u9246\u9247\u9248\u9249\u924a\u924b\u924c\u924d\u924e\u924f\u9250\u9251\u9252\u9253\u9254\u9255\u9256\u9257\u9258\u9259\u925a\u925b\u925c\u925d\u925e\u925f\u9260\u9261\u9262\u9263\u9264\u9265\u9266\u9267\u9268\u9269\u926a\u926b\u926c\u926d\u926e\u926f\u9270\u9271\u9272\u9273\u9274\u9275\u9276\u9277\u9278\u9279\u927a\u927b\u927c\u927d\u927e\u927f\u9280\u9281\u9282\u9283\u9284\u9285\u9286\u9287\u9288\u9289\u928a\u928b\u928c\u928d\u928e\u928f\u9290\u9291\u9292\u9293\u9294\u9295\u9296\u9297\u9298\u9299\u929a\u929b\u929c\u929d\u929e\u929f\u92a0\u92a1\u92a2\u92a3\u92a4\u92a5\u92a6\u92a7\u92a8\u92a9\u92aa\u92ab\u92ac\u92ad\u92ae\u92af\u92b0\u92b1\u92b2\u92b3\u92b4\u92b5\u92b6\u92b7\u92b8\u92b9\u92ba\u92bb\u92bc\u92bd\u92be\u92bf\u92c0\u92c1\u92c2\u92c3\u92c4\u92c5\u92c6\u92c7\u92c8\u92c9\u92ca\u92cb\u92cc\u92cd\u92ce\u92cf\u92d0\u92d1\u92d2\u92d3\u92d4\u92d5\u92d6\u92d7\u92d8\u92d9\u92da\u92db\u92dc\u92dd\u92de\u92df\u92e0\u92e1\u92e2\u92e3\u92e4\u92e5\u92e6\u92e7\u92e8\u92e9\u92ea\u92eb\u92ec\u92ed\u92ee\u92ef\u92f0\u92f1\u92f2\u92f3\u92f4\u92f5\u92f6\u92f7\u92f8\u92f9\u92fa\u92fb\u92fc\u92fd\u92fe\u92ff\u9300\u9301\u9302\u9303\u9304\u9305\u9306\u9307\u9308\u9309\u930a\u930b\u930c\u930d\u930e\u930f\u9310\u9311\u9312\u9313\u9314\u9315\u9316\u9317\u9318\u9319\u931a\u931b\u931c\u931d\u931e\u931f\u9320\u9321\u9322\u9323\u9324\u9325\u9326\u9327\u9328\u9329\u932a\u932b\u932c\u932d\u932e\u932f\u9330\u9331\u9332\u9333\u9334\u9335\u9336\u9337\u9338\u9339\u933a\u933b\u933c\u933d\u933e\u933f\u9340\u9341\u9342\u9343\u9344\u9345\u9346\u9347\u9348\u9349\u934a\u934b\u934c\u934d\u934e\u934f\u9350\u9351\u9352\u9353\u9354\u9355\u9356\u9357\u9358\u9359\u935a\u935b\u935c\u935d\u935e\u935f\u9360\u9361\u9362\u9363\u9364\u9365\u9366\u9367\u9368\u9369\u936a\u936b\u936c\u936d\u936e\u936f\u9370\u9371\u9372\u9373\u9374\u9375\u9376\u9377\u9378\u9379\u937a\u937b\u937c\u937d\u937e\u937f\u9380\u9381\u9382\u9383\u9384\u9385\u9386\u9387\u9388\u9389\u938a\u938b\u938c\u938d\u938e\u938f\u9390\u9391\u9392\u9393\u9394\u9395\u9396\u9397\u9398\u9399\u939a\u939b\u939c\u939d\u939e\u939f\u93a0\u93a1\u93a2\u93a3\u93a4\u93a5\u93a6\u93a7\u93a8\u93a9\u93aa\u93ab\u93ac\u93ad\u93ae\u93af\u93b0\u93b1\u93b2\u93b3\u93b4\u93b5\u93b6\u93b7\u93b8\u93b9\u93ba\u93bb\u93bc\u93bd\u93be\u93bf\u93c0\u93c1\u93c2\u93c3\u93c4\u93c5\u93c6\u93c7\u93c8\u93c9\u93ca\u93cb\u93cc\u93cd\u93ce\u93cf\u93d0\u93d1\u93d2\u93d3\u93d4\u93d5\u93d6\u93d7\u93d8\u93d9\u93da\u93db\u93dc\u93dd\u93de\u93df\u93e0\u93e1\u93e2\u93e3\u93e4\u93e5\u93e6\u93e7\u93e8\u93e9\u93ea\u93eb\u93ec\u93ed\u93ee\u93ef\u93f0\u93f1\u93f2\u93f3\u93f4\u93f5\u93f6\u93f7\u93f8\u93f9\u93fa\u93fb\u93fc\u93fd\u93fe\u93ff\u9400\u9401\u9402\u9403\u9404\u9405\u9406\u9407\u9408\u9409\u940a\u940b\u940c\u940d\u940e\u940f\u9410\u9411\u9412\u9413\u9414\u9415\u9416\u9417\u9418\u9419\u941a\u941b\u941c\u941d\u941e\u941f\u9420\u9421\u9422\u9423\u9424\u9425\u9426\u9427\u9428\u9429\u942a\u942b\u942c\u942d\u942e\u942f\u9430\u9431\u9432\u9433\u9434\u9435\u9436\u9437\u9438\u9439\u943a\u943b\u943c\u943d\u943e\u943f\u9440\u9441\u9442\u9443\u9444\u9445\u9446\u9447\u9448\u9449\u944a\u944b\u944c\u944d\u944e\u944f\u9450\u9451\u9452\u9453\u9454\u9455\u9456\u9457\u9458\u9459\u945a\u945b\u945c\u945d\u945e\u945f\u9460\u9461\u9462\u9463\u9464\u9465\u9466\u9467\u9468\u9469\u946a\u946b\u946c\u946d\u946e\u946f\u9470\u9471\u9472\u9473\u9474\u9475\u9476\u9477\u9478\u9479\u947a\u947b\u947c\u947d\u947e\u947f\u9480\u9481\u9482\u9483\u9484\u9485\u9486\u9487\u9488\u9489\u948a\u948b\u948c\u948d\u948e\u948f\u9490\u9491\u9492\u9493\u9494\u9495\u9496\u9497\u9498\u9499\u949a\u949b\u949c\u949d\u949e\u949f\u94a0\u94a1\u94a2\u94a3\u94a4\u94a5\u94a6\u94a7\u94a8\u94a9\u94aa\u94ab\u94ac\u94ad\u94ae\u94af\u94b0\u94b1\u94b2\u94b3\u94b4\u94b5\u94b6\u94b7\u94b8\u94b9\u94ba\u94bb\u94bc\u94bd\u94be\u94bf\u94c0\u94c1\u94c2\u94c3\u94c4\u94c5\u94c6\u94c7\u94c8\u94c9\u94ca\u94cb\u94cc\u94cd\u94ce\u94cf\u94d0\u94d1\u94d2\u94d3\u94d4\u94d5\u94d6\u94d7\u94d8\u94d9\u94da\u94db\u94dc\u94dd\u94de\u94df\u94e0\u94e1\u94e2\u94e3\u94e4\u94e5\u94e6\u94e7\u94e8\u94e9\u94ea\u94eb\u94ec\u94ed\u94ee\u94ef\u94f0\u94f1\u94f2\u94f3\u94f4\u94f5\u94f6\u94f7\u94f8\u94f9\u94fa\u94fb\u94fc\u94fd\u94fe\u94ff\u9500\u9501\u9502\u9503\u9504\u9505\u9506\u9507\u9508\u9509\u950a\u950b\u950c\u950d\u950e\u950f\u9510\u9511\u9512\u9513\u9514\u9515\u9516\u9517\u9518\u9519\u951a\u951b\u951c\u951d\u951e\u951f\u9520\u9521\u9522\u9523\u9524\u9525\u9526\u9527\u9528\u9529\u952a\u952b\u952c\u952d\u952e\u952f\u9530\u9531\u9532\u9533\u9534\u9535\u9536\u9537\u9538\u9539\u953a\u953b\u953c\u953d\u953e\u953f\u9540\u9541\u9542\u9543\u9544\u9545\u9546\u9547\u9548\u9549\u954a\u954b\u954c\u954d\u954e\u954f\u9550\u9551\u9552\u9553\u9554\u9555\u9556\u9557\u9558\u9559\u955a\u955b\u955c\u955d\u955e\u955f\u9560\u9561\u9562\u9563\u9564\u9565\u9566\u9567\u9568\u9569\u956a\u956b\u956c\u956d\u956e\u956f\u9570\u9571\u9572\u9573\u9574\u9575\u9576\u9577\u9578\u9579\u957a\u957b\u957c\u957d\u957e\u957f\u9580\u9581\u9582\u9583\u9584\u9585\u9586\u9587\u9588\u9589\u958a\u958b\u958c\u958d\u958e\u958f\u9590\u9591\u9592\u9593\u9594\u9595\u9596\u9597\u9598\u9599\u959a\u959b\u959c\u959d\u959e\u959f\u95a0\u95a1\u95a2\u95a3\u95a4\u95a5\u95a6\u95a7\u95a8\u95a9\u95aa\u95ab\u95ac\u95ad\u95ae\u95af\u95b0\u95b1\u95b2\u95b3\u95b4\u95b5\u95b6\u95b7\u95b8\u95b9\u95ba\u95bb\u95bc\u95bd\u95be\u95bf\u95c0\u95c1\u95c2\u95c3\u95c4\u95c5\u95c6\u95c7\u95c8\u95c9\u95ca\u95cb\u95cc\u95cd\u95ce\u95cf\u95d0\u95d1\u95d2\u95d3\u95d4\u95d5\u95d6\u95d7\u95d8\u95d9\u95da\u95db\u95dc\u95dd\u95de\u95df\u95e0\u95e1\u95e2\u95e3\u95e4\u95e5\u95e6\u95e7\u95e8\u95e9\u95ea\u95eb\u95ec\u95ed\u95ee\u95ef\u95f0\u95f1\u95f2\u95f3\u95f4\u95f5\u95f6\u95f7\u95f8\u95f9\u95fa\u95fb\u95fc\u95fd\u95fe\u95ff\u9600\u9601\u9602\u9603\u9604\u9605\u9606\u9607\u9608\u9609\u960a\u960b\u960c\u960d\u960e\u960f\u9610\u9611\u9612\u9613\u9614\u9615\u9616\u9617\u9618\u9619\u961a\u961b\u961c\u961d\u961e\u961f\u9620\u9621\u9622\u9623\u9624\u9625\u9626\u9627\u9628\u9629\u962a\u962b\u962c\u962d\u962e\u962f\u9630\u9631\u9632\u9633\u9634\u9635\u9636\u9637\u9638\u9639\u963a\u963b\u963c\u963d\u963e\u963f\u9640\u9641\u9642\u9643\u9644\u9645\u9646\u9647\u9648\u9649\u964a\u964b\u964c\u964d\u964e\u964f\u9650\u9651\u9652\u9653\u9654\u9655\u9656\u9657\u9658\u9659\u965a\u965b\u965c\u965d\u965e\u965f\u9660\u9661\u9662\u9663\u9664\u9665\u9666\u9667\u9668\u9669\u966a\u966b\u966c\u966d\u966e\u966f\u9670\u9671\u9672\u9673\u9674\u9675\u9676\u9677\u9678\u9679\u967a\u967b\u967c\u967d\u967e\u967f\u9680\u9681\u9682\u9683\u9684\u9685\u9686\u9687\u9688\u9689\u968a\u968b\u968c\u968d\u968e\u968f\u9690\u9691\u9692\u9693\u9694\u9695\u9696\u9697\u9698\u9699\u969a\u969b\u969c\u969d\u969e\u969f\u96a0\u96a1\u96a2\u96a3\u96a4\u96a5\u96a6\u96a7\u96a8\u96a9\u96aa\u96ab\u96ac\u96ad\u96ae\u96af\u96b0\u96b1\u96b2\u96b3\u96b4\u96b5\u96b6\u96b7\u96b8\u96b9\u96ba\u96bb\u96bc\u96bd\u96be\u96bf\u96c0\u96c1\u96c2\u96c3\u96c4\u96c5\u96c6\u96c7\u96c8\u96c9\u96ca\u96cb\u96cc\u96cd\u96ce\u96cf\u96d0\u96d1\u96d2\u96d3\u96d4\u96d5\u96d6\u96d7\u96d8\u96d9\u96da\u96db\u96dc\u96dd\u96de\u96df\u96e0\u96e1\u96e2\u96e3\u96e4\u96e5\u96e6\u96e7\u96e8\u96e9\u96ea\u96eb\u96ec\u96ed\u96ee\u96ef\u96f0\u96f1\u96f2\u96f3\u96f4\u96f5\u96f6\u96f7\u96f8\u96f9\u96fa\u96fb\u96fc\u96fd\u96fe\u96ff\u9700\u9701\u9702\u9703\u9704\u9705\u9706\u9707\u9708\u9709\u970a\u970b\u970c\u970d\u970e\u970f\u9710\u9711\u9712\u9713\u9714\u9715\u9716\u9717\u9718\u9719\u971a\u971b\u971c\u971d\u971e\u971f\u9720\u9721\u9722\u9723\u9724\u9725\u9726\u9727\u9728\u9729\u972a\u972b\u972c\u972d\u972e\u972f\u9730\u9731\u9732\u9733\u9734\u9735\u9736\u9737\u9738\u9739\u973a\u973b\u973c\u973d\u973e\u973f\u9740\u9741\u9742\u9743\u9744\u9745\u9746\u9747\u9748\u9749\u974a\u974b\u974c\u974d\u974e\u974f\u9750\u9751\u9752\u9753\u9754\u9755\u9756\u9757\u9758\u9759\u975a\u975b\u975c\u975d\u975e\u975f\u9760\u9761\u9762\u9763\u9764\u9765\u9766\u9767\u9768\u9769\u976a\u976b\u976c\u976d\u976e\u976f\u9770\u9771\u9772\u9773\u9774\u9775\u9776\u9777\u9778\u9779\u977a\u977b\u977c\u977d\u977e\u977f\u9780\u9781\u9782\u9783\u9784\u9785\u9786\u9787\u9788\u9789\u978a\u978b\u978c\u978d\u978e\u978f\u9790\u9791\u9792\u9793\u9794\u9795\u9796\u9797\u9798\u9799\u979a\u979b\u979c\u979d\u979e\u979f\u97a0\u97a1\u97a2\u97a3\u97a4\u97a5\u97a6\u97a7\u97a8\u97a9\u97aa\u97ab\u97ac\u97ad\u97ae\u97af\u97b0\u97b1\u97b2\u97b3\u97b4\u97b5\u97b6\u97b7\u97b8\u97b9\u97ba\u97bb\u97bc\u97bd\u97be\u97bf\u97c0\u97c1\u97c2\u97c3\u97c4\u97c5\u97c6\u97c7\u97c8\u97c9\u97ca\u97cb\u97cc\u97cd\u97ce\u97cf\u97d0\u97d1\u97d2\u97d3\u97d4\u97d5\u97d6\u97d7\u97d8\u97d9\u97da\u97db\u97dc\u97dd\u97de\u97df\u97e0\u97e1\u97e2\u97e3\u97e4\u97e5\u97e6\u97e7\u97e8\u97e9\u97ea\u97eb\u97ec\u97ed\u97ee\u97ef\u97f0\u97f1\u97f2\u97f3\u97f4\u97f5\u97f6\u97f7\u97f8\u97f9\u97fa\u97fb\u97fc\u97fd\u97fe\u97ff\u9800\u9801\u9802\u9803\u9804\u9805\u9806\u9807\u9808\u9809\u980a\u980b\u980c\u980d\u980e\u980f\u9810\u9811\u9812\u9813\u9814\u9815\u9816\u9817\u9818\u9819\u981a\u981b\u981c\u981d\u981e\u981f\u9820\u9821\u9822\u9823\u9824\u9825\u9826\u9827\u9828\u9829\u982a\u982b\u982c\u982d\u982e\u982f\u9830\u9831\u9832\u9833\u9834\u9835\u9836\u9837\u9838\u9839\u983a\u983b\u983c\u983d\u983e\u983f\u9840\u9841\u9842\u9843\u9844\u9845\u9846\u9847\u9848\u9849\u984a\u984b\u984c\u984d\u984e\u984f\u9850\u9851\u9852\u9853\u9854\u9855\u9856\u9857\u9858\u9859\u985a\u985b\u985c\u985d\u985e\u985f\u9860\u9861\u9862\u9863\u9864\u9865\u9866\u9867\u9868\u9869\u986a\u986b\u986c\u986d\u986e\u986f\u9870\u9871\u9872\u9873\u9874\u9875\u9876\u9877\u9878\u9879\u987a\u987b\u987c\u987d\u987e\u987f\u9880\u9881\u9882\u9883\u9884\u9885\u9886\u9887\u9888\u9889\u988a\u988b\u988c\u988d\u988e\u988f\u9890\u9891\u9892\u9893\u9894\u9895\u9896\u9897\u9898\u9899\u989a\u989b\u989c\u989d\u989e\u989f\u98a0\u98a1\u98a2\u98a3\u98a4\u98a5\u98a6\u98a7\u98a8\u98a9\u98aa\u98ab\u98ac\u98ad\u98ae\u98af\u98b0\u98b1\u98b2\u98b3\u98b4\u98b5\u98b6\u98b7\u98b8\u98b9\u98ba\u98bb\u98bc\u98bd\u98be\u98bf\u98c0\u98c1\u98c2\u98c3\u98c4\u98c5\u98c6\u98c7\u98c8\u98c9\u98ca\u98cb\u98cc\u98cd\u98ce\u98cf\u98d0\u98d1\u98d2\u98d3\u98d4\u98d5\u98d6\u98d7\u98d8\u98d9\u98da\u98db\u98dc\u98dd\u98de\u98df\u98e0\u98e1\u98e2\u98e3\u98e4\u98e5\u98e6\u98e7\u98e8\u98e9\u98ea\u98eb\u98ec\u98ed\u98ee\u98ef\u98f0\u98f1\u98f2\u98f3\u98f4\u98f5\u98f6\u98f7\u98f8\u98f9\u98fa\u98fb\u98fc\u98fd\u98fe\u98ff\u9900\u9901\u9902\u9903\u9904\u9905\u9906\u9907\u9908\u9909\u990a\u990b\u990c\u990d\u990e\u990f\u9910\u9911\u9912\u9913\u9914\u9915\u9916\u9917\u9918\u9919\u991a\u991b\u991c\u991d\u991e\u991f\u9920\u9921\u9922\u9923\u9924\u9925\u9926\u9927\u9928\u9929\u992a\u992b\u992c\u992d\u992e\u992f\u9930\u9931\u9932\u9933\u9934\u9935\u9936\u9937\u9938\u9939\u993a\u993b\u993c\u993d\u993e\u993f\u9940\u9941\u9942\u9943\u9944\u9945\u9946\u9947\u9948\u9949\u994a\u994b\u994c\u994d\u994e\u994f\u9950\u9951\u9952\u9953\u9954\u9955\u9956\u9957\u9958\u9959\u995a\u995b\u995c\u995d\u995e\u995f\u9960\u9961\u9962\u9963\u9964\u9965\u9966\u9967\u9968\u9969\u996a\u996b\u996c\u996d\u996e\u996f\u9970\u9971\u9972\u9973\u9974\u9975\u9976\u9977\u9978\u9979\u997a\u997b\u997c\u997d\u997e\u997f\u9980\u9981\u9982\u9983\u9984\u9985\u9986\u9987\u9988\u9989\u998a\u998b\u998c\u998d\u998e\u998f\u9990\u9991\u9992\u9993\u9994\u9995\u9996\u9997\u9998\u9999\u999a\u999b\u999c\u999d\u999e\u999f\u99a0\u99a1\u99a2\u99a3\u99a4\u99a5\u99a6\u99a7\u99a8\u99a9\u99aa\u99ab\u99ac\u99ad\u99ae\u99af\u99b0\u99b1\u99b2\u99b3\u99b4\u99b5\u99b6\u99b7\u99b8\u99b9\u99ba\u99bb\u99bc\u99bd\u99be\u99bf\u99c0\u99c1\u99c2\u99c3\u99c4\u99c5\u99c6\u99c7\u99c8\u99c9\u99ca\u99cb\u99cc\u99cd\u99ce\u99cf\u99d0\u99d1\u99d2\u99d3\u99d4\u99d5\u99d6\u99d7\u99d8\u99d9\u99da\u99db\u99dc\u99dd\u99de\u99df\u99e0\u99e1\u99e2\u99e3\u99e4\u99e5\u99e6\u99e7\u99e8\u99e9\u99ea\u99eb\u99ec\u99ed\u99ee\u99ef\u99f0\u99f1\u99f2\u99f3\u99f4\u99f5\u99f6\u99f7\u99f8\u99f9\u99fa\u99fb\u99fc\u99fd\u99fe\u99ff\u9a00\u9a01\u9a02\u9a03\u9a04\u9a05\u9a06\u9a07\u9a08\u9a09\u9a0a\u9a0b\u9a0c\u9a0d\u9a0e\u9a0f\u9a10\u9a11\u9a12\u9a13\u9a14\u9a15\u9a16\u9a17\u9a18\u9a19\u9a1a\u9a1b\u9a1c\u9a1d\u9a1e\u9a1f\u9a20\u9a21\u9a22\u9a23\u9a24\u9a25\u9a26\u9a27\u9a28\u9a29\u9a2a\u9a2b\u9a2c\u9a2d\u9a2e\u9a2f\u9a30\u9a31\u9a32\u9a33\u9a34\u9a35\u9a36\u9a37\u9a38\u9a39\u9a3a\u9a3b\u9a3c\u9a3d\u9a3e\u9a3f\u9a40\u9a41\u9a42\u9a43\u9a44\u9a45\u9a46\u9a47\u9a48\u9a49\u9a4a\u9a4b\u9a4c\u9a4d\u9a4e\u9a4f\u9a50\u9a51\u9a52\u9a53\u9a54\u9a55\u9a56\u9a57\u9a58\u9a59\u9a5a\u9a5b\u9a5c\u9a5d\u9a5e\u9a5f\u9a60\u9a61\u9a62\u9a63\u9a64\u9a65\u9a66\u9a67\u9a68\u9a69\u9a6a\u9a6b\u9a6c\u9a6d\u9a6e\u9a6f\u9a70\u9a71\u9a72\u9a73\u9a74\u9a75\u9a76\u9a77\u9a78\u9a79\u9a7a\u9a7b\u9a7c\u9a7d\u9a7e\u9a7f\u9a80\u9a81\u9a82\u9a83\u9a84\u9a85\u9a86\u9a87\u9a88\u9a89\u9a8a\u9a8b\u9a8c\u9a8d\u9a8e\u9a8f\u9a90\u9a91\u9a92\u9a93\u9a94\u9a95\u9a96\u9a97\u9a98\u9a99\u9a9a\u9a9b\u9a9c\u9a9d\u9a9e\u9a9f\u9aa0\u9aa1\u9aa2\u9aa3\u9aa4\u9aa5\u9aa6\u9aa7\u9aa8\u9aa9\u9aaa\u9aab\u9aac\u9aad\u9aae\u9aaf\u9ab0\u9ab1\u9ab2\u9ab3\u9ab4\u9ab5\u9ab6\u9ab7\u9ab8\u9ab9\u9aba\u9abb\u9abc\u9abd\u9abe\u9abf\u9ac0\u9ac1\u9ac2\u9ac3\u9ac4\u9ac5\u9ac6\u9ac7\u9ac8\u9ac9\u9aca\u9acb\u9acc\u9acd\u9ace\u9acf\u9ad0\u9ad1\u9ad2\u9ad3\u9ad4\u9ad5\u9ad6\u9ad7\u9ad8\u9ad9\u9ada\u9adb\u9adc\u9add\u9ade\u9adf\u9ae0\u9ae1\u9ae2\u9ae3\u9ae4\u9ae5\u9ae6\u9ae7\u9ae8\u9ae9\u9aea\u9aeb\u9aec\u9aed\u9aee\u9aef\u9af0\u9af1\u9af2\u9af3\u9af4\u9af5\u9af6\u9af7\u9af8\u9af9\u9afa\u9afb\u9afc\u9afd\u9afe\u9aff\u9b00\u9b01\u9b02\u9b03\u9b04\u9b05\u9b06\u9b07\u9b08\u9b09\u9b0a\u9b0b\u9b0c\u9b0d\u9b0e\u9b0f\u9b10\u9b11\u9b12\u9b13\u9b14\u9b15\u9b16\u9b17\u9b18\u9b19\u9b1a\u9b1b\u9b1c\u9b1d\u9b1e\u9b1f\u9b20\u9b21\u9b22\u9b23\u9b24\u9b25\u9b26\u9b27\u9b28\u9b29\u9b2a\u9b2b\u9b2c\u9b2d\u9b2e\u9b2f\u9b30\u9b31\u9b32\u9b33\u9b34\u9b35\u9b36\u9b37\u9b38\u9b39\u9b3a\u9b3b\u9b3c\u9b3d\u9b3e\u9b3f\u9b40\u9b41\u9b42\u9b43\u9b44\u9b45\u9b46\u9b47\u9b48\u9b49\u9b4a\u9b4b\u9b4c\u9b4d\u9b4e\u9b4f\u9b50\u9b51\u9b52\u9b53\u9b54\u9b55\u9b56\u9b57\u9b58\u9b59\u9b5a\u9b5b\u9b5c\u9b5d\u9b5e\u9b5f\u9b60\u9b61\u9b62\u9b63\u9b64\u9b65\u9b66\u9b67\u9b68\u9b69\u9b6a\u9b6b\u9b6c\u9b6d\u9b6e\u9b6f\u9b70\u9b71\u9b72\u9b73\u9b74\u9b75\u9b76\u9b77\u9b78\u9b79\u9b7a\u9b7b\u9b7c\u9b7d\u9b7e\u9b7f\u9b80\u9b81\u9b82\u9b83\u9b84\u9b85\u9b86\u9b87\u9b88\u9b89\u9b8a\u9b8b\u9b8c\u9b8d\u9b8e\u9b8f\u9b90\u9b91\u9b92\u9b93\u9b94\u9b95\u9b96\u9b97\u9b98\u9b99\u9b9a\u9b9b\u9b9c\u9b9d\u9b9e\u9b9f\u9ba0\u9ba1\u9ba2\u9ba3\u9ba4\u9ba5\u9ba6\u9ba7\u9ba8\u9ba9\u9baa\u9bab\u9bac\u9bad\u9bae\u9baf\u9bb0\u9bb1\u9bb2\u9bb3\u9bb4\u9bb5\u9bb6\u9bb7\u9bb8\u9bb9\u9bba\u9bbb\u9bbc\u9bbd\u9bbe\u9bbf\u9bc0\u9bc1\u9bc2\u9bc3\u9bc4\u9bc5\u9bc6\u9bc7\u9bc8\u9bc9\u9bca\u9bcb\u9bcc\u9bcd\u9bce\u9bcf\u9bd0\u9bd1\u9bd2\u9bd3\u9bd4\u9bd5\u9bd6\u9bd7\u9bd8\u9bd9\u9bda\u9bdb\u9bdc\u9bdd\u9bde\u9bdf\u9be0\u9be1\u9be2\u9be3\u9be4\u9be5\u9be6\u9be7\u9be8\u9be9\u9bea\u9beb\u9bec\u9bed\u9bee\u9bef\u9bf0\u9bf1\u9bf2\u9bf3\u9bf4\u9bf5\u9bf6\u9bf7\u9bf8\u9bf9\u9bfa\u9bfb\u9bfc\u9bfd\u9bfe\u9bff\u9c00\u9c01\u9c02\u9c03\u9c04\u9c05\u9c06\u9c07\u9c08\u9c09\u9c0a\u9c0b\u9c0c\u9c0d\u9c0e\u9c0f\u9c10\u9c11\u9c12\u9c13\u9c14\u9c15\u9c16\u9c17\u9c18\u9c19\u9c1a\u9c1b\u9c1c\u9c1d\u9c1e\u9c1f\u9c20\u9c21\u9c22\u9c23\u9c24\u9c25\u9c26\u9c27\u9c28\u9c29\u9c2a\u9c2b\u9c2c\u9c2d\u9c2e\u9c2f\u9c30\u9c31\u9c32\u9c33\u9c34\u9c35\u9c36\u9c37\u9c38\u9c39\u9c3a\u9c3b\u9c3c\u9c3d\u9c3e\u9c3f\u9c40\u9c41\u9c42\u9c43\u9c44\u9c45\u9c46\u9c47\u9c48\u9c49\u9c4a\u9c4b\u9c4c\u9c4d\u9c4e\u9c4f\u9c50\u9c51\u9c52\u9c53\u9c54\u9c55\u9c56\u9c57\u9c58\u9c59\u9c5a\u9c5b\u9c5c\u9c5d\u9c5e\u9c5f\u9c60\u9c61\u9c62\u9c63\u9c64\u9c65\u9c66\u9c67\u9c68\u9c69\u9c6a\u9c6b\u9c6c\u9c6d\u9c6e\u9c6f\u9c70\u9c71\u9c72\u9c73\u9c74\u9c75\u9c76\u9c77\u9c78\u9c79\u9c7a\u9c7b\u9c7c\u9c7d\u9c7e\u9c7f\u9c80\u9c81\u9c82\u9c83\u9c84\u9c85\u9c86\u9c87\u9c88\u9c89\u9c8a\u9c8b\u9c8c\u9c8d\u9c8e\u9c8f\u9c90\u9c91\u9c92\u9c93\u9c94\u9c95\u9c96\u9c97\u9c98\u9c99\u9c9a\u9c9b\u9c9c\u9c9d\u9c9e\u9c9f\u9ca0\u9ca1\u9ca2\u9ca3\u9ca4\u9ca5\u9ca6\u9ca7\u9ca8\u9ca9\u9caa\u9cab\u9cac\u9cad\u9cae\u9caf\u9cb0\u9cb1\u9cb2\u9cb3\u9cb4\u9cb5\u9cb6\u9cb7\u9cb8\u9cb9\u9cba\u9cbb\u9cbc\u9cbd\u9cbe\u9cbf\u9cc0\u9cc1\u9cc2\u9cc3\u9cc4\u9cc5\u9cc6\u9cc7\u9cc8\u9cc9\u9cca\u9ccb\u9ccc\u9ccd\u9cce\u9ccf\u9cd0\u9cd1\u9cd2\u9cd3\u9cd4\u9cd5\u9cd6\u9cd7\u9cd8\u9cd9\u9cda\u9cdb\u9cdc\u9cdd\u9cde\u9cdf\u9ce0\u9ce1\u9ce2\u9ce3\u9ce4\u9ce5\u9ce6\u9ce7\u9ce8\u9ce9\u9cea\u9ceb\u9cec\u9ced\u9cee\u9cef\u9cf0\u9cf1\u9cf2\u9cf3\u9cf4\u9cf5\u9cf6\u9cf7\u9cf8\u9cf9\u9cfa\u9cfb\u9cfc\u9cfd\u9cfe\u9cff\u9d00\u9d01\u9d02\u9d03\u9d04\u9d05\u9d06\u9d07\u9d08\u9d09\u9d0a\u9d0b\u9d0c\u9d0d\u9d0e\u9d0f\u9d10\u9d11\u9d12\u9d13\u9d14\u9d15\u9d16\u9d17\u9d18\u9d19\u9d1a\u9d1b\u9d1c\u9d1d\u9d1e\u9d1f\u9d20\u9d21\u9d22\u9d23\u9d24\u9d25\u9d26\u9d27\u9d28\u9d29\u9d2a\u9d2b\u9d2c\u9d2d\u9d2e\u9d2f\u9d30\u9d31\u9d32\u9d33\u9d34\u9d35\u9d36\u9d37\u9d38\u9d39\u9d3a\u9d3b\u9d3c\u9d3d\u9d3e\u9d3f\u9d40\u9d41\u9d42\u9d43\u9d44\u9d45\u9d46\u9d47\u9d48\u9d49\u9d4a\u9d4b\u9d4c\u9d4d\u9d4e\u9d4f\u9d50\u9d51\u9d52\u9d53\u9d54\u9d55\u9d56\u9d57\u9d58\u9d59\u9d5a\u9d5b\u9d5c\u9d5d\u9d5e\u9d5f\u9d60\u9d61\u9d62\u9d63\u9d64\u9d65\u9d66\u9d67\u9d68\u9d69\u9d6a\u9d6b\u9d6c\u9d6d\u9d6e\u9d6f\u9d70\u9d71\u9d72\u9d73\u9d74\u9d75\u9d76\u9d77\u9d78\u9d79\u9d7a\u9d7b\u9d7c\u9d7d\u9d7e\u9d7f\u9d80\u9d81\u9d82\u9d83\u9d84\u9d85\u9d86\u9d87\u9d88\u9d89\u9d8a\u9d8b\u9d8c\u9d8d\u9d8e\u9d8f\u9d90\u9d91\u9d92\u9d93\u9d94\u9d95\u9d96\u9d97\u9d98\u9d99\u9d9a\u9d9b\u9d9c\u9d9d\u9d9e\u9d9f\u9da0\u9da1\u9da2\u9da3\u9da4\u9da5\u9da6\u9da7\u9da8\u9da9\u9daa\u9dab\u9dac\u9dad\u9dae\u9daf\u9db0\u9db1\u9db2\u9db3\u9db4\u9db5\u9db6\u9db7\u9db8\u9db9\u9dba\u9dbb\u9dbc\u9dbd\u9dbe\u9dbf\u9dc0\u9dc1\u9dc2\u9dc3\u9dc4\u9dc5\u9dc6\u9dc7\u9dc8\u9dc9\u9dca\u9dcb\u9dcc\u9dcd\u9dce\u9dcf\u9dd0\u9dd1\u9dd2\u9dd3\u9dd4\u9dd5\u9dd6\u9dd7\u9dd8\u9dd9\u9dda\u9ddb\u9ddc\u9ddd\u9dde\u9ddf\u9de0\u9de1\u9de2\u9de3\u9de4\u9de5\u9de6\u9de7\u9de8\u9de9\u9dea\u9deb\u9dec\u9ded\u9dee\u9def\u9df0\u9df1\u9df2\u9df3\u9df4\u9df5\u9df6\u9df7\u9df8\u9df9\u9dfa\u9dfb\u9dfc\u9dfd\u9dfe\u9dff\u9e00\u9e01\u9e02\u9e03\u9e04\u9e05\u9e06\u9e07\u9e08\u9e09\u9e0a\u9e0b\u9e0c\u9e0d\u9e0e\u9e0f\u9e10\u9e11\u9e12\u9e13\u9e14\u9e15\u9e16\u9e17\u9e18\u9e19\u9e1a\u9e1b\u9e1c\u9e1d\u9e1e\u9e1f\u9e20\u9e21\u9e22\u9e23\u9e24\u9e25\u9e26\u9e27\u9e28\u9e29\u9e2a\u9e2b\u9e2c\u9e2d\u9e2e\u9e2f\u9e30\u9e31\u9e32\u9e33\u9e34\u9e35\u9e36\u9e37\u9e38\u9e39\u9e3a\u9e3b\u9e3c\u9e3d\u9e3e\u9e3f\u9e40\u9e41\u9e42\u9e43\u9e44\u9e45\u9e46\u9e47\u9e48\u9e49\u9e4a\u9e4b\u9e4c\u9e4d\u9e4e\u9e4f\u9e50\u9e51\u9e52\u9e53\u9e54\u9e55\u9e56\u9e57\u9e58\u9e59\u9e5a\u9e5b\u9e5c\u9e5d\u9e5e\u9e5f\u9e60\u9e61\u9e62\u9e63\u9e64\u9e65\u9e66\u9e67\u9e68\u9e69\u9e6a\u9e6b\u9e6c\u9e6d\u9e6e\u9e6f\u9e70\u9e71\u9e72\u9e73\u9e74\u9e75\u9e76\u9e77\u9e78\u9e79\u9e7a\u9e7b\u9e7c\u9e7d\u9e7e\u9e7f\u9e80\u9e81\u9e82\u9e83\u9e84\u9e85\u9e86\u9e87\u9e88\u9e89\u9e8a\u9e8b\u9e8c\u9e8d\u9e8e\u9e8f\u9e90\u9e91\u9e92\u9e93\u9e94\u9e95\u9e96\u9e97\u9e98\u9e99\u9e9a\u9e9b\u9e9c\u9e9d\u9e9e\u9e9f\u9ea0\u9ea1\u9ea2\u9ea3\u9ea4\u9ea5\u9ea6\u9ea7\u9ea8\u9ea9\u9eaa\u9eab\u9eac\u9ead\u9eae\u9eaf\u9eb0\u9eb1\u9eb2\u9eb3\u9eb4\u9eb5\u9eb6\u9eb7\u9eb8\u9eb9\u9eba\u9ebb\u9ebc\u9ebd\u9ebe\u9ebf\u9ec0\u9ec1\u9ec2\u9ec3\u9ec4\u9ec5\u9ec6\u9ec7\u9ec8\u9ec9\u9eca\u9ecb\u9ecc\u9ecd\u9ece\u9ecf\u9ed0\u9ed1\u9ed2\u9ed3\u9ed4\u9ed5\u9ed6\u9ed7\u9ed8\u9ed9\u9eda\u9edb\u9edc\u9edd\u9ede\u9edf\u9ee0\u9ee1\u9ee2\u9ee3\u9ee4\u9ee5\u9ee6\u9ee7\u9ee8\u9ee9\u9eea\u9eeb\u9eec\u9eed\u9eee\u9eef\u9ef0\u9ef1\u9ef2\u9ef3\u9ef4\u9ef5\u9ef6\u9ef7\u9ef8\u9ef9\u9efa\u9efb\u9efc\u9efd\u9efe\u9eff\u9f00\u9f01\u9f02\u9f03\u9f04\u9f05\u9f06\u9f07\u9f08\u9f09\u9f0a\u9f0b\u9f0c\u9f0d\u9f0e\u9f0f\u9f10\u9f11\u9f12\u9f13\u9f14\u9f15\u9f16\u9f17\u9f18\u9f19\u9f1a\u9f1b\u9f1c\u9f1d\u9f1e\u9f1f\u9f20\u9f21\u9f22\u9f23\u9f24\u9f25\u9f26\u9f27\u9f28\u9f29\u9f2a\u9f2b\u9f2c\u9f2d\u9f2e\u9f2f\u9f30\u9f31\u9f32\u9f33\u9f34\u9f35\u9f36\u9f37\u9f38\u9f39\u9f3a\u9f3b\u9f3c\u9f3d\u9f3e\u9f3f\u9f40\u9f41\u9f42\u9f43\u9f44\u9f45\u9f46\u9f47\u9f48\u9f49\u9f4a\u9f4b\u9f4c\u9f4d\u9f4e\u9f4f\u9f50\u9f51\u9f52\u9f53\u9f54\u9f55\u9f56\u9f57\u9f58\u9f59\u9f5a\u9f5b\u9f5c\u9f5d\u9f5e\u9f5f\u9f60\u9f61\u9f62\u9f63\u9f64\u9f65\u9f66\u9f67\u9f68\u9f69\u9f6a\u9f6b\u9f6c\u9f6d\u9f6e\u9f6f\u9f70\u9f71\u9f72\u9f73\u9f74\u9f75\u9f76\u9f77\u9f78\u9f79\u9f7a\u9f7b\u9f7c\u9f7d\u9f7e\u9f7f\u9f80\u9f81\u9f82\u9f83\u9f84\u9f85\u9f86\u9f87\u9f88\u9f89\u9f8a\u9f8b\u9f8c\u9f8d\u9f8e\u9f8f\u9f90\u9f91\u9f92\u9f93\u9f94\u9f95\u9f96\u9f97\u9f98\u9f99\u9f9a\u9f9b\u9f9c\u9f9d\u9f9e\u9f9f\u9fa0\u9fa1\u9fa2\u9fa3\u9fa4\u9fa5\u9fa6\u9fa7\u9fa8\u9fa9\u9faa\u9fab\u9fac\u9fad\u9fae\u9faf\u9fb0\u9fb1\u9fb2\u9fb3\u9fb4\u9fb5\u9fb6\u9fb7\u9fb8\u9fb9\u9fba\u9fbb\ua000\ua001\ua002\ua003\ua004\ua005\ua006\ua007\ua008\ua009\ua00a\ua00b\ua00c\ua00d\ua00e\ua00f\ua010\ua011\ua012\ua013\ua014\ua016\ua017\ua018\ua019\ua01a\ua01b\ua01c\ua01d\ua01e\ua01f\ua020\ua021\ua022\ua023\ua024\ua025\ua026\ua027\ua028\ua029\ua02a\ua02b\ua02c\ua02d\ua02e\ua02f\ua030\ua031\ua032\ua033\ua034\ua035\ua036\ua037\ua038\ua039\ua03a\ua03b\ua03c\ua03d\ua03e\ua03f\ua040\ua041\ua042\ua043\ua044\ua045\ua046\ua047\ua048\ua049\ua04a\ua04b\ua04c\ua04d\ua04e\ua04f\ua050\ua051\ua052\ua053\ua054\ua055\ua056\ua057\ua058\ua059\ua05a\ua05b\ua05c\ua05d\ua05e\ua05f\ua060\ua061\ua062\ua063\ua064\ua065\ua066\ua067\ua068\ua069\ua06a\ua06b\ua06c\ua06d\ua06e\ua06f\ua070\ua071\ua072\ua073\ua074\ua075\ua076\ua077\ua078\ua079\ua07a\ua07b\ua07c\ua07d\ua07e\ua07f\ua080\ua081\ua082\ua083\ua084\ua085\ua086\ua087\ua088\ua089\ua08a\ua08b\ua08c\ua08d\ua08e\ua08f\ua090\ua091\ua092\ua093\ua094\ua095\ua096\ua097\ua098\ua099\ua09a\ua09b\ua09c\ua09d\ua09e\ua09f\ua0a0\ua0a1\ua0a2\ua0a3\ua0a4\ua0a5\ua0a6\ua0a7\ua0a8\ua0a9\ua0aa\ua0ab\ua0ac\ua0ad\ua0ae\ua0af\ua0b0\ua0b1\ua0b2\ua0b3\ua0b4\ua0b5\ua0b6\ua0b7\ua0b8\ua0b9\ua0ba\ua0bb\ua0bc\ua0bd\ua0be\ua0bf\ua0c0\ua0c1\ua0c2\ua0c3\ua0c4\ua0c5\ua0c6\ua0c7\ua0c8\ua0c9\ua0ca\ua0cb\ua0cc\ua0cd\ua0ce\ua0cf\ua0d0\ua0d1\ua0d2\ua0d3\ua0d4\ua0d5\ua0d6\ua0d7\ua0d8\ua0d9\ua0da\ua0db\ua0dc\ua0dd\ua0de\ua0df\ua0e0\ua0e1\ua0e2\ua0e3\ua0e4\ua0e5\ua0e6\ua0e7\ua0e8\ua0e9\ua0ea\ua0eb\ua0ec\ua0ed\ua0ee\ua0ef\ua0f0\ua0f1\ua0f2\ua0f3\ua0f4\ua0f5\ua0f6\ua0f7\ua0f8\ua0f9\ua0fa\ua0fb\ua0fc\ua0fd\ua0fe\ua0ff\ua100\ua101\ua102\ua103\ua104\ua105\ua106\ua107\ua108\ua109\ua10a\ua10b\ua10c\ua10d\ua10e\ua10f\ua110\ua111\ua112\ua113\ua114\ua115\ua116\ua117\ua118\ua119\ua11a\ua11b\ua11c\ua11d\ua11e\ua11f\ua120\ua121\ua122\ua123\ua124\ua125\ua126\ua127\ua128\ua129\ua12a\ua12b\ua12c\ua12d\ua12e\ua12f\ua130\ua131\ua132\ua133\ua134\ua135\ua136\ua137\ua138\ua139\ua13a\ua13b\ua13c\ua13d\ua13e\ua13f\ua140\ua141\ua142\ua143\ua144\ua145\ua146\ua147\ua148\ua149\ua14a\ua14b\ua14c\ua14d\ua14e\ua14f\ua150\ua151\ua152\ua153\ua154\ua155\ua156\ua157\ua158\ua159\ua15a\ua15b\ua15c\ua15d\ua15e\ua15f\ua160\ua161\ua162\ua163\ua164\ua165\ua166\ua167\ua168\ua169\ua16a\ua16b\ua16c\ua16d\ua16e\ua16f\ua170\ua171\ua172\ua173\ua174\ua175\ua176\ua177\ua178\ua179\ua17a\ua17b\ua17c\ua17d\ua17e\ua17f\ua180\ua181\ua182\ua183\ua184\ua185\ua186\ua187\ua188\ua189\ua18a\ua18b\ua18c\ua18d\ua18e\ua18f\ua190\ua191\ua192\ua193\ua194\ua195\ua196\ua197\ua198\ua199\ua19a\ua19b\ua19c\ua19d\ua19e\ua19f\ua1a0\ua1a1\ua1a2\ua1a3\ua1a4\ua1a5\ua1a6\ua1a7\ua1a8\ua1a9\ua1aa\ua1ab\ua1ac\ua1ad\ua1ae\ua1af\ua1b0\ua1b1\ua1b2\ua1b3\ua1b4\ua1b5\ua1b6\ua1b7\ua1b8\ua1b9\ua1ba\ua1bb\ua1bc\ua1bd\ua1be\ua1bf\ua1c0\ua1c1\ua1c2\ua1c3\ua1c4\ua1c5\ua1c6\ua1c7\ua1c8\ua1c9\ua1ca\ua1cb\ua1cc\ua1cd\ua1ce\ua1cf\ua1d0\ua1d1\ua1d2\ua1d3\ua1d4\ua1d5\ua1d6\ua1d7\ua1d8\ua1d9\ua1da\ua1db\ua1dc\ua1dd\ua1de\ua1df\ua1e0\ua1e1\ua1e2\ua1e3\ua1e4\ua1e5\ua1e6\ua1e7\ua1e8\ua1e9\ua1ea\ua1eb\ua1ec\ua1ed\ua1ee\ua1ef\ua1f0\ua1f1\ua1f2\ua1f3\ua1f4\ua1f5\ua1f6\ua1f7\ua1f8\ua1f9\ua1fa\ua1fb\ua1fc\ua1fd\ua1fe\ua1ff\ua200\ua201\ua202\ua203\ua204\ua205\ua206\ua207\ua208\ua209\ua20a\ua20b\ua20c\ua20d\ua20e\ua20f\ua210\ua211\ua212\ua213\ua214\ua215\ua216\ua217\ua218\ua219\ua21a\ua21b\ua21c\ua21d\ua21e\ua21f\ua220\ua221\ua222\ua223\ua224\ua225\ua226\ua227\ua228\ua229\ua22a\ua22b\ua22c\ua22d\ua22e\ua22f\ua230\ua231\ua232\ua233\ua234\ua235\ua236\ua237\ua238\ua239\ua23a\ua23b\ua23c\ua23d\ua23e\ua23f\ua240\ua241\ua242\ua243\ua244\ua245\ua246\ua247\ua248\ua249\ua24a\ua24b\ua24c\ua24d\ua24e\ua24f\ua250\ua251\ua252\ua253\ua254\ua255\ua256\ua257\ua258\ua259\ua25a\ua25b\ua25c\ua25d\ua25e\ua25f\ua260\ua261\ua262\ua263\ua264\ua265\ua266\ua267\ua268\ua269\ua26a\ua26b\ua26c\ua26d\ua26e\ua26f\ua270\ua271\ua272\ua273\ua274\ua275\ua276\ua277\ua278\ua279\ua27a\ua27b\ua27c\ua27d\ua27e\ua27f\ua280\ua281\ua282\ua283\ua284\ua285\ua286\ua287\ua288\ua289\ua28a\ua28b\ua28c\ua28d\ua28e\ua28f\ua290\ua291\ua292\ua293\ua294\ua295\ua296\ua297\ua298\ua299\ua29a\ua29b\ua29c\ua29d\ua29e\ua29f\ua2a0\ua2a1\ua2a2\ua2a3\ua2a4\ua2a5\ua2a6\ua2a7\ua2a8\ua2a9\ua2aa\ua2ab\ua2ac\ua2ad\ua2ae\ua2af\ua2b0\ua2b1\ua2b2\ua2b3\ua2b4\ua2b5\ua2b6\ua2b7\ua2b8\ua2b9\ua2ba\ua2bb\ua2bc\ua2bd\ua2be\ua2bf\ua2c0\ua2c1\ua2c2\ua2c3\ua2c4\ua2c5\ua2c6\ua2c7\ua2c8\ua2c9\ua2ca\ua2cb\ua2cc\ua2cd\ua2ce\ua2cf\ua2d0\ua2d1\ua2d2\ua2d3\ua2d4\ua2d5\ua2d6\ua2d7\ua2d8\ua2d9\ua2da\ua2db\ua2dc\ua2dd\ua2de\ua2df\ua2e0\ua2e1\ua2e2\ua2e3\ua2e4\ua2e5\ua2e6\ua2e7\ua2e8\ua2e9\ua2ea\ua2eb\ua2ec\ua2ed\ua2ee\ua2ef\ua2f0\ua2f1\ua2f2\ua2f3\ua2f4\ua2f5\ua2f6\ua2f7\ua2f8\ua2f9\ua2fa\ua2fb\ua2fc\ua2fd\ua2fe\ua2ff\ua300\ua301\ua302\ua303\ua304\ua305\ua306\ua307\ua308\ua309\ua30a\ua30b\ua30c\ua30d\ua30e\ua30f\ua310\ua311\ua312\ua313\ua314\ua315\ua316\ua317\ua318\ua319\ua31a\ua31b\ua31c\ua31d\ua31e\ua31f\ua320\ua321\ua322\ua323\ua324\ua325\ua326\ua327\ua328\ua329\ua32a\ua32b\ua32c\ua32d\ua32e\ua32f\ua330\ua331\ua332\ua333\ua334\ua335\ua336\ua337\ua338\ua339\ua33a\ua33b\ua33c\ua33d\ua33e\ua33f\ua340\ua341\ua342\ua343\ua344\ua345\ua346\ua347\ua348\ua349\ua34a\ua34b\ua34c\ua34d\ua34e\ua34f\ua350\ua351\ua352\ua353\ua354\ua355\ua356\ua357\ua358\ua359\ua35a\ua35b\ua35c\ua35d\ua35e\ua35f\ua360\ua361\ua362\ua363\ua364\ua365\ua366\ua367\ua368\ua369\ua36a\ua36b\ua36c\ua36d\ua36e\ua36f\ua370\ua371\ua372\ua373\ua374\ua375\ua376\ua377\ua378\ua379\ua37a\ua37b\ua37c\ua37d\ua37e\ua37f\ua380\ua381\ua382\ua383\ua384\ua385\ua386\ua387\ua388\ua389\ua38a\ua38b\ua38c\ua38d\ua38e\ua38f\ua390\ua391\ua392\ua393\ua394\ua395\ua396\ua397\ua398\ua399\ua39a\ua39b\ua39c\ua39d\ua39e\ua39f\ua3a0\ua3a1\ua3a2\ua3a3\ua3a4\ua3a5\ua3a6\ua3a7\ua3a8\ua3a9\ua3aa\ua3ab\ua3ac\ua3ad\ua3ae\ua3af\ua3b0\ua3b1\ua3b2\ua3b3\ua3b4\ua3b5\ua3b6\ua3b7\ua3b8\ua3b9\ua3ba\ua3bb\ua3bc\ua3bd\ua3be\ua3bf\ua3c0\ua3c1\ua3c2\ua3c3\ua3c4\ua3c5\ua3c6\ua3c7\ua3c8\ua3c9\ua3ca\ua3cb\ua3cc\ua3cd\ua3ce\ua3cf\ua3d0\ua3d1\ua3d2\ua3d3\ua3d4\ua3d5\ua3d6\ua3d7\ua3d8\ua3d9\ua3da\ua3db\ua3dc\ua3dd\ua3de\ua3df\ua3e0\ua3e1\ua3e2\ua3e3\ua3e4\ua3e5\ua3e6\ua3e7\ua3e8\ua3e9\ua3ea\ua3eb\ua3ec\ua3ed\ua3ee\ua3ef\ua3f0\ua3f1\ua3f2\ua3f3\ua3f4\ua3f5\ua3f6\ua3f7\ua3f8\ua3f9\ua3fa\ua3fb\ua3fc\ua3fd\ua3fe\ua3ff\ua400\ua401\ua402\ua403\ua404\ua405\ua406\ua407\ua408\ua409\ua40a\ua40b\ua40c\ua40d\ua40e\ua40f\ua410\ua411\ua412\ua413\ua414\ua415\ua416\ua417\ua418\ua419\ua41a\ua41b\ua41c\ua41d\ua41e\ua41f\ua420\ua421\ua422\ua423\ua424\ua425\ua426\ua427\ua428\ua429\ua42a\ua42b\ua42c\ua42d\ua42e\ua42f\ua430\ua431\ua432\ua433\ua434\ua435\ua436\ua437\ua438\ua439\ua43a\ua43b\ua43c\ua43d\ua43e\ua43f\ua440\ua441\ua442\ua443\ua444\ua445\ua446\ua447\ua448\ua449\ua44a\ua44b\ua44c\ua44d\ua44e\ua44f\ua450\ua451\ua452\ua453\ua454\ua455\ua456\ua457\ua458\ua459\ua45a\ua45b\ua45c\ua45d\ua45e\ua45f\ua460\ua461\ua462\ua463\ua464\ua465\ua466\ua467\ua468\ua469\ua46a\ua46b\ua46c\ua46d\ua46e\ua46f\ua470\ua471\ua472\ua473\ua474\ua475\ua476\ua477\ua478\ua479\ua47a\ua47b\ua47c\ua47d\ua47e\ua47f\ua480\ua481\ua482\ua483\ua484\ua485\ua486\ua487\ua488\ua489\ua48a\ua48b\ua48c\ua800\ua801\ua803\ua804\ua805\ua807\ua808\ua809\ua80a\ua80c\ua80d\ua80e\ua80f\ua810\ua811\ua812\ua813\ua814\ua815\ua816\ua817\ua818\ua819\ua81a\ua81b\ua81c\ua81d\ua81e\ua81f\ua820\ua821\ua822\uac00\uac01\uac02\uac03\uac04\uac05\uac06\uac07\uac08\uac09\uac0a\uac0b\uac0c\uac0d\uac0e\uac0f\uac10\uac11\uac12\uac13\uac14\uac15\uac16\uac17\uac18\uac19\uac1a\uac1b\uac1c\uac1d\uac1e\uac1f\uac20\uac21\uac22\uac23\uac24\uac25\uac26\uac27\uac28\uac29\uac2a\uac2b\uac2c\uac2d\uac2e\uac2f\uac30\uac31\uac32\uac33\uac34\uac35\uac36\uac37\uac38\uac39\uac3a\uac3b\uac3c\uac3d\uac3e\uac3f\uac40\uac41\uac42\uac43\uac44\uac45\uac46\uac47\uac48\uac49\uac4a\uac4b\uac4c\uac4d\uac4e\uac4f\uac50\uac51\uac52\uac53\uac54\uac55\uac56\uac57\uac58\uac59\uac5a\uac5b\uac5c\uac5d\uac5e\uac5f\uac60\uac61\uac62\uac63\uac64\uac65\uac66\uac67\uac68\uac69\uac6a\uac6b\uac6c\uac6d\uac6e\uac6f\uac70\uac71\uac72\uac73\uac74\uac75\uac76\uac77\uac78\uac79\uac7a\uac7b\uac7c\uac7d\uac7e\uac7f\uac80\uac81\uac82\uac83\uac84\uac85\uac86\uac87\uac88\uac89\uac8a\uac8b\uac8c\uac8d\uac8e\uac8f\uac90\uac91\uac92\uac93\uac94\uac95\uac96\uac97\uac98\uac99\uac9a\uac9b\uac9c\uac9d\uac9e\uac9f\uaca0\uaca1\uaca2\uaca3\uaca4\uaca5\uaca6\uaca7\uaca8\uaca9\uacaa\uacab\uacac\uacad\uacae\uacaf\uacb0\uacb1\uacb2\uacb3\uacb4\uacb5\uacb6\uacb7\uacb8\uacb9\uacba\uacbb\uacbc\uacbd\uacbe\uacbf\uacc0\uacc1\uacc2\uacc3\uacc4\uacc5\uacc6\uacc7\uacc8\uacc9\uacca\uaccb\uaccc\uaccd\uacce\uaccf\uacd0\uacd1\uacd2\uacd3\uacd4\uacd5\uacd6\uacd7\uacd8\uacd9\uacda\uacdb\uacdc\uacdd\uacde\uacdf\uace0\uace1\uace2\uace3\uace4\uace5\uace6\uace7\uace8\uace9\uacea\uaceb\uacec\uaced\uacee\uacef\uacf0\uacf1\uacf2\uacf3\uacf4\uacf5\uacf6\uacf7\uacf8\uacf9\uacfa\uacfb\uacfc\uacfd\uacfe\uacff\uad00\uad01\uad02\uad03\uad04\uad05\uad06\uad07\uad08\uad09\uad0a\uad0b\uad0c\uad0d\uad0e\uad0f\uad10\uad11\uad12\uad13\uad14\uad15\uad16\uad17\uad18\uad19\uad1a\uad1b\uad1c\uad1d\uad1e\uad1f\uad20\uad21\uad22\uad23\uad24\uad25\uad26\uad27\uad28\uad29\uad2a\uad2b\uad2c\uad2d\uad2e\uad2f\uad30\uad31\uad32\uad33\uad34\uad35\uad36\uad37\uad38\uad39\uad3a\uad3b\uad3c\uad3d\uad3e\uad3f\uad40\uad41\uad42\uad43\uad44\uad45\uad46\uad47\uad48\uad49\uad4a\uad4b\uad4c\uad4d\uad4e\uad4f\uad50\uad51\uad52\uad53\uad54\uad55\uad56\uad57\uad58\uad59\uad5a\uad5b\uad5c\uad5d\uad5e\uad5f\uad60\uad61\uad62\uad63\uad64\uad65\uad66\uad67\uad68\uad69\uad6a\uad6b\uad6c\uad6d\uad6e\uad6f\uad70\uad71\uad72\uad73\uad74\uad75\uad76\uad77\uad78\uad79\uad7a\uad7b\uad7c\uad7d\uad7e\uad7f\uad80\uad81\uad82\uad83\uad84\uad85\uad86\uad87\uad88\uad89\uad8a\uad8b\uad8c\uad8d\uad8e\uad8f\uad90\uad91\uad92\uad93\uad94\uad95\uad96\uad97\uad98\uad99\uad9a\uad9b\uad9c\uad9d\uad9e\uad9f\uada0\uada1\uada2\uada3\uada4\uada5\uada6\uada7\uada8\uada9\uadaa\uadab\uadac\uadad\uadae\uadaf\uadb0\uadb1\uadb2\uadb3\uadb4\uadb5\uadb6\uadb7\uadb8\uadb9\uadba\uadbb\uadbc\uadbd\uadbe\uadbf\uadc0\uadc1\uadc2\uadc3\uadc4\uadc5\uadc6\uadc7\uadc8\uadc9\uadca\uadcb\uadcc\uadcd\uadce\uadcf\uadd0\uadd1\uadd2\uadd3\uadd4\uadd5\uadd6\uadd7\uadd8\uadd9\uadda\uaddb\uaddc\uaddd\uadde\uaddf\uade0\uade1\uade2\uade3\uade4\uade5\uade6\uade7\uade8\uade9\uadea\uadeb\uadec\uaded\uadee\uadef\uadf0\uadf1\uadf2\uadf3\uadf4\uadf5\uadf6\uadf7\uadf8\uadf9\uadfa\uadfb\uadfc\uadfd\uadfe\uadff\uae00\uae01\uae02\uae03\uae04\uae05\uae06\uae07\uae08\uae09\uae0a\uae0b\uae0c\uae0d\uae0e\uae0f\uae10\uae11\uae12\uae13\uae14\uae15\uae16\uae17\uae18\uae19\uae1a\uae1b\uae1c\uae1d\uae1e\uae1f\uae20\uae21\uae22\uae23\uae24\uae25\uae26\uae27\uae28\uae29\uae2a\uae2b\uae2c\uae2d\uae2e\uae2f\uae30\uae31\uae32\uae33\uae34\uae35\uae36\uae37\uae38\uae39\uae3a\uae3b\uae3c\uae3d\uae3e\uae3f\uae40\uae41\uae42\uae43\uae44\uae45\uae46\uae47\uae48\uae49\uae4a\uae4b\uae4c\uae4d\uae4e\uae4f\uae50\uae51\uae52\uae53\uae54\uae55\uae56\uae57\uae58\uae59\uae5a\uae5b\uae5c\uae5d\uae5e\uae5f\uae60\uae61\uae62\uae63\uae64\uae65\uae66\uae67\uae68\uae69\uae6a\uae6b\uae6c\uae6d\uae6e\uae6f\uae70\uae71\uae72\uae73\uae74\uae75\uae76\uae77\uae78\uae79\uae7a\uae7b\uae7c\uae7d\uae7e\uae7f\uae80\uae81\uae82\uae83\uae84\uae85\uae86\uae87\uae88\uae89\uae8a\uae8b\uae8c\uae8d\uae8e\uae8f\uae90\uae91\uae92\uae93\uae94\uae95\uae96\uae97\uae98\uae99\uae9a\uae9b\uae9c\uae9d\uae9e\uae9f\uaea0\uaea1\uaea2\uaea3\uaea4\uaea5\uaea6\uaea7\uaea8\uaea9\uaeaa\uaeab\uaeac\uaead\uaeae\uaeaf\uaeb0\uaeb1\uaeb2\uaeb3\uaeb4\uaeb5\uaeb6\uaeb7\uaeb8\uaeb9\uaeba\uaebb\uaebc\uaebd\uaebe\uaebf\uaec0\uaec1\uaec2\uaec3\uaec4\uaec5\uaec6\uaec7\uaec8\uaec9\uaeca\uaecb\uaecc\uaecd\uaece\uaecf\uaed0\uaed1\uaed2\uaed3\uaed4\uaed5\uaed6\uaed7\uaed8\uaed9\uaeda\uaedb\uaedc\uaedd\uaede\uaedf\uaee0\uaee1\uaee2\uaee3\uaee4\uaee5\uaee6\uaee7\uaee8\uaee9\uaeea\uaeeb\uaeec\uaeed\uaeee\uaeef\uaef0\uaef1\uaef2\uaef3\uaef4\uaef5\uaef6\uaef7\uaef8\uaef9\uaefa\uaefb\uaefc\uaefd\uaefe\uaeff\uaf00\uaf01\uaf02\uaf03\uaf04\uaf05\uaf06\uaf07\uaf08\uaf09\uaf0a\uaf0b\uaf0c\uaf0d\uaf0e\uaf0f\uaf10\uaf11\uaf12\uaf13\uaf14\uaf15\uaf16\uaf17\uaf18\uaf19\uaf1a\uaf1b\uaf1c\uaf1d\uaf1e\uaf1f\uaf20\uaf21\uaf22\uaf23\uaf24\uaf25\uaf26\uaf27\uaf28\uaf29\uaf2a\uaf2b\uaf2c\uaf2d\uaf2e\uaf2f\uaf30\uaf31\uaf32\uaf33\uaf34\uaf35\uaf36\uaf37\uaf38\uaf39\uaf3a\uaf3b\uaf3c\uaf3d\uaf3e\uaf3f\uaf40\uaf41\uaf42\uaf43\uaf44\uaf45\uaf46\uaf47\uaf48\uaf49\uaf4a\uaf4b\uaf4c\uaf4d\uaf4e\uaf4f\uaf50\uaf51\uaf52\uaf53\uaf54\uaf55\uaf56\uaf57\uaf58\uaf59\uaf5a\uaf5b\uaf5c\uaf5d\uaf5e\uaf5f\uaf60\uaf61\uaf62\uaf63\uaf64\uaf65\uaf66\uaf67\uaf68\uaf69\uaf6a\uaf6b\uaf6c\uaf6d\uaf6e\uaf6f\uaf70\uaf71\uaf72\uaf73\uaf74\uaf75\uaf76\uaf77\uaf78\uaf79\uaf7a\uaf7b\uaf7c\uaf7d\uaf7e\uaf7f\uaf80\uaf81\uaf82\uaf83\uaf84\uaf85\uaf86\uaf87\uaf88\uaf89\uaf8a\uaf8b\uaf8c\uaf8d\uaf8e\uaf8f\uaf90\uaf91\uaf92\uaf93\uaf94\uaf95\uaf96\uaf97\uaf98\uaf99\uaf9a\uaf9b\uaf9c\uaf9d\uaf9e\uaf9f\uafa0\uafa1\uafa2\uafa3\uafa4\uafa5\uafa6\uafa7\uafa8\uafa9\uafaa\uafab\uafac\uafad\uafae\uafaf\uafb0\uafb1\uafb2\uafb3\uafb4\uafb5\uafb6\uafb7\uafb8\uafb9\uafba\uafbb\uafbc\uafbd\uafbe\uafbf\uafc0\uafc1\uafc2\uafc3\uafc4\uafc5\uafc6\uafc7\uafc8\uafc9\uafca\uafcb\uafcc\uafcd\uafce\uafcf\uafd0\uafd1\uafd2\uafd3\uafd4\uafd5\uafd6\uafd7\uafd8\uafd9\uafda\uafdb\uafdc\uafdd\uafde\uafdf\uafe0\uafe1\uafe2\uafe3\uafe4\uafe5\uafe6\uafe7\uafe8\uafe9\uafea\uafeb\uafec\uafed\uafee\uafef\uaff0\uaff1\uaff2\uaff3\uaff4\uaff5\uaff6\uaff7\uaff8\uaff9\uaffa\uaffb\uaffc\uaffd\uaffe\uafff\ub000\ub001\ub002\ub003\ub004\ub005\ub006\ub007\ub008\ub009\ub00a\ub00b\ub00c\ub00d\ub00e\ub00f\ub010\ub011\ub012\ub013\ub014\ub015\ub016\ub017\ub018\ub019\ub01a\ub01b\ub01c\ub01d\ub01e\ub01f\ub020\ub021\ub022\ub023\ub024\ub025\ub026\ub027\ub028\ub029\ub02a\ub02b\ub02c\ub02d\ub02e\ub02f\ub030\ub031\ub032\ub033\ub034\ub035\ub036\ub037\ub038\ub039\ub03a\ub03b\ub03c\ub03d\ub03e\ub03f\ub040\ub041\ub042\ub043\ub044\ub045\ub046\ub047\ub048\ub049\ub04a\ub04b\ub04c\ub04d\ub04e\ub04f\ub050\ub051\ub052\ub053\ub054\ub055\ub056\ub057\ub058\ub059\ub05a\ub05b\ub05c\ub05d\ub05e\ub05f\ub060\ub061\ub062\ub063\ub064\ub065\ub066\ub067\ub068\ub069\ub06a\ub06b\ub06c\ub06d\ub06e\ub06f\ub070\ub071\ub072\ub073\ub074\ub075\ub076\ub077\ub078\ub079\ub07a\ub07b\ub07c\ub07d\ub07e\ub07f\ub080\ub081\ub082\ub083\ub084\ub085\ub086\ub087\ub088\ub089\ub08a\ub08b\ub08c\ub08d\ub08e\ub08f\ub090\ub091\ub092\ub093\ub094\ub095\ub096\ub097\ub098\ub099\ub09a\ub09b\ub09c\ub09d\ub09e\ub09f\ub0a0\ub0a1\ub0a2\ub0a3\ub0a4\ub0a5\ub0a6\ub0a7\ub0a8\ub0a9\ub0aa\ub0ab\ub0ac\ub0ad\ub0ae\ub0af\ub0b0\ub0b1\ub0b2\ub0b3\ub0b4\ub0b5\ub0b6\ub0b7\ub0b8\ub0b9\ub0ba\ub0bb\ub0bc\ub0bd\ub0be\ub0bf\ub0c0\ub0c1\ub0c2\ub0c3\ub0c4\ub0c5\ub0c6\ub0c7\ub0c8\ub0c9\ub0ca\ub0cb\ub0cc\ub0cd\ub0ce\ub0cf\ub0d0\ub0d1\ub0d2\ub0d3\ub0d4\ub0d5\ub0d6\ub0d7\ub0d8\ub0d9\ub0da\ub0db\ub0dc\ub0dd\ub0de\ub0df\ub0e0\ub0e1\ub0e2\ub0e3\ub0e4\ub0e5\ub0e6\ub0e7\ub0e8\ub0e9\ub0ea\ub0eb\ub0ec\ub0ed\ub0ee\ub0ef\ub0f0\ub0f1\ub0f2\ub0f3\ub0f4\ub0f5\ub0f6\ub0f7\ub0f8\ub0f9\ub0fa\ub0fb\ub0fc\ub0fd\ub0fe\ub0ff\ub100\ub101\ub102\ub103\ub104\ub105\ub106\ub107\ub108\ub109\ub10a\ub10b\ub10c\ub10d\ub10e\ub10f\ub110\ub111\ub112\ub113\ub114\ub115\ub116\ub117\ub118\ub119\ub11a\ub11b\ub11c\ub11d\ub11e\ub11f\ub120\ub121\ub122\ub123\ub124\ub125\ub126\ub127\ub128\ub129\ub12a\ub12b\ub12c\ub12d\ub12e\ub12f\ub130\ub131\ub132\ub133\ub134\ub135\ub136\ub137\ub138\ub139\ub13a\ub13b\ub13c\ub13d\ub13e\ub13f\ub140\ub141\ub142\ub143\ub144\ub145\ub146\ub147\ub148\ub149\ub14a\ub14b\ub14c\ub14d\ub14e\ub14f\ub150\ub151\ub152\ub153\ub154\ub155\ub156\ub157\ub158\ub159\ub15a\ub15b\ub15c\ub15d\ub15e\ub15f\ub160\ub161\ub162\ub163\ub164\ub165\ub166\ub167\ub168\ub169\ub16a\ub16b\ub16c\ub16d\ub16e\ub16f\ub170\ub171\ub172\ub173\ub174\ub175\ub176\ub177\ub178\ub179\ub17a\ub17b\ub17c\ub17d\ub17e\ub17f\ub180\ub181\ub182\ub183\ub184\ub185\ub186\ub187\ub188\ub189\ub18a\ub18b\ub18c\ub18d\ub18e\ub18f\ub190\ub191\ub192\ub193\ub194\ub195\ub196\ub197\ub198\ub199\ub19a\ub19b\ub19c\ub19d\ub19e\ub19f\ub1a0\ub1a1\ub1a2\ub1a3\ub1a4\ub1a5\ub1a6\ub1a7\ub1a8\ub1a9\ub1aa\ub1ab\ub1ac\ub1ad\ub1ae\ub1af\ub1b0\ub1b1\ub1b2\ub1b3\ub1b4\ub1b5\ub1b6\ub1b7\ub1b8\ub1b9\ub1ba\ub1bb\ub1bc\ub1bd\ub1be\ub1bf\ub1c0\ub1c1\ub1c2\ub1c3\ub1c4\ub1c5\ub1c6\ub1c7\ub1c8\ub1c9\ub1ca\ub1cb\ub1cc\ub1cd\ub1ce\ub1cf\ub1d0\ub1d1\ub1d2\ub1d3\ub1d4\ub1d5\ub1d6\ub1d7\ub1d8\ub1d9\ub1da\ub1db\ub1dc\ub1dd\ub1de\ub1df\ub1e0\ub1e1\ub1e2\ub1e3\ub1e4\ub1e5\ub1e6\ub1e7\ub1e8\ub1e9\ub1ea\ub1eb\ub1ec\ub1ed\ub1ee\ub1ef\ub1f0\ub1f1\ub1f2\ub1f3\ub1f4\ub1f5\ub1f6\ub1f7\ub1f8\ub1f9\ub1fa\ub1fb\ub1fc\ub1fd\ub1fe\ub1ff\ub200\ub201\ub202\ub203\ub204\ub205\ub206\ub207\ub208\ub209\ub20a\ub20b\ub20c\ub20d\ub20e\ub20f\ub210\ub211\ub212\ub213\ub214\ub215\ub216\ub217\ub218\ub219\ub21a\ub21b\ub21c\ub21d\ub21e\ub21f\ub220\ub221\ub222\ub223\ub224\ub225\ub226\ub227\ub228\ub229\ub22a\ub22b\ub22c\ub22d\ub22e\ub22f\ub230\ub231\ub232\ub233\ub234\ub235\ub236\ub237\ub238\ub239\ub23a\ub23b\ub23c\ub23d\ub23e\ub23f\ub240\ub241\ub242\ub243\ub244\ub245\ub246\ub247\ub248\ub249\ub24a\ub24b\ub24c\ub24d\ub24e\ub24f\ub250\ub251\ub252\ub253\ub254\ub255\ub256\ub257\ub258\ub259\ub25a\ub25b\ub25c\ub25d\ub25e\ub25f\ub260\ub261\ub262\ub263\ub264\ub265\ub266\ub267\ub268\ub269\ub26a\ub26b\ub26c\ub26d\ub26e\ub26f\ub270\ub271\ub272\ub273\ub274\ub275\ub276\ub277\ub278\ub279\ub27a\ub27b\ub27c\ub27d\ub27e\ub27f\ub280\ub281\ub282\ub283\ub284\ub285\ub286\ub287\ub288\ub289\ub28a\ub28b\ub28c\ub28d\ub28e\ub28f\ub290\ub291\ub292\ub293\ub294\ub295\ub296\ub297\ub298\ub299\ub29a\ub29b\ub29c\ub29d\ub29e\ub29f\ub2a0\ub2a1\ub2a2\ub2a3\ub2a4\ub2a5\ub2a6\ub2a7\ub2a8\ub2a9\ub2aa\ub2ab\ub2ac\ub2ad\ub2ae\ub2af\ub2b0\ub2b1\ub2b2\ub2b3\ub2b4\ub2b5\ub2b6\ub2b7\ub2b8\ub2b9\ub2ba\ub2bb\ub2bc\ub2bd\ub2be\ub2bf\ub2c0\ub2c1\ub2c2\ub2c3\ub2c4\ub2c5\ub2c6\ub2c7\ub2c8\ub2c9\ub2ca\ub2cb\ub2cc\ub2cd\ub2ce\ub2cf\ub2d0\ub2d1\ub2d2\ub2d3\ub2d4\ub2d5\ub2d6\ub2d7\ub2d8\ub2d9\ub2da\ub2db\ub2dc\ub2dd\ub2de\ub2df\ub2e0\ub2e1\ub2e2\ub2e3\ub2e4\ub2e5\ub2e6\ub2e7\ub2e8\ub2e9\ub2ea\ub2eb\ub2ec\ub2ed\ub2ee\ub2ef\ub2f0\ub2f1\ub2f2\ub2f3\ub2f4\ub2f5\ub2f6\ub2f7\ub2f8\ub2f9\ub2fa\ub2fb\ub2fc\ub2fd\ub2fe\ub2ff\ub300\ub301\ub302\ub303\ub304\ub305\ub306\ub307\ub308\ub309\ub30a\ub30b\ub30c\ub30d\ub30e\ub30f\ub310\ub311\ub312\ub313\ub314\ub315\ub316\ub317\ub318\ub319\ub31a\ub31b\ub31c\ub31d\ub31e\ub31f\ub320\ub321\ub322\ub323\ub324\ub325\ub326\ub327\ub328\ub329\ub32a\ub32b\ub32c\ub32d\ub32e\ub32f\ub330\ub331\ub332\ub333\ub334\ub335\ub336\ub337\ub338\ub339\ub33a\ub33b\ub33c\ub33d\ub33e\ub33f\ub340\ub341\ub342\ub343\ub344\ub345\ub346\ub347\ub348\ub349\ub34a\ub34b\ub34c\ub34d\ub34e\ub34f\ub350\ub351\ub352\ub353\ub354\ub355\ub356\ub357\ub358\ub359\ub35a\ub35b\ub35c\ub35d\ub35e\ub35f\ub360\ub361\ub362\ub363\ub364\ub365\ub366\ub367\ub368\ub369\ub36a\ub36b\ub36c\ub36d\ub36e\ub36f\ub370\ub371\ub372\ub373\ub374\ub375\ub376\ub377\ub378\ub379\ub37a\ub37b\ub37c\ub37d\ub37e\ub37f\ub380\ub381\ub382\ub383\ub384\ub385\ub386\ub387\ub388\ub389\ub38a\ub38b\ub38c\ub38d\ub38e\ub38f\ub390\ub391\ub392\ub393\ub394\ub395\ub396\ub397\ub398\ub399\ub39a\ub39b\ub39c\ub39d\ub39e\ub39f\ub3a0\ub3a1\ub3a2\ub3a3\ub3a4\ub3a5\ub3a6\ub3a7\ub3a8\ub3a9\ub3aa\ub3ab\ub3ac\ub3ad\ub3ae\ub3af\ub3b0\ub3b1\ub3b2\ub3b3\ub3b4\ub3b5\ub3b6\ub3b7\ub3b8\ub3b9\ub3ba\ub3bb\ub3bc\ub3bd\ub3be\ub3bf\ub3c0\ub3c1\ub3c2\ub3c3\ub3c4\ub3c5\ub3c6\ub3c7\ub3c8\ub3c9\ub3ca\ub3cb\ub3cc\ub3cd\ub3ce\ub3cf\ub3d0\ub3d1\ub3d2\ub3d3\ub3d4\ub3d5\ub3d6\ub3d7\ub3d8\ub3d9\ub3da\ub3db\ub3dc\ub3dd\ub3de\ub3df\ub3e0\ub3e1\ub3e2\ub3e3\ub3e4\ub3e5\ub3e6\ub3e7\ub3e8\ub3e9\ub3ea\ub3eb\ub3ec\ub3ed\ub3ee\ub3ef\ub3f0\ub3f1\ub3f2\ub3f3\ub3f4\ub3f5\ub3f6\ub3f7\ub3f8\ub3f9\ub3fa\ub3fb\ub3fc\ub3fd\ub3fe\ub3ff\ub400\ub401\ub402\ub403\ub404\ub405\ub406\ub407\ub408\ub409\ub40a\ub40b\ub40c\ub40d\ub40e\ub40f\ub410\ub411\ub412\ub413\ub414\ub415\ub416\ub417\ub418\ub419\ub41a\ub41b\ub41c\ub41d\ub41e\ub41f\ub420\ub421\ub422\ub423\ub424\ub425\ub426\ub427\ub428\ub429\ub42a\ub42b\ub42c\ub42d\ub42e\ub42f\ub430\ub431\ub432\ub433\ub434\ub435\ub436\ub437\ub438\ub439\ub43a\ub43b\ub43c\ub43d\ub43e\ub43f\ub440\ub441\ub442\ub443\ub444\ub445\ub446\ub447\ub448\ub449\ub44a\ub44b\ub44c\ub44d\ub44e\ub44f\ub450\ub451\ub452\ub453\ub454\ub455\ub456\ub457\ub458\ub459\ub45a\ub45b\ub45c\ub45d\ub45e\ub45f\ub460\ub461\ub462\ub463\ub464\ub465\ub466\ub467\ub468\ub469\ub46a\ub46b\ub46c\ub46d\ub46e\ub46f\ub470\ub471\ub472\ub473\ub474\ub475\ub476\ub477\ub478\ub479\ub47a\ub47b\ub47c\ub47d\ub47e\ub47f\ub480\ub481\ub482\ub483\ub484\ub485\ub486\ub487\ub488\ub489\ub48a\ub48b\ub48c\ub48d\ub48e\ub48f\ub490\ub491\ub492\ub493\ub494\ub495\ub496\ub497\ub498\ub499\ub49a\ub49b\ub49c\ub49d\ub49e\ub49f\ub4a0\ub4a1\ub4a2\ub4a3\ub4a4\ub4a5\ub4a6\ub4a7\ub4a8\ub4a9\ub4aa\ub4ab\ub4ac\ub4ad\ub4ae\ub4af\ub4b0\ub4b1\ub4b2\ub4b3\ub4b4\ub4b5\ub4b6\ub4b7\ub4b8\ub4b9\ub4ba\ub4bb\ub4bc\ub4bd\ub4be\ub4bf\ub4c0\ub4c1\ub4c2\ub4c3\ub4c4\ub4c5\ub4c6\ub4c7\ub4c8\ub4c9\ub4ca\ub4cb\ub4cc\ub4cd\ub4ce\ub4cf\ub4d0\ub4d1\ub4d2\ub4d3\ub4d4\ub4d5\ub4d6\ub4d7\ub4d8\ub4d9\ub4da\ub4db\ub4dc\ub4dd\ub4de\ub4df\ub4e0\ub4e1\ub4e2\ub4e3\ub4e4\ub4e5\ub4e6\ub4e7\ub4e8\ub4e9\ub4ea\ub4eb\ub4ec\ub4ed\ub4ee\ub4ef\ub4f0\ub4f1\ub4f2\ub4f3\ub4f4\ub4f5\ub4f6\ub4f7\ub4f8\ub4f9\ub4fa\ub4fb\ub4fc\ub4fd\ub4fe\ub4ff\ub500\ub501\ub502\ub503\ub504\ub505\ub506\ub507\ub508\ub509\ub50a\ub50b\ub50c\ub50d\ub50e\ub50f\ub510\ub511\ub512\ub513\ub514\ub515\ub516\ub517\ub518\ub519\ub51a\ub51b\ub51c\ub51d\ub51e\ub51f\ub520\ub521\ub522\ub523\ub524\ub525\ub526\ub527\ub528\ub529\ub52a\ub52b\ub52c\ub52d\ub52e\ub52f\ub530\ub531\ub532\ub533\ub534\ub535\ub536\ub537\ub538\ub539\ub53a\ub53b\ub53c\ub53d\ub53e\ub53f\ub540\ub541\ub542\ub543\ub544\ub545\ub546\ub547\ub548\ub549\ub54a\ub54b\ub54c\ub54d\ub54e\ub54f\ub550\ub551\ub552\ub553\ub554\ub555\ub556\ub557\ub558\ub559\ub55a\ub55b\ub55c\ub55d\ub55e\ub55f\ub560\ub561\ub562\ub563\ub564\ub565\ub566\ub567\ub568\ub569\ub56a\ub56b\ub56c\ub56d\ub56e\ub56f\ub570\ub571\ub572\ub573\ub574\ub575\ub576\ub577\ub578\ub579\ub57a\ub57b\ub57c\ub57d\ub57e\ub57f\ub580\ub581\ub582\ub583\ub584\ub585\ub586\ub587\ub588\ub589\ub58a\ub58b\ub58c\ub58d\ub58e\ub58f\ub590\ub591\ub592\ub593\ub594\ub595\ub596\ub597\ub598\ub599\ub59a\ub59b\ub59c\ub59d\ub59e\ub59f\ub5a0\ub5a1\ub5a2\ub5a3\ub5a4\ub5a5\ub5a6\ub5a7\ub5a8\ub5a9\ub5aa\ub5ab\ub5ac\ub5ad\ub5ae\ub5af\ub5b0\ub5b1\ub5b2\ub5b3\ub5b4\ub5b5\ub5b6\ub5b7\ub5b8\ub5b9\ub5ba\ub5bb\ub5bc\ub5bd\ub5be\ub5bf\ub5c0\ub5c1\ub5c2\ub5c3\ub5c4\ub5c5\ub5c6\ub5c7\ub5c8\ub5c9\ub5ca\ub5cb\ub5cc\ub5cd\ub5ce\ub5cf\ub5d0\ub5d1\ub5d2\ub5d3\ub5d4\ub5d5\ub5d6\ub5d7\ub5d8\ub5d9\ub5da\ub5db\ub5dc\ub5dd\ub5de\ub5df\ub5e0\ub5e1\ub5e2\ub5e3\ub5e4\ub5e5\ub5e6\ub5e7\ub5e8\ub5e9\ub5ea\ub5eb\ub5ec\ub5ed\ub5ee\ub5ef\ub5f0\ub5f1\ub5f2\ub5f3\ub5f4\ub5f5\ub5f6\ub5f7\ub5f8\ub5f9\ub5fa\ub5fb\ub5fc\ub5fd\ub5fe\ub5ff\ub600\ub601\ub602\ub603\ub604\ub605\ub606\ub607\ub608\ub609\ub60a\ub60b\ub60c\ub60d\ub60e\ub60f\ub610\ub611\ub612\ub613\ub614\ub615\ub616\ub617\ub618\ub619\ub61a\ub61b\ub61c\ub61d\ub61e\ub61f\ub620\ub621\ub622\ub623\ub624\ub625\ub626\ub627\ub628\ub629\ub62a\ub62b\ub62c\ub62d\ub62e\ub62f\ub630\ub631\ub632\ub633\ub634\ub635\ub636\ub637\ub638\ub639\ub63a\ub63b\ub63c\ub63d\ub63e\ub63f\ub640\ub641\ub642\ub643\ub644\ub645\ub646\ub647\ub648\ub649\ub64a\ub64b\ub64c\ub64d\ub64e\ub64f\ub650\ub651\ub652\ub653\ub654\ub655\ub656\ub657\ub658\ub659\ub65a\ub65b\ub65c\ub65d\ub65e\ub65f\ub660\ub661\ub662\ub663\ub664\ub665\ub666\ub667\ub668\ub669\ub66a\ub66b\ub66c\ub66d\ub66e\ub66f\ub670\ub671\ub672\ub673\ub674\ub675\ub676\ub677\ub678\ub679\ub67a\ub67b\ub67c\ub67d\ub67e\ub67f\ub680\ub681\ub682\ub683\ub684\ub685\ub686\ub687\ub688\ub689\ub68a\ub68b\ub68c\ub68d\ub68e\ub68f\ub690\ub691\ub692\ub693\ub694\ub695\ub696\ub697\ub698\ub699\ub69a\ub69b\ub69c\ub69d\ub69e\ub69f\ub6a0\ub6a1\ub6a2\ub6a3\ub6a4\ub6a5\ub6a6\ub6a7\ub6a8\ub6a9\ub6aa\ub6ab\ub6ac\ub6ad\ub6ae\ub6af\ub6b0\ub6b1\ub6b2\ub6b3\ub6b4\ub6b5\ub6b6\ub6b7\ub6b8\ub6b9\ub6ba\ub6bb\ub6bc\ub6bd\ub6be\ub6bf\ub6c0\ub6c1\ub6c2\ub6c3\ub6c4\ub6c5\ub6c6\ub6c7\ub6c8\ub6c9\ub6ca\ub6cb\ub6cc\ub6cd\ub6ce\ub6cf\ub6d0\ub6d1\ub6d2\ub6d3\ub6d4\ub6d5\ub6d6\ub6d7\ub6d8\ub6d9\ub6da\ub6db\ub6dc\ub6dd\ub6de\ub6df\ub6e0\ub6e1\ub6e2\ub6e3\ub6e4\ub6e5\ub6e6\ub6e7\ub6e8\ub6e9\ub6ea\ub6eb\ub6ec\ub6ed\ub6ee\ub6ef\ub6f0\ub6f1\ub6f2\ub6f3\ub6f4\ub6f5\ub6f6\ub6f7\ub6f8\ub6f9\ub6fa\ub6fb\ub6fc\ub6fd\ub6fe\ub6ff\ub700\ub701\ub702\ub703\ub704\ub705\ub706\ub707\ub708\ub709\ub70a\ub70b\ub70c\ub70d\ub70e\ub70f\ub710\ub711\ub712\ub713\ub714\ub715\ub716\ub717\ub718\ub719\ub71a\ub71b\ub71c\ub71d\ub71e\ub71f\ub720\ub721\ub722\ub723\ub724\ub725\ub726\ub727\ub728\ub729\ub72a\ub72b\ub72c\ub72d\ub72e\ub72f\ub730\ub731\ub732\ub733\ub734\ub735\ub736\ub737\ub738\ub739\ub73a\ub73b\ub73c\ub73d\ub73e\ub73f\ub740\ub741\ub742\ub743\ub744\ub745\ub746\ub747\ub748\ub749\ub74a\ub74b\ub74c\ub74d\ub74e\ub74f\ub750\ub751\ub752\ub753\ub754\ub755\ub756\ub757\ub758\ub759\ub75a\ub75b\ub75c\ub75d\ub75e\ub75f\ub760\ub761\ub762\ub763\ub764\ub765\ub766\ub767\ub768\ub769\ub76a\ub76b\ub76c\ub76d\ub76e\ub76f\ub770\ub771\ub772\ub773\ub774\ub775\ub776\ub777\ub778\ub779\ub77a\ub77b\ub77c\ub77d\ub77e\ub77f\ub780\ub781\ub782\ub783\ub784\ub785\ub786\ub787\ub788\ub789\ub78a\ub78b\ub78c\ub78d\ub78e\ub78f\ub790\ub791\ub792\ub793\ub794\ub795\ub796\ub797\ub798\ub799\ub79a\ub79b\ub79c\ub79d\ub79e\ub79f\ub7a0\ub7a1\ub7a2\ub7a3\ub7a4\ub7a5\ub7a6\ub7a7\ub7a8\ub7a9\ub7aa\ub7ab\ub7ac\ub7ad\ub7ae\ub7af\ub7b0\ub7b1\ub7b2\ub7b3\ub7b4\ub7b5\ub7b6\ub7b7\ub7b8\ub7b9\ub7ba\ub7bb\ub7bc\ub7bd\ub7be\ub7bf\ub7c0\ub7c1\ub7c2\ub7c3\ub7c4\ub7c5\ub7c6\ub7c7\ub7c8\ub7c9\ub7ca\ub7cb\ub7cc\ub7cd\ub7ce\ub7cf\ub7d0\ub7d1\ub7d2\ub7d3\ub7d4\ub7d5\ub7d6\ub7d7\ub7d8\ub7d9\ub7da\ub7db\ub7dc\ub7dd\ub7de\ub7df\ub7e0\ub7e1\ub7e2\ub7e3\ub7e4\ub7e5\ub7e6\ub7e7\ub7e8\ub7e9\ub7ea\ub7eb\ub7ec\ub7ed\ub7ee\ub7ef\ub7f0\ub7f1\ub7f2\ub7f3\ub7f4\ub7f5\ub7f6\ub7f7\ub7f8\ub7f9\ub7fa\ub7fb\ub7fc\ub7fd\ub7fe\ub7ff\ub800\ub801\ub802\ub803\ub804\ub805\ub806\ub807\ub808\ub809\ub80a\ub80b\ub80c\ub80d\ub80e\ub80f\ub810\ub811\ub812\ub813\ub814\ub815\ub816\ub817\ub818\ub819\ub81a\ub81b\ub81c\ub81d\ub81e\ub81f\ub820\ub821\ub822\ub823\ub824\ub825\ub826\ub827\ub828\ub829\ub82a\ub82b\ub82c\ub82d\ub82e\ub82f\ub830\ub831\ub832\ub833\ub834\ub835\ub836\ub837\ub838\ub839\ub83a\ub83b\ub83c\ub83d\ub83e\ub83f\ub840\ub841\ub842\ub843\ub844\ub845\ub846\ub847\ub848\ub849\ub84a\ub84b\ub84c\ub84d\ub84e\ub84f\ub850\ub851\ub852\ub853\ub854\ub855\ub856\ub857\ub858\ub859\ub85a\ub85b\ub85c\ub85d\ub85e\ub85f\ub860\ub861\ub862\ub863\ub864\ub865\ub866\ub867\ub868\ub869\ub86a\ub86b\ub86c\ub86d\ub86e\ub86f\ub870\ub871\ub872\ub873\ub874\ub875\ub876\ub877\ub878\ub879\ub87a\ub87b\ub87c\ub87d\ub87e\ub87f\ub880\ub881\ub882\ub883\ub884\ub885\ub886\ub887\ub888\ub889\ub88a\ub88b\ub88c\ub88d\ub88e\ub88f\ub890\ub891\ub892\ub893\ub894\ub895\ub896\ub897\ub898\ub899\ub89a\ub89b\ub89c\ub89d\ub89e\ub89f\ub8a0\ub8a1\ub8a2\ub8a3\ub8a4\ub8a5\ub8a6\ub8a7\ub8a8\ub8a9\ub8aa\ub8ab\ub8ac\ub8ad\ub8ae\ub8af\ub8b0\ub8b1\ub8b2\ub8b3\ub8b4\ub8b5\ub8b6\ub8b7\ub8b8\ub8b9\ub8ba\ub8bb\ub8bc\ub8bd\ub8be\ub8bf\ub8c0\ub8c1\ub8c2\ub8c3\ub8c4\ub8c5\ub8c6\ub8c7\ub8c8\ub8c9\ub8ca\ub8cb\ub8cc\ub8cd\ub8ce\ub8cf\ub8d0\ub8d1\ub8d2\ub8d3\ub8d4\ub8d5\ub8d6\ub8d7\ub8d8\ub8d9\ub8da\ub8db\ub8dc\ub8dd\ub8de\ub8df\ub8e0\ub8e1\ub8e2\ub8e3\ub8e4\ub8e5\ub8e6\ub8e7\ub8e8\ub8e9\ub8ea\ub8eb\ub8ec\ub8ed\ub8ee\ub8ef\ub8f0\ub8f1\ub8f2\ub8f3\ub8f4\ub8f5\ub8f6\ub8f7\ub8f8\ub8f9\ub8fa\ub8fb\ub8fc\ub8fd\ub8fe\ub8ff\ub900\ub901\ub902\ub903\ub904\ub905\ub906\ub907\ub908\ub909\ub90a\ub90b\ub90c\ub90d\ub90e\ub90f\ub910\ub911\ub912\ub913\ub914\ub915\ub916\ub917\ub918\ub919\ub91a\ub91b\ub91c\ub91d\ub91e\ub91f\ub920\ub921\ub922\ub923\ub924\ub925\ub926\ub927\ub928\ub929\ub92a\ub92b\ub92c\ub92d\ub92e\ub92f\ub930\ub931\ub932\ub933\ub934\ub935\ub936\ub937\ub938\ub939\ub93a\ub93b\ub93c\ub93d\ub93e\ub93f\ub940\ub941\ub942\ub943\ub944\ub945\ub946\ub947\ub948\ub949\ub94a\ub94b\ub94c\ub94d\ub94e\ub94f\ub950\ub951\ub952\ub953\ub954\ub955\ub956\ub957\ub958\ub959\ub95a\ub95b\ub95c\ub95d\ub95e\ub95f\ub960\ub961\ub962\ub963\ub964\ub965\ub966\ub967\ub968\ub969\ub96a\ub96b\ub96c\ub96d\ub96e\ub96f\ub970\ub971\ub972\ub973\ub974\ub975\ub976\ub977\ub978\ub979\ub97a\ub97b\ub97c\ub97d\ub97e\ub97f\ub980\ub981\ub982\ub983\ub984\ub985\ub986\ub987\ub988\ub989\ub98a\ub98b\ub98c\ub98d\ub98e\ub98f\ub990\ub991\ub992\ub993\ub994\ub995\ub996\ub997\ub998\ub999\ub99a\ub99b\ub99c\ub99d\ub99e\ub99f\ub9a0\ub9a1\ub9a2\ub9a3\ub9a4\ub9a5\ub9a6\ub9a7\ub9a8\ub9a9\ub9aa\ub9ab\ub9ac\ub9ad\ub9ae\ub9af\ub9b0\ub9b1\ub9b2\ub9b3\ub9b4\ub9b5\ub9b6\ub9b7\ub9b8\ub9b9\ub9ba\ub9bb\ub9bc\ub9bd\ub9be\ub9bf\ub9c0\ub9c1\ub9c2\ub9c3\ub9c4\ub9c5\ub9c6\ub9c7\ub9c8\ub9c9\ub9ca\ub9cb\ub9cc\ub9cd\ub9ce\ub9cf\ub9d0\ub9d1\ub9d2\ub9d3\ub9d4\ub9d5\ub9d6\ub9d7\ub9d8\ub9d9\ub9da\ub9db\ub9dc\ub9dd\ub9de\ub9df\ub9e0\ub9e1\ub9e2\ub9e3\ub9e4\ub9e5\ub9e6\ub9e7\ub9e8\ub9e9\ub9ea\ub9eb\ub9ec\ub9ed\ub9ee\ub9ef\ub9f0\ub9f1\ub9f2\ub9f3\ub9f4\ub9f5\ub9f6\ub9f7\ub9f8\ub9f9\ub9fa\ub9fb\ub9fc\ub9fd\ub9fe\ub9ff\uba00\uba01\uba02\uba03\uba04\uba05\uba06\uba07\uba08\uba09\uba0a\uba0b\uba0c\uba0d\uba0e\uba0f\uba10\uba11\uba12\uba13\uba14\uba15\uba16\uba17\uba18\uba19\uba1a\uba1b\uba1c\uba1d\uba1e\uba1f\uba20\uba21\uba22\uba23\uba24\uba25\uba26\uba27\uba28\uba29\uba2a\uba2b\uba2c\uba2d\uba2e\uba2f\uba30\uba31\uba32\uba33\uba34\uba35\uba36\uba37\uba38\uba39\uba3a\uba3b\uba3c\uba3d\uba3e\uba3f\uba40\uba41\uba42\uba43\uba44\uba45\uba46\uba47\uba48\uba49\uba4a\uba4b\uba4c\uba4d\uba4e\uba4f\uba50\uba51\uba52\uba53\uba54\uba55\uba56\uba57\uba58\uba59\uba5a\uba5b\uba5c\uba5d\uba5e\uba5f\uba60\uba61\uba62\uba63\uba64\uba65\uba66\uba67\uba68\uba69\uba6a\uba6b\uba6c\uba6d\uba6e\uba6f\uba70\uba71\uba72\uba73\uba74\uba75\uba76\uba77\uba78\uba79\uba7a\uba7b\uba7c\uba7d\uba7e\uba7f\uba80\uba81\uba82\uba83\uba84\uba85\uba86\uba87\uba88\uba89\uba8a\uba8b\uba8c\uba8d\uba8e\uba8f\uba90\uba91\uba92\uba93\uba94\uba95\uba96\uba97\uba98\uba99\uba9a\uba9b\uba9c\uba9d\uba9e\uba9f\ubaa0\ubaa1\ubaa2\ubaa3\ubaa4\ubaa5\ubaa6\ubaa7\ubaa8\ubaa9\ubaaa\ubaab\ubaac\ubaad\ubaae\ubaaf\ubab0\ubab1\ubab2\ubab3\ubab4\ubab5\ubab6\ubab7\ubab8\ubab9\ubaba\ubabb\ubabc\ubabd\ubabe\ubabf\ubac0\ubac1\ubac2\ubac3\ubac4\ubac5\ubac6\ubac7\ubac8\ubac9\ubaca\ubacb\ubacc\ubacd\ubace\ubacf\ubad0\ubad1\ubad2\ubad3\ubad4\ubad5\ubad6\ubad7\ubad8\ubad9\ubada\ubadb\ubadc\ubadd\ubade\ubadf\ubae0\ubae1\ubae2\ubae3\ubae4\ubae5\ubae6\ubae7\ubae8\ubae9\ubaea\ubaeb\ubaec\ubaed\ubaee\ubaef\ubaf0\ubaf1\ubaf2\ubaf3\ubaf4\ubaf5\ubaf6\ubaf7\ubaf8\ubaf9\ubafa\ubafb\ubafc\ubafd\ubafe\ubaff\ubb00\ubb01\ubb02\ubb03\ubb04\ubb05\ubb06\ubb07\ubb08\ubb09\ubb0a\ubb0b\ubb0c\ubb0d\ubb0e\ubb0f\ubb10\ubb11\ubb12\ubb13\ubb14\ubb15\ubb16\ubb17\ubb18\ubb19\ubb1a\ubb1b\ubb1c\ubb1d\ubb1e\ubb1f\ubb20\ubb21\ubb22\ubb23\ubb24\ubb25\ubb26\ubb27\ubb28\ubb29\ubb2a\ubb2b\ubb2c\ubb2d\ubb2e\ubb2f\ubb30\ubb31\ubb32\ubb33\ubb34\ubb35\ubb36\ubb37\ubb38\ubb39\ubb3a\ubb3b\ubb3c\ubb3d\ubb3e\ubb3f\ubb40\ubb41\ubb42\ubb43\ubb44\ubb45\ubb46\ubb47\ubb48\ubb49\ubb4a\ubb4b\ubb4c\ubb4d\ubb4e\ubb4f\ubb50\ubb51\ubb52\ubb53\ubb54\ubb55\ubb56\ubb57\ubb58\ubb59\ubb5a\ubb5b\ubb5c\ubb5d\ubb5e\ubb5f\ubb60\ubb61\ubb62\ubb63\ubb64\ubb65\ubb66\ubb67\ubb68\ubb69\ubb6a\ubb6b\ubb6c\ubb6d\ubb6e\ubb6f\ubb70\ubb71\ubb72\ubb73\ubb74\ubb75\ubb76\ubb77\ubb78\ubb79\ubb7a\ubb7b\ubb7c\ubb7d\ubb7e\ubb7f\ubb80\ubb81\ubb82\ubb83\ubb84\ubb85\ubb86\ubb87\ubb88\ubb89\ubb8a\ubb8b\ubb8c\ubb8d\ubb8e\ubb8f\ubb90\ubb91\ubb92\ubb93\ubb94\ubb95\ubb96\ubb97\ubb98\ubb99\ubb9a\ubb9b\ubb9c\ubb9d\ubb9e\ubb9f\ubba0\ubba1\ubba2\ubba3\ubba4\ubba5\ubba6\ubba7\ubba8\ubba9\ubbaa\ubbab\ubbac\ubbad\ubbae\ubbaf\ubbb0\ubbb1\ubbb2\ubbb3\ubbb4\ubbb5\ubbb6\ubbb7\ubbb8\ubbb9\ubbba\ubbbb\ubbbc\ubbbd\ubbbe\ubbbf\ubbc0\ubbc1\ubbc2\ubbc3\ubbc4\ubbc5\ubbc6\ubbc7\ubbc8\ubbc9\ubbca\ubbcb\ubbcc\ubbcd\ubbce\ubbcf\ubbd0\ubbd1\ubbd2\ubbd3\ubbd4\ubbd5\ubbd6\ubbd7\ubbd8\ubbd9\ubbda\ubbdb\ubbdc\ubbdd\ubbde\ubbdf\ubbe0\ubbe1\ubbe2\ubbe3\ubbe4\ubbe5\ubbe6\ubbe7\ubbe8\ubbe9\ubbea\ubbeb\ubbec\ubbed\ubbee\ubbef\ubbf0\ubbf1\ubbf2\ubbf3\ubbf4\ubbf5\ubbf6\ubbf7\ubbf8\ubbf9\ubbfa\ubbfb\ubbfc\ubbfd\ubbfe\ubbff\ubc00\ubc01\ubc02\ubc03\ubc04\ubc05\ubc06\ubc07\ubc08\ubc09\ubc0a\ubc0b\ubc0c\ubc0d\ubc0e\ubc0f\ubc10\ubc11\ubc12\ubc13\ubc14\ubc15\ubc16\ubc17\ubc18\ubc19\ubc1a\ubc1b\ubc1c\ubc1d\ubc1e\ubc1f\ubc20\ubc21\ubc22\ubc23\ubc24\ubc25\ubc26\ubc27\ubc28\ubc29\ubc2a\ubc2b\ubc2c\ubc2d\ubc2e\ubc2f\ubc30\ubc31\ubc32\ubc33\ubc34\ubc35\ubc36\ubc37\ubc38\ubc39\ubc3a\ubc3b\ubc3c\ubc3d\ubc3e\ubc3f\ubc40\ubc41\ubc42\ubc43\ubc44\ubc45\ubc46\ubc47\ubc48\ubc49\ubc4a\ubc4b\ubc4c\ubc4d\ubc4e\ubc4f\ubc50\ubc51\ubc52\ubc53\ubc54\ubc55\ubc56\ubc57\ubc58\ubc59\ubc5a\ubc5b\ubc5c\ubc5d\ubc5e\ubc5f\ubc60\ubc61\ubc62\ubc63\ubc64\ubc65\ubc66\ubc67\ubc68\ubc69\ubc6a\ubc6b\ubc6c\ubc6d\ubc6e\ubc6f\ubc70\ubc71\ubc72\ubc73\ubc74\ubc75\ubc76\ubc77\ubc78\ubc79\ubc7a\ubc7b\ubc7c\ubc7d\ubc7e\ubc7f\ubc80\ubc81\ubc82\ubc83\ubc84\ubc85\ubc86\ubc87\ubc88\ubc89\ubc8a\ubc8b\ubc8c\ubc8d\ubc8e\ubc8f\ubc90\ubc91\ubc92\ubc93\ubc94\ubc95\ubc96\ubc97\ubc98\ubc99\ubc9a\ubc9b\ubc9c\ubc9d\ubc9e\ubc9f\ubca0\ubca1\ubca2\ubca3\ubca4\ubca5\ubca6\ubca7\ubca8\ubca9\ubcaa\ubcab\ubcac\ubcad\ubcae\ubcaf\ubcb0\ubcb1\ubcb2\ubcb3\ubcb4\ubcb5\ubcb6\ubcb7\ubcb8\ubcb9\ubcba\ubcbb\ubcbc\ubcbd\ubcbe\ubcbf\ubcc0\ubcc1\ubcc2\ubcc3\ubcc4\ubcc5\ubcc6\ubcc7\ubcc8\ubcc9\ubcca\ubccb\ubccc\ubccd\ubcce\ubccf\ubcd0\ubcd1\ubcd2\ubcd3\ubcd4\ubcd5\ubcd6\ubcd7\ubcd8\ubcd9\ubcda\ubcdb\ubcdc\ubcdd\ubcde\ubcdf\ubce0\ubce1\ubce2\ubce3\ubce4\ubce5\ubce6\ubce7\ubce8\ubce9\ubcea\ubceb\ubcec\ubced\ubcee\ubcef\ubcf0\ubcf1\ubcf2\ubcf3\ubcf4\ubcf5\ubcf6\ubcf7\ubcf8\ubcf9\ubcfa\ubcfb\ubcfc\ubcfd\ubcfe\ubcff\ubd00\ubd01\ubd02\ubd03\ubd04\ubd05\ubd06\ubd07\ubd08\ubd09\ubd0a\ubd0b\ubd0c\ubd0d\ubd0e\ubd0f\ubd10\ubd11\ubd12\ubd13\ubd14\ubd15\ubd16\ubd17\ubd18\ubd19\ubd1a\ubd1b\ubd1c\ubd1d\ubd1e\ubd1f\ubd20\ubd21\ubd22\ubd23\ubd24\ubd25\ubd26\ubd27\ubd28\ubd29\ubd2a\ubd2b\ubd2c\ubd2d\ubd2e\ubd2f\ubd30\ubd31\ubd32\ubd33\ubd34\ubd35\ubd36\ubd37\ubd38\ubd39\ubd3a\ubd3b\ubd3c\ubd3d\ubd3e\ubd3f\ubd40\ubd41\ubd42\ubd43\ubd44\ubd45\ubd46\ubd47\ubd48\ubd49\ubd4a\ubd4b\ubd4c\ubd4d\ubd4e\ubd4f\ubd50\ubd51\ubd52\ubd53\ubd54\ubd55\ubd56\ubd57\ubd58\ubd59\ubd5a\ubd5b\ubd5c\ubd5d\ubd5e\ubd5f\ubd60\ubd61\ubd62\ubd63\ubd64\ubd65\ubd66\ubd67\ubd68\ubd69\ubd6a\ubd6b\ubd6c\ubd6d\ubd6e\ubd6f\ubd70\ubd71\ubd72\ubd73\ubd74\ubd75\ubd76\ubd77\ubd78\ubd79\ubd7a\ubd7b\ubd7c\ubd7d\ubd7e\ubd7f\ubd80\ubd81\ubd82\ubd83\ubd84\ubd85\ubd86\ubd87\ubd88\ubd89\ubd8a\ubd8b\ubd8c\ubd8d\ubd8e\ubd8f\ubd90\ubd91\ubd92\ubd93\ubd94\ubd95\ubd96\ubd97\ubd98\ubd99\ubd9a\ubd9b\ubd9c\ubd9d\ubd9e\ubd9f\ubda0\ubda1\ubda2\ubda3\ubda4\ubda5\ubda6\ubda7\ubda8\ubda9\ubdaa\ubdab\ubdac\ubdad\ubdae\ubdaf\ubdb0\ubdb1\ubdb2\ubdb3\ubdb4\ubdb5\ubdb6\ubdb7\ubdb8\ubdb9\ubdba\ubdbb\ubdbc\ubdbd\ubdbe\ubdbf\ubdc0\ubdc1\ubdc2\ubdc3\ubdc4\ubdc5\ubdc6\ubdc7\ubdc8\ubdc9\ubdca\ubdcb\ubdcc\ubdcd\ubdce\ubdcf\ubdd0\ubdd1\ubdd2\ubdd3\ubdd4\ubdd5\ubdd6\ubdd7\ubdd8\ubdd9\ubdda\ubddb\ubddc\ubddd\ubdde\ubddf\ubde0\ubde1\ubde2\ubde3\ubde4\ubde5\ubde6\ubde7\ubde8\ubde9\ubdea\ubdeb\ubdec\ubded\ubdee\ubdef\ubdf0\ubdf1\ubdf2\ubdf3\ubdf4\ubdf5\ubdf6\ubdf7\ubdf8\ubdf9\ubdfa\ubdfb\ubdfc\ubdfd\ubdfe\ubdff\ube00\ube01\ube02\ube03\ube04\ube05\ube06\ube07\ube08\ube09\ube0a\ube0b\ube0c\ube0d\ube0e\ube0f\ube10\ube11\ube12\ube13\ube14\ube15\ube16\ube17\ube18\ube19\ube1a\ube1b\ube1c\ube1d\ube1e\ube1f\ube20\ube21\ube22\ube23\ube24\ube25\ube26\ube27\ube28\ube29\ube2a\ube2b\ube2c\ube2d\ube2e\ube2f\ube30\ube31\ube32\ube33\ube34\ube35\ube36\ube37\ube38\ube39\ube3a\ube3b\ube3c\ube3d\ube3e\ube3f\ube40\ube41\ube42\ube43\ube44\ube45\ube46\ube47\ube48\ube49\ube4a\ube4b\ube4c\ube4d\ube4e\ube4f\ube50\ube51\ube52\ube53\ube54\ube55\ube56\ube57\ube58\ube59\ube5a\ube5b\ube5c\ube5d\ube5e\ube5f\ube60\ube61\ube62\ube63\ube64\ube65\ube66\ube67\ube68\ube69\ube6a\ube6b\ube6c\ube6d\ube6e\ube6f\ube70\ube71\ube72\ube73\ube74\ube75\ube76\ube77\ube78\ube79\ube7a\ube7b\ube7c\ube7d\ube7e\ube7f\ube80\ube81\ube82\ube83\ube84\ube85\ube86\ube87\ube88\ube89\ube8a\ube8b\ube8c\ube8d\ube8e\ube8f\ube90\ube91\ube92\ube93\ube94\ube95\ube96\ube97\ube98\ube99\ube9a\ube9b\ube9c\ube9d\ube9e\ube9f\ubea0\ubea1\ubea2\ubea3\ubea4\ubea5\ubea6\ubea7\ubea8\ubea9\ubeaa\ubeab\ubeac\ubead\ubeae\ubeaf\ubeb0\ubeb1\ubeb2\ubeb3\ubeb4\ubeb5\ubeb6\ubeb7\ubeb8\ubeb9\ubeba\ubebb\ubebc\ubebd\ubebe\ubebf\ubec0\ubec1\ubec2\ubec3\ubec4\ubec5\ubec6\ubec7\ubec8\ubec9\ubeca\ubecb\ubecc\ubecd\ubece\ubecf\ubed0\ubed1\ubed2\ubed3\ubed4\ubed5\ubed6\ubed7\ubed8\ubed9\ubeda\ubedb\ubedc\ubedd\ubede\ubedf\ubee0\ubee1\ubee2\ubee3\ubee4\ubee5\ubee6\ubee7\ubee8\ubee9\ubeea\ubeeb\ubeec\ubeed\ubeee\ubeef\ubef0\ubef1\ubef2\ubef3\ubef4\ubef5\ubef6\ubef7\ubef8\ubef9\ubefa\ubefb\ubefc\ubefd\ubefe\ubeff\ubf00\ubf01\ubf02\ubf03\ubf04\ubf05\ubf06\ubf07\ubf08\ubf09\ubf0a\ubf0b\ubf0c\ubf0d\ubf0e\ubf0f\ubf10\ubf11\ubf12\ubf13\ubf14\ubf15\ubf16\ubf17\ubf18\ubf19\ubf1a\ubf1b\ubf1c\ubf1d\ubf1e\ubf1f\ubf20\ubf21\ubf22\ubf23\ubf24\ubf25\ubf26\ubf27\ubf28\ubf29\ubf2a\ubf2b\ubf2c\ubf2d\ubf2e\ubf2f\ubf30\ubf31\ubf32\ubf33\ubf34\ubf35\ubf36\ubf37\ubf38\ubf39\ubf3a\ubf3b\ubf3c\ubf3d\ubf3e\ubf3f\ubf40\ubf41\ubf42\ubf43\ubf44\ubf45\ubf46\ubf47\ubf48\ubf49\ubf4a\ubf4b\ubf4c\ubf4d\ubf4e\ubf4f\ubf50\ubf51\ubf52\ubf53\ubf54\ubf55\ubf56\ubf57\ubf58\ubf59\ubf5a\ubf5b\ubf5c\ubf5d\ubf5e\ubf5f\ubf60\ubf61\ubf62\ubf63\ubf64\ubf65\ubf66\ubf67\ubf68\ubf69\ubf6a\ubf6b\ubf6c\ubf6d\ubf6e\ubf6f\ubf70\ubf71\ubf72\ubf73\ubf74\ubf75\ubf76\ubf77\ubf78\ubf79\ubf7a\ubf7b\ubf7c\ubf7d\ubf7e\ubf7f\ubf80\ubf81\ubf82\ubf83\ubf84\ubf85\ubf86\ubf87\ubf88\ubf89\ubf8a\ubf8b\ubf8c\ubf8d\ubf8e\ubf8f\ubf90\ubf91\ubf92\ubf93\ubf94\ubf95\ubf96\ubf97\ubf98\ubf99\ubf9a\ubf9b\ubf9c\ubf9d\ubf9e\ubf9f\ubfa0\ubfa1\ubfa2\ubfa3\ubfa4\ubfa5\ubfa6\ubfa7\ubfa8\ubfa9\ubfaa\ubfab\ubfac\ubfad\ubfae\ubfaf\ubfb0\ubfb1\ubfb2\ubfb3\ubfb4\ubfb5\ubfb6\ubfb7\ubfb8\ubfb9\ubfba\ubfbb\ubfbc\ubfbd\ubfbe\ubfbf\ubfc0\ubfc1\ubfc2\ubfc3\ubfc4\ubfc5\ubfc6\ubfc7\ubfc8\ubfc9\ubfca\ubfcb\ubfcc\ubfcd\ubfce\ubfcf\ubfd0\ubfd1\ubfd2\ubfd3\ubfd4\ubfd5\ubfd6\ubfd7\ubfd8\ubfd9\ubfda\ubfdb\ubfdc\ubfdd\ubfde\ubfdf\ubfe0\ubfe1\ubfe2\ubfe3\ubfe4\ubfe5\ubfe6\ubfe7\ubfe8\ubfe9\ubfea\ubfeb\ubfec\ubfed\ubfee\ubfef\ubff0\ubff1\ubff2\ubff3\ubff4\ubff5\ubff6\ubff7\ubff8\ubff9\ubffa\ubffb\ubffc\ubffd\ubffe\ubfff\uc000\uc001\uc002\uc003\uc004\uc005\uc006\uc007\uc008\uc009\uc00a\uc00b\uc00c\uc00d\uc00e\uc00f\uc010\uc011\uc012\uc013\uc014\uc015\uc016\uc017\uc018\uc019\uc01a\uc01b\uc01c\uc01d\uc01e\uc01f\uc020\uc021\uc022\uc023\uc024\uc025\uc026\uc027\uc028\uc029\uc02a\uc02b\uc02c\uc02d\uc02e\uc02f\uc030\uc031\uc032\uc033\uc034\uc035\uc036\uc037\uc038\uc039\uc03a\uc03b\uc03c\uc03d\uc03e\uc03f\uc040\uc041\uc042\uc043\uc044\uc045\uc046\uc047\uc048\uc049\uc04a\uc04b\uc04c\uc04d\uc04e\uc04f\uc050\uc051\uc052\uc053\uc054\uc055\uc056\uc057\uc058\uc059\uc05a\uc05b\uc05c\uc05d\uc05e\uc05f\uc060\uc061\uc062\uc063\uc064\uc065\uc066\uc067\uc068\uc069\uc06a\uc06b\uc06c\uc06d\uc06e\uc06f\uc070\uc071\uc072\uc073\uc074\uc075\uc076\uc077\uc078\uc079\uc07a\uc07b\uc07c\uc07d\uc07e\uc07f\uc080\uc081\uc082\uc083\uc084\uc085\uc086\uc087\uc088\uc089\uc08a\uc08b\uc08c\uc08d\uc08e\uc08f\uc090\uc091\uc092\uc093\uc094\uc095\uc096\uc097\uc098\uc099\uc09a\uc09b\uc09c\uc09d\uc09e\uc09f\uc0a0\uc0a1\uc0a2\uc0a3\uc0a4\uc0a5\uc0a6\uc0a7\uc0a8\uc0a9\uc0aa\uc0ab\uc0ac\uc0ad\uc0ae\uc0af\uc0b0\uc0b1\uc0b2\uc0b3\uc0b4\uc0b5\uc0b6\uc0b7\uc0b8\uc0b9\uc0ba\uc0bb\uc0bc\uc0bd\uc0be\uc0bf\uc0c0\uc0c1\uc0c2\uc0c3\uc0c4\uc0c5\uc0c6\uc0c7\uc0c8\uc0c9\uc0ca\uc0cb\uc0cc\uc0cd\uc0ce\uc0cf\uc0d0\uc0d1\uc0d2\uc0d3\uc0d4\uc0d5\uc0d6\uc0d7\uc0d8\uc0d9\uc0da\uc0db\uc0dc\uc0dd\uc0de\uc0df\uc0e0\uc0e1\uc0e2\uc0e3\uc0e4\uc0e5\uc0e6\uc0e7\uc0e8\uc0e9\uc0ea\uc0eb\uc0ec\uc0ed\uc0ee\uc0ef\uc0f0\uc0f1\uc0f2\uc0f3\uc0f4\uc0f5\uc0f6\uc0f7\uc0f8\uc0f9\uc0fa\uc0fb\uc0fc\uc0fd\uc0fe\uc0ff\uc100\uc101\uc102\uc103\uc104\uc105\uc106\uc107\uc108\uc109\uc10a\uc10b\uc10c\uc10d\uc10e\uc10f\uc110\uc111\uc112\uc113\uc114\uc115\uc116\uc117\uc118\uc119\uc11a\uc11b\uc11c\uc11d\uc11e\uc11f\uc120\uc121\uc122\uc123\uc124\uc125\uc126\uc127\uc128\uc129\uc12a\uc12b\uc12c\uc12d\uc12e\uc12f\uc130\uc131\uc132\uc133\uc134\uc135\uc136\uc137\uc138\uc139\uc13a\uc13b\uc13c\uc13d\uc13e\uc13f\uc140\uc141\uc142\uc143\uc144\uc145\uc146\uc147\uc148\uc149\uc14a\uc14b\uc14c\uc14d\uc14e\uc14f\uc150\uc151\uc152\uc153\uc154\uc155\uc156\uc157\uc158\uc159\uc15a\uc15b\uc15c\uc15d\uc15e\uc15f\uc160\uc161\uc162\uc163\uc164\uc165\uc166\uc167\uc168\uc169\uc16a\uc16b\uc16c\uc16d\uc16e\uc16f\uc170\uc171\uc172\uc173\uc174\uc175\uc176\uc177\uc178\uc179\uc17a\uc17b\uc17c\uc17d\uc17e\uc17f\uc180\uc181\uc182\uc183\uc184\uc185\uc186\uc187\uc188\uc189\uc18a\uc18b\uc18c\uc18d\uc18e\uc18f\uc190\uc191\uc192\uc193\uc194\uc195\uc196\uc197\uc198\uc199\uc19a\uc19b\uc19c\uc19d\uc19e\uc19f\uc1a0\uc1a1\uc1a2\uc1a3\uc1a4\uc1a5\uc1a6\uc1a7\uc1a8\uc1a9\uc1aa\uc1ab\uc1ac\uc1ad\uc1ae\uc1af\uc1b0\uc1b1\uc1b2\uc1b3\uc1b4\uc1b5\uc1b6\uc1b7\uc1b8\uc1b9\uc1ba\uc1bb\uc1bc\uc1bd\uc1be\uc1bf\uc1c0\uc1c1\uc1c2\uc1c3\uc1c4\uc1c5\uc1c6\uc1c7\uc1c8\uc1c9\uc1ca\uc1cb\uc1cc\uc1cd\uc1ce\uc1cf\uc1d0\uc1d1\uc1d2\uc1d3\uc1d4\uc1d5\uc1d6\uc1d7\uc1d8\uc1d9\uc1da\uc1db\uc1dc\uc1dd\uc1de\uc1df\uc1e0\uc1e1\uc1e2\uc1e3\uc1e4\uc1e5\uc1e6\uc1e7\uc1e8\uc1e9\uc1ea\uc1eb\uc1ec\uc1ed\uc1ee\uc1ef\uc1f0\uc1f1\uc1f2\uc1f3\uc1f4\uc1f5\uc1f6\uc1f7\uc1f8\uc1f9\uc1fa\uc1fb\uc1fc\uc1fd\uc1fe\uc1ff\uc200\uc201\uc202\uc203\uc204\uc205\uc206\uc207\uc208\uc209\uc20a\uc20b\uc20c\uc20d\uc20e\uc20f\uc210\uc211\uc212\uc213\uc214\uc215\uc216\uc217\uc218\uc219\uc21a\uc21b\uc21c\uc21d\uc21e\uc21f\uc220\uc221\uc222\uc223\uc224\uc225\uc226\uc227\uc228\uc229\uc22a\uc22b\uc22c\uc22d\uc22e\uc22f\uc230\uc231\uc232\uc233\uc234\uc235\uc236\uc237\uc238\uc239\uc23a\uc23b\uc23c\uc23d\uc23e\uc23f\uc240\uc241\uc242\uc243\uc244\uc245\uc246\uc247\uc248\uc249\uc24a\uc24b\uc24c\uc24d\uc24e\uc24f\uc250\uc251\uc252\uc253\uc254\uc255\uc256\uc257\uc258\uc259\uc25a\uc25b\uc25c\uc25d\uc25e\uc25f\uc260\uc261\uc262\uc263\uc264\uc265\uc266\uc267\uc268\uc269\uc26a\uc26b\uc26c\uc26d\uc26e\uc26f\uc270\uc271\uc272\uc273\uc274\uc275\uc276\uc277\uc278\uc279\uc27a\uc27b\uc27c\uc27d\uc27e\uc27f\uc280\uc281\uc282\uc283\uc284\uc285\uc286\uc287\uc288\uc289\uc28a\uc28b\uc28c\uc28d\uc28e\uc28f\uc290\uc291\uc292\uc293\uc294\uc295\uc296\uc297\uc298\uc299\uc29a\uc29b\uc29c\uc29d\uc29e\uc29f\uc2a0\uc2a1\uc2a2\uc2a3\uc2a4\uc2a5\uc2a6\uc2a7\uc2a8\uc2a9\uc2aa\uc2ab\uc2ac\uc2ad\uc2ae\uc2af\uc2b0\uc2b1\uc2b2\uc2b3\uc2b4\uc2b5\uc2b6\uc2b7\uc2b8\uc2b9\uc2ba\uc2bb\uc2bc\uc2bd\uc2be\uc2bf\uc2c0\uc2c1\uc2c2\uc2c3\uc2c4\uc2c5\uc2c6\uc2c7\uc2c8\uc2c9\uc2ca\uc2cb\uc2cc\uc2cd\uc2ce\uc2cf\uc2d0\uc2d1\uc2d2\uc2d3\uc2d4\uc2d5\uc2d6\uc2d7\uc2d8\uc2d9\uc2da\uc2db\uc2dc\uc2dd\uc2de\uc2df\uc2e0\uc2e1\uc2e2\uc2e3\uc2e4\uc2e5\uc2e6\uc2e7\uc2e8\uc2e9\uc2ea\uc2eb\uc2ec\uc2ed\uc2ee\uc2ef\uc2f0\uc2f1\uc2f2\uc2f3\uc2f4\uc2f5\uc2f6\uc2f7\uc2f8\uc2f9\uc2fa\uc2fb\uc2fc\uc2fd\uc2fe\uc2ff\uc300\uc301\uc302\uc303\uc304\uc305\uc306\uc307\uc308\uc309\uc30a\uc30b\uc30c\uc30d\uc30e\uc30f\uc310\uc311\uc312\uc313\uc314\uc315\uc316\uc317\uc318\uc319\uc31a\uc31b\uc31c\uc31d\uc31e\uc31f\uc320\uc321\uc322\uc323\uc324\uc325\uc326\uc327\uc328\uc329\uc32a\uc32b\uc32c\uc32d\uc32e\uc32f\uc330\uc331\uc332\uc333\uc334\uc335\uc336\uc337\uc338\uc339\uc33a\uc33b\uc33c\uc33d\uc33e\uc33f\uc340\uc341\uc342\uc343\uc344\uc345\uc346\uc347\uc348\uc349\uc34a\uc34b\uc34c\uc34d\uc34e\uc34f\uc350\uc351\uc352\uc353\uc354\uc355\uc356\uc357\uc358\uc359\uc35a\uc35b\uc35c\uc35d\uc35e\uc35f\uc360\uc361\uc362\uc363\uc364\uc365\uc366\uc367\uc368\uc369\uc36a\uc36b\uc36c\uc36d\uc36e\uc36f\uc370\uc371\uc372\uc373\uc374\uc375\uc376\uc377\uc378\uc379\uc37a\uc37b\uc37c\uc37d\uc37e\uc37f\uc380\uc381\uc382\uc383\uc384\uc385\uc386\uc387\uc388\uc389\uc38a\uc38b\uc38c\uc38d\uc38e\uc38f\uc390\uc391\uc392\uc393\uc394\uc395\uc396\uc397\uc398\uc399\uc39a\uc39b\uc39c\uc39d\uc39e\uc39f\uc3a0\uc3a1\uc3a2\uc3a3\uc3a4\uc3a5\uc3a6\uc3a7\uc3a8\uc3a9\uc3aa\uc3ab\uc3ac\uc3ad\uc3ae\uc3af\uc3b0\uc3b1\uc3b2\uc3b3\uc3b4\uc3b5\uc3b6\uc3b7\uc3b8\uc3b9\uc3ba\uc3bb\uc3bc\uc3bd\uc3be\uc3bf\uc3c0\uc3c1\uc3c2\uc3c3\uc3c4\uc3c5\uc3c6\uc3c7\uc3c8\uc3c9\uc3ca\uc3cb\uc3cc\uc3cd\uc3ce\uc3cf\uc3d0\uc3d1\uc3d2\uc3d3\uc3d4\uc3d5\uc3d6\uc3d7\uc3d8\uc3d9\uc3da\uc3db\uc3dc\uc3dd\uc3de\uc3df\uc3e0\uc3e1\uc3e2\uc3e3\uc3e4\uc3e5\uc3e6\uc3e7\uc3e8\uc3e9\uc3ea\uc3eb\uc3ec\uc3ed\uc3ee\uc3ef\uc3f0\uc3f1\uc3f2\uc3f3\uc3f4\uc3f5\uc3f6\uc3f7\uc3f8\uc3f9\uc3fa\uc3fb\uc3fc\uc3fd\uc3fe\uc3ff\uc400\uc401\uc402\uc403\uc404\uc405\uc406\uc407\uc408\uc409\uc40a\uc40b\uc40c\uc40d\uc40e\uc40f\uc410\uc411\uc412\uc413\uc414\uc415\uc416\uc417\uc418\uc419\uc41a\uc41b\uc41c\uc41d\uc41e\uc41f\uc420\uc421\uc422\uc423\uc424\uc425\uc426\uc427\uc428\uc429\uc42a\uc42b\uc42c\uc42d\uc42e\uc42f\uc430\uc431\uc432\uc433\uc434\uc435\uc436\uc437\uc438\uc439\uc43a\uc43b\uc43c\uc43d\uc43e\uc43f\uc440\uc441\uc442\uc443\uc444\uc445\uc446\uc447\uc448\uc449\uc44a\uc44b\uc44c\uc44d\uc44e\uc44f\uc450\uc451\uc452\uc453\uc454\uc455\uc456\uc457\uc458\uc459\uc45a\uc45b\uc45c\uc45d\uc45e\uc45f\uc460\uc461\uc462\uc463\uc464\uc465\uc466\uc467\uc468\uc469\uc46a\uc46b\uc46c\uc46d\uc46e\uc46f\uc470\uc471\uc472\uc473\uc474\uc475\uc476\uc477\uc478\uc479\uc47a\uc47b\uc47c\uc47d\uc47e\uc47f\uc480\uc481\uc482\uc483\uc484\uc485\uc486\uc487\uc488\uc489\uc48a\uc48b\uc48c\uc48d\uc48e\uc48f\uc490\uc491\uc492\uc493\uc494\uc495\uc496\uc497\uc498\uc499\uc49a\uc49b\uc49c\uc49d\uc49e\uc49f\uc4a0\uc4a1\uc4a2\uc4a3\uc4a4\uc4a5\uc4a6\uc4a7\uc4a8\uc4a9\uc4aa\uc4ab\uc4ac\uc4ad\uc4ae\uc4af\uc4b0\uc4b1\uc4b2\uc4b3\uc4b4\uc4b5\uc4b6\uc4b7\uc4b8\uc4b9\uc4ba\uc4bb\uc4bc\uc4bd\uc4be\uc4bf\uc4c0\uc4c1\uc4c2\uc4c3\uc4c4\uc4c5\uc4c6\uc4c7\uc4c8\uc4c9\uc4ca\uc4cb\uc4cc\uc4cd\uc4ce\uc4cf\uc4d0\uc4d1\uc4d2\uc4d3\uc4d4\uc4d5\uc4d6\uc4d7\uc4d8\uc4d9\uc4da\uc4db\uc4dc\uc4dd\uc4de\uc4df\uc4e0\uc4e1\uc4e2\uc4e3\uc4e4\uc4e5\uc4e6\uc4e7\uc4e8\uc4e9\uc4ea\uc4eb\uc4ec\uc4ed\uc4ee\uc4ef\uc4f0\uc4f1\uc4f2\uc4f3\uc4f4\uc4f5\uc4f6\uc4f7\uc4f8\uc4f9\uc4fa\uc4fb\uc4fc\uc4fd\uc4fe\uc4ff\uc500\uc501\uc502\uc503\uc504\uc505\uc506\uc507\uc508\uc509\uc50a\uc50b\uc50c\uc50d\uc50e\uc50f\uc510\uc511\uc512\uc513\uc514\uc515\uc516\uc517\uc518\uc519\uc51a\uc51b\uc51c\uc51d\uc51e\uc51f\uc520\uc521\uc522\uc523\uc524\uc525\uc526\uc527\uc528\uc529\uc52a\uc52b\uc52c\uc52d\uc52e\uc52f\uc530\uc531\uc532\uc533\uc534\uc535\uc536\uc537\uc538\uc539\uc53a\uc53b\uc53c\uc53d\uc53e\uc53f\uc540\uc541\uc542\uc543\uc544\uc545\uc546\uc547\uc548\uc549\uc54a\uc54b\uc54c\uc54d\uc54e\uc54f\uc550\uc551\uc552\uc553\uc554\uc555\uc556\uc557\uc558\uc559\uc55a\uc55b\uc55c\uc55d\uc55e\uc55f\uc560\uc561\uc562\uc563\uc564\uc565\uc566\uc567\uc568\uc569\uc56a\uc56b\uc56c\uc56d\uc56e\uc56f\uc570\uc571\uc572\uc573\uc574\uc575\uc576\uc577\uc578\uc579\uc57a\uc57b\uc57c\uc57d\uc57e\uc57f\uc580\uc581\uc582\uc583\uc584\uc585\uc586\uc587\uc588\uc589\uc58a\uc58b\uc58c\uc58d\uc58e\uc58f\uc590\uc591\uc592\uc593\uc594\uc595\uc596\uc597\uc598\uc599\uc59a\uc59b\uc59c\uc59d\uc59e\uc59f\uc5a0\uc5a1\uc5a2\uc5a3\uc5a4\uc5a5\uc5a6\uc5a7\uc5a8\uc5a9\uc5aa\uc5ab\uc5ac\uc5ad\uc5ae\uc5af\uc5b0\uc5b1\uc5b2\uc5b3\uc5b4\uc5b5\uc5b6\uc5b7\uc5b8\uc5b9\uc5ba\uc5bb\uc5bc\uc5bd\uc5be\uc5bf\uc5c0\uc5c1\uc5c2\uc5c3\uc5c4\uc5c5\uc5c6\uc5c7\uc5c8\uc5c9\uc5ca\uc5cb\uc5cc\uc5cd\uc5ce\uc5cf\uc5d0\uc5d1\uc5d2\uc5d3\uc5d4\uc5d5\uc5d6\uc5d7\uc5d8\uc5d9\uc5da\uc5db\uc5dc\uc5dd\uc5de\uc5df\uc5e0\uc5e1\uc5e2\uc5e3\uc5e4\uc5e5\uc5e6\uc5e7\uc5e8\uc5e9\uc5ea\uc5eb\uc5ec\uc5ed\uc5ee\uc5ef\uc5f0\uc5f1\uc5f2\uc5f3\uc5f4\uc5f5\uc5f6\uc5f7\uc5f8\uc5f9\uc5fa\uc5fb\uc5fc\uc5fd\uc5fe\uc5ff\uc600\uc601\uc602\uc603\uc604\uc605\uc606\uc607\uc608\uc609\uc60a\uc60b\uc60c\uc60d\uc60e\uc60f\uc610\uc611\uc612\uc613\uc614\uc615\uc616\uc617\uc618\uc619\uc61a\uc61b\uc61c\uc61d\uc61e\uc61f\uc620\uc621\uc622\uc623\uc624\uc625\uc626\uc627\uc628\uc629\uc62a\uc62b\uc62c\uc62d\uc62e\uc62f\uc630\uc631\uc632\uc633\uc634\uc635\uc636\uc637\uc638\uc639\uc63a\uc63b\uc63c\uc63d\uc63e\uc63f\uc640\uc641\uc642\uc643\uc644\uc645\uc646\uc647\uc648\uc649\uc64a\uc64b\uc64c\uc64d\uc64e\uc64f\uc650\uc651\uc652\uc653\uc654\uc655\uc656\uc657\uc658\uc659\uc65a\uc65b\uc65c\uc65d\uc65e\uc65f\uc660\uc661\uc662\uc663\uc664\uc665\uc666\uc667\uc668\uc669\uc66a\uc66b\uc66c\uc66d\uc66e\uc66f\uc670\uc671\uc672\uc673\uc674\uc675\uc676\uc677\uc678\uc679\uc67a\uc67b\uc67c\uc67d\uc67e\uc67f\uc680\uc681\uc682\uc683\uc684\uc685\uc686\uc687\uc688\uc689\uc68a\uc68b\uc68c\uc68d\uc68e\uc68f\uc690\uc691\uc692\uc693\uc694\uc695\uc696\uc697\uc698\uc699\uc69a\uc69b\uc69c\uc69d\uc69e\uc69f\uc6a0\uc6a1\uc6a2\uc6a3\uc6a4\uc6a5\uc6a6\uc6a7\uc6a8\uc6a9\uc6aa\uc6ab\uc6ac\uc6ad\uc6ae\uc6af\uc6b0\uc6b1\uc6b2\uc6b3\uc6b4\uc6b5\uc6b6\uc6b7\uc6b8\uc6b9\uc6ba\uc6bb\uc6bc\uc6bd\uc6be\uc6bf\uc6c0\uc6c1\uc6c2\uc6c3\uc6c4\uc6c5\uc6c6\uc6c7\uc6c8\uc6c9\uc6ca\uc6cb\uc6cc\uc6cd\uc6ce\uc6cf\uc6d0\uc6d1\uc6d2\uc6d3\uc6d4\uc6d5\uc6d6\uc6d7\uc6d8\uc6d9\uc6da\uc6db\uc6dc\uc6dd\uc6de\uc6df\uc6e0\uc6e1\uc6e2\uc6e3\uc6e4\uc6e5\uc6e6\uc6e7\uc6e8\uc6e9\uc6ea\uc6eb\uc6ec\uc6ed\uc6ee\uc6ef\uc6f0\uc6f1\uc6f2\uc6f3\uc6f4\uc6f5\uc6f6\uc6f7\uc6f8\uc6f9\uc6fa\uc6fb\uc6fc\uc6fd\uc6fe\uc6ff\uc700\uc701\uc702\uc703\uc704\uc705\uc706\uc707\uc708\uc709\uc70a\uc70b\uc70c\uc70d\uc70e\uc70f\uc710\uc711\uc712\uc713\uc714\uc715\uc716\uc717\uc718\uc719\uc71a\uc71b\uc71c\uc71d\uc71e\uc71f\uc720\uc721\uc722\uc723\uc724\uc725\uc726\uc727\uc728\uc729\uc72a\uc72b\uc72c\uc72d\uc72e\uc72f\uc730\uc731\uc732\uc733\uc734\uc735\uc736\uc737\uc738\uc739\uc73a\uc73b\uc73c\uc73d\uc73e\uc73f\uc740\uc741\uc742\uc743\uc744\uc745\uc746\uc747\uc748\uc749\uc74a\uc74b\uc74c\uc74d\uc74e\uc74f\uc750\uc751\uc752\uc753\uc754\uc755\uc756\uc757\uc758\uc759\uc75a\uc75b\uc75c\uc75d\uc75e\uc75f\uc760\uc761\uc762\uc763\uc764\uc765\uc766\uc767\uc768\uc769\uc76a\uc76b\uc76c\uc76d\uc76e\uc76f\uc770\uc771\uc772\uc773\uc774\uc775\uc776\uc777\uc778\uc779\uc77a\uc77b\uc77c\uc77d\uc77e\uc77f\uc780\uc781\uc782\uc783\uc784\uc785\uc786\uc787\uc788\uc789\uc78a\uc78b\uc78c\uc78d\uc78e\uc78f\uc790\uc791\uc792\uc793\uc794\uc795\uc796\uc797\uc798\uc799\uc79a\uc79b\uc79c\uc79d\uc79e\uc79f\uc7a0\uc7a1\uc7a2\uc7a3\uc7a4\uc7a5\uc7a6\uc7a7\uc7a8\uc7a9\uc7aa\uc7ab\uc7ac\uc7ad\uc7ae\uc7af\uc7b0\uc7b1\uc7b2\uc7b3\uc7b4\uc7b5\uc7b6\uc7b7\uc7b8\uc7b9\uc7ba\uc7bb\uc7bc\uc7bd\uc7be\uc7bf\uc7c0\uc7c1\uc7c2\uc7c3\uc7c4\uc7c5\uc7c6\uc7c7\uc7c8\uc7c9\uc7ca\uc7cb\uc7cc\uc7cd\uc7ce\uc7cf\uc7d0\uc7d1\uc7d2\uc7d3\uc7d4\uc7d5\uc7d6\uc7d7\uc7d8\uc7d9\uc7da\uc7db\uc7dc\uc7dd\uc7de\uc7df\uc7e0\uc7e1\uc7e2\uc7e3\uc7e4\uc7e5\uc7e6\uc7e7\uc7e8\uc7e9\uc7ea\uc7eb\uc7ec\uc7ed\uc7ee\uc7ef\uc7f0\uc7f1\uc7f2\uc7f3\uc7f4\uc7f5\uc7f6\uc7f7\uc7f8\uc7f9\uc7fa\uc7fb\uc7fc\uc7fd\uc7fe\uc7ff\uc800\uc801\uc802\uc803\uc804\uc805\uc806\uc807\uc808\uc809\uc80a\uc80b\uc80c\uc80d\uc80e\uc80f\uc810\uc811\uc812\uc813\uc814\uc815\uc816\uc817\uc818\uc819\uc81a\uc81b\uc81c\uc81d\uc81e\uc81f\uc820\uc821\uc822\uc823\uc824\uc825\uc826\uc827\uc828\uc829\uc82a\uc82b\uc82c\uc82d\uc82e\uc82f\uc830\uc831\uc832\uc833\uc834\uc835\uc836\uc837\uc838\uc839\uc83a\uc83b\uc83c\uc83d\uc83e\uc83f\uc840\uc841\uc842\uc843\uc844\uc845\uc846\uc847\uc848\uc849\uc84a\uc84b\uc84c\uc84d\uc84e\uc84f\uc850\uc851\uc852\uc853\uc854\uc855\uc856\uc857\uc858\uc859\uc85a\uc85b\uc85c\uc85d\uc85e\uc85f\uc860\uc861\uc862\uc863\uc864\uc865\uc866\uc867\uc868\uc869\uc86a\uc86b\uc86c\uc86d\uc86e\uc86f\uc870\uc871\uc872\uc873\uc874\uc875\uc876\uc877\uc878\uc879\uc87a\uc87b\uc87c\uc87d\uc87e\uc87f\uc880\uc881\uc882\uc883\uc884\uc885\uc886\uc887\uc888\uc889\uc88a\uc88b\uc88c\uc88d\uc88e\uc88f\uc890\uc891\uc892\uc893\uc894\uc895\uc896\uc897\uc898\uc899\uc89a\uc89b\uc89c\uc89d\uc89e\uc89f\uc8a0\uc8a1\uc8a2\uc8a3\uc8a4\uc8a5\uc8a6\uc8a7\uc8a8\uc8a9\uc8aa\uc8ab\uc8ac\uc8ad\uc8ae\uc8af\uc8b0\uc8b1\uc8b2\uc8b3\uc8b4\uc8b5\uc8b6\uc8b7\uc8b8\uc8b9\uc8ba\uc8bb\uc8bc\uc8bd\uc8be\uc8bf\uc8c0\uc8c1\uc8c2\uc8c3\uc8c4\uc8c5\uc8c6\uc8c7\uc8c8\uc8c9\uc8ca\uc8cb\uc8cc\uc8cd\uc8ce\uc8cf\uc8d0\uc8d1\uc8d2\uc8d3\uc8d4\uc8d5\uc8d6\uc8d7\uc8d8\uc8d9\uc8da\uc8db\uc8dc\uc8dd\uc8de\uc8df\uc8e0\uc8e1\uc8e2\uc8e3\uc8e4\uc8e5\uc8e6\uc8e7\uc8e8\uc8e9\uc8ea\uc8eb\uc8ec\uc8ed\uc8ee\uc8ef\uc8f0\uc8f1\uc8f2\uc8f3\uc8f4\uc8f5\uc8f6\uc8f7\uc8f8\uc8f9\uc8fa\uc8fb\uc8fc\uc8fd\uc8fe\uc8ff\uc900\uc901\uc902\uc903\uc904\uc905\uc906\uc907\uc908\uc909\uc90a\uc90b\uc90c\uc90d\uc90e\uc90f\uc910\uc911\uc912\uc913\uc914\uc915\uc916\uc917\uc918\uc919\uc91a\uc91b\uc91c\uc91d\uc91e\uc91f\uc920\uc921\uc922\uc923\uc924\uc925\uc926\uc927\uc928\uc929\uc92a\uc92b\uc92c\uc92d\uc92e\uc92f\uc930\uc931\uc932\uc933\uc934\uc935\uc936\uc937\uc938\uc939\uc93a\uc93b\uc93c\uc93d\uc93e\uc93f\uc940\uc941\uc942\uc943\uc944\uc945\uc946\uc947\uc948\uc949\uc94a\uc94b\uc94c\uc94d\uc94e\uc94f\uc950\uc951\uc952\uc953\uc954\uc955\uc956\uc957\uc958\uc959\uc95a\uc95b\uc95c\uc95d\uc95e\uc95f\uc960\uc961\uc962\uc963\uc964\uc965\uc966\uc967\uc968\uc969\uc96a\uc96b\uc96c\uc96d\uc96e\uc96f\uc970\uc971\uc972\uc973\uc974\uc975\uc976\uc977\uc978\uc979\uc97a\uc97b\uc97c\uc97d\uc97e\uc97f\uc980\uc981\uc982\uc983\uc984\uc985\uc986\uc987\uc988\uc989\uc98a\uc98b\uc98c\uc98d\uc98e\uc98f\uc990\uc991\uc992\uc993\uc994\uc995\uc996\uc997\uc998\uc999\uc99a\uc99b\uc99c\uc99d\uc99e\uc99f\uc9a0\uc9a1\uc9a2\uc9a3\uc9a4\uc9a5\uc9a6\uc9a7\uc9a8\uc9a9\uc9aa\uc9ab\uc9ac\uc9ad\uc9ae\uc9af\uc9b0\uc9b1\uc9b2\uc9b3\uc9b4\uc9b5\uc9b6\uc9b7\uc9b8\uc9b9\uc9ba\uc9bb\uc9bc\uc9bd\uc9be\uc9bf\uc9c0\uc9c1\uc9c2\uc9c3\uc9c4\uc9c5\uc9c6\uc9c7\uc9c8\uc9c9\uc9ca\uc9cb\uc9cc\uc9cd\uc9ce\uc9cf\uc9d0\uc9d1\uc9d2\uc9d3\uc9d4\uc9d5\uc9d6\uc9d7\uc9d8\uc9d9\uc9da\uc9db\uc9dc\uc9dd\uc9de\uc9df\uc9e0\uc9e1\uc9e2\uc9e3\uc9e4\uc9e5\uc9e6\uc9e7\uc9e8\uc9e9\uc9ea\uc9eb\uc9ec\uc9ed\uc9ee\uc9ef\uc9f0\uc9f1\uc9f2\uc9f3\uc9f4\uc9f5\uc9f6\uc9f7\uc9f8\uc9f9\uc9fa\uc9fb\uc9fc\uc9fd\uc9fe\uc9ff\uca00\uca01\uca02\uca03\uca04\uca05\uca06\uca07\uca08\uca09\uca0a\uca0b\uca0c\uca0d\uca0e\uca0f\uca10\uca11\uca12\uca13\uca14\uca15\uca16\uca17\uca18\uca19\uca1a\uca1b\uca1c\uca1d\uca1e\uca1f\uca20\uca21\uca22\uca23\uca24\uca25\uca26\uca27\uca28\uca29\uca2a\uca2b\uca2c\uca2d\uca2e\uca2f\uca30\uca31\uca32\uca33\uca34\uca35\uca36\uca37\uca38\uca39\uca3a\uca3b\uca3c\uca3d\uca3e\uca3f\uca40\uca41\uca42\uca43\uca44\uca45\uca46\uca47\uca48\uca49\uca4a\uca4b\uca4c\uca4d\uca4e\uca4f\uca50\uca51\uca52\uca53\uca54\uca55\uca56\uca57\uca58\uca59\uca5a\uca5b\uca5c\uca5d\uca5e\uca5f\uca60\uca61\uca62\uca63\uca64\uca65\uca66\uca67\uca68\uca69\uca6a\uca6b\uca6c\uca6d\uca6e\uca6f\uca70\uca71\uca72\uca73\uca74\uca75\uca76\uca77\uca78\uca79\uca7a\uca7b\uca7c\uca7d\uca7e\uca7f\uca80\uca81\uca82\uca83\uca84\uca85\uca86\uca87\uca88\uca89\uca8a\uca8b\uca8c\uca8d\uca8e\uca8f\uca90\uca91\uca92\uca93\uca94\uca95\uca96\uca97\uca98\uca99\uca9a\uca9b\uca9c\uca9d\uca9e\uca9f\ucaa0\ucaa1\ucaa2\ucaa3\ucaa4\ucaa5\ucaa6\ucaa7\ucaa8\ucaa9\ucaaa\ucaab\ucaac\ucaad\ucaae\ucaaf\ucab0\ucab1\ucab2\ucab3\ucab4\ucab5\ucab6\ucab7\ucab8\ucab9\ucaba\ucabb\ucabc\ucabd\ucabe\ucabf\ucac0\ucac1\ucac2\ucac3\ucac4\ucac5\ucac6\ucac7\ucac8\ucac9\ucaca\ucacb\ucacc\ucacd\ucace\ucacf\ucad0\ucad1\ucad2\ucad3\ucad4\ucad5\ucad6\ucad7\ucad8\ucad9\ucada\ucadb\ucadc\ucadd\ucade\ucadf\ucae0\ucae1\ucae2\ucae3\ucae4\ucae5\ucae6\ucae7\ucae8\ucae9\ucaea\ucaeb\ucaec\ucaed\ucaee\ucaef\ucaf0\ucaf1\ucaf2\ucaf3\ucaf4\ucaf5\ucaf6\ucaf7\ucaf8\ucaf9\ucafa\ucafb\ucafc\ucafd\ucafe\ucaff\ucb00\ucb01\ucb02\ucb03\ucb04\ucb05\ucb06\ucb07\ucb08\ucb09\ucb0a\ucb0b\ucb0c\ucb0d\ucb0e\ucb0f\ucb10\ucb11\ucb12\ucb13\ucb14\ucb15\ucb16\ucb17\ucb18\ucb19\ucb1a\ucb1b\ucb1c\ucb1d\ucb1e\ucb1f\ucb20\ucb21\ucb22\ucb23\ucb24\ucb25\ucb26\ucb27\ucb28\ucb29\ucb2a\ucb2b\ucb2c\ucb2d\ucb2e\ucb2f\ucb30\ucb31\ucb32\ucb33\ucb34\ucb35\ucb36\ucb37\ucb38\ucb39\ucb3a\ucb3b\ucb3c\ucb3d\ucb3e\ucb3f\ucb40\ucb41\ucb42\ucb43\ucb44\ucb45\ucb46\ucb47\ucb48\ucb49\ucb4a\ucb4b\ucb4c\ucb4d\ucb4e\ucb4f\ucb50\ucb51\ucb52\ucb53\ucb54\ucb55\ucb56\ucb57\ucb58\ucb59\ucb5a\ucb5b\ucb5c\ucb5d\ucb5e\ucb5f\ucb60\ucb61\ucb62\ucb63\ucb64\ucb65\ucb66\ucb67\ucb68\ucb69\ucb6a\ucb6b\ucb6c\ucb6d\ucb6e\ucb6f\ucb70\ucb71\ucb72\ucb73\ucb74\ucb75\ucb76\ucb77\ucb78\ucb79\ucb7a\ucb7b\ucb7c\ucb7d\ucb7e\ucb7f\ucb80\ucb81\ucb82\ucb83\ucb84\ucb85\ucb86\ucb87\ucb88\ucb89\ucb8a\ucb8b\ucb8c\ucb8d\ucb8e\ucb8f\ucb90\ucb91\ucb92\ucb93\ucb94\ucb95\ucb96\ucb97\ucb98\ucb99\ucb9a\ucb9b\ucb9c\ucb9d\ucb9e\ucb9f\ucba0\ucba1\ucba2\ucba3\ucba4\ucba5\ucba6\ucba7\ucba8\ucba9\ucbaa\ucbab\ucbac\ucbad\ucbae\ucbaf\ucbb0\ucbb1\ucbb2\ucbb3\ucbb4\ucbb5\ucbb6\ucbb7\ucbb8\ucbb9\ucbba\ucbbb\ucbbc\ucbbd\ucbbe\ucbbf\ucbc0\ucbc1\ucbc2\ucbc3\ucbc4\ucbc5\ucbc6\ucbc7\ucbc8\ucbc9\ucbca\ucbcb\ucbcc\ucbcd\ucbce\ucbcf\ucbd0\ucbd1\ucbd2\ucbd3\ucbd4\ucbd5\ucbd6\ucbd7\ucbd8\ucbd9\ucbda\ucbdb\ucbdc\ucbdd\ucbde\ucbdf\ucbe0\ucbe1\ucbe2\ucbe3\ucbe4\ucbe5\ucbe6\ucbe7\ucbe8\ucbe9\ucbea\ucbeb\ucbec\ucbed\ucbee\ucbef\ucbf0\ucbf1\ucbf2\ucbf3\ucbf4\ucbf5\ucbf6\ucbf7\ucbf8\ucbf9\ucbfa\ucbfb\ucbfc\ucbfd\ucbfe\ucbff\ucc00\ucc01\ucc02\ucc03\ucc04\ucc05\ucc06\ucc07\ucc08\ucc09\ucc0a\ucc0b\ucc0c\ucc0d\ucc0e\ucc0f\ucc10\ucc11\ucc12\ucc13\ucc14\ucc15\ucc16\ucc17\ucc18\ucc19\ucc1a\ucc1b\ucc1c\ucc1d\ucc1e\ucc1f\ucc20\ucc21\ucc22\ucc23\ucc24\ucc25\ucc26\ucc27\ucc28\ucc29\ucc2a\ucc2b\ucc2c\ucc2d\ucc2e\ucc2f\ucc30\ucc31\ucc32\ucc33\ucc34\ucc35\ucc36\ucc37\ucc38\ucc39\ucc3a\ucc3b\ucc3c\ucc3d\ucc3e\ucc3f\ucc40\ucc41\ucc42\ucc43\ucc44\ucc45\ucc46\ucc47\ucc48\ucc49\ucc4a\ucc4b\ucc4c\ucc4d\ucc4e\ucc4f\ucc50\ucc51\ucc52\ucc53\ucc54\ucc55\ucc56\ucc57\ucc58\ucc59\ucc5a\ucc5b\ucc5c\ucc5d\ucc5e\ucc5f\ucc60\ucc61\ucc62\ucc63\ucc64\ucc65\ucc66\ucc67\ucc68\ucc69\ucc6a\ucc6b\ucc6c\ucc6d\ucc6e\ucc6f\ucc70\ucc71\ucc72\ucc73\ucc74\ucc75\ucc76\ucc77\ucc78\ucc79\ucc7a\ucc7b\ucc7c\ucc7d\ucc7e\ucc7f\ucc80\ucc81\ucc82\ucc83\ucc84\ucc85\ucc86\ucc87\ucc88\ucc89\ucc8a\ucc8b\ucc8c\ucc8d\ucc8e\ucc8f\ucc90\ucc91\ucc92\ucc93\ucc94\ucc95\ucc96\ucc97\ucc98\ucc99\ucc9a\ucc9b\ucc9c\ucc9d\ucc9e\ucc9f\ucca0\ucca1\ucca2\ucca3\ucca4\ucca5\ucca6\ucca7\ucca8\ucca9\uccaa\uccab\uccac\uccad\uccae\uccaf\uccb0\uccb1\uccb2\uccb3\uccb4\uccb5\uccb6\uccb7\uccb8\uccb9\uccba\uccbb\uccbc\uccbd\uccbe\uccbf\uccc0\uccc1\uccc2\uccc3\uccc4\uccc5\uccc6\uccc7\uccc8\uccc9\uccca\ucccb\ucccc\ucccd\uccce\ucccf\uccd0\uccd1\uccd2\uccd3\uccd4\uccd5\uccd6\uccd7\uccd8\uccd9\uccda\uccdb\uccdc\uccdd\uccde\uccdf\ucce0\ucce1\ucce2\ucce3\ucce4\ucce5\ucce6\ucce7\ucce8\ucce9\uccea\ucceb\uccec\ucced\uccee\uccef\uccf0\uccf1\uccf2\uccf3\uccf4\uccf5\uccf6\uccf7\uccf8\uccf9\uccfa\uccfb\uccfc\uccfd\uccfe\uccff\ucd00\ucd01\ucd02\ucd03\ucd04\ucd05\ucd06\ucd07\ucd08\ucd09\ucd0a\ucd0b\ucd0c\ucd0d\ucd0e\ucd0f\ucd10\ucd11\ucd12\ucd13\ucd14\ucd15\ucd16\ucd17\ucd18\ucd19\ucd1a\ucd1b\ucd1c\ucd1d\ucd1e\ucd1f\ucd20\ucd21\ucd22\ucd23\ucd24\ucd25\ucd26\ucd27\ucd28\ucd29\ucd2a\ucd2b\ucd2c\ucd2d\ucd2e\ucd2f\ucd30\ucd31\ucd32\ucd33\ucd34\ucd35\ucd36\ucd37\ucd38\ucd39\ucd3a\ucd3b\ucd3c\ucd3d\ucd3e\ucd3f\ucd40\ucd41\ucd42\ucd43\ucd44\ucd45\ucd46\ucd47\ucd48\ucd49\ucd4a\ucd4b\ucd4c\ucd4d\ucd4e\ucd4f\ucd50\ucd51\ucd52\ucd53\ucd54\ucd55\ucd56\ucd57\ucd58\ucd59\ucd5a\ucd5b\ucd5c\ucd5d\ucd5e\ucd5f\ucd60\ucd61\ucd62\ucd63\ucd64\ucd65\ucd66\ucd67\ucd68\ucd69\ucd6a\ucd6b\ucd6c\ucd6d\ucd6e\ucd6f\ucd70\ucd71\ucd72\ucd73\ucd74\ucd75\ucd76\ucd77\ucd78\ucd79\ucd7a\ucd7b\ucd7c\ucd7d\ucd7e\ucd7f\ucd80\ucd81\ucd82\ucd83\ucd84\ucd85\ucd86\ucd87\ucd88\ucd89\ucd8a\ucd8b\ucd8c\ucd8d\ucd8e\ucd8f\ucd90\ucd91\ucd92\ucd93\ucd94\ucd95\ucd96\ucd97\ucd98\ucd99\ucd9a\ucd9b\ucd9c\ucd9d\ucd9e\ucd9f\ucda0\ucda1\ucda2\ucda3\ucda4\ucda5\ucda6\ucda7\ucda8\ucda9\ucdaa\ucdab\ucdac\ucdad\ucdae\ucdaf\ucdb0\ucdb1\ucdb2\ucdb3\ucdb4\ucdb5\ucdb6\ucdb7\ucdb8\ucdb9\ucdba\ucdbb\ucdbc\ucdbd\ucdbe\ucdbf\ucdc0\ucdc1\ucdc2\ucdc3\ucdc4\ucdc5\ucdc6\ucdc7\ucdc8\ucdc9\ucdca\ucdcb\ucdcc\ucdcd\ucdce\ucdcf\ucdd0\ucdd1\ucdd2\ucdd3\ucdd4\ucdd5\ucdd6\ucdd7\ucdd8\ucdd9\ucdda\ucddb\ucddc\ucddd\ucdde\ucddf\ucde0\ucde1\ucde2\ucde3\ucde4\ucde5\ucde6\ucde7\ucde8\ucde9\ucdea\ucdeb\ucdec\ucded\ucdee\ucdef\ucdf0\ucdf1\ucdf2\ucdf3\ucdf4\ucdf5\ucdf6\ucdf7\ucdf8\ucdf9\ucdfa\ucdfb\ucdfc\ucdfd\ucdfe\ucdff\uce00\uce01\uce02\uce03\uce04\uce05\uce06\uce07\uce08\uce09\uce0a\uce0b\uce0c\uce0d\uce0e\uce0f\uce10\uce11\uce12\uce13\uce14\uce15\uce16\uce17\uce18\uce19\uce1a\uce1b\uce1c\uce1d\uce1e\uce1f\uce20\uce21\uce22\uce23\uce24\uce25\uce26\uce27\uce28\uce29\uce2a\uce2b\uce2c\uce2d\uce2e\uce2f\uce30\uce31\uce32\uce33\uce34\uce35\uce36\uce37\uce38\uce39\uce3a\uce3b\uce3c\uce3d\uce3e\uce3f\uce40\uce41\uce42\uce43\uce44\uce45\uce46\uce47\uce48\uce49\uce4a\uce4b\uce4c\uce4d\uce4e\uce4f\uce50\uce51\uce52\uce53\uce54\uce55\uce56\uce57\uce58\uce59\uce5a\uce5b\uce5c\uce5d\uce5e\uce5f\uce60\uce61\uce62\uce63\uce64\uce65\uce66\uce67\uce68\uce69\uce6a\uce6b\uce6c\uce6d\uce6e\uce6f\uce70\uce71\uce72\uce73\uce74\uce75\uce76\uce77\uce78\uce79\uce7a\uce7b\uce7c\uce7d\uce7e\uce7f\uce80\uce81\uce82\uce83\uce84\uce85\uce86\uce87\uce88\uce89\uce8a\uce8b\uce8c\uce8d\uce8e\uce8f\uce90\uce91\uce92\uce93\uce94\uce95\uce96\uce97\uce98\uce99\uce9a\uce9b\uce9c\uce9d\uce9e\uce9f\ucea0\ucea1\ucea2\ucea3\ucea4\ucea5\ucea6\ucea7\ucea8\ucea9\uceaa\uceab\uceac\ucead\uceae\uceaf\uceb0\uceb1\uceb2\uceb3\uceb4\uceb5\uceb6\uceb7\uceb8\uceb9\uceba\ucebb\ucebc\ucebd\ucebe\ucebf\ucec0\ucec1\ucec2\ucec3\ucec4\ucec5\ucec6\ucec7\ucec8\ucec9\uceca\ucecb\ucecc\ucecd\ucece\ucecf\uced0\uced1\uced2\uced3\uced4\uced5\uced6\uced7\uced8\uced9\uceda\ucedb\ucedc\ucedd\ucede\ucedf\ucee0\ucee1\ucee2\ucee3\ucee4\ucee5\ucee6\ucee7\ucee8\ucee9\uceea\uceeb\uceec\uceed\uceee\uceef\ucef0\ucef1\ucef2\ucef3\ucef4\ucef5\ucef6\ucef7\ucef8\ucef9\ucefa\ucefb\ucefc\ucefd\ucefe\uceff\ucf00\ucf01\ucf02\ucf03\ucf04\ucf05\ucf06\ucf07\ucf08\ucf09\ucf0a\ucf0b\ucf0c\ucf0d\ucf0e\ucf0f\ucf10\ucf11\ucf12\ucf13\ucf14\ucf15\ucf16\ucf17\ucf18\ucf19\ucf1a\ucf1b\ucf1c\ucf1d\ucf1e\ucf1f\ucf20\ucf21\ucf22\ucf23\ucf24\ucf25\ucf26\ucf27\ucf28\ucf29\ucf2a\ucf2b\ucf2c\ucf2d\ucf2e\ucf2f\ucf30\ucf31\ucf32\ucf33\ucf34\ucf35\ucf36\ucf37\ucf38\ucf39\ucf3a\ucf3b\ucf3c\ucf3d\ucf3e\ucf3f\ucf40\ucf41\ucf42\ucf43\ucf44\ucf45\ucf46\ucf47\ucf48\ucf49\ucf4a\ucf4b\ucf4c\ucf4d\ucf4e\ucf4f\ucf50\ucf51\ucf52\ucf53\ucf54\ucf55\ucf56\ucf57\ucf58\ucf59\ucf5a\ucf5b\ucf5c\ucf5d\ucf5e\ucf5f\ucf60\ucf61\ucf62\ucf63\ucf64\ucf65\ucf66\ucf67\ucf68\ucf69\ucf6a\ucf6b\ucf6c\ucf6d\ucf6e\ucf6f\ucf70\ucf71\ucf72\ucf73\ucf74\ucf75\ucf76\ucf77\ucf78\ucf79\ucf7a\ucf7b\ucf7c\ucf7d\ucf7e\ucf7f\ucf80\ucf81\ucf82\ucf83\ucf84\ucf85\ucf86\ucf87\ucf88\ucf89\ucf8a\ucf8b\ucf8c\ucf8d\ucf8e\ucf8f\ucf90\ucf91\ucf92\ucf93\ucf94\ucf95\ucf96\ucf97\ucf98\ucf99\ucf9a\ucf9b\ucf9c\ucf9d\ucf9e\ucf9f\ucfa0\ucfa1\ucfa2\ucfa3\ucfa4\ucfa5\ucfa6\ucfa7\ucfa8\ucfa9\ucfaa\ucfab\ucfac\ucfad\ucfae\ucfaf\ucfb0\ucfb1\ucfb2\ucfb3\ucfb4\ucfb5\ucfb6\ucfb7\ucfb8\ucfb9\ucfba\ucfbb\ucfbc\ucfbd\ucfbe\ucfbf\ucfc0\ucfc1\ucfc2\ucfc3\ucfc4\ucfc5\ucfc6\ucfc7\ucfc8\ucfc9\ucfca\ucfcb\ucfcc\ucfcd\ucfce\ucfcf\ucfd0\ucfd1\ucfd2\ucfd3\ucfd4\ucfd5\ucfd6\ucfd7\ucfd8\ucfd9\ucfda\ucfdb\ucfdc\ucfdd\ucfde\ucfdf\ucfe0\ucfe1\ucfe2\ucfe3\ucfe4\ucfe5\ucfe6\ucfe7\ucfe8\ucfe9\ucfea\ucfeb\ucfec\ucfed\ucfee\ucfef\ucff0\ucff1\ucff2\ucff3\ucff4\ucff5\ucff6\ucff7\ucff8\ucff9\ucffa\ucffb\ucffc\ucffd\ucffe\ucfff\ud000\ud001\ud002\ud003\ud004\ud005\ud006\ud007\ud008\ud009\ud00a\ud00b\ud00c\ud00d\ud00e\ud00f\ud010\ud011\ud012\ud013\ud014\ud015\ud016\ud017\ud018\ud019\ud01a\ud01b\ud01c\ud01d\ud01e\ud01f\ud020\ud021\ud022\ud023\ud024\ud025\ud026\ud027\ud028\ud029\ud02a\ud02b\ud02c\ud02d\ud02e\ud02f\ud030\ud031\ud032\ud033\ud034\ud035\ud036\ud037\ud038\ud039\ud03a\ud03b\ud03c\ud03d\ud03e\ud03f\ud040\ud041\ud042\ud043\ud044\ud045\ud046\ud047\ud048\ud049\ud04a\ud04b\ud04c\ud04d\ud04e\ud04f\ud050\ud051\ud052\ud053\ud054\ud055\ud056\ud057\ud058\ud059\ud05a\ud05b\ud05c\ud05d\ud05e\ud05f\ud060\ud061\ud062\ud063\ud064\ud065\ud066\ud067\ud068\ud069\ud06a\ud06b\ud06c\ud06d\ud06e\ud06f\ud070\ud071\ud072\ud073\ud074\ud075\ud076\ud077\ud078\ud079\ud07a\ud07b\ud07c\ud07d\ud07e\ud07f\ud080\ud081\ud082\ud083\ud084\ud085\ud086\ud087\ud088\ud089\ud08a\ud08b\ud08c\ud08d\ud08e\ud08f\ud090\ud091\ud092\ud093\ud094\ud095\ud096\ud097\ud098\ud099\ud09a\ud09b\ud09c\ud09d\ud09e\ud09f\ud0a0\ud0a1\ud0a2\ud0a3\ud0a4\ud0a5\ud0a6\ud0a7\ud0a8\ud0a9\ud0aa\ud0ab\ud0ac\ud0ad\ud0ae\ud0af\ud0b0\ud0b1\ud0b2\ud0b3\ud0b4\ud0b5\ud0b6\ud0b7\ud0b8\ud0b9\ud0ba\ud0bb\ud0bc\ud0bd\ud0be\ud0bf\ud0c0\ud0c1\ud0c2\ud0c3\ud0c4\ud0c5\ud0c6\ud0c7\ud0c8\ud0c9\ud0ca\ud0cb\ud0cc\ud0cd\ud0ce\ud0cf\ud0d0\ud0d1\ud0d2\ud0d3\ud0d4\ud0d5\ud0d6\ud0d7\ud0d8\ud0d9\ud0da\ud0db\ud0dc\ud0dd\ud0de\ud0df\ud0e0\ud0e1\ud0e2\ud0e3\ud0e4\ud0e5\ud0e6\ud0e7\ud0e8\ud0e9\ud0ea\ud0eb\ud0ec\ud0ed\ud0ee\ud0ef\ud0f0\ud0f1\ud0f2\ud0f3\ud0f4\ud0f5\ud0f6\ud0f7\ud0f8\ud0f9\ud0fa\ud0fb\ud0fc\ud0fd\ud0fe\ud0ff\ud100\ud101\ud102\ud103\ud104\ud105\ud106\ud107\ud108\ud109\ud10a\ud10b\ud10c\ud10d\ud10e\ud10f\ud110\ud111\ud112\ud113\ud114\ud115\ud116\ud117\ud118\ud119\ud11a\ud11b\ud11c\ud11d\ud11e\ud11f\ud120\ud121\ud122\ud123\ud124\ud125\ud126\ud127\ud128\ud129\ud12a\ud12b\ud12c\ud12d\ud12e\ud12f\ud130\ud131\ud132\ud133\ud134\ud135\ud136\ud137\ud138\ud139\ud13a\ud13b\ud13c\ud13d\ud13e\ud13f\ud140\ud141\ud142\ud143\ud144\ud145\ud146\ud147\ud148\ud149\ud14a\ud14b\ud14c\ud14d\ud14e\ud14f\ud150\ud151\ud152\ud153\ud154\ud155\ud156\ud157\ud158\ud159\ud15a\ud15b\ud15c\ud15d\ud15e\ud15f\ud160\ud161\ud162\ud163\ud164\ud165\ud166\ud167\ud168\ud169\ud16a\ud16b\ud16c\ud16d\ud16e\ud16f\ud170\ud171\ud172\ud173\ud174\ud175\ud176\ud177\ud178\ud179\ud17a\ud17b\ud17c\ud17d\ud17e\ud17f\ud180\ud181\ud182\ud183\ud184\ud185\ud186\ud187\ud188\ud189\ud18a\ud18b\ud18c\ud18d\ud18e\ud18f\ud190\ud191\ud192\ud193\ud194\ud195\ud196\ud197\ud198\ud199\ud19a\ud19b\ud19c\ud19d\ud19e\ud19f\ud1a0\ud1a1\ud1a2\ud1a3\ud1a4\ud1a5\ud1a6\ud1a7\ud1a8\ud1a9\ud1aa\ud1ab\ud1ac\ud1ad\ud1ae\ud1af\ud1b0\ud1b1\ud1b2\ud1b3\ud1b4\ud1b5\ud1b6\ud1b7\ud1b8\ud1b9\ud1ba\ud1bb\ud1bc\ud1bd\ud1be\ud1bf\ud1c0\ud1c1\ud1c2\ud1c3\ud1c4\ud1c5\ud1c6\ud1c7\ud1c8\ud1c9\ud1ca\ud1cb\ud1cc\ud1cd\ud1ce\ud1cf\ud1d0\ud1d1\ud1d2\ud1d3\ud1d4\ud1d5\ud1d6\ud1d7\ud1d8\ud1d9\ud1da\ud1db\ud1dc\ud1dd\ud1de\ud1df\ud1e0\ud1e1\ud1e2\ud1e3\ud1e4\ud1e5\ud1e6\ud1e7\ud1e8\ud1e9\ud1ea\ud1eb\ud1ec\ud1ed\ud1ee\ud1ef\ud1f0\ud1f1\ud1f2\ud1f3\ud1f4\ud1f5\ud1f6\ud1f7\ud1f8\ud1f9\ud1fa\ud1fb\ud1fc\ud1fd\ud1fe\ud1ff\ud200\ud201\ud202\ud203\ud204\ud205\ud206\ud207\ud208\ud209\ud20a\ud20b\ud20c\ud20d\ud20e\ud20f\ud210\ud211\ud212\ud213\ud214\ud215\ud216\ud217\ud218\ud219\ud21a\ud21b\ud21c\ud21d\ud21e\ud21f\ud220\ud221\ud222\ud223\ud224\ud225\ud226\ud227\ud228\ud229\ud22a\ud22b\ud22c\ud22d\ud22e\ud22f\ud230\ud231\ud232\ud233\ud234\ud235\ud236\ud237\ud238\ud239\ud23a\ud23b\ud23c\ud23d\ud23e\ud23f\ud240\ud241\ud242\ud243\ud244\ud245\ud246\ud247\ud248\ud249\ud24a\ud24b\ud24c\ud24d\ud24e\ud24f\ud250\ud251\ud252\ud253\ud254\ud255\ud256\ud257\ud258\ud259\ud25a\ud25b\ud25c\ud25d\ud25e\ud25f\ud260\ud261\ud262\ud263\ud264\ud265\ud266\ud267\ud268\ud269\ud26a\ud26b\ud26c\ud26d\ud26e\ud26f\ud270\ud271\ud272\ud273\ud274\ud275\ud276\ud277\ud278\ud279\ud27a\ud27b\ud27c\ud27d\ud27e\ud27f\ud280\ud281\ud282\ud283\ud284\ud285\ud286\ud287\ud288\ud289\ud28a\ud28b\ud28c\ud28d\ud28e\ud28f\ud290\ud291\ud292\ud293\ud294\ud295\ud296\ud297\ud298\ud299\ud29a\ud29b\ud29c\ud29d\ud29e\ud29f\ud2a0\ud2a1\ud2a2\ud2a3\ud2a4\ud2a5\ud2a6\ud2a7\ud2a8\ud2a9\ud2aa\ud2ab\ud2ac\ud2ad\ud2ae\ud2af\ud2b0\ud2b1\ud2b2\ud2b3\ud2b4\ud2b5\ud2b6\ud2b7\ud2b8\ud2b9\ud2ba\ud2bb\ud2bc\ud2bd\ud2be\ud2bf\ud2c0\ud2c1\ud2c2\ud2c3\ud2c4\ud2c5\ud2c6\ud2c7\ud2c8\ud2c9\ud2ca\ud2cb\ud2cc\ud2cd\ud2ce\ud2cf\ud2d0\ud2d1\ud2d2\ud2d3\ud2d4\ud2d5\ud2d6\ud2d7\ud2d8\ud2d9\ud2da\ud2db\ud2dc\ud2dd\ud2de\ud2df\ud2e0\ud2e1\ud2e2\ud2e3\ud2e4\ud2e5\ud2e6\ud2e7\ud2e8\ud2e9\ud2ea\ud2eb\ud2ec\ud2ed\ud2ee\ud2ef\ud2f0\ud2f1\ud2f2\ud2f3\ud2f4\ud2f5\ud2f6\ud2f7\ud2f8\ud2f9\ud2fa\ud2fb\ud2fc\ud2fd\ud2fe\ud2ff\ud300\ud301\ud302\ud303\ud304\ud305\ud306\ud307\ud308\ud309\ud30a\ud30b\ud30c\ud30d\ud30e\ud30f\ud310\ud311\ud312\ud313\ud314\ud315\ud316\ud317\ud318\ud319\ud31a\ud31b\ud31c\ud31d\ud31e\ud31f\ud320\ud321\ud322\ud323\ud324\ud325\ud326\ud327\ud328\ud329\ud32a\ud32b\ud32c\ud32d\ud32e\ud32f\ud330\ud331\ud332\ud333\ud334\ud335\ud336\ud337\ud338\ud339\ud33a\ud33b\ud33c\ud33d\ud33e\ud33f\ud340\ud341\ud342\ud343\ud344\ud345\ud346\ud347\ud348\ud349\ud34a\ud34b\ud34c\ud34d\ud34e\ud34f\ud350\ud351\ud352\ud353\ud354\ud355\ud356\ud357\ud358\ud359\ud35a\ud35b\ud35c\ud35d\ud35e\ud35f\ud360\ud361\ud362\ud363\ud364\ud365\ud366\ud367\ud368\ud369\ud36a\ud36b\ud36c\ud36d\ud36e\ud36f\ud370\ud371\ud372\ud373\ud374\ud375\ud376\ud377\ud378\ud379\ud37a\ud37b\ud37c\ud37d\ud37e\ud37f\ud380\ud381\ud382\ud383\ud384\ud385\ud386\ud387\ud388\ud389\ud38a\ud38b\ud38c\ud38d\ud38e\ud38f\ud390\ud391\ud392\ud393\ud394\ud395\ud396\ud397\ud398\ud399\ud39a\ud39b\ud39c\ud39d\ud39e\ud39f\ud3a0\ud3a1\ud3a2\ud3a3\ud3a4\ud3a5\ud3a6\ud3a7\ud3a8\ud3a9\ud3aa\ud3ab\ud3ac\ud3ad\ud3ae\ud3af\ud3b0\ud3b1\ud3b2\ud3b3\ud3b4\ud3b5\ud3b6\ud3b7\ud3b8\ud3b9\ud3ba\ud3bb\ud3bc\ud3bd\ud3be\ud3bf\ud3c0\ud3c1\ud3c2\ud3c3\ud3c4\ud3c5\ud3c6\ud3c7\ud3c8\ud3c9\ud3ca\ud3cb\ud3cc\ud3cd\ud3ce\ud3cf\ud3d0\ud3d1\ud3d2\ud3d3\ud3d4\ud3d5\ud3d6\ud3d7\ud3d8\ud3d9\ud3da\ud3db\ud3dc\ud3dd\ud3de\ud3df\ud3e0\ud3e1\ud3e2\ud3e3\ud3e4\ud3e5\ud3e6\ud3e7\ud3e8\ud3e9\ud3ea\ud3eb\ud3ec\ud3ed\ud3ee\ud3ef\ud3f0\ud3f1\ud3f2\ud3f3\ud3f4\ud3f5\ud3f6\ud3f7\ud3f8\ud3f9\ud3fa\ud3fb\ud3fc\ud3fd\ud3fe\ud3ff\ud400\ud401\ud402\ud403\ud404\ud405\ud406\ud407\ud408\ud409\ud40a\ud40b\ud40c\ud40d\ud40e\ud40f\ud410\ud411\ud412\ud413\ud414\ud415\ud416\ud417\ud418\ud419\ud41a\ud41b\ud41c\ud41d\ud41e\ud41f\ud420\ud421\ud422\ud423\ud424\ud425\ud426\ud427\ud428\ud429\ud42a\ud42b\ud42c\ud42d\ud42e\ud42f\ud430\ud431\ud432\ud433\ud434\ud435\ud436\ud437\ud438\ud439\ud43a\ud43b\ud43c\ud43d\ud43e\ud43f\ud440\ud441\ud442\ud443\ud444\ud445\ud446\ud447\ud448\ud449\ud44a\ud44b\ud44c\ud44d\ud44e\ud44f\ud450\ud451\ud452\ud453\ud454\ud455\ud456\ud457\ud458\ud459\ud45a\ud45b\ud45c\ud45d\ud45e\ud45f\ud460\ud461\ud462\ud463\ud464\ud465\ud466\ud467\ud468\ud469\ud46a\ud46b\ud46c\ud46d\ud46e\ud46f\ud470\ud471\ud472\ud473\ud474\ud475\ud476\ud477\ud478\ud479\ud47a\ud47b\ud47c\ud47d\ud47e\ud47f\ud480\ud481\ud482\ud483\ud484\ud485\ud486\ud487\ud488\ud489\ud48a\ud48b\ud48c\ud48d\ud48e\ud48f\ud490\ud491\ud492\ud493\ud494\ud495\ud496\ud497\ud498\ud499\ud49a\ud49b\ud49c\ud49d\ud49e\ud49f\ud4a0\ud4a1\ud4a2\ud4a3\ud4a4\ud4a5\ud4a6\ud4a7\ud4a8\ud4a9\ud4aa\ud4ab\ud4ac\ud4ad\ud4ae\ud4af\ud4b0\ud4b1\ud4b2\ud4b3\ud4b4\ud4b5\ud4b6\ud4b7\ud4b8\ud4b9\ud4ba\ud4bb\ud4bc\ud4bd\ud4be\ud4bf\ud4c0\ud4c1\ud4c2\ud4c3\ud4c4\ud4c5\ud4c6\ud4c7\ud4c8\ud4c9\ud4ca\ud4cb\ud4cc\ud4cd\ud4ce\ud4cf\ud4d0\ud4d1\ud4d2\ud4d3\ud4d4\ud4d5\ud4d6\ud4d7\ud4d8\ud4d9\ud4da\ud4db\ud4dc\ud4dd\ud4de\ud4df\ud4e0\ud4e1\ud4e2\ud4e3\ud4e4\ud4e5\ud4e6\ud4e7\ud4e8\ud4e9\ud4ea\ud4eb\ud4ec\ud4ed\ud4ee\ud4ef\ud4f0\ud4f1\ud4f2\ud4f3\ud4f4\ud4f5\ud4f6\ud4f7\ud4f8\ud4f9\ud4fa\ud4fb\ud4fc\ud4fd\ud4fe\ud4ff\ud500\ud501\ud502\ud503\ud504\ud505\ud506\ud507\ud508\ud509\ud50a\ud50b\ud50c\ud50d\ud50e\ud50f\ud510\ud511\ud512\ud513\ud514\ud515\ud516\ud517\ud518\ud519\ud51a\ud51b\ud51c\ud51d\ud51e\ud51f\ud520\ud521\ud522\ud523\ud524\ud525\ud526\ud527\ud528\ud529\ud52a\ud52b\ud52c\ud52d\ud52e\ud52f\ud530\ud531\ud532\ud533\ud534\ud535\ud536\ud537\ud538\ud539\ud53a\ud53b\ud53c\ud53d\ud53e\ud53f\ud540\ud541\ud542\ud543\ud544\ud545\ud546\ud547\ud548\ud549\ud54a\ud54b\ud54c\ud54d\ud54e\ud54f\ud550\ud551\ud552\ud553\ud554\ud555\ud556\ud557\ud558\ud559\ud55a\ud55b\ud55c\ud55d\ud55e\ud55f\ud560\ud561\ud562\ud563\ud564\ud565\ud566\ud567\ud568\ud569\ud56a\ud56b\ud56c\ud56d\ud56e\ud56f\ud570\ud571\ud572\ud573\ud574\ud575\ud576\ud577\ud578\ud579\ud57a\ud57b\ud57c\ud57d\ud57e\ud57f\ud580\ud581\ud582\ud583\ud584\ud585\ud586\ud587\ud588\ud589\ud58a\ud58b\ud58c\ud58d\ud58e\ud58f\ud590\ud591\ud592\ud593\ud594\ud595\ud596\ud597\ud598\ud599\ud59a\ud59b\ud59c\ud59d\ud59e\ud59f\ud5a0\ud5a1\ud5a2\ud5a3\ud5a4\ud5a5\ud5a6\ud5a7\ud5a8\ud5a9\ud5aa\ud5ab\ud5ac\ud5ad\ud5ae\ud5af\ud5b0\ud5b1\ud5b2\ud5b3\ud5b4\ud5b5\ud5b6\ud5b7\ud5b8\ud5b9\ud5ba\ud5bb\ud5bc\ud5bd\ud5be\ud5bf\ud5c0\ud5c1\ud5c2\ud5c3\ud5c4\ud5c5\ud5c6\ud5c7\ud5c8\ud5c9\ud5ca\ud5cb\ud5cc\ud5cd\ud5ce\ud5cf\ud5d0\ud5d1\ud5d2\ud5d3\ud5d4\ud5d5\ud5d6\ud5d7\ud5d8\ud5d9\ud5da\ud5db\ud5dc\ud5dd\ud5de\ud5df\ud5e0\ud5e1\ud5e2\ud5e3\ud5e4\ud5e5\ud5e6\ud5e7\ud5e8\ud5e9\ud5ea\ud5eb\ud5ec\ud5ed\ud5ee\ud5ef\ud5f0\ud5f1\ud5f2\ud5f3\ud5f4\ud5f5\ud5f6\ud5f7\ud5f8\ud5f9\ud5fa\ud5fb\ud5fc\ud5fd\ud5fe\ud5ff\ud600\ud601\ud602\ud603\ud604\ud605\ud606\ud607\ud608\ud609\ud60a\ud60b\ud60c\ud60d\ud60e\ud60f\ud610\ud611\ud612\ud613\ud614\ud615\ud616\ud617\ud618\ud619\ud61a\ud61b\ud61c\ud61d\ud61e\ud61f\ud620\ud621\ud622\ud623\ud624\ud625\ud626\ud627\ud628\ud629\ud62a\ud62b\ud62c\ud62d\ud62e\ud62f\ud630\ud631\ud632\ud633\ud634\ud635\ud636\ud637\ud638\ud639\ud63a\ud63b\ud63c\ud63d\ud63e\ud63f\ud640\ud641\ud642\ud643\ud644\ud645\ud646\ud647\ud648\ud649\ud64a\ud64b\ud64c\ud64d\ud64e\ud64f\ud650\ud651\ud652\ud653\ud654\ud655\ud656\ud657\ud658\ud659\ud65a\ud65b\ud65c\ud65d\ud65e\ud65f\ud660\ud661\ud662\ud663\ud664\ud665\ud666\ud667\ud668\ud669\ud66a\ud66b\ud66c\ud66d\ud66e\ud66f\ud670\ud671\ud672\ud673\ud674\ud675\ud676\ud677\ud678\ud679\ud67a\ud67b\ud67c\ud67d\ud67e\ud67f\ud680\ud681\ud682\ud683\ud684\ud685\ud686\ud687\ud688\ud689\ud68a\ud68b\ud68c\ud68d\ud68e\ud68f\ud690\ud691\ud692\ud693\ud694\ud695\ud696\ud697\ud698\ud699\ud69a\ud69b\ud69c\ud69d\ud69e\ud69f\ud6a0\ud6a1\ud6a2\ud6a3\ud6a4\ud6a5\ud6a6\ud6a7\ud6a8\ud6a9\ud6aa\ud6ab\ud6ac\ud6ad\ud6ae\ud6af\ud6b0\ud6b1\ud6b2\ud6b3\ud6b4\ud6b5\ud6b6\ud6b7\ud6b8\ud6b9\ud6ba\ud6bb\ud6bc\ud6bd\ud6be\ud6bf\ud6c0\ud6c1\ud6c2\ud6c3\ud6c4\ud6c5\ud6c6\ud6c7\ud6c8\ud6c9\ud6ca\ud6cb\ud6cc\ud6cd\ud6ce\ud6cf\ud6d0\ud6d1\ud6d2\ud6d3\ud6d4\ud6d5\ud6d6\ud6d7\ud6d8\ud6d9\ud6da\ud6db\ud6dc\ud6dd\ud6de\ud6df\ud6e0\ud6e1\ud6e2\ud6e3\ud6e4\ud6e5\ud6e6\ud6e7\ud6e8\ud6e9\ud6ea\ud6eb\ud6ec\ud6ed\ud6ee\ud6ef\ud6f0\ud6f1\ud6f2\ud6f3\ud6f4\ud6f5\ud6f6\ud6f7\ud6f8\ud6f9\ud6fa\ud6fb\ud6fc\ud6fd\ud6fe\ud6ff\ud700\ud701\ud702\ud703\ud704\ud705\ud706\ud707\ud708\ud709\ud70a\ud70b\ud70c\ud70d\ud70e\ud70f\ud710\ud711\ud712\ud713\ud714\ud715\ud716\ud717\ud718\ud719\ud71a\ud71b\ud71c\ud71d\ud71e\ud71f\ud720\ud721\ud722\ud723\ud724\ud725\ud726\ud727\ud728\ud729\ud72a\ud72b\ud72c\ud72d\ud72e\ud72f\ud730\ud731\ud732\ud733\ud734\ud735\ud736\ud737\ud738\ud739\ud73a\ud73b\ud73c\ud73d\ud73e\ud73f\ud740\ud741\ud742\ud743\ud744\ud745\ud746\ud747\ud748\ud749\ud74a\ud74b\ud74c\ud74d\ud74e\ud74f\ud750\ud751\ud752\ud753\ud754\ud755\ud756\ud757\ud758\ud759\ud75a\ud75b\ud75c\ud75d\ud75e\ud75f\ud760\ud761\ud762\ud763\ud764\ud765\ud766\ud767\ud768\ud769\ud76a\ud76b\ud76c\ud76d\ud76e\ud76f\ud770\ud771\ud772\ud773\ud774\ud775\ud776\ud777\ud778\ud779\ud77a\ud77b\ud77c\ud77d\ud77e\ud77f\ud780\ud781\ud782\ud783\ud784\ud785\ud786\ud787\ud788\ud789\ud78a\ud78b\ud78c\ud78d\ud78e\ud78f\ud790\ud791\ud792\ud793\ud794\ud795\ud796\ud797\ud798\ud799\ud79a\ud79b\ud79c\ud79d\ud79e\ud79f\ud7a0\ud7a1\ud7a2\ud7a3\uf900\uf901\uf902\uf903\uf904\uf905\uf906\uf907\uf908\uf909\uf90a\uf90b\uf90c\uf90d\uf90e\uf90f\uf910\uf911\uf912\uf913\uf914\uf915\uf916\uf917\uf918\uf919\uf91a\uf91b\uf91c\uf91d\uf91e\uf91f\uf920\uf921\uf922\uf923\uf924\uf925\uf926\uf927\uf928\uf929\uf92a\uf92b\uf92c\uf92d\uf92e\uf92f\uf930\uf931\uf932\uf933\uf934\uf935\uf936\uf937\uf938\uf939\uf93a\uf93b\uf93c\uf93d\uf93e\uf93f\uf940\uf941\uf942\uf943\uf944\uf945\uf946\uf947\uf948\uf949\uf94a\uf94b\uf94c\uf94d\uf94e\uf94f\uf950\uf951\uf952\uf953\uf954\uf955\uf956\uf957\uf958\uf959\uf95a\uf95b\uf95c\uf95d\uf95e\uf95f\uf960\uf961\uf962\uf963\uf964\uf965\uf966\uf967\uf968\uf969\uf96a\uf96b\uf96c\uf96d\uf96e\uf96f\uf970\uf971\uf972\uf973\uf974\uf975\uf976\uf977\uf978\uf979\uf97a\uf97b\uf97c\uf97d\uf97e\uf97f\uf980\uf981\uf982\uf983\uf984\uf985\uf986\uf987\uf988\uf989\uf98a\uf98b\uf98c\uf98d\uf98e\uf98f\uf990\uf991\uf992\uf993\uf994\uf995\uf996\uf997\uf998\uf999\uf99a\uf99b\uf99c\uf99d\uf99e\uf99f\uf9a0\uf9a1\uf9a2\uf9a3\uf9a4\uf9a5\uf9a6\uf9a7\uf9a8\uf9a9\uf9aa\uf9ab\uf9ac\uf9ad\uf9ae\uf9af\uf9b0\uf9b1\uf9b2\uf9b3\uf9b4\uf9b5\uf9b6\uf9b7\uf9b8\uf9b9\uf9ba\uf9bb\uf9bc\uf9bd\uf9be\uf9bf\uf9c0\uf9c1\uf9c2\uf9c3\uf9c4\uf9c5\uf9c6\uf9c7\uf9c8\uf9c9\uf9ca\uf9cb\uf9cc\uf9cd\uf9ce\uf9cf\uf9d0\uf9d1\uf9d2\uf9d3\uf9d4\uf9d5\uf9d6\uf9d7\uf9d8\uf9d9\uf9da\uf9db\uf9dc\uf9dd\uf9de\uf9df\uf9e0\uf9e1\uf9e2\uf9e3\uf9e4\uf9e5\uf9e6\uf9e7\uf9e8\uf9e9\uf9ea\uf9eb\uf9ec\uf9ed\uf9ee\uf9ef\uf9f0\uf9f1\uf9f2\uf9f3\uf9f4\uf9f5\uf9f6\uf9f7\uf9f8\uf9f9\uf9fa\uf9fb\uf9fc\uf9fd\uf9fe\uf9ff\ufa00\ufa01\ufa02\ufa03\ufa04\ufa05\ufa06\ufa07\ufa08\ufa09\ufa0a\ufa0b\ufa0c\ufa0d\ufa0e\ufa0f\ufa10\ufa11\ufa12\ufa13\ufa14\ufa15\ufa16\ufa17\ufa18\ufa19\ufa1a\ufa1b\ufa1c\ufa1d\ufa1e\ufa1f\ufa20\ufa21\ufa22\ufa23\ufa24\ufa25\ufa26\ufa27\ufa28\ufa29\ufa2a\ufa2b\ufa2c\ufa2d\ufa30\ufa31\ufa32\ufa33\ufa34\ufa35\ufa36\ufa37\ufa38\ufa39\ufa3a\ufa3b\ufa3c\ufa3d\ufa3e\ufa3f\ufa40\ufa41\ufa42\ufa43\ufa44\ufa45\ufa46\ufa47\ufa48\ufa49\ufa4a\ufa4b\ufa4c\ufa4d\ufa4e\ufa4f\ufa50\ufa51\ufa52\ufa53\ufa54\ufa55\ufa56\ufa57\ufa58\ufa59\ufa5a\ufa5b\ufa5c\ufa5d\ufa5e\ufa5f\ufa60\ufa61\ufa62\ufa63\ufa64\ufa65\ufa66\ufa67\ufa68\ufa69\ufa6a\ufa70\ufa71\ufa72\ufa73\ufa74\ufa75\ufa76\ufa77\ufa78\ufa79\ufa7a\ufa7b\ufa7c\ufa7d\ufa7e\ufa7f\ufa80\ufa81\ufa82\ufa83\ufa84\ufa85\ufa86\ufa87\ufa88\ufa89\ufa8a\ufa8b\ufa8c\ufa8d\ufa8e\ufa8f\ufa90\ufa91\ufa92\ufa93\ufa94\ufa95\ufa96\ufa97\ufa98\ufa99\ufa9a\ufa9b\ufa9c\ufa9d\ufa9e\ufa9f\ufaa0\ufaa1\ufaa2\ufaa3\ufaa4\ufaa5\ufaa6\ufaa7\ufaa8\ufaa9\ufaaa\ufaab\ufaac\ufaad\ufaae\ufaaf\ufab0\ufab1\ufab2\ufab3\ufab4\ufab5\ufab6\ufab7\ufab8\ufab9\ufaba\ufabb\ufabc\ufabd\ufabe\ufabf\ufac0\ufac1\ufac2\ufac3\ufac4\ufac5\ufac6\ufac7\ufac8\ufac9\ufaca\ufacb\ufacc\ufacd\uface\ufacf\ufad0\ufad1\ufad2\ufad3\ufad4\ufad5\ufad6\ufad7\ufad8\ufad9\ufb1d\ufb1f\ufb20\ufb21\ufb22\ufb23\ufb24\ufb25\ufb26\ufb27\ufb28\ufb2a\ufb2b\ufb2c\ufb2d\ufb2e\ufb2f\ufb30\ufb31\ufb32\ufb33\ufb34\ufb35\ufb36\ufb38\ufb39\ufb3a\ufb3b\ufb3c\ufb3e\ufb40\ufb41\ufb43\ufb44\ufb46\ufb47\ufb48\ufb49\ufb4a\ufb4b\ufb4c\ufb4d\ufb4e\ufb4f\ufb50\ufb51\ufb52\ufb53\ufb54\ufb55\ufb56\ufb57\ufb58\ufb59\ufb5a\ufb5b\ufb5c\ufb5d\ufb5e\ufb5f\ufb60\ufb61\ufb62\ufb63\ufb64\ufb65\ufb66\ufb67\ufb68\ufb69\ufb6a\ufb6b\ufb6c\ufb6d\ufb6e\ufb6f\ufb70\ufb71\ufb72\ufb73\ufb74\ufb75\ufb76\ufb77\ufb78\ufb79\ufb7a\ufb7b\ufb7c\ufb7d\ufb7e\ufb7f\ufb80\ufb81\ufb82\ufb83\ufb84\ufb85\ufb86\ufb87\ufb88\ufb89\ufb8a\ufb8b\ufb8c\ufb8d\ufb8e\ufb8f\ufb90\ufb91\ufb92\ufb93\ufb94\ufb95\ufb96\ufb97\ufb98\ufb99\ufb9a\ufb9b\ufb9c\ufb9d\ufb9e\ufb9f\ufba0\ufba1\ufba2\ufba3\ufba4\ufba5\ufba6\ufba7\ufba8\ufba9\ufbaa\ufbab\ufbac\ufbad\ufbae\ufbaf\ufbb0\ufbb1\ufbd3\ufbd4\ufbd5\ufbd6\ufbd7\ufbd8\ufbd9\ufbda\ufbdb\ufbdc\ufbdd\ufbde\ufbdf\ufbe0\ufbe1\ufbe2\ufbe3\ufbe4\ufbe5\ufbe6\ufbe7\ufbe8\ufbe9\ufbea\ufbeb\ufbec\ufbed\ufbee\ufbef\ufbf0\ufbf1\ufbf2\ufbf3\ufbf4\ufbf5\ufbf6\ufbf7\ufbf8\ufbf9\ufbfa\ufbfb\ufbfc\ufbfd\ufbfe\ufbff\ufc00\ufc01\ufc02\ufc03\ufc04\ufc05\ufc06\ufc07\ufc08\ufc09\ufc0a\ufc0b\ufc0c\ufc0d\ufc0e\ufc0f\ufc10\ufc11\ufc12\ufc13\ufc14\ufc15\ufc16\ufc17\ufc18\ufc19\ufc1a\ufc1b\ufc1c\ufc1d\ufc1e\ufc1f\ufc20\ufc21\ufc22\ufc23\ufc24\ufc25\ufc26\ufc27\ufc28\ufc29\ufc2a\ufc2b\ufc2c\ufc2d\ufc2e\ufc2f\ufc30\ufc31\ufc32\ufc33\ufc34\ufc35\ufc36\ufc37\ufc38\ufc39\ufc3a\ufc3b\ufc3c\ufc3d\ufc3e\ufc3f\ufc40\ufc41\ufc42\ufc43\ufc44\ufc45\ufc46\ufc47\ufc48\ufc49\ufc4a\ufc4b\ufc4c\ufc4d\ufc4e\ufc4f\ufc50\ufc51\ufc52\ufc53\ufc54\ufc55\ufc56\ufc57\ufc58\ufc59\ufc5a\ufc5b\ufc5c\ufc5d\ufc5e\ufc5f\ufc60\ufc61\ufc62\ufc63\ufc64\ufc65\ufc66\ufc67\ufc68\ufc69\ufc6a\ufc6b\ufc6c\ufc6d\ufc6e\ufc6f\ufc70\ufc71\ufc72\ufc73\ufc74\ufc75\ufc76\ufc77\ufc78\ufc79\ufc7a\ufc7b\ufc7c\ufc7d\ufc7e\ufc7f\ufc80\ufc81\ufc82\ufc83\ufc84\ufc85\ufc86\ufc87\ufc88\ufc89\ufc8a\ufc8b\ufc8c\ufc8d\ufc8e\ufc8f\ufc90\ufc91\ufc92\ufc93\ufc94\ufc95\ufc96\ufc97\ufc98\ufc99\ufc9a\ufc9b\ufc9c\ufc9d\ufc9e\ufc9f\ufca0\ufca1\ufca2\ufca3\ufca4\ufca5\ufca6\ufca7\ufca8\ufca9\ufcaa\ufcab\ufcac\ufcad\ufcae\ufcaf\ufcb0\ufcb1\ufcb2\ufcb3\ufcb4\ufcb5\ufcb6\ufcb7\ufcb8\ufcb9\ufcba\ufcbb\ufcbc\ufcbd\ufcbe\ufcbf\ufcc0\ufcc1\ufcc2\ufcc3\ufcc4\ufcc5\ufcc6\ufcc7\ufcc8\ufcc9\ufcca\ufccb\ufccc\ufccd\ufcce\ufccf\ufcd0\ufcd1\ufcd2\ufcd3\ufcd4\ufcd5\ufcd6\ufcd7\ufcd8\ufcd9\ufcda\ufcdb\ufcdc\ufcdd\ufcde\ufcdf\ufce0\ufce1\ufce2\ufce3\ufce4\ufce5\ufce6\ufce7\ufce8\ufce9\ufcea\ufceb\ufcec\ufced\ufcee\ufcef\ufcf0\ufcf1\ufcf2\ufcf3\ufcf4\ufcf5\ufcf6\ufcf7\ufcf8\ufcf9\ufcfa\ufcfb\ufcfc\ufcfd\ufcfe\ufcff\ufd00\ufd01\ufd02\ufd03\ufd04\ufd05\ufd06\ufd07\ufd08\ufd09\ufd0a\ufd0b\ufd0c\ufd0d\ufd0e\ufd0f\ufd10\ufd11\ufd12\ufd13\ufd14\ufd15\ufd16\ufd17\ufd18\ufd19\ufd1a\ufd1b\ufd1c\ufd1d\ufd1e\ufd1f\ufd20\ufd21\ufd22\ufd23\ufd24\ufd25\ufd26\ufd27\ufd28\ufd29\ufd2a\ufd2b\ufd2c\ufd2d\ufd2e\ufd2f\ufd30\ufd31\ufd32\ufd33\ufd34\ufd35\ufd36\ufd37\ufd38\ufd39\ufd3a\ufd3b\ufd3c\ufd3d\ufd50\ufd51\ufd52\ufd53\ufd54\ufd55\ufd56\ufd57\ufd58\ufd59\ufd5a\ufd5b\ufd5c\ufd5d\ufd5e\ufd5f\ufd60\ufd61\ufd62\ufd63\ufd64\ufd65\ufd66\ufd67\ufd68\ufd69\ufd6a\ufd6b\ufd6c\ufd6d\ufd6e\ufd6f\ufd70\ufd71\ufd72\ufd73\ufd74\ufd75\ufd76\ufd77\ufd78\ufd79\ufd7a\ufd7b\ufd7c\ufd7d\ufd7e\ufd7f\ufd80\ufd81\ufd82\ufd83\ufd84\ufd85\ufd86\ufd87\ufd88\ufd89\ufd8a\ufd8b\ufd8c\ufd8d\ufd8e\ufd8f\ufd92\ufd93\ufd94\ufd95\ufd96\ufd97\ufd98\ufd99\ufd9a\ufd9b\ufd9c\ufd9d\ufd9e\ufd9f\ufda0\ufda1\ufda2\ufda3\ufda4\ufda5\ufda6\ufda7\ufda8\ufda9\ufdaa\ufdab\ufdac\ufdad\ufdae\ufdaf\ufdb0\ufdb1\ufdb2\ufdb3\ufdb4\ufdb5\ufdb6\ufdb7\ufdb8\ufdb9\ufdba\ufdbb\ufdbc\ufdbd\ufdbe\ufdbf\ufdc0\ufdc1\ufdc2\ufdc3\ufdc4\ufdc5\ufdc6\ufdc7\ufdf0\ufdf1\ufdf2\ufdf3\ufdf4\ufdf5\ufdf6\ufdf7\ufdf8\ufdf9\ufdfa\ufdfb\ufe70\ufe71\ufe72\ufe73\ufe74\ufe76\ufe77\ufe78\ufe79\ufe7a\ufe7b\ufe7c\ufe7d\ufe7e\ufe7f\ufe80\ufe81\ufe82\ufe83\ufe84\ufe85\ufe86\ufe87\ufe88\ufe89\ufe8a\ufe8b\ufe8c\ufe8d\ufe8e\ufe8f\ufe90\ufe91\ufe92\ufe93\ufe94\ufe95\ufe96\ufe97\ufe98\ufe99\ufe9a\ufe9b\ufe9c\ufe9d\ufe9e\ufe9f\ufea0\ufea1\ufea2\ufea3\ufea4\ufea5\ufea6\ufea7\ufea8\ufea9\ufeaa\ufeab\ufeac\ufead\ufeae\ufeaf\ufeb0\ufeb1\ufeb2\ufeb3\ufeb4\ufeb5\ufeb6\ufeb7\ufeb8\ufeb9\ufeba\ufebb\ufebc\ufebd\ufebe\ufebf\ufec0\ufec1\ufec2\ufec3\ufec4\ufec5\ufec6\ufec7\ufec8\ufec9\ufeca\ufecb\ufecc\ufecd\ufece\ufecf\ufed0\ufed1\ufed2\ufed3\ufed4\ufed5\ufed6\ufed7\ufed8\ufed9\ufeda\ufedb\ufedc\ufedd\ufede\ufedf\ufee0\ufee1\ufee2\ufee3\ufee4\ufee5\ufee6\ufee7\ufee8\ufee9\ufeea\ufeeb\ufeec\ufeed\ufeee\ufeef\ufef0\ufef1\ufef2\ufef3\ufef4\ufef5\ufef6\ufef7\ufef8\ufef9\ufefa\ufefb\ufefc\uff66\uff67\uff68\uff69\uff6a\uff6b\uff6c\uff6d\uff6e\uff6f\uff71\uff72\uff73\uff74\uff75\uff76\uff77\uff78\uff79\uff7a\uff7b\uff7c\uff7d\uff7e\uff7f\uff80\uff81\uff82\uff83\uff84\uff85\uff86\uff87\uff88\uff89\uff8a\uff8b\uff8c\uff8d\uff8e\uff8f\uff90\uff91\uff92\uff93\uff94\uff95\uff96\uff97\uff98\uff99\uff9a\uff9b\uff9c\uff9d\uffa0\uffa1\uffa2\uffa3\uffa4\uffa5\uffa6\uffa7\uffa8\uffa9\uffaa\uffab\uffac\uffad\uffae\uffaf\uffb0\uffb1\uffb2\uffb3\uffb4\uffb5\uffb6\uffb7\uffb8\uffb9\uffba\uffbb\uffbc\uffbd\uffbe\uffc2\uffc3\uffc4\uffc5\uffc6\uffc7\uffca\uffcb\uffcc\uffcd\uffce\uffcf\uffd2\uffd3\uffd4\uffd5\uffd6\uffd7\uffda\uffdb\uffdc'
+
+Lt = u'\u01c5\u01c8\u01cb\u01f2\u1f88\u1f89\u1f8a\u1f8b\u1f8c\u1f8d\u1f8e\u1f8f\u1f98\u1f99\u1f9a\u1f9b\u1f9c\u1f9d\u1f9e\u1f9f\u1fa8\u1fa9\u1faa\u1fab\u1fac\u1fad\u1fae\u1faf\u1fbc\u1fcc\u1ffc'
+
+Lu = u'ABCDEFGHIJKLMNOPQRSTUVWXYZ\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd8\xd9\xda\xdb\xdc\xdd\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178\u0179\u017b\u017d\u0181\u0182\u0184\u0186\u0187\u0189\u018a\u018b\u018e\u018f\u0190\u0191\u0193\u0194\u0196\u0197\u0198\u019c\u019d\u019f\u01a0\u01a2\u01a4\u01a6\u01a7\u01a9\u01ac\u01ae\u01af\u01b1\u01b2\u01b3\u01b5\u01b7\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6\u01f7\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a\u023b\u023d\u023e\u0241\u0386\u0388\u0389\u038a\u038c\u038e\u038f\u0391\u0392\u0393\u0394\u0395\u0396\u0397\u0398\u0399\u039a\u039b\u039c\u039d\u039e\u039f\u03a0\u03a1\u03a3\u03a4\u03a5\u03a6\u03a7\u03a8\u03a9\u03aa\u03ab\u03d2\u03d3\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9\u03fa\u03fd\u03fe\u03ff\u0400\u0401\u0402\u0403\u0404\u0405\u0406\u0407\u0408\u0409\u040a\u040b\u040c\u040d\u040e\u040f\u0410\u0411\u0412\u0413\u0414\u0415\u0416\u0417\u0418\u0419\u041a\u041b\u041c\u041d\u041e\u041f\u0420\u0421\u0422\u0423\u0424\u0425\u0426\u0427\u0428\u0429\u042a\u042b\u042c\u042d\u042e\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0531\u0532\u0533\u0534\u0535\u0536\u0537\u0538\u0539\u053a\u053b\u053c\u053d\u053e\u053f\u0540\u0541\u0542\u0543\u0544\u0545\u0546\u0547\u0548\u0549\u054a\u054b\u054c\u054d\u054e\u054f\u0550\u0551\u0552\u0553\u0554\u0555\u0556\u10a0\u10a1\u10a2\u10a3\u10a4\u10a5\u10a6\u10a7\u10a8\u10a9\u10aa\u10ab\u10ac\u10ad\u10ae\u10af\u10b0\u10b1\u10b2\u10b3\u10b4\u10b5\u10b6\u10b7\u10b8\u10b9\u10ba\u10bb\u10bc\u10bd\u10be\u10bf\u10c0\u10c1\u10c2\u10c3\u10c4\u10c5\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1f08\u1f09\u1f0a\u1f0b\u1f0c\u1f0d\u1f0e\u1f0f\u1f18\u1f19\u1f1a\u1f1b\u1f1c\u1f1d\u1f28\u1f29\u1f2a\u1f2b\u1f2c\u1f2d\u1f2e\u1f2f\u1f38\u1f39\u1f3a\u1f3b\u1f3c\u1f3d\u1f3e\u1f3f\u1f48\u1f49\u1f4a\u1f4b\u1f4c\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68\u1f69\u1f6a\u1f6b\u1f6c\u1f6d\u1f6e\u1f6f\u1fb8\u1fb9\u1fba\u1fbb\u1fc8\u1fc9\u1fca\u1fcb\u1fd8\u1fd9\u1fda\u1fdb\u1fe8\u1fe9\u1fea\u1feb\u1fec\u1ff8\u1ff9\u1ffa\u1ffb\u2102\u2107\u210b\u210c\u210d\u2110\u2111\u2112\u2115\u2119\u211a\u211b\u211c\u211d\u2124\u2126\u2128\u212a\u212b\u212c\u212d\u2130\u2131\u2133\u213e\u213f\u2145\u2c00\u2c01\u2c02\u2c03\u2c04\u2c05\u2c06\u2c07\u2c08\u2c09\u2c0a\u2c0b\u2c0c\u2c0d\u2c0e\u2c0f\u2c10\u2c11\u2c12\u2c13\u2c14\u2c15\u2c16\u2c17\u2c18\u2c19\u2c1a\u2c1b\u2c1c\u2c1d\u2c1e\u2c1f\u2c20\u2c21\u2c22\u2c23\u2c24\u2c25\u2c26\u2c27\u2c28\u2c29\u2c2a\u2c2b\u2c2c\u2c2d\u2c2e\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\uff21\uff22\uff23\uff24\uff25\uff26\uff27\uff28\uff29\uff2a\uff2b\uff2c\uff2d\uff2e\uff2f\uff30\uff31\uff32\uff33\uff34\uff35\uff36\uff37\uff38\uff39\uff3a'
+
+Mc = u'\u0903\u093e\u093f\u0940\u0949\u094a\u094b\u094c\u0982\u0983\u09be\u09bf\u09c0\u09c7\u09c8\u09cb\u09cc\u09d7\u0a03\u0a3e\u0a3f\u0a40\u0a83\u0abe\u0abf\u0ac0\u0ac9\u0acb\u0acc\u0b02\u0b03\u0b3e\u0b40\u0b47\u0b48\u0b4b\u0b4c\u0b57\u0bbe\u0bbf\u0bc1\u0bc2\u0bc6\u0bc7\u0bc8\u0bca\u0bcb\u0bcc\u0bd7\u0c01\u0c02\u0c03\u0c41\u0c42\u0c43\u0c44\u0c82\u0c83\u0cbe\u0cc0\u0cc1\u0cc2\u0cc3\u0cc4\u0cc7\u0cc8\u0cca\u0ccb\u0cd5\u0cd6\u0d02\u0d03\u0d3e\u0d3f\u0d40\u0d46\u0d47\u0d48\u0d4a\u0d4b\u0d4c\u0d57\u0d82\u0d83\u0dcf\u0dd0\u0dd1\u0dd8\u0dd9\u0dda\u0ddb\u0ddc\u0ddd\u0dde\u0ddf\u0df2\u0df3\u0f3e\u0f3f\u0f7f\u102c\u1031\u1038\u1056\u1057\u17b6\u17be\u17bf\u17c0\u17c1\u17c2\u17c3\u17c4\u17c5\u17c7\u17c8\u1923\u1924\u1925\u1926\u1929\u192a\u192b\u1930\u1931\u1933\u1934\u1935\u1936\u1937\u1938\u19b0\u19b1\u19b2\u19b3\u19b4\u19b5\u19b6\u19b7\u19b8\u19b9\u19ba\u19bb\u19bc\u19bd\u19be\u19bf\u19c0\u19c8\u19c9\u1a19\u1a1a\u1a1b\ua802\ua823\ua824\ua827'
+
+Me = u'\u0488\u0489\u06de\u20dd\u20de\u20df\u20e0\u20e2\u20e3\u20e4'
+
+Mn = u'\u0300\u0301\u0302\u0303\u0304\u0305\u0306\u0307\u0308\u0309\u030a\u030b\u030c\u030d\u030e\u030f\u0310\u0311\u0312\u0313\u0314\u0315\u0316\u0317\u0318\u0319\u031a\u031b\u031c\u031d\u031e\u031f\u0320\u0321\u0322\u0323\u0324\u0325\u0326\u0327\u0328\u0329\u032a\u032b\u032c\u032d\u032e\u032f\u0330\u0331\u0332\u0333\u0334\u0335\u0336\u0337\u0338\u0339\u033a\u033b\u033c\u033d\u033e\u033f\u0340\u0341\u0342\u0343\u0344\u0345\u0346\u0347\u0348\u0349\u034a\u034b\u034c\u034d\u034e\u034f\u0350\u0351\u0352\u0353\u0354\u0355\u0356\u0357\u0358\u0359\u035a\u035b\u035c\u035d\u035e\u035f\u0360\u0361\u0362\u0363\u0364\u0365\u0366\u0367\u0368\u0369\u036a\u036b\u036c\u036d\u036e\u036f\u0483\u0484\u0485\u0486\u0591\u0592\u0593\u0594\u0595\u0596\u0597\u0598\u0599\u059a\u059b\u059c\u059d\u059e\u059f\u05a0\u05a1\u05a2\u05a3\u05a4\u05a5\u05a6\u05a7\u05a8\u05a9\u05aa\u05ab\u05ac\u05ad\u05ae\u05af\u05b0\u05b1\u05b2\u05b3\u05b4\u05b5\u05b6\u05b7\u05b8\u05b9\u05bb\u05bc\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610\u0611\u0612\u0613\u0614\u0615\u064b\u064c\u064d\u064e\u064f\u0650\u0651\u0652\u0653\u0654\u0655\u0656\u0657\u0658\u0659\u065a\u065b\u065c\u065d\u065e\u0670\u06d6\u06d7\u06d8\u06d9\u06da\u06db\u06dc\u06df\u06e0\u06e1\u06e2\u06e3\u06e4\u06e7\u06e8\u06ea\u06eb\u06ec\u06ed\u0711\u0730\u0731\u0732\u0733\u0734\u0735\u0736\u0737\u0738\u0739\u073a\u073b\u073c\u073d\u073e\u073f\u0740\u0741\u0742\u0743\u0744\u0745\u0746\u0747\u0748\u0749\u074a\u07a6\u07a7\u07a8\u07a9\u07aa\u07ab\u07ac\u07ad\u07ae\u07af\u07b0\u0901\u0902\u093c\u0941\u0942\u0943\u0944\u0945\u0946\u0947\u0948\u094d\u0951\u0952\u0953\u0954\u0962\u0963\u0981\u09bc\u09c1\u09c2\u09c3\u09c4\u09cd\u09e2\u09e3\u0a01\u0a02\u0a3c\u0a41\u0a42\u0a47\u0a48\u0a4b\u0a4c\u0a4d\u0a70\u0a71\u0a81\u0a82\u0abc\u0ac1\u0ac2\u0ac3\u0ac4\u0ac5\u0ac7\u0ac8\u0acd\u0ae2\u0ae3\u0b01\u0b3c\u0b3f\u0b41\u0b42\u0b43\u0b4d\u0b56\u0b82\u0bc0\u0bcd\u0c3e\u0c3f\u0c40\u0c46\u0c47\u0c48\u0c4a\u0c4b\u0c4c\u0c4d\u0c55\u0c56\u0cbc\u0cbf\u0cc6\u0ccc\u0ccd\u0d41\u0d42\u0d43\u0d4d\u0dca\u0dd2\u0dd3\u0dd4\u0dd6\u0e31\u0e34\u0e35\u0e36\u0e37\u0e38\u0e39\u0e3a\u0e47\u0e48\u0e49\u0e4a\u0e4b\u0e4c\u0e4d\u0e4e\u0eb1\u0eb4\u0eb5\u0eb6\u0eb7\u0eb8\u0eb9\u0ebb\u0ebc\u0ec8\u0ec9\u0eca\u0ecb\u0ecc\u0ecd\u0f18\u0f19\u0f35\u0f37\u0f39\u0f71\u0f72\u0f73\u0f74\u0f75\u0f76\u0f77\u0f78\u0f79\u0f7a\u0f7b\u0f7c\u0f7d\u0f7e\u0f80\u0f81\u0f82\u0f83\u0f84\u0f86\u0f87\u0f90\u0f91\u0f92\u0f93\u0f94\u0f95\u0f96\u0f97\u0f99\u0f9a\u0f9b\u0f9c\u0f9d\u0f9e\u0f9f\u0fa0\u0fa1\u0fa2\u0fa3\u0fa4\u0fa5\u0fa6\u0fa7\u0fa8\u0fa9\u0faa\u0fab\u0fac\u0fad\u0fae\u0faf\u0fb0\u0fb1\u0fb2\u0fb3\u0fb4\u0fb5\u0fb6\u0fb7\u0fb8\u0fb9\u0fba\u0fbb\u0fbc\u0fc6\u102d\u102e\u102f\u1030\u1032\u1036\u1037\u1039\u1058\u1059\u135f\u1712\u1713\u1714\u1732\u1733\u1734\u1752\u1753\u1772\u1773\u17b7\u17b8\u17b9\u17ba\u17bb\u17bc\u17bd\u17c6\u17c9\u17ca\u17cb\u17cc\u17cd\u17ce\u17cf\u17d0\u17d1\u17d2\u17d3\u17dd\u180b\u180c\u180d\u18a9\u1920\u1921\u1922\u1927\u1928\u1932\u1939\u193a\u193b\u1a17\u1a18\u1dc0\u1dc1\u1dc2\u1dc3\u20d0\u20d1\u20d2\u20d3\u20d4\u20d5\u20d6\u20d7\u20d8\u20d9\u20da\u20db\u20dc\u20e1\u20e5\u20e6\u20e7\u20e8\u20e9\u20ea\u20eb\u302a\u302b\u302c\u302d\u302e\u302f\u3099\u309a\ua806\ua80b\ua825\ua826\ufb1e\ufe00\ufe01\ufe02\ufe03\ufe04\ufe05\ufe06\ufe07\ufe08\ufe09\ufe0a\ufe0b\ufe0c\ufe0d\ufe0e\ufe0f\ufe20\ufe21\ufe22\ufe23'
+
+Nd = u'0123456789\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669\u06f0\u06f1\u06f2\u06f3\u06f4\u06f5\u06f6\u06f7\u06f8\u06f9\u0966\u0967\u0968\u0969\u096a\u096b\u096c\u096d\u096e\u096f\u09e6\u09e7\u09e8\u09e9\u09ea\u09eb\u09ec\u09ed\u09ee\u09ef\u0a66\u0a67\u0a68\u0a69\u0a6a\u0a6b\u0a6c\u0a6d\u0a6e\u0a6f\u0ae6\u0ae7\u0ae8\u0ae9\u0aea\u0aeb\u0aec\u0aed\u0aee\u0aef\u0b66\u0b67\u0b68\u0b69\u0b6a\u0b6b\u0b6c\u0b6d\u0b6e\u0b6f\u0be6\u0be7\u0be8\u0be9\u0bea\u0beb\u0bec\u0bed\u0bee\u0bef\u0c66\u0c67\u0c68\u0c69\u0c6a\u0c6b\u0c6c\u0c6d\u0c6e\u0c6f\u0ce6\u0ce7\u0ce8\u0ce9\u0cea\u0ceb\u0cec\u0ced\u0cee\u0cef\u0d66\u0d67\u0d68\u0d69\u0d6a\u0d6b\u0d6c\u0d6d\u0d6e\u0d6f\u0e50\u0e51\u0e52\u0e53\u0e54\u0e55\u0e56\u0e57\u0e58\u0e59\u0ed0\u0ed1\u0ed2\u0ed3\u0ed4\u0ed5\u0ed6\u0ed7\u0ed8\u0ed9\u0f20\u0f21\u0f22\u0f23\u0f24\u0f25\u0f26\u0f27\u0f28\u0f29\u1040\u1041\u1042\u1043\u1044\u1045\u1046\u1047\u1048\u1049\u17e0\u17e1\u17e2\u17e3\u17e4\u17e5\u17e6\u17e7\u17e8\u17e9\u1810\u1811\u1812\u1813\u1814\u1815\u1816\u1817\u1818\u1819\u1946\u1947\u1948\u1949\u194a\u194b\u194c\u194d\u194e\u194f\u19d0\u19d1\u19d2\u19d3\u19d4\u19d5\u19d6\u19d7\u19d8\u19d9\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19'
+
+Nl = u'\u16ee\u16ef\u16f0\u2160\u2161\u2162\u2163\u2164\u2165\u2166\u2167\u2168\u2169\u216a\u216b\u216c\u216d\u216e\u216f\u2170\u2171\u2172\u2173\u2174\u2175\u2176\u2177\u2178\u2179\u217a\u217b\u217c\u217d\u217e\u217f\u2180\u2181\u2182\u2183\u3007\u3021\u3022\u3023\u3024\u3025\u3026\u3027\u3028\u3029\u3038\u3039\u303a'
+
+No = u'\xb2\xb3\xb9\xbc\xbd\xbe\u09f4\u09f5\u09f6\u09f7\u09f8\u09f9\u0bf0\u0bf1\u0bf2\u0f2a\u0f2b\u0f2c\u0f2d\u0f2e\u0f2f\u0f30\u0f31\u0f32\u0f33\u1369\u136a\u136b\u136c\u136d\u136e\u136f\u1370\u1371\u1372\u1373\u1374\u1375\u1376\u1377\u1378\u1379\u137a\u137b\u137c\u17f0\u17f1\u17f2\u17f3\u17f4\u17f5\u17f6\u17f7\u17f8\u17f9\u2070\u2074\u2075\u2076\u2077\u2078\u2079\u2080\u2081\u2082\u2083\u2084\u2085\u2086\u2087\u2088\u2089\u2153\u2154\u2155\u2156\u2157\u2158\u2159\u215a\u215b\u215c\u215d\u215e\u215f\u2460\u2461\u2462\u2463\u2464\u2465\u2466\u2467\u2468\u2469\u246a\u246b\u246c\u246d\u246e\u246f\u2470\u2471\u2472\u2473\u2474\u2475\u2476\u2477\u2478\u2479\u247a\u247b\u247c\u247d\u247e\u247f\u2480\u2481\u2482\u2483\u2484\u2485\u2486\u2487\u2488\u2489\u248a\u248b\u248c\u248d\u248e\u248f\u2490\u2491\u2492\u2493\u2494\u2495\u2496\u2497\u2498\u2499\u249a\u249b\u24ea\u24eb\u24ec\u24ed\u24ee\u24ef\u24f0\u24f1\u24f2\u24f3\u24f4\u24f5\u24f6\u24f7\u24f8\u24f9\u24fa\u24fb\u24fc\u24fd\u24fe\u24ff\u2776\u2777\u2778\u2779\u277a\u277b\u277c\u277d\u277e\u277f\u2780\u2781\u2782\u2783\u2784\u2785\u2786\u2787\u2788\u2789\u278a\u278b\u278c\u278d\u278e\u278f\u2790\u2791\u2792\u2793\u2cfd\u3192\u3193\u3194\u3195\u3220\u3221\u3222\u3223\u3224\u3225\u3226\u3227\u3228\u3229\u3251\u3252\u3253\u3254\u3255\u3256\u3257\u3258\u3259\u325a\u325b\u325c\u325d\u325e\u325f\u3280\u3281\u3282\u3283\u3284\u3285\u3286\u3287\u3288\u3289\u32b1\u32b2\u32b3\u32b4\u32b5\u32b6\u32b7\u32b8\u32b9\u32ba\u32bb\u32bc\u32bd\u32be\u32bf'
+
+Pc = u'_\u203f\u2040\u2054\ufe33\ufe34\ufe4d\ufe4e\ufe4f\uff3f'
+
+Pd = u'-\u058a\u1806\u2010\u2011\u2012\u2013\u2014\u2015\u2e17\u301c\u3030\u30a0\ufe31\ufe32\ufe58\ufe63\uff0d'
+
+Pe = u')]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u232a\u23b5\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63'
+
+Pf = u'\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d\u2e1d'
+
+Pi = u'\xab\u2018\u201b\u201c\u201f\u2039\u2e02\u2e04\u2e09\u2e0c\u2e1c'
+
+Po = u'!"#%&\'*,./:;?@\\\xa1\xb7\xbf\u037e\u0387\u055a\u055b\u055c\u055d\u055e\u055f\u0589\u05be\u05c0\u05c3\u05c6\u05f3\u05f4\u060c\u060d\u061b\u061e\u061f\u066a\u066b\u066c\u066d\u06d4\u0700\u0701\u0702\u0703\u0704\u0705\u0706\u0707\u0708\u0709\u070a\u070b\u070c\u070d\u0964\u0965\u0970\u0df4\u0e4f\u0e5a\u0e5b\u0f04\u0f05\u0f06\u0f07\u0f08\u0f09\u0f0a\u0f0b\u0f0c\u0f0d\u0f0e\u0f0f\u0f10\u0f11\u0f12\u0f85\u0fd0\u0fd1\u104a\u104b\u104c\u104d\u104e\u104f\u10fb\u1361\u1362\u1363\u1364\u1365\u1366\u1367\u1368\u166d\u166e\u16eb\u16ec\u16ed\u1735\u1736\u17d4\u17d5\u17d6\u17d8\u17d9\u17da\u1800\u1801\u1802\u1803\u1804\u1805\u1807\u1808\u1809\u180a\u1944\u1945\u19de\u19df\u1a1e\u1a1f\u2016\u2017\u2020\u2021\u2022\u2023\u2024\u2025\u2026\u2027\u2030\u2031\u2032\u2033\u2034\u2035\u2036\u2037\u2038\u203b\u203c\u203d\u203e\u2041\u2042\u2043\u2047\u2048\u2049\u204a\u204b\u204c\u204d\u204e\u204f\u2050\u2051\u2053\u2055\u2056\u2057\u2058\u2059\u205a\u205b\u205c\u205d\u205e\u23b6\u2cf9\u2cfa\u2cfb\u2cfc\u2cfe\u2cff\u2e00\u2e01\u2e06\u2e07\u2e08\u2e0b\u2e0e\u2e0f\u2e10\u2e11\u2e12\u2e13\u2e14\u2e15\u2e16\u3001\u3002\u3003\u303d\u30fb\ufe10\ufe11\ufe12\ufe13\ufe14\ufe15\ufe16\ufe19\ufe30\ufe45\ufe46\ufe49\ufe4a\ufe4b\ufe4c\ufe50\ufe51\ufe52\ufe54\ufe55\ufe56\ufe57\ufe5f\ufe60\ufe61\ufe68\ufe6a\ufe6b\uff01\uff02\uff03\uff05\uff06\uff07\uff0a\uff0c\uff0e\uff0f\uff1a\uff1b\uff1f\uff20\uff3c\uff61\uff64\uff65'
+
+Ps = u'([{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2329\u23b4\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62'
+
+Sc = u'$\xa2\xa3\xa4\xa5\u060b\u09f2\u09f3\u0af1\u0bf9\u0e3f\u17db\u20a0\u20a1\u20a2\u20a3\u20a4\u20a5\u20a6\u20a7\u20a8\u20a9\u20aa\u20ab\u20ac\u20ad\u20ae\u20af\u20b0\u20b1\u20b2\u20b3\u20b4\u20b5\ufdfc\ufe69\uff04\uffe0\uffe1\uffe5\uffe6'
+
+Sk = u'^`\xa8\xaf\xb4\xb8\u02c2\u02c3\u02c4\u02c5\u02d2\u02d3\u02d4\u02d5\u02d6\u02d7\u02d8\u02d9\u02da\u02db\u02dc\u02dd\u02de\u02df\u02e5\u02e6\u02e7\u02e8\u02e9\u02ea\u02eb\u02ec\u02ed\u02ef\u02f0\u02f1\u02f2\u02f3\u02f4\u02f5\u02f6\u02f7\u02f8\u02f9\u02fa\u02fb\u02fc\u02fd\u02fe\u02ff\u0374\u0375\u0384\u0385\u1fbd\u1fbf\u1fc0\u1fc1\u1fcd\u1fce\u1fcf\u1fdd\u1fde\u1fdf\u1fed\u1fee\u1fef\u1ffd\u1ffe\u309b\u309c\ua700\ua701\ua702\ua703\ua704\ua705\ua706\ua707\ua708\ua709\ua70a\ua70b\ua70c\ua70d\ua70e\ua70f\ua710\ua711\ua712\ua713\ua714\ua715\ua716\uff3e\uff40\uffe3'
+
+Sm = u'+<=>|~\xac\xb1\xd7\xf7\u03f6\u2044\u2052\u207a\u207b\u207c\u208a\u208b\u208c\u2140\u2141\u2142\u2143\u2144\u214b\u2190\u2191\u2192\u2193\u2194\u219a\u219b\u21a0\u21a3\u21a6\u21ae\u21ce\u21cf\u21d2\u21d4\u21f4\u21f5\u21f6\u21f7\u21f8\u21f9\u21fa\u21fb\u21fc\u21fd\u21fe\u21ff\u2200\u2201\u2202\u2203\u2204\u2205\u2206\u2207\u2208\u2209\u220a\u220b\u220c\u220d\u220e\u220f\u2210\u2211\u2212\u2213\u2214\u2215\u2216\u2217\u2218\u2219\u221a\u221b\u221c\u221d\u221e\u221f\u2220\u2221\u2222\u2223\u2224\u2225\u2226\u2227\u2228\u2229\u222a\u222b\u222c\u222d\u222e\u222f\u2230\u2231\u2232\u2233\u2234\u2235\u2236\u2237\u2238\u2239\u223a\u223b\u223c\u223d\u223e\u223f\u2240\u2241\u2242\u2243\u2244\u2245\u2246\u2247\u2248\u2249\u224a\u224b\u224c\u224d\u224e\u224f\u2250\u2251\u2252\u2253\u2254\u2255\u2256\u2257\u2258\u2259\u225a\u225b\u225c\u225d\u225e\u225f\u2260\u2261\u2262\u2263\u2264\u2265\u2266\u2267\u2268\u2269\u226a\u226b\u226c\u226d\u226e\u226f\u2270\u2271\u2272\u2273\u2274\u2275\u2276\u2277\u2278\u2279\u227a\u227b\u227c\u227d\u227e\u227f\u2280\u2281\u2282\u2283\u2284\u2285\u2286\u2287\u2288\u2289\u228a\u228b\u228c\u228d\u228e\u228f\u2290\u2291\u2292\u2293\u2294\u2295\u2296\u2297\u2298\u2299\u229a\u229b\u229c\u229d\u229e\u229f\u22a0\u22a1\u22a2\u22a3\u22a4\u22a5\u22a6\u22a7\u22a8\u22a9\u22aa\u22ab\u22ac\u22ad\u22ae\u22af\u22b0\u22b1\u22b2\u22b3\u22b4\u22b5\u22b6\u22b7\u22b8\u22b9\u22ba\u22bb\u22bc\u22bd\u22be\u22bf\u22c0\u22c1\u22c2\u22c3\u22c4\u22c5\u22c6\u22c7\u22c8\u22c9\u22ca\u22cb\u22cc\u22cd\u22ce\u22cf\u22d0\u22d1\u22d2\u22d3\u22d4\u22d5\u22d6\u22d7\u22d8\u22d9\u22da\u22db\u22dc\u22dd\u22de\u22df\u22e0\u22e1\u22e2\u22e3\u22e4\u22e5\u22e6\u22e7\u22e8\u22e9\u22ea\u22eb\u22ec\u22ed\u22ee\u22ef\u22f0\u22f1\u22f2\u22f3\u22f4\u22f5\u22f6\u22f7\u22f8\u22f9\u22fa\u22fb\u22fc\u22fd\u22fe\u22ff\u2308\u2309\u230a\u230b\u2320\u2321\u237c\u239b\u239c\u239d\u239e\u239f\u23a0\u23a1\u23a2\u23a3\u23a4\u23a5\u23a6\u23a7\u23a8\u23a9\u23aa\u23ab\u23ac\u23ad\u23ae\u23af\u23b0\u23b1\u23b2\u23b3\u25b7\u25c1\u25f8\u25f9\u25fa\u25fb\u25fc\u25fd\u25fe\u25ff\u266f\u27c0\u27c1\u27c2\u27c3\u27c4\u27d0\u27d1\u27d2\u27d3\u27d4\u27d5\u27d6\u27d7\u27d8\u27d9\u27da\u27db\u27dc\u27dd\u27de\u27df\u27e0\u27e1\u27e2\u27e3\u27e4\u27e5\u27f0\u27f1\u27f2\u27f3\u27f4\u27f5\u27f6\u27f7\u27f8\u27f9\u27fa\u27fb\u27fc\u27fd\u27fe\u27ff\u2900\u2901\u2902\u2903\u2904\u2905\u2906\u2907\u2908\u2909\u290a\u290b\u290c\u290d\u290e\u290f\u2910\u2911\u2912\u2913\u2914\u2915\u2916\u2917\u2918\u2919\u291a\u291b\u291c\u291d\u291e\u291f\u2920\u2921\u2922\u2923\u2924\u2925\u2926\u2927\u2928\u2929\u292a\u292b\u292c\u292d\u292e\u292f\u2930\u2931\u2932\u2933\u2934\u2935\u2936\u2937\u2938\u2939\u293a\u293b\u293c\u293d\u293e\u293f\u2940\u2941\u2942\u2943\u2944\u2945\u2946\u2947\u2948\u2949\u294a\u294b\u294c\u294d\u294e\u294f\u2950\u2951\u2952\u2953\u2954\u2955\u2956\u2957\u2958\u2959\u295a\u295b\u295c\u295d\u295e\u295f\u2960\u2961\u2962\u2963\u2964\u2965\u2966\u2967\u2968\u2969\u296a\u296b\u296c\u296d\u296e\u296f\u2970\u2971\u2972\u2973\u2974\u2975\u2976\u2977\u2978\u2979\u297a\u297b\u297c\u297d\u297e\u297f\u2980\u2981\u2982\u2999\u299a\u299b\u299c\u299d\u299e\u299f\u29a0\u29a1\u29a2\u29a3\u29a4\u29a5\u29a6\u29a7\u29a8\u29a9\u29aa\u29ab\u29ac\u29ad\u29ae\u29af\u29b0\u29b1\u29b2\u29b3\u29b4\u29b5\u29b6\u29b7\u29b8\u29b9\u29ba\u29bb\u29bc\u29bd\u29be\u29bf\u29c0\u29c1\u29c2\u29c3\u29c4\u29c5\u29c6\u29c7\u29c8\u29c9\u29ca\u29cb\u29cc\u29cd\u29ce\u29cf\u29d0\u29d1\u29d2\u29d3\u29d4\u29d5\u29d6\u29d7\u29dc\u29dd\u29de\u29df\u29e0\u29e1\u29e2\u29e3\u29e4\u29e5\u29e6\u29e7\u29e8\u29e9\u29ea\u29eb\u29ec\u29ed\u29ee\u29ef\u29f0\u29f1\u29f2\u29f3\u29f4\u29f5\u29f6\u29f7\u29f8\u29f9\u29fa\u29fb\u29fe\u29ff\u2a00\u2a01\u2a02\u2a03\u2a04\u2a05\u2a06\u2a07\u2a08\u2a09\u2a0a\u2a0b\u2a0c\u2a0d\u2a0e\u2a0f\u2a10\u2a11\u2a12\u2a13\u2a14\u2a15\u2a16\u2a17\u2a18\u2a19\u2a1a\u2a1b\u2a1c\u2a1d\u2a1e\u2a1f\u2a20\u2a21\u2a22\u2a23\u2a24\u2a25\u2a26\u2a27\u2a28\u2a29\u2a2a\u2a2b\u2a2c\u2a2d\u2a2e\u2a2f\u2a30\u2a31\u2a32\u2a33\u2a34\u2a35\u2a36\u2a37\u2a38\u2a39\u2a3a\u2a3b\u2a3c\u2a3d\u2a3e\u2a3f\u2a40\u2a41\u2a42\u2a43\u2a44\u2a45\u2a46\u2a47\u2a48\u2a49\u2a4a\u2a4b\u2a4c\u2a4d\u2a4e\u2a4f\u2a50\u2a51\u2a52\u2a53\u2a54\u2a55\u2a56\u2a57\u2a58\u2a59\u2a5a\u2a5b\u2a5c\u2a5d\u2a5e\u2a5f\u2a60\u2a61\u2a62\u2a63\u2a64\u2a65\u2a66\u2a67\u2a68\u2a69\u2a6a\u2a6b\u2a6c\u2a6d\u2a6e\u2a6f\u2a70\u2a71\u2a72\u2a73\u2a74\u2a75\u2a76\u2a77\u2a78\u2a79\u2a7a\u2a7b\u2a7c\u2a7d\u2a7e\u2a7f\u2a80\u2a81\u2a82\u2a83\u2a84\u2a85\u2a86\u2a87\u2a88\u2a89\u2a8a\u2a8b\u2a8c\u2a8d\u2a8e\u2a8f\u2a90\u2a91\u2a92\u2a93\u2a94\u2a95\u2a96\u2a97\u2a98\u2a99\u2a9a\u2a9b\u2a9c\u2a9d\u2a9e\u2a9f\u2aa0\u2aa1\u2aa2\u2aa3\u2aa4\u2aa5\u2aa6\u2aa7\u2aa8\u2aa9\u2aaa\u2aab\u2aac\u2aad\u2aae\u2aaf\u2ab0\u2ab1\u2ab2\u2ab3\u2ab4\u2ab5\u2ab6\u2ab7\u2ab8\u2ab9\u2aba\u2abb\u2abc\u2abd\u2abe\u2abf\u2ac0\u2ac1\u2ac2\u2ac3\u2ac4\u2ac5\u2ac6\u2ac7\u2ac8\u2ac9\u2aca\u2acb\u2acc\u2acd\u2ace\u2acf\u2ad0\u2ad1\u2ad2\u2ad3\u2ad4\u2ad5\u2ad6\u2ad7\u2ad8\u2ad9\u2ada\u2adb\u2adc\u2add\u2ade\u2adf\u2ae0\u2ae1\u2ae2\u2ae3\u2ae4\u2ae5\u2ae6\u2ae7\u2ae8\u2ae9\u2aea\u2aeb\u2aec\u2aed\u2aee\u2aef\u2af0\u2af1\u2af2\u2af3\u2af4\u2af5\u2af6\u2af7\u2af8\u2af9\u2afa\u2afb\u2afc\u2afd\u2afe\u2aff\ufb29\ufe62\ufe64\ufe65\ufe66\uff0b\uff1c\uff1d\uff1e\uff5c\uff5e\uffe2\uffe9\uffea\uffeb\uffec'
+
+So = u'\xa6\xa7\xa9\xae\xb0\xb6\u0482\u060e\u060f\u06e9\u06fd\u06fe\u09fa\u0b70\u0bf3\u0bf4\u0bf5\u0bf6\u0bf7\u0bf8\u0bfa\u0f01\u0f02\u0f03\u0f13\u0f14\u0f15\u0f16\u0f17\u0f1a\u0f1b\u0f1c\u0f1d\u0f1e\u0f1f\u0f34\u0f36\u0f38\u0fbe\u0fbf\u0fc0\u0fc1\u0fc2\u0fc3\u0fc4\u0fc5\u0fc7\u0fc8\u0fc9\u0fca\u0fcb\u0fcc\u0fcf\u1360\u1390\u1391\u1392\u1393\u1394\u1395\u1396\u1397\u1398\u1399\u1940\u19e0\u19e1\u19e2\u19e3\u19e4\u19e5\u19e6\u19e7\u19e8\u19e9\u19ea\u19eb\u19ec\u19ed\u19ee\u19ef\u19f0\u19f1\u19f2\u19f3\u19f4\u19f5\u19f6\u19f7\u19f8\u19f9\u19fa\u19fb\u19fc\u19fd\u19fe\u19ff\u2100\u2101\u2103\u2104\u2105\u2106\u2108\u2109\u2114\u2116\u2117\u2118\u211e\u211f\u2120\u2121\u2122\u2123\u2125\u2127\u2129\u212e\u2132\u213a\u213b\u214a\u214c\u2195\u2196\u2197\u2198\u2199\u219c\u219d\u219e\u219f\u21a1\u21a2\u21a4\u21a5\u21a7\u21a8\u21a9\u21aa\u21ab\u21ac\u21ad\u21af\u21b0\u21b1\u21b2\u21b3\u21b4\u21b5\u21b6\u21b7\u21b8\u21b9\u21ba\u21bb\u21bc\u21bd\u21be\u21bf\u21c0\u21c1\u21c2\u21c3\u21c4\u21c5\u21c6\u21c7\u21c8\u21c9\u21ca\u21cb\u21cc\u21cd\u21d0\u21d1\u21d3\u21d5\u21d6\u21d7\u21d8\u21d9\u21da\u21db\u21dc\u21dd\u21de\u21df\u21e0\u21e1\u21e2\u21e3\u21e4\u21e5\u21e6\u21e7\u21e8\u21e9\u21ea\u21eb\u21ec\u21ed\u21ee\u21ef\u21f0\u21f1\u21f2\u21f3\u2300\u2301\u2302\u2303\u2304\u2305\u2306\u2307\u230c\u230d\u230e\u230f\u2310\u2311\u2312\u2313\u2314\u2315\u2316\u2317\u2318\u2319\u231a\u231b\u231c\u231d\u231e\u231f\u2322\u2323\u2324\u2325\u2326\u2327\u2328\u232b\u232c\u232d\u232e\u232f\u2330\u2331\u2332\u2333\u2334\u2335\u2336\u2337\u2338\u2339\u233a\u233b\u233c\u233d\u233e\u233f\u2340\u2341\u2342\u2343\u2344\u2345\u2346\u2347\u2348\u2349\u234a\u234b\u234c\u234d\u234e\u234f\u2350\u2351\u2352\u2353\u2354\u2355\u2356\u2357\u2358\u2359\u235a\u235b\u235c\u235d\u235e\u235f\u2360\u2361\u2362\u2363\u2364\u2365\u2366\u2367\u2368\u2369\u236a\u236b\u236c\u236d\u236e\u236f\u2370\u2371\u2372\u2373\u2374\u2375\u2376\u2377\u2378\u2379\u237a\u237b\u237d\u237e\u237f\u2380\u2381\u2382\u2383\u2384\u2385\u2386\u2387\u2388\u2389\u238a\u238b\u238c\u238d\u238e\u238f\u2390\u2391\u2392\u2393\u2394\u2395\u2396\u2397\u2398\u2399\u239a\u23b7\u23b8\u23b9\u23ba\u23bb\u23bc\u23bd\u23be\u23bf\u23c0\u23c1\u23c2\u23c3\u23c4\u23c5\u23c6\u23c7\u23c8\u23c9\u23ca\u23cb\u23cc\u23cd\u23ce\u23cf\u23d0\u23d1\u23d2\u23d3\u23d4\u23d5\u23d6\u23d7\u23d8\u23d9\u23da\u23db\u2400\u2401\u2402\u2403\u2404\u2405\u2406\u2407\u2408\u2409\u240a\u240b\u240c\u240d\u240e\u240f\u2410\u2411\u2412\u2413\u2414\u2415\u2416\u2417\u2418\u2419\u241a\u241b\u241c\u241d\u241e\u241f\u2420\u2421\u2422\u2423\u2424\u2425\u2426\u2440\u2441\u2442\u2443\u2444\u2445\u2446\u2447\u2448\u2449\u244a\u249c\u249d\u249e\u249f\u24a0\u24a1\u24a2\u24a3\u24a4\u24a5\u24a6\u24a7\u24a8\u24a9\u24aa\u24ab\u24ac\u24ad\u24ae\u24af\u24b0\u24b1\u24b2\u24b3\u24b4\u24b5\u24b6\u24b7\u24b8\u24b9\u24ba\u24bb\u24bc\u24bd\u24be\u24bf\u24c0\u24c1\u24c2\u24c3\u24c4\u24c5\u24c6\u24c7\u24c8\u24c9\u24ca\u24cb\u24cc\u24cd\u24ce\u24cf\u24d0\u24d1\u24d2\u24d3\u24d4\u24d5\u24d6\u24d7\u24d8\u24d9\u24da\u24db\u24dc\u24dd\u24de\u24df\u24e0\u24e1\u24e2\u24e3\u24e4\u24e5\u24e6\u24e7\u24e8\u24e9\u2500\u2501\u2502\u2503\u2504\u2505\u2506\u2507\u2508\u2509\u250a\u250b\u250c\u250d\u250e\u250f\u2510\u2511\u2512\u2513\u2514\u2515\u2516\u2517\u2518\u2519\u251a\u251b\u251c\u251d\u251e\u251f\u2520\u2521\u2522\u2523\u2524\u2525\u2526\u2527\u2528\u2529\u252a\u252b\u252c\u252d\u252e\u252f\u2530\u2531\u2532\u2533\u2534\u2535\u2536\u2537\u2538\u2539\u253a\u253b\u253c\u253d\u253e\u253f\u2540\u2541\u2542\u2543\u2544\u2545\u2546\u2547\u2548\u2549\u254a\u254b\u254c\u254d\u254e\u254f\u2550\u2551\u2552\u2553\u2554\u2555\u2556\u2557\u2558\u2559\u255a\u255b\u255c\u255d\u255e\u255f\u2560\u2561\u2562\u2563\u2564\u2565\u2566\u2567\u2568\u2569\u256a\u256b\u256c\u256d\u256e\u256f\u2570\u2571\u2572\u2573\u2574\u2575\u2576\u2577\u2578\u2579\u257a\u257b\u257c\u257d\u257e\u257f\u2580\u2581\u2582\u2583\u2584\u2585\u2586\u2587\u2588\u2589\u258a\u258b\u258c\u258d\u258e\u258f\u2590\u2591\u2592\u2593\u2594\u2595\u2596\u2597\u2598\u2599\u259a\u259b\u259c\u259d\u259e\u259f\u25a0\u25a1\u25a2\u25a3\u25a4\u25a5\u25a6\u25a7\u25a8\u25a9\u25aa\u25ab\u25ac\u25ad\u25ae\u25af\u25b0\u25b1\u25b2\u25b3\u25b4\u25b5\u25b6\u25b8\u25b9\u25ba\u25bb\u25bc\u25bd\u25be\u25bf\u25c0\u25c2\u25c3\u25c4\u25c5\u25c6\u25c7\u25c8\u25c9\u25ca\u25cb\u25cc\u25cd\u25ce\u25cf\u25d0\u25d1\u25d2\u25d3\u25d4\u25d5\u25d6\u25d7\u25d8\u25d9\u25da\u25db\u25dc\u25dd\u25de\u25df\u25e0\u25e1\u25e2\u25e3\u25e4\u25e5\u25e6\u25e7\u25e8\u25e9\u25ea\u25eb\u25ec\u25ed\u25ee\u25ef\u25f0\u25f1\u25f2\u25f3\u25f4\u25f5\u25f6\u25f7\u2600\u2601\u2602\u2603\u2604\u2605\u2606\u2607\u2608\u2609\u260a\u260b\u260c\u260d\u260e\u260f\u2610\u2611\u2612\u2613\u2614\u2615\u2616\u2617\u2618\u2619\u261a\u261b\u261c\u261d\u261e\u261f\u2620\u2621\u2622\u2623\u2624\u2625\u2626\u2627\u2628\u2629\u262a\u262b\u262c\u262d\u262e\u262f\u2630\u2631\u2632\u2633\u2634\u2635\u2636\u2637\u2638\u2639\u263a\u263b\u263c\u263d\u263e\u263f\u2640\u2641\u2642\u2643\u2644\u2645\u2646\u2647\u2648\u2649\u264a\u264b\u264c\u264d\u264e\u264f\u2650\u2651\u2652\u2653\u2654\u2655\u2656\u2657\u2658\u2659\u265a\u265b\u265c\u265d\u265e\u265f\u2660\u2661\u2662\u2663\u2664\u2665\u2666\u2667\u2668\u2669\u266a\u266b\u266c\u266d\u266e\u2670\u2671\u2672\u2673\u2674\u2675\u2676\u2677\u2678\u2679\u267a\u267b\u267c\u267d\u267e\u267f\u2680\u2681\u2682\u2683\u2684\u2685\u2686\u2687\u2688\u2689\u268a\u268b\u268c\u268d\u268e\u268f\u2690\u2691\u2692\u2693\u2694\u2695\u2696\u2697\u2698\u2699\u269a\u269b\u269c\u26a0\u26a1\u26a2\u26a3\u26a4\u26a5\u26a6\u26a7\u26a8\u26a9\u26aa\u26ab\u26ac\u26ad\u26ae\u26af\u26b0\u26b1\u2701\u2702\u2703\u2704\u2706\u2707\u2708\u2709\u270c\u270d\u270e\u270f\u2710\u2711\u2712\u2713\u2714\u2715\u2716\u2717\u2718\u2719\u271a\u271b\u271c\u271d\u271e\u271f\u2720\u2721\u2722\u2723\u2724\u2725\u2726\u2727\u2729\u272a\u272b\u272c\u272d\u272e\u272f\u2730\u2731\u2732\u2733\u2734\u2735\u2736\u2737\u2738\u2739\u273a\u273b\u273c\u273d\u273e\u273f\u2740\u2741\u2742\u2743\u2744\u2745\u2746\u2747\u2748\u2749\u274a\u274b\u274d\u274f\u2750\u2751\u2752\u2756\u2758\u2759\u275a\u275b\u275c\u275d\u275e\u2761\u2762\u2763\u2764\u2765\u2766\u2767\u2794\u2798\u2799\u279a\u279b\u279c\u279d\u279e\u279f\u27a0\u27a1\u27a2\u27a3\u27a4\u27a5\u27a6\u27a7\u27a8\u27a9\u27aa\u27ab\u27ac\u27ad\u27ae\u27af\u27b1\u27b2\u27b3\u27b4\u27b5\u27b6\u27b7\u27b8\u27b9\u27ba\u27bb\u27bc\u27bd\u27be\u2800\u2801\u2802\u2803\u2804\u2805\u2806\u2807\u2808\u2809\u280a\u280b\u280c\u280d\u280e\u280f\u2810\u2811\u2812\u2813\u2814\u2815\u2816\u2817\u2818\u2819\u281a\u281b\u281c\u281d\u281e\u281f\u2820\u2821\u2822\u2823\u2824\u2825\u2826\u2827\u2828\u2829\u282a\u282b\u282c\u282d\u282e\u282f\u2830\u2831\u2832\u2833\u2834\u2835\u2836\u2837\u2838\u2839\u283a\u283b\u283c\u283d\u283e\u283f\u2840\u2841\u2842\u2843\u2844\u2845\u2846\u2847\u2848\u2849\u284a\u284b\u284c\u284d\u284e\u284f\u2850\u2851\u2852\u2853\u2854\u2855\u2856\u2857\u2858\u2859\u285a\u285b\u285c\u285d\u285e\u285f\u2860\u2861\u2862\u2863\u2864\u2865\u2866\u2867\u2868\u2869\u286a\u286b\u286c\u286d\u286e\u286f\u2870\u2871\u2872\u2873\u2874\u2875\u2876\u2877\u2878\u2879\u287a\u287b\u287c\u287d\u287e\u287f\u2880\u2881\u2882\u2883\u2884\u2885\u2886\u2887\u2888\u2889\u288a\u288b\u288c\u288d\u288e\u288f\u2890\u2891\u2892\u2893\u2894\u2895\u2896\u2897\u2898\u2899\u289a\u289b\u289c\u289d\u289e\u289f\u28a0\u28a1\u28a2\u28a3\u28a4\u28a5\u28a6\u28a7\u28a8\u28a9\u28aa\u28ab\u28ac\u28ad\u28ae\u28af\u28b0\u28b1\u28b2\u28b3\u28b4\u28b5\u28b6\u28b7\u28b8\u28b9\u28ba\u28bb\u28bc\u28bd\u28be\u28bf\u28c0\u28c1\u28c2\u28c3\u28c4\u28c5\u28c6\u28c7\u28c8\u28c9\u28ca\u28cb\u28cc\u28cd\u28ce\u28cf\u28d0\u28d1\u28d2\u28d3\u28d4\u28d5\u28d6\u28d7\u28d8\u28d9\u28da\u28db\u28dc\u28dd\u28de\u28df\u28e0\u28e1\u28e2\u28e3\u28e4\u28e5\u28e6\u28e7\u28e8\u28e9\u28ea\u28eb\u28ec\u28ed\u28ee\u28ef\u28f0\u28f1\u28f2\u28f3\u28f4\u28f5\u28f6\u28f7\u28f8\u28f9\u28fa\u28fb\u28fc\u28fd\u28fe\u28ff\u2b00\u2b01\u2b02\u2b03\u2b04\u2b05\u2b06\u2b07\u2b08\u2b09\u2b0a\u2b0b\u2b0c\u2b0d\u2b0e\u2b0f\u2b10\u2b11\u2b12\u2b13\u2ce5\u2ce6\u2ce7\u2ce8\u2ce9\u2cea\u2e80\u2e81\u2e82\u2e83\u2e84\u2e85\u2e86\u2e87\u2e88\u2e89\u2e8a\u2e8b\u2e8c\u2e8d\u2e8e\u2e8f\u2e90\u2e91\u2e92\u2e93\u2e94\u2e95\u2e96\u2e97\u2e98\u2e99\u2e9b\u2e9c\u2e9d\u2e9e\u2e9f\u2ea0\u2ea1\u2ea2\u2ea3\u2ea4\u2ea5\u2ea6\u2ea7\u2ea8\u2ea9\u2eaa\u2eab\u2eac\u2ead\u2eae\u2eaf\u2eb0\u2eb1\u2eb2\u2eb3\u2eb4\u2eb5\u2eb6\u2eb7\u2eb8\u2eb9\u2eba\u2ebb\u2ebc\u2ebd\u2ebe\u2ebf\u2ec0\u2ec1\u2ec2\u2ec3\u2ec4\u2ec5\u2ec6\u2ec7\u2ec8\u2ec9\u2eca\u2ecb\u2ecc\u2ecd\u2ece\u2ecf\u2ed0\u2ed1\u2ed2\u2ed3\u2ed4\u2ed5\u2ed6\u2ed7\u2ed8\u2ed9\u2eda\u2edb\u2edc\u2edd\u2ede\u2edf\u2ee0\u2ee1\u2ee2\u2ee3\u2ee4\u2ee5\u2ee6\u2ee7\u2ee8\u2ee9\u2eea\u2eeb\u2eec\u2eed\u2eee\u2eef\u2ef0\u2ef1\u2ef2\u2ef3\u2f00\u2f01\u2f02\u2f03\u2f04\u2f05\u2f06\u2f07\u2f08\u2f09\u2f0a\u2f0b\u2f0c\u2f0d\u2f0e\u2f0f\u2f10\u2f11\u2f12\u2f13\u2f14\u2f15\u2f16\u2f17\u2f18\u2f19\u2f1a\u2f1b\u2f1c\u2f1d\u2f1e\u2f1f\u2f20\u2f21\u2f22\u2f23\u2f24\u2f25\u2f26\u2f27\u2f28\u2f29\u2f2a\u2f2b\u2f2c\u2f2d\u2f2e\u2f2f\u2f30\u2f31\u2f32\u2f33\u2f34\u2f35\u2f36\u2f37\u2f38\u2f39\u2f3a\u2f3b\u2f3c\u2f3d\u2f3e\u2f3f\u2f40\u2f41\u2f42\u2f43\u2f44\u2f45\u2f46\u2f47\u2f48\u2f49\u2f4a\u2f4b\u2f4c\u2f4d\u2f4e\u2f4f\u2f50\u2f51\u2f52\u2f53\u2f54\u2f55\u2f56\u2f57\u2f58\u2f59\u2f5a\u2f5b\u2f5c\u2f5d\u2f5e\u2f5f\u2f60\u2f61\u2f62\u2f63\u2f64\u2f65\u2f66\u2f67\u2f68\u2f69\u2f6a\u2f6b\u2f6c\u2f6d\u2f6e\u2f6f\u2f70\u2f71\u2f72\u2f73\u2f74\u2f75\u2f76\u2f77\u2f78\u2f79\u2f7a\u2f7b\u2f7c\u2f7d\u2f7e\u2f7f\u2f80\u2f81\u2f82\u2f83\u2f84\u2f85\u2f86\u2f87\u2f88\u2f89\u2f8a\u2f8b\u2f8c\u2f8d\u2f8e\u2f8f\u2f90\u2f91\u2f92\u2f93\u2f94\u2f95\u2f96\u2f97\u2f98\u2f99\u2f9a\u2f9b\u2f9c\u2f9d\u2f9e\u2f9f\u2fa0\u2fa1\u2fa2\u2fa3\u2fa4\u2fa5\u2fa6\u2fa7\u2fa8\u2fa9\u2faa\u2fab\u2fac\u2fad\u2fae\u2faf\u2fb0\u2fb1\u2fb2\u2fb3\u2fb4\u2fb5\u2fb6\u2fb7\u2fb8\u2fb9\u2fba\u2fbb\u2fbc\u2fbd\u2fbe\u2fbf\u2fc0\u2fc1\u2fc2\u2fc3\u2fc4\u2fc5\u2fc6\u2fc7\u2fc8\u2fc9\u2fca\u2fcb\u2fcc\u2fcd\u2fce\u2fcf\u2fd0\u2fd1\u2fd2\u2fd3\u2fd4\u2fd5\u2ff0\u2ff1\u2ff2\u2ff3\u2ff4\u2ff5\u2ff6\u2ff7\u2ff8\u2ff9\u2ffa\u2ffb\u3004\u3012\u3013\u3020\u3036\u3037\u303e\u303f\u3190\u3191\u3196\u3197\u3198\u3199\u319a\u319b\u319c\u319d\u319e\u319f\u31c0\u31c1\u31c2\u31c3\u31c4\u31c5\u31c6\u31c7\u31c8\u31c9\u31ca\u31cb\u31cc\u31cd\u31ce\u31cf\u3200\u3201\u3202\u3203\u3204\u3205\u3206\u3207\u3208\u3209\u320a\u320b\u320c\u320d\u320e\u320f\u3210\u3211\u3212\u3213\u3214\u3215\u3216\u3217\u3218\u3219\u321a\u321b\u321c\u321d\u321e\u322a\u322b\u322c\u322d\u322e\u322f\u3230\u3231\u3232\u3233\u3234\u3235\u3236\u3237\u3238\u3239\u323a\u323b\u323c\u323d\u323e\u323f\u3240\u3241\u3242\u3243\u3250\u3260\u3261\u3262\u3263\u3264\u3265\u3266\u3267\u3268\u3269\u326a\u326b\u326c\u326d\u326e\u326f\u3270\u3271\u3272\u3273\u3274\u3275\u3276\u3277\u3278\u3279\u327a\u327b\u327c\u327d\u327e\u327f\u328a\u328b\u328c\u328d\u328e\u328f\u3290\u3291\u3292\u3293\u3294\u3295\u3296\u3297\u3298\u3299\u329a\u329b\u329c\u329d\u329e\u329f\u32a0\u32a1\u32a2\u32a3\u32a4\u32a5\u32a6\u32a7\u32a8\u32a9\u32aa\u32ab\u32ac\u32ad\u32ae\u32af\u32b0\u32c0\u32c1\u32c2\u32c3\u32c4\u32c5\u32c6\u32c7\u32c8\u32c9\u32ca\u32cb\u32cc\u32cd\u32ce\u32cf\u32d0\u32d1\u32d2\u32d3\u32d4\u32d5\u32d6\u32d7\u32d8\u32d9\u32da\u32db\u32dc\u32dd\u32de\u32df\u32e0\u32e1\u32e2\u32e3\u32e4\u32e5\u32e6\u32e7\u32e8\u32e9\u32ea\u32eb\u32ec\u32ed\u32ee\u32ef\u32f0\u32f1\u32f2\u32f3\u32f4\u32f5\u32f6\u32f7\u32f8\u32f9\u32fa\u32fb\u32fc\u32fd\u32fe\u3300\u3301\u3302\u3303\u3304\u3305\u3306\u3307\u3308\u3309\u330a\u330b\u330c\u330d\u330e\u330f\u3310\u3311\u3312\u3313\u3314\u3315\u3316\u3317\u3318\u3319\u331a\u331b\u331c\u331d\u331e\u331f\u3320\u3321\u3322\u3323\u3324\u3325\u3326\u3327\u3328\u3329\u332a\u332b\u332c\u332d\u332e\u332f\u3330\u3331\u3332\u3333\u3334\u3335\u3336\u3337\u3338\u3339\u333a\u333b\u333c\u333d\u333e\u333f\u3340\u3341\u3342\u3343\u3344\u3345\u3346\u3347\u3348\u3349\u334a\u334b\u334c\u334d\u334e\u334f\u3350\u3351\u3352\u3353\u3354\u3355\u3356\u3357\u3358\u3359\u335a\u335b\u335c\u335d\u335e\u335f\u3360\u3361\u3362\u3363\u3364\u3365\u3366\u3367\u3368\u3369\u336a\u336b\u336c\u336d\u336e\u336f\u3370\u3371\u3372\u3373\u3374\u3375\u3376\u3377\u3378\u3379\u337a\u337b\u337c\u337d\u337e\u337f\u3380\u3381\u3382\u3383\u3384\u3385\u3386\u3387\u3388\u3389\u338a\u338b\u338c\u338d\u338e\u338f\u3390\u3391\u3392\u3393\u3394\u3395\u3396\u3397\u3398\u3399\u339a\u339b\u339c\u339d\u339e\u339f\u33a0\u33a1\u33a2\u33a3\u33a4\u33a5\u33a6\u33a7\u33a8\u33a9\u33aa\u33ab\u33ac\u33ad\u33ae\u33af\u33b0\u33b1\u33b2\u33b3\u33b4\u33b5\u33b6\u33b7\u33b8\u33b9\u33ba\u33bb\u33bc\u33bd\u33be\u33bf\u33c0\u33c1\u33c2\u33c3\u33c4\u33c5\u33c6\u33c7\u33c8\u33c9\u33ca\u33cb\u33cc\u33cd\u33ce\u33cf\u33d0\u33d1\u33d2\u33d3\u33d4\u33d5\u33d6\u33d7\u33d8\u33d9\u33da\u33db\u33dc\u33dd\u33de\u33df\u33e0\u33e1\u33e2\u33e3\u33e4\u33e5\u33e6\u33e7\u33e8\u33e9\u33ea\u33eb\u33ec\u33ed\u33ee\u33ef\u33f0\u33f1\u33f2\u33f3\u33f4\u33f5\u33f6\u33f7\u33f8\u33f9\u33fa\u33fb\u33fc\u33fd\u33fe\u33ff\u4dc0\u4dc1\u4dc2\u4dc3\u4dc4\u4dc5\u4dc6\u4dc7\u4dc8\u4dc9\u4dca\u4dcb\u4dcc\u4dcd\u4dce\u4dcf\u4dd0\u4dd1\u4dd2\u4dd3\u4dd4\u4dd5\u4dd6\u4dd7\u4dd8\u4dd9\u4dda\u4ddb\u4ddc\u4ddd\u4dde\u4ddf\u4de0\u4de1\u4de2\u4de3\u4de4\u4de5\u4de6\u4de7\u4de8\u4de9\u4dea\u4deb\u4dec\u4ded\u4dee\u4def\u4df0\u4df1\u4df2\u4df3\u4df4\u4df5\u4df6\u4df7\u4df8\u4df9\u4dfa\u4dfb\u4dfc\u4dfd\u4dfe\u4dff\ua490\ua491\ua492\ua493\ua494\ua495\ua496\ua497\ua498\ua499\ua49a\ua49b\ua49c\ua49d\ua49e\ua49f\ua4a0\ua4a1\ua4a2\ua4a3\ua4a4\ua4a5\ua4a6\ua4a7\ua4a8\ua4a9\ua4aa\ua4ab\ua4ac\ua4ad\ua4ae\ua4af\ua4b0\ua4b1\ua4b2\ua4b3\ua4b4\ua4b5\ua4b6\ua4b7\ua4b8\ua4b9\ua4ba\ua4bb\ua4bc\ua4bd\ua4be\ua4bf\ua4c0\ua4c1\ua4c2\ua4c3\ua4c4\ua4c5\ua4c6\ua828\ua829\ua82a\ua82b\ufdfd\uffe4\uffe8\uffed\uffee\ufffc\ufffd'
+
+Zl = u'\u2028'
+
+Zp = u'\u2029'
+
+Zs = u' \xa0\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000'
+
+cats = ['Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu', 'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs']
+
+def combine(*args):
+ return u''.join([globals()[cat] for cat in args])
+
+xid_start = u'\u0041-\u005A\u005F\u0061-\u007A\u00AA\u00B5\u00BA\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u01BA\u01BB\u01BC-\u01BF\u01C0-\u01C3\u01C4-\u0241\u0250-\u02AF\u02B0-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EE\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03CE\u03D0-\u03F5\u03F7-\u0481\u048A-\u04CE\u04D0-\u04F9\u0500-\u050F\u0531-\u0556\u0559\u0561-\u0587\u05D0-\u05EA\u05F0-\u05F2\u0621-\u063A\u0640\u0641-\u064A\u066E-\u066F\u0671-\u06D3\u06D5\u06E5-\u06E6\u06EE-\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u076D\u0780-\u07A5\u07B1\u0904-\u0939\u093D\u0950\u0958-\u0961\u097D\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC-\u09DD\u09DF-\u09E1\u09F0-\u09F1\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33\u0A35-\u0A36\u0A38-\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0-\u0AE1\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32-\u0B33\u0B35-\u0B39\u0B3D\u0B5C-\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C33\u0C35-\u0C39\u0C60-\u0C61\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE\u0CE0-\u0CE1\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D28\u0D2A-\u0D39\u0D60-\u0D61\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32\u0E40-\u0E45\u0E46\u0E81-\u0E82\u0E84\u0E87-\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA-\u0EAB\u0EAD-\u0EB0\u0EB2\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDD\u0F00\u0F40-\u0F47\u0F49-\u0F6A\u0F88-\u0F8B\u1000-\u1021\u1023-\u1027\u1029-\u102A\u1050-\u1055\u10A0-\u10C5\u10D0-\u10FA\u10FC\u1100-\u1159\u115F-\u11A2\u11A8-\u11F9\u1200-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F4\u1401-\u166C\u166F-\u1676\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F0\u1700-\u170C\u170E-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1842\u1843\u1844-\u1877\u1880-\u18A8\u1900-\u191C\u1950-\u196D\u1970-\u1974\u1980-\u19A9\u19C1-\u19C7\u1A00-\u1A16\u1D00-\u1D2B\u1D2C-\u1D61\u1D62-\u1D77\u1D78\u1D79-\u1D9A\u1D9B-\u1DBF\u1E00-\u1E9B\u1EA0-\u1EF9\u1F00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u2094\u2102\u2107\u210A-\u2113\u2115\u2118\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212E\u212F-\u2131\u2133-\u2134\u2135-\u2138\u2139\u213C-\u213F\u2145-\u2149\u2160-\u2183\u2C00-\u2C2E\u2C30-\u2C5E\u2C80-\u2CE4\u2D00-\u2D25\u2D30-\u2D65\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u3005\u3006\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303A\u303B\u303C\u3041-\u3096\u309D-\u309E\u309F\u30A1-\u30FA\u30FC-\u30FE\u30FF\u3105-\u312C\u3131-\u318E\u31A0-\u31B7\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FBB\uA000-\uA014\uA015\uA016-\uA48C\uA800-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uAC00-\uD7A3\uF900-\uFA2D\uFA30-\uFA6A\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFC5D\uFC64-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDF9\uFE71\uFE73\uFE77\uFE79\uFE7B\uFE7D\uFE7F-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFF6F\uFF70\uFF71-\uFF9D\uFFA0-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC'
+
+xid_continue = u'\u0030-\u0039\u0041-\u005A\u005F\u0061-\u007A\u00AA\u00B5\u00B7\u00BA\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u01BA\u01BB\u01BC-\u01BF\u01C0-\u01C3\u01C4-\u0241\u0250-\u02AF\u02B0-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EE\u0300-\u036F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03CE\u03D0-\u03F5\u03F7-\u0481\u0483-\u0486\u048A-\u04CE\u04D0-\u04F9\u0500-\u050F\u0531-\u0556\u0559\u0561-\u0587\u0591-\u05B9\u05BB-\u05BD\u05BF\u05C1-\u05C2\u05C4-\u05C5\u05C7\u05D0-\u05EA\u05F0-\u05F2\u0610-\u0615\u0621-\u063A\u0640\u0641-\u064A\u064B-\u065E\u0660-\u0669\u066E-\u066F\u0670\u0671-\u06D3\u06D5\u06D6-\u06DC\u06DF-\u06E4\u06E5-\u06E6\u06E7-\u06E8\u06EA-\u06ED\u06EE-\u06EF\u06F0-\u06F9\u06FA-\u06FC\u06FF\u0710\u0711\u0712-\u072F\u0730-\u074A\u074D-\u076D\u0780-\u07A5\u07A6-\u07B0\u07B1\u0901-\u0902\u0903\u0904-\u0939\u093C\u093D\u093E-\u0940\u0941-\u0948\u0949-\u094C\u094D\u0950\u0951-\u0954\u0958-\u0961\u0962-\u0963\u0966-\u096F\u097D\u0981\u0982-\u0983\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BC\u09BD\u09BE-\u09C0\u09C1-\u09C4\u09C7-\u09C8\u09CB-\u09CC\u09CD\u09CE\u09D7\u09DC-\u09DD\u09DF-\u09E1\u09E2-\u09E3\u09E6-\u09EF\u09F0-\u09F1\u0A01-\u0A02\u0A03\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33\u0A35-\u0A36\u0A38-\u0A39\u0A3C\u0A3E-\u0A40\u0A41-\u0A42\u0A47-\u0A48\u0A4B-\u0A4D\u0A59-\u0A5C\u0A5E\u0A66-\u0A6F\u0A70-\u0A71\u0A72-\u0A74\u0A81-\u0A82\u0A83\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABC\u0ABD\u0ABE-\u0AC0\u0AC1-\u0AC5\u0AC7-\u0AC8\u0AC9\u0ACB-\u0ACC\u0ACD\u0AD0\u0AE0-\u0AE1\u0AE2-\u0AE3\u0AE6-\u0AEF\u0B01\u0B02-\u0B03\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32-\u0B33\u0B35-\u0B39\u0B3C\u0B3D\u0B3E\u0B3F\u0B40\u0B41-\u0B43\u0B47-\u0B48\u0B4B-\u0B4C\u0B4D\u0B56\u0B57\u0B5C-\u0B5D\u0B5F-\u0B61\u0B66-\u0B6F\u0B71\u0B82\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BBE-\u0BBF\u0BC0\u0BC1-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCC\u0BCD\u0BD7\u0BE6-\u0BEF\u0C01-\u0C03\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C33\u0C35-\u0C39\u0C3E-\u0C40\u0C41-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55-\u0C56\u0C60-\u0C61\u0C66-\u0C6F\u0C82-\u0C83\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBC\u0CBD\u0CBE\u0CBF\u0CC0-\u0CC4\u0CC6\u0CC7-\u0CC8\u0CCA-\u0CCB\u0CCC-\u0CCD\u0CD5-\u0CD6\u0CDE\u0CE0-\u0CE1\u0CE6-\u0CEF\u0D02-\u0D03\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D28\u0D2A-\u0D39\u0D3E-\u0D40\u0D41-\u0D43\u0D46-\u0D48\u0D4A-\u0D4C\u0D4D\u0D57\u0D60-\u0D61\u0D66-\u0D6F\u0D82-\u0D83\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DCA\u0DCF-\u0DD1\u0DD2-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2-\u0DF3\u0E01-\u0E30\u0E31\u0E32-\u0E33\u0E34-\u0E3A\u0E40-\u0E45\u0E46\u0E47-\u0E4E\u0E50-\u0E59\u0E81-\u0E82\u0E84\u0E87-\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA-\u0EAB\u0EAD-\u0EB0\u0EB1\u0EB2-\u0EB3\u0EB4-\u0EB9\u0EBB-\u0EBC\u0EBD\u0EC0-\u0EC4\u0EC6\u0EC8-\u0ECD\u0ED0-\u0ED9\u0EDC-\u0EDD\u0F00\u0F18-\u0F19\u0F20-\u0F29\u0F35\u0F37\u0F39\u0F3E-\u0F3F\u0F40-\u0F47\u0F49-\u0F6A\u0F71-\u0F7E\u0F7F\u0F80-\u0F84\u0F86-\u0F87\u0F88-\u0F8B\u0F90-\u0F97\u0F99-\u0FBC\u0FC6\u1000-\u1021\u1023-\u1027\u1029-\u102A\u102C\u102D-\u1030\u1031\u1032\u1036-\u1037\u1038\u1039\u1040-\u1049\u1050-\u1055\u1056-\u1057\u1058-\u1059\u10A0-\u10C5\u10D0-\u10FA\u10FC\u1100-\u1159\u115F-\u11A2\u11A8-\u11F9\u1200-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u135F\u1369-\u1371\u1380-\u138F\u13A0-\u13F4\u1401-\u166C\u166F-\u1676\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F0\u1700-\u170C\u170E-\u1711\u1712-\u1714\u1720-\u1731\u1732-\u1734\u1740-\u1751\u1752-\u1753\u1760-\u176C\u176E-\u1770\u1772-\u1773\u1780-\u17B3\u17B6\u17B7-\u17BD\u17BE-\u17C5\u17C6\u17C7-\u17C8\u17C9-\u17D3\u17D7\u17DC\u17DD\u17E0-\u17E9\u180B-\u180D\u1810-\u1819\u1820-\u1842\u1843\u1844-\u1877\u1880-\u18A8\u18A9\u1900-\u191C\u1920-\u1922\u1923-\u1926\u1927-\u1928\u1929-\u192B\u1930-\u1931\u1932\u1933-\u1938\u1939-\u193B\u1946-\u194F\u1950-\u196D\u1970-\u1974\u1980-\u19A9\u19B0-\u19C0\u19C1-\u19C7\u19C8-\u19C9\u19D0-\u19D9\u1A00-\u1A16\u1A17-\u1A18\u1A19-\u1A1B\u1D00-\u1D2B\u1D2C-\u1D61\u1D62-\u1D77\u1D78\u1D79-\u1D9A\u1D9B-\u1DBF\u1DC0-\u1DC3\u1E00-\u1E9B\u1EA0-\u1EF9\u1F00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u203F-\u2040\u2054\u2071\u207F\u2090-\u2094\u20D0-\u20DC\u20E1\u20E5-\u20EB\u2102\u2107\u210A-\u2113\u2115\u2118\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212E\u212F-\u2131\u2133-\u2134\u2135-\u2138\u2139\u213C-\u213F\u2145-\u2149\u2160-\u2183\u2C00-\u2C2E\u2C30-\u2C5E\u2C80-\u2CE4\u2D00-\u2D25\u2D30-\u2D65\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u3005\u3006\u3007\u3021-\u3029\u302A-\u302F\u3031-\u3035\u3038-\u303A\u303B\u303C\u3041-\u3096\u3099-\u309A\u309D-\u309E\u309F\u30A1-\u30FA\u30FC-\u30FE\u30FF\u3105-\u312C\u3131-\u318E\u31A0-\u31B7\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FBB\uA000-\uA014\uA015\uA016-\uA48C\uA800-\uA801\uA802\uA803-\uA805\uA806\uA807-\uA80A\uA80B\uA80C-\uA822\uA823-\uA824\uA825-\uA826\uA827\uAC00-\uD7A3\uF900-\uFA2D\uFA30-\uFA6A\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1E\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFC5D\uFC64-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDF9\uFE00-\uFE0F\uFE20-\uFE23\uFE33-\uFE34\uFE4D-\uFE4F\uFE71\uFE73\uFE77\uFE79\uFE7B\uFE7D\uFE7F-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF3F\uFF41-\uFF5A\uFF66-\uFF6F\uFF70\uFF71-\uFF9D\uFF9E-\uFF9F\uFFA0-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC'
+
+def allexcept(*args):
+ newcats = cats[:]
+ for arg in args:
+ newcats.remove(arg)
+ return u''.join([globals()[cat] for cat in newcats])
+
+if __name__ == '__main__':
+ import unicodedata
+
+ categories = {}
+
+ f = open(__file__.rstrip('co'))
+ try:
+ content = f.read()
+ finally:
+ f.close()
+
+ header = content[:content.find('Cc =')]
+ footer = content[content.find("def combine("):]
+
+ for code in range(65535):
+ c = unichr(code)
+ cat = unicodedata.category(c)
+ categories.setdefault(cat, []).append(c)
+
+ f = open(__file__, 'w')
+ f.write(header)
+
+ for cat in sorted(categories):
+ val = u''.join(categories[cat])
+ if cat == 'Cs':
+ # Jython can't handle isolated surrogates
+ f.write("""\
+try:
+ Cs = eval(r"%r")
+except UnicodeDecodeError:
+ Cs = '' # Jython can't handle isolated surrogates\n\n""" % val)
+ else:
+ f.write('%s = %r\n\n' % (cat, val))
+ f.write('cats = %r\n\n' % sorted(categories.keys()))
+
+ f.write(footer)
+ f.close()
diff --git a/slider-agent/src/main/python/jinja2/jinja2/bccache.py b/slider-agent/src/main/python/jinja2/jinja2/bccache.py
new file mode 100644
index 0000000..1e2236c
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/bccache.py
@@ -0,0 +1,280 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.bccache
+ ~~~~~~~~~~~~~~
+
+ This module implements the bytecode cache system Jinja is optionally
+ using. This is useful if you have very complex template situations and
+ the compiliation of all those templates slow down your application too
+ much.
+
+ Situations where this is useful are often forking web applications that
+ are initialized on the first request.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD.
+"""
+from os import path, listdir
+import marshal
+import tempfile
+import cPickle as pickle
+import fnmatch
+from cStringIO import StringIO
+try:
+ from hashlib import sha1
+except ImportError:
+ from sha import new as sha1
+from jinja2.utils import open_if_exists
+
+
+bc_version = 1
+bc_magic = 'j2'.encode('ascii') + pickle.dumps(bc_version, 2)
+
+
+class Bucket(object):
+ """Buckets are used to store the bytecode for one template. It's created
+ and initialized by the bytecode cache and passed to the loading functions.
+
+ The buckets get an internal checksum from the cache assigned and use this
+ to automatically reject outdated cache material. Individual bytecode
+ cache subclasses don't have to care about cache invalidation.
+ """
+
+ def __init__(self, environment, key, checksum):
+ self.environment = environment
+ self.key = key
+ self.checksum = checksum
+ self.reset()
+
+ def reset(self):
+ """Resets the bucket (unloads the bytecode)."""
+ self.code = None
+
+ def load_bytecode(self, f):
+ """Loads bytecode from a file or file like object."""
+ # make sure the magic header is correct
+ magic = f.read(len(bc_magic))
+ if magic != bc_magic:
+ self.reset()
+ return
+ # the source code of the file changed, we need to reload
+ checksum = pickle.load(f)
+ if self.checksum != checksum:
+ self.reset()
+ return
+ # now load the code. Because marshal is not able to load
+ # from arbitrary streams we have to work around that
+ if isinstance(f, file):
+ self.code = marshal.load(f)
+ else:
+ self.code = marshal.loads(f.read())
+
+ def write_bytecode(self, f):
+ """Dump the bytecode into the file or file like object passed."""
+ if self.code is None:
+ raise TypeError('can\'t write empty bucket')
+ f.write(bc_magic)
+ pickle.dump(self.checksum, f, 2)
+ if isinstance(f, file):
+ marshal.dump(self.code, f)
+ else:
+ f.write(marshal.dumps(self.code))
+
+ def bytecode_from_string(self, string):
+ """Load bytecode from a string."""
+ self.load_bytecode(StringIO(string))
+
+ def bytecode_to_string(self):
+ """Return the bytecode as string."""
+ out = StringIO()
+ self.write_bytecode(out)
+ return out.getvalue()
+
+
+class BytecodeCache(object):
+ """To implement your own bytecode cache you have to subclass this class
+ and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of
+ these methods are passed a :class:`~jinja2.bccache.Bucket`.
+
+ A very basic bytecode cache that saves the bytecode on the file system::
+
+ from os import path
+
+ class MyCache(BytecodeCache):
+
+ def __init__(self, directory):
+ self.directory = directory
+
+ def load_bytecode(self, bucket):
+ filename = path.join(self.directory, bucket.key)
+ if path.exists(filename):
+ with open(filename, 'rb') as f:
+ bucket.load_bytecode(f)
+
+ def dump_bytecode(self, bucket):
+ filename = path.join(self.directory, bucket.key)
+ with open(filename, 'wb') as f:
+ bucket.write_bytecode(f)
+
+ A more advanced version of a filesystem based bytecode cache is part of
+ Jinja2.
+ """
+
+ def load_bytecode(self, bucket):
+ """Subclasses have to override this method to load bytecode into a
+ bucket. If they are not able to find code in the cache for the
+ bucket, it must not do anything.
+ """
+ raise NotImplementedError()
+
+ def dump_bytecode(self, bucket):
+ """Subclasses have to override this method to write the bytecode
+ from a bucket back to the cache. If it unable to do so it must not
+ fail silently but raise an exception.
+ """
+ raise NotImplementedError()
+
+ def clear(self):
+ """Clears the cache. This method is not used by Jinja2 but should be
+ implemented to allow applications to clear the bytecode cache used
+ by a particular environment.
+ """
+
+ def get_cache_key(self, name, filename=None):
+ """Returns the unique hash key for this template name."""
+ hash = sha1(name.encode('utf-8'))
+ if filename is not None:
+ if isinstance(filename, unicode):
+ filename = filename.encode('utf-8')
+ hash.update('|' + filename)
+ return hash.hexdigest()
+
+ def get_source_checksum(self, source):
+ """Returns a checksum for the source."""
+ return sha1(source.encode('utf-8')).hexdigest()
+
+ def get_bucket(self, environment, name, filename, source):
+ """Return a cache bucket for the given template. All arguments are
+ mandatory but filename may be `None`.
+ """
+ key = self.get_cache_key(name, filename)
+ checksum = self.get_source_checksum(source)
+ bucket = Bucket(environment, key, checksum)
+ self.load_bytecode(bucket)
+ return bucket
+
+ def set_bucket(self, bucket):
+ """Put the bucket into the cache."""
+ self.dump_bytecode(bucket)
+
+
+class FileSystemBytecodeCache(BytecodeCache):
+ """A bytecode cache that stores bytecode on the filesystem. It accepts
+ two arguments: The directory where the cache items are stored and a
+ pattern string that is used to build the filename.
+
+ If no directory is specified the system temporary items folder is used.
+
+ The pattern can be used to have multiple separate caches operate on the
+ same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s``
+ is replaced with the cache key.
+
+ >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
+
+ This bytecode cache supports clearing of the cache using the clear method.
+ """
+
+ def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
+ if directory is None:
+ directory = tempfile.gettempdir()
+ self.directory = directory
+ self.pattern = pattern
+
+ def _get_cache_filename(self, bucket):
+ return path.join(self.directory, self.pattern % bucket.key)
+
+ def load_bytecode(self, bucket):
+ f = open_if_exists(self._get_cache_filename(bucket), 'rb')
+ if f is not None:
+ try:
+ bucket.load_bytecode(f)
+ finally:
+ f.close()
+
+ def dump_bytecode(self, bucket):
+ f = open(self._get_cache_filename(bucket), 'wb')
+ try:
+ bucket.write_bytecode(f)
+ finally:
+ f.close()
+
+ def clear(self):
+ # imported lazily here because google app-engine doesn't support
+ # write access on the file system and the function does not exist
+ # normally.
+ from os import remove
+ files = fnmatch.filter(listdir(self.directory), self.pattern % '*')
+ for filename in files:
+ try:
+ remove(path.join(self.directory, filename))
+ except OSError:
+ pass
+
+
+class MemcachedBytecodeCache(BytecodeCache):
+ """This class implements a bytecode cache that uses a memcache cache for
+ storing the information. It does not enforce a specific memcache library
+ (tummy's memcache or cmemcache) but will accept any class that provides
+ the minimal interface required.
+
+ Libraries compatible with this class:
+
+ - `werkzeug <http://werkzeug.pocoo.org/>`_.contrib.cache
+ - `python-memcached <http://www.tummy.com/Community/software/python-memcached/>`_
+ - `cmemcache <http://gijsbert.org/cmemcache/>`_
+
+ (Unfortunately the django cache interface is not compatible because it
+ does not support storing binary data, only unicode. You can however pass
+ the underlying cache client to the bytecode cache which is available
+ as `django.core.cache.cache._client`.)
+
+ The minimal interface for the client passed to the constructor is this:
+
+ .. class:: MinimalClientInterface
+
+ .. method:: set(key, value[, timeout])
+
+ Stores the bytecode in the cache. `value` is a string and
+ `timeout` the timeout of the key. If timeout is not provided
+ a default timeout or no timeout should be assumed, if it's
+ provided it's an integer with the number of seconds the cache
+ item should exist.
+
+ .. method:: get(key)
+
+ Returns the value for the cache key. If the item does not
+ exist in the cache the return value must be `None`.
+
+ The other arguments to the constructor are the prefix for all keys that
+ is added before the actual cache key and the timeout for the bytecode in
+ the cache system. We recommend a high (or no) timeout.
+
+ This bytecode cache does not support clearing of used items in the cache.
+ The clear method is a no-operation function.
+ """
+
+ def __init__(self, client, prefix='jinja2/bytecode/', timeout=None):
+ self.client = client
+ self.prefix = prefix
+ self.timeout = timeout
+
+ def load_bytecode(self, bucket):
+ code = self.client.get(self.prefix + bucket.key)
+ if code is not None:
+ bucket.bytecode_from_string(code)
+
+ def dump_bytecode(self, bucket):
+ args = (self.prefix + bucket.key, bucket.bytecode_to_string())
+ if self.timeout is not None:
+ args += (self.timeout,)
+ self.client.set(*args)
diff --git a/slider-agent/src/main/python/jinja2/jinja2/compiler.py b/slider-agent/src/main/python/jinja2/jinja2/compiler.py
new file mode 100644
index 0000000..5764159
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/compiler.py
@@ -0,0 +1,1640 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.compiler
+ ~~~~~~~~~~~~~~~
+
+ Compiles nodes into python code.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+from cStringIO import StringIO
+from itertools import chain
+from copy import deepcopy
+from jinja2 import nodes
+from jinja2.nodes import EvalContext
+from jinja2.visitor import NodeVisitor, NodeTransformer
+from jinja2.exceptions import TemplateAssertionError
+from jinja2.utils import Markup, concat, escape, is_python_keyword, next
+
+
+operators = {
+ 'eq': '==',
+ 'ne': '!=',
+ 'gt': '>',
+ 'gteq': '>=',
+ 'lt': '<',
+ 'lteq': '<=',
+ 'in': 'in',
+ 'notin': 'not in'
+}
+
+try:
+ exec '(0 if 0 else 0)'
+except SyntaxError:
+ have_condexpr = False
+else:
+ have_condexpr = True
+
+
+# what method to iterate over items do we want to use for dict iteration
+# in generated code? on 2.x let's go with iteritems, on 3.x with items
+if hasattr(dict, 'iteritems'):
+ dict_item_iter = 'iteritems'
+else:
+ dict_item_iter = 'items'
+
+
+# does if 0: dummy(x) get us x into the scope?
+def unoptimize_before_dead_code():
+ x = 42
+ def f():
+ if 0: dummy(x)
+ return f
+unoptimize_before_dead_code = bool(unoptimize_before_dead_code().func_closure)
+
+
+def generate(node, environment, name, filename, stream=None,
+ defer_init=False):
+ """Generate the python source for a node tree."""
+ if not isinstance(node, nodes.Template):
+ raise TypeError('Can\'t compile non template nodes')
+ generator = CodeGenerator(environment, name, filename, stream, defer_init)
+ generator.visit(node)
+ if stream is None:
+ return generator.stream.getvalue()
+
+
+def has_safe_repr(value):
+ """Does the node have a safe representation?"""
+ if value is None or value is NotImplemented or value is Ellipsis:
+ return True
+ if isinstance(value, (bool, int, long, float, complex, basestring,
+ xrange, Markup)):
+ return True
+ if isinstance(value, (tuple, list, set, frozenset)):
+ for item in value:
+ if not has_safe_repr(item):
+ return False
+ return True
+ elif isinstance(value, dict):
+ for key, value in value.iteritems():
+ if not has_safe_repr(key):
+ return False
+ if not has_safe_repr(value):
+ return False
+ return True
+ return False
+
+
+def find_undeclared(nodes, names):
+ """Check if the names passed are accessed undeclared. The return value
+ is a set of all the undeclared names from the sequence of names found.
+ """
+ visitor = UndeclaredNameVisitor(names)
+ try:
+ for node in nodes:
+ visitor.visit(node)
+ except VisitorExit:
+ pass
+ return visitor.undeclared
+
+
+class Identifiers(object):
+ """Tracks the status of identifiers in frames."""
+
+ def __init__(self):
+ # variables that are known to be declared (probably from outer
+ # frames or because they are special for the frame)
+ self.declared = set()
+
+ # undeclared variables from outer scopes
+ self.outer_undeclared = set()
+
+ # names that are accessed without being explicitly declared by
+ # this one or any of the outer scopes. Names can appear both in
+ # declared and undeclared.
+ self.undeclared = set()
+
+ # names that are declared locally
+ self.declared_locally = set()
+
+ # names that are declared by parameters
+ self.declared_parameter = set()
+
+ def add_special(self, name):
+ """Register a special name like `loop`."""
+ self.undeclared.discard(name)
+ self.declared.add(name)
+
+ def is_declared(self, name, local_only=False):
+ """Check if a name is declared in this or an outer scope."""
+ if name in self.declared_locally or name in self.declared_parameter:
+ return True
+ if local_only:
+ return False
+ return name in self.declared
+
+ def copy(self):
+ return deepcopy(self)
+
+
+class Frame(object):
+ """Holds compile time information for us."""
+
+ def __init__(self, eval_ctx, parent=None):
+ self.eval_ctx = eval_ctx
+ self.identifiers = Identifiers()
+
+ # a toplevel frame is the root + soft frames such as if conditions.
+ self.toplevel = False
+
+ # the root frame is basically just the outermost frame, so no if
+ # conditions. This information is used to optimize inheritance
+ # situations.
+ self.rootlevel = False
+
+ # in some dynamic inheritance situations the compiler needs to add
+ # write tests around output statements.
+ self.require_output_check = parent and parent.require_output_check
+
+ # inside some tags we are using a buffer rather than yield statements.
+ # this for example affects {% filter %} or {% macro %}. If a frame
+ # is buffered this variable points to the name of the list used as
+ # buffer.
+ self.buffer = None
+
+ # the name of the block we're in, otherwise None.
+ self.block = parent and parent.block or None
+
+ # a set of actually assigned names
+ self.assigned_names = set()
+
+ # the parent of this frame
+ self.parent = parent
+
+ if parent is not None:
+ self.identifiers.declared.update(
+ parent.identifiers.declared |
+ parent.identifiers.declared_parameter |
+ parent.assigned_names
+ )
+ self.identifiers.outer_undeclared.update(
+ parent.identifiers.undeclared -
+ self.identifiers.declared
+ )
+ self.buffer = parent.buffer
+
+ def copy(self):
+ """Create a copy of the current one."""
+ rv = object.__new__(self.__class__)
+ rv.__dict__.update(self.__dict__)
+ rv.identifiers = object.__new__(self.identifiers.__class__)
+ rv.identifiers.__dict__.update(self.identifiers.__dict__)
+ return rv
+
+ def inspect(self, nodes, hard_scope=False):
+ """Walk the node and check for identifiers. If the scope is hard (eg:
+ enforce on a python level) overrides from outer scopes are tracked
+ differently.
+ """
+ visitor = FrameIdentifierVisitor(self.identifiers, hard_scope)
+ for node in nodes:
+ visitor.visit(node)
+
+ def find_shadowed(self, extra=()):
+ """Find all the shadowed names. extra is an iterable of variables
+ that may be defined with `add_special` which may occour scoped.
+ """
+ i = self.identifiers
+ return (i.declared | i.outer_undeclared) & \
+ (i.declared_locally | i.declared_parameter) | \
+ set(x for x in extra if i.is_declared(x))
+
+ def inner(self):
+ """Return an inner frame."""
+ return Frame(self.eval_ctx, self)
+
+ def soft(self):
+ """Return a soft frame. A soft frame may not be modified as
+ standalone thing as it shares the resources with the frame it
+ was created of, but it's not a rootlevel frame any longer.
+ """
+ rv = self.copy()
+ rv.rootlevel = False
+ return rv
+
+ __copy__ = copy
+
+
+class VisitorExit(RuntimeError):
+ """Exception used by the `UndeclaredNameVisitor` to signal a stop."""
+
+
+class DependencyFinderVisitor(NodeVisitor):
+ """A visitor that collects filter and test calls."""
+
+ def __init__(self):
+ self.filters = set()
+ self.tests = set()
+
+ def visit_Filter(self, node):
+ self.generic_visit(node)
+ self.filters.add(node.name)
+
+ def visit_Test(self, node):
+ self.generic_visit(node)
+ self.tests.add(node.name)
+
+ def visit_Block(self, node):
+ """Stop visiting at blocks."""
+
+
+class UndeclaredNameVisitor(NodeVisitor):
+ """A visitor that checks if a name is accessed without being
+ declared. This is different from the frame visitor as it will
+ not stop at closure frames.
+ """
+
+ def __init__(self, names):
+ self.names = set(names)
+ self.undeclared = set()
+
+ def visit_Name(self, node):
+ if node.ctx == 'load' and node.name in self.names:
+ self.undeclared.add(node.name)
+ if self.undeclared == self.names:
+ raise VisitorExit()
+ else:
+ self.names.discard(node.name)
+
+ def visit_Block(self, node):
+ """Stop visiting a blocks."""
+
+
+class FrameIdentifierVisitor(NodeVisitor):
+ """A visitor for `Frame.inspect`."""
+
+ def __init__(self, identifiers, hard_scope):
+ self.identifiers = identifiers
+ self.hard_scope = hard_scope
+
+ def visit_Name(self, node):
+ """All assignments to names go through this function."""
+ if node.ctx == 'store':
+ self.identifiers.declared_locally.add(node.name)
+ elif node.ctx == 'param':
+ self.identifiers.declared_parameter.add(node.name)
+ elif node.ctx == 'load' and not \
+ self.identifiers.is_declared(node.name, self.hard_scope):
+ self.identifiers.undeclared.add(node.name)
+
+ def visit_If(self, node):
+ self.visit(node.test)
+ real_identifiers = self.identifiers
+
+ old_names = real_identifiers.declared_locally | \
+ real_identifiers.declared_parameter
+
+ def inner_visit(nodes):
+ if not nodes:
+ return set()
+ self.identifiers = real_identifiers.copy()
+ for subnode in nodes:
+ self.visit(subnode)
+ rv = self.identifiers.declared_locally - old_names
+ # we have to remember the undeclared variables of this branch
+ # because we will have to pull them.
+ real_identifiers.undeclared.update(self.identifiers.undeclared)
+ self.identifiers = real_identifiers
+ return rv
+
+ body = inner_visit(node.body)
+ else_ = inner_visit(node.else_ or ())
+
+ # the differences between the two branches are also pulled as
+ # undeclared variables
+ real_identifiers.undeclared.update(body.symmetric_difference(else_) -
+ real_identifiers.declared)
+
+ # remember those that are declared.
+ real_identifiers.declared_locally.update(body | else_)
+
+ def visit_Macro(self, node):
+ self.identifiers.declared_locally.add(node.name)
+
+ def visit_Import(self, node):
+ self.generic_visit(node)
+ self.identifiers.declared_locally.add(node.target)
+
+ def visit_FromImport(self, node):
+ self.generic_visit(node)
+ for name in node.names:
+ if isinstance(name, tuple):
+ self.identifiers.declared_locally.add(name[1])
+ else:
+ self.identifiers.declared_locally.add(name)
+
+ def visit_Assign(self, node):
+ """Visit assignments in the correct order."""
+ self.visit(node.node)
+ self.visit(node.target)
+
+ def visit_For(self, node):
+ """Visiting stops at for blocks. However the block sequence
+ is visited as part of the outer scope.
+ """
+ self.visit(node.iter)
+
+ def visit_CallBlock(self, node):
+ self.visit(node.call)
+
+ def visit_FilterBlock(self, node):
+ self.visit(node.filter)
+
+ def visit_Scope(self, node):
+ """Stop visiting at scopes."""
+
+ def visit_Block(self, node):
+ """Stop visiting at blocks."""
+
+
+class CompilerExit(Exception):
+ """Raised if the compiler encountered a situation where it just
+ doesn't make sense to further process the code. Any block that
+ raises such an exception is not further processed.
+ """
+
+
+class CodeGenerator(NodeVisitor):
+
+ def __init__(self, environment, name, filename, stream=None,
+ defer_init=False):
+ if stream is None:
+ stream = StringIO()
+ self.environment = environment
+ self.name = name
+ self.filename = filename
+ self.stream = stream
+ self.created_block_context = False
+ self.defer_init = defer_init
+
+ # aliases for imports
+ self.import_aliases = {}
+
+ # a registry for all blocks. Because blocks are moved out
+ # into the global python scope they are registered here
+ self.blocks = {}
+
+ # the number of extends statements so far
+ self.extends_so_far = 0
+
+ # some templates have a rootlevel extends. In this case we
+ # can safely assume that we're a child template and do some
+ # more optimizations.
+ self.has_known_extends = False
+
+ # the current line number
+ self.code_lineno = 1
+
+ # registry of all filters and tests (global, not block local)
+ self.tests = {}
+ self.filters = {}
+
+ # the debug information
+ self.debug_info = []
+ self._write_debug_info = None
+
+ # the number of new lines before the next write()
+ self._new_lines = 0
+
+ # the line number of the last written statement
+ self._last_line = 0
+
+ # true if nothing was written so far.
+ self._first_write = True
+
+ # used by the `temporary_identifier` method to get new
+ # unique, temporary identifier
+ self._last_identifier = 0
+
+ # the current indentation
+ self._indentation = 0
+
+ # -- Various compilation helpers
+
+ def fail(self, msg, lineno):
+ """Fail with a :exc:`TemplateAssertionError`."""
+ raise TemplateAssertionError(msg, lineno, self.name, self.filename)
+
+ def temporary_identifier(self):
+ """Get a new unique identifier."""
+ self._last_identifier += 1
+ return 't_%d' % self._last_identifier
+
+ def buffer(self, frame):
+ """Enable buffering for the frame from that point onwards."""
+ frame.buffer = self.temporary_identifier()
+ self.writeline('%s = []' % frame.buffer)
+
+ def return_buffer_contents(self, frame):
+ """Return the buffer contents of the frame."""
+ if frame.eval_ctx.volatile:
+ self.writeline('if context.eval_ctx.autoescape:')
+ self.indent()
+ self.writeline('return Markup(concat(%s))' % frame.buffer)
+ self.outdent()
+ self.writeline('else:')
+ self.indent()
+ self.writeline('return concat(%s)' % frame.buffer)
+ self.outdent()
+ elif frame.eval_ctx.autoescape:
+ self.writeline('return Markup(concat(%s))' % frame.buffer)
+ else:
+ self.writeline('return concat(%s)' % frame.buffer)
+
+ def indent(self):
+ """Indent by one."""
+ self._indentation += 1
+
+ def outdent(self, step=1):
+ """Outdent by step."""
+ self._indentation -= step
+
+ def start_write(self, frame, node=None):
+ """Yield or write into the frame buffer."""
+ if frame.buffer is None:
+ self.writeline('yield ', node)
+ else:
+ self.writeline('%s.append(' % frame.buffer, node)
+
+ def end_write(self, frame):
+ """End the writing process started by `start_write`."""
+ if frame.buffer is not None:
+ self.write(')')
+
+ def simple_write(self, s, frame, node=None):
+ """Simple shortcut for start_write + write + end_write."""
+ self.start_write(frame, node)
+ self.write(s)
+ self.end_write(frame)
+
+ def blockvisit(self, nodes, frame):
+ """Visit a list of nodes as block in a frame. If the current frame
+ is no buffer a dummy ``if 0: yield None`` is written automatically
+ unless the force_generator parameter is set to False.
+ """
+ if frame.buffer is None:
+ self.writeline('if 0: yield None')
+ else:
+ self.writeline('pass')
+ try:
+ for node in nodes:
+ self.visit(node, frame)
+ except CompilerExit:
+ pass
+
+ def write(self, x):
+ """Write a string into the output stream."""
+ if self._new_lines:
+ if not self._first_write:
+ self.stream.write('\n' * self._new_lines)
+ self.code_lineno += self._new_lines
+ if self._write_debug_info is not None:
+ self.debug_info.append((self._write_debug_info,
+ self.code_lineno))
+ self._write_debug_info = None
+ self._first_write = False
+ self.stream.write(' ' * self._indentation)
+ self._new_lines = 0
+ self.stream.write(x)
+
+ def writeline(self, x, node=None, extra=0):
+ """Combination of newline and write."""
+ self.newline(node, extra)
+ self.write(x)
+
+ def newline(self, node=None, extra=0):
+ """Add one or more newlines before the next write."""
+ self._new_lines = max(self._new_lines, 1 + extra)
+ if node is not None and node.lineno != self._last_line:
+ self._write_debug_info = node.lineno
+ self._last_line = node.lineno
+
+ def signature(self, node, frame, extra_kwargs=None):
+ """Writes a function call to the stream for the current node.
+ A leading comma is added automatically. The extra keyword
+ arguments may not include python keywords otherwise a syntax
+ error could occour. The extra keyword arguments should be given
+ as python dict.
+ """
+ # if any of the given keyword arguments is a python keyword
+ # we have to make sure that no invalid call is created.
+ kwarg_workaround = False
+ for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()):
+ if is_python_keyword(kwarg):
+ kwarg_workaround = True
+ break
+
+ for arg in node.args:
+ self.write(', ')
+ self.visit(arg, frame)
+
+ if not kwarg_workaround:
+ for kwarg in node.kwargs:
+ self.write(', ')
+ self.visit(kwarg, frame)
+ if extra_kwargs is not None:
+ for key, value in extra_kwargs.iteritems():
+ self.write(', %s=%s' % (key, value))
+ if node.dyn_args:
+ self.write(', *')
+ self.visit(node.dyn_args, frame)
+
+ if kwarg_workaround:
+ if node.dyn_kwargs is not None:
+ self.write(', **dict({')
+ else:
+ self.write(', **{')
+ for kwarg in node.kwargs:
+ self.write('%r: ' % kwarg.key)
+ self.visit(kwarg.value, frame)
+ self.write(', ')
+ if extra_kwargs is not None:
+ for key, value in extra_kwargs.iteritems():
+ self.write('%r: %s, ' % (key, value))
+ if node.dyn_kwargs is not None:
+ self.write('}, **')
+ self.visit(node.dyn_kwargs, frame)
+ self.write(')')
+ else:
+ self.write('}')
+
+ elif node.dyn_kwargs is not None:
+ self.write(', **')
+ self.visit(node.dyn_kwargs, frame)
+
+ def pull_locals(self, frame):
+ """Pull all the references identifiers into the local scope."""
+ for name in frame.identifiers.undeclared:
+ self.writeline('l_%s = context.resolve(%r)' % (name, name))
+
+ def pull_dependencies(self, nodes):
+ """Pull all the dependencies."""
+ visitor = DependencyFinderVisitor()
+ for node in nodes:
+ visitor.visit(node)
+ for dependency in 'filters', 'tests':
+ mapping = getattr(self, dependency)
+ for name in getattr(visitor, dependency):
+ if name not in mapping:
+ mapping[name] = self.temporary_identifier()
+ self.writeline('%s = environment.%s[%r]' %
+ (mapping[name], dependency, name))
+
+ def unoptimize_scope(self, frame):
+ """Disable Python optimizations for the frame."""
+ # XXX: this is not that nice but it has no real overhead. It
+ # mainly works because python finds the locals before dead code
+ # is removed. If that breaks we have to add a dummy function
+ # that just accepts the arguments and does nothing.
+ if frame.identifiers.declared:
+ self.writeline('%sdummy(%s)' % (
+ unoptimize_before_dead_code and 'if 0: ' or '',
+ ', '.join('l_' + name for name in frame.identifiers.declared)
+ ))
+
+ def push_scope(self, frame, extra_vars=()):
+ """This function returns all the shadowed variables in a dict
+ in the form name: alias and will write the required assignments
+ into the current scope. No indentation takes place.
+
+ This also predefines locally declared variables from the loop
+ body because under some circumstances it may be the case that
+
+ `extra_vars` is passed to `Frame.find_shadowed`.
+ """
+ aliases = {}
+ for name in frame.find_shadowed(extra_vars):
+ aliases[name] = ident = self.temporary_identifier()
+ self.writeline('%s = l_%s' % (ident, name))
+ to_declare = set()
+ for name in frame.identifiers.declared_locally:
+ if name not in aliases:
+ to_declare.add('l_' + name)
+ if to_declare:
+ self.writeline(' = '.join(to_declare) + ' = missing')
+ return aliases
+
+ def pop_scope(self, aliases, frame):
+ """Restore all aliases and delete unused variables."""
+ for name, alias in aliases.iteritems():
+ self.writeline('l_%s = %s' % (name, alias))
+ to_delete = set()
+ for name in frame.identifiers.declared_locally:
+ if name not in aliases:
+ to_delete.add('l_' + name)
+ if to_delete:
+ # we cannot use the del statement here because enclosed
+ # scopes can trigger a SyntaxError:
+ # a = 42; b = lambda: a; del a
+ self.writeline(' = '.join(to_delete) + ' = missing')
+
+ def function_scoping(self, node, frame, children=None,
+ find_special=True):
+ """In Jinja a few statements require the help of anonymous
+ functions. Those are currently macros and call blocks and in
+ the future also recursive loops. As there is currently
+ technical limitation that doesn't allow reading and writing a
+ variable in a scope where the initial value is coming from an
+ outer scope, this function tries to fall back with a common
+ error message. Additionally the frame passed is modified so
+ that the argumetns are collected and callers are looked up.
+
+ This will return the modified frame.
+ """
+ # we have to iterate twice over it, make sure that works
+ if children is None:
+ children = node.iter_child_nodes()
+ children = list(children)
+ func_frame = frame.inner()
+ func_frame.inspect(children, hard_scope=True)
+
+ # variables that are undeclared (accessed before declaration) and
+ # declared locally *and* part of an outside scope raise a template
+ # assertion error. Reason: we can't generate reasonable code from
+ # it without aliasing all the variables.
+ # this could be fixed in Python 3 where we have the nonlocal
+ # keyword or if we switch to bytecode generation
+ overriden_closure_vars = (
+ func_frame.identifiers.undeclared &
+ func_frame.identifiers.declared &
+ (func_frame.identifiers.declared_locally |
+ func_frame.identifiers.declared_parameter)
+ )
+ if overriden_closure_vars:
+ self.fail('It\'s not possible to set and access variables '
+ 'derived from an outer scope! (affects: %s)' %
+ ', '.join(sorted(overriden_closure_vars)), node.lineno)
+
+ # remove variables from a closure from the frame's undeclared
+ # identifiers.
+ func_frame.identifiers.undeclared -= (
+ func_frame.identifiers.undeclared &
+ func_frame.identifiers.declared
+ )
+
+ # no special variables for this scope, abort early
+ if not find_special:
+ return func_frame
+
+ func_frame.accesses_kwargs = False
+ func_frame.accesses_varargs = False
+ func_frame.accesses_caller = False
+ func_frame.arguments = args = ['l_' + x.name for x in node.args]
+
+ undeclared = find_undeclared(children, ('caller', 'kwargs', 'varargs'))
+
+ if 'caller' in undeclared:
+ func_frame.accesses_caller = True
+ func_frame.identifiers.add_special('caller')
+ args.append('l_caller')
+ if 'kwargs' in undeclared:
+ func_frame.accesses_kwargs = True
+ func_frame.identifiers.add_special('kwargs')
+ args.append('l_kwargs')
+ if 'varargs' in undeclared:
+ func_frame.accesses_varargs = True
+ func_frame.identifiers.add_special('varargs')
+ args.append('l_varargs')
+ return func_frame
+
+ def macro_body(self, node, frame, children=None):
+ """Dump the function def of a macro or call block."""
+ frame = self.function_scoping(node, frame, children)
+ # macros are delayed, they never require output checks
+ frame.require_output_check = False
+ args = frame.arguments
+ # XXX: this is an ugly fix for the loop nesting bug
+ # (tests.test_old_bugs.test_loop_call_bug). This works around
+ # a identifier nesting problem we have in general. It's just more
+ # likely to happen in loops which is why we work around it. The
+ # real solution would be "nonlocal" all the identifiers that are
+ # leaking into a new python frame and might be used both unassigned
+ # and assigned.
+ if 'loop' in frame.identifiers.declared:
+ args = args + ['l_loop=l_loop']
+ self.writeline('def macro(%s):' % ', '.join(args), node)
+ self.indent()
+ self.buffer(frame)
+ self.pull_locals(frame)
+ self.blockvisit(node.body, frame)
+ self.return_buffer_contents(frame)
+ self.outdent()
+ return frame
+
+ def macro_def(self, node, frame):
+ """Dump the macro definition for the def created by macro_body."""
+ arg_tuple = ', '.join(repr(x.name) for x in node.args)
+ name = getattr(node, 'name', None)
+ if len(node.args) == 1:
+ arg_tuple += ','
+ self.write('Macro(environment, macro, %r, (%s), (' %
+ (name, arg_tuple))
+ for arg in node.defaults:
+ self.visit(arg, frame)
+ self.write(', ')
+ self.write('), %r, %r, %r)' % (
+ bool(frame.accesses_kwargs),
+ bool(frame.accesses_varargs),
+ bool(frame.accesses_caller)
+ ))
+
+ def position(self, node):
+ """Return a human readable position for the node."""
+ rv = 'line %d' % node.lineno
+ if self.name is not None:
+ rv += ' in ' + repr(self.name)
+ return rv
+
+ # -- Statement Visitors
+
+ def visit_Template(self, node, frame=None):
+ assert frame is None, 'no root frame allowed'
+ eval_ctx = EvalContext(self.environment, self.name)
+
+ from jinja2.runtime import __all__ as exported
+ self.writeline('from __future__ import division')
+ self.writeline('from jinja2.runtime import ' + ', '.join(exported))
+ if not unoptimize_before_dead_code:
+ self.writeline('dummy = lambda *x: None')
+
+ # if we want a deferred initialization we cannot move the
+ # environment into a local name
+ envenv = not self.defer_init and ', environment=environment' or ''
+
+ # do we have an extends tag at all? If not, we can save some
+ # overhead by just not processing any inheritance code.
+ have_extends = node.find(nodes.Extends) is not None
+
+ # find all blocks
+ for block in node.find_all(nodes.Block):
+ if block.name in self.blocks:
+ self.fail('block %r defined twice' % block.name, block.lineno)
+ self.blocks[block.name] = block
+
+ # find all imports and import them
+ for import_ in node.find_all(nodes.ImportedName):
+ if import_.importname not in self.import_aliases:
+ imp = import_.importname
+ self.import_aliases[imp] = alias = self.temporary_identifier()
+ if '.' in imp:
+ module, obj = imp.rsplit('.', 1)
+ self.writeline('from %s import %s as %s' %
+ (module, obj, alias))
+ else:
+ self.writeline('import %s as %s' % (imp, alias))
+
+ # add the load name
+ self.writeline('name = %r' % self.name)
+
+ # generate the root render function.
+ self.writeline('def root(context%s):' % envenv, extra=1)
+
+ # process the root
+ frame = Frame(eval_ctx)
+ frame.inspect(node.body)
+ frame.toplevel = frame.rootlevel = True
+ frame.require_output_check = have_extends and not self.has_known_extends
+ self.indent()
+ if have_extends:
+ self.writeline('parent_template = None')
+ if 'self' in find_undeclared(node.body, ('self',)):
+ frame.identifiers.add_special('self')
+ self.writeline('l_self = TemplateReference(context)')
+ self.pull_locals(frame)
+ self.pull_dependencies(node.body)
+ self.blockvisit(node.body, frame)
+ self.outdent()
+
+ # make sure that the parent root is called.
+ if have_extends:
+ if not self.has_known_extends:
+ self.indent()
+ self.writeline('if parent_template is not None:')
+ self.indent()
+ self.writeline('for event in parent_template.'
+ 'root_render_func(context):')
+ self.indent()
+ self.writeline('yield event')
+ self.outdent(2 + (not self.has_known_extends))
+
+ # at this point we now have the blocks collected and can visit them too.
+ for name, block in self.blocks.iteritems():
+ block_frame = Frame(eval_ctx)
+ block_frame.inspect(block.body)
+ block_frame.block = name
+ self.writeline('def block_%s(context%s):' % (name, envenv),
+ block, 1)
+ self.indent()
+ undeclared = find_undeclared(block.body, ('self', 'super'))
+ if 'self' in undeclared:
+ block_frame.identifiers.add_special('self')
+ self.writeline('l_self = TemplateReference(context)')
+ if 'super' in undeclared:
+ block_frame.identifiers.add_special('super')
+ self.writeline('l_super = context.super(%r, '
+ 'block_%s)' % (name, name))
+ self.pull_locals(block_frame)
+ self.pull_dependencies(block.body)
+ self.blockvisit(block.body, block_frame)
+ self.outdent()
+
+ self.writeline('blocks = {%s}' % ', '.join('%r: block_%s' % (x, x)
+ for x in self.blocks),
+ extra=1)
+
+ # add a function that returns the debug info
+ self.writeline('debug_info = %r' % '&'.join('%s=%s' % x for x
+ in self.debug_info))
+
+ def visit_Block(self, node, frame):
+ """Call a block and register it for the template."""
+ level = 1
+ if frame.toplevel:
+ # if we know that we are a child template, there is no need to
+ # check if we are one
+ if self.has_known_extends:
+ return
+ if self.extends_so_far > 0:
+ self.writeline('if parent_template is None:')
+ self.indent()
+ level += 1
+ context = node.scoped and 'context.derived(locals())' or 'context'
+ self.writeline('for event in context.blocks[%r][0](%s):' % (
+ node.name, context), node)
+ self.indent()
+ self.simple_write('event', frame)
+ self.outdent(level)
+
+ def visit_Extends(self, node, frame):
+ """Calls the extender."""
+ if not frame.toplevel:
+ self.fail('cannot use extend from a non top-level scope',
+ node.lineno)
+
+ # if the number of extends statements in general is zero so
+ # far, we don't have to add a check if something extended
+ # the template before this one.
+ if self.extends_so_far > 0:
+
+ # if we have a known extends we just add a template runtime
+ # error into the generated code. We could catch that at compile
+ # time too, but i welcome it not to confuse users by throwing the
+ # same error at different times just "because we can".
+ if not self.has_known_extends:
+ self.writeline('if parent_template is not None:')
+ self.indent()
+ self.writeline('raise TemplateRuntimeError(%r)' %
+ 'extended multiple times')
+ self.outdent()
+
+ # if we have a known extends already we don't need that code here
+ # as we know that the template execution will end here.
+ if self.has_known_extends:
+ raise CompilerExit()
+
+ self.writeline('parent_template = environment.get_template(', node)
+ self.visit(node.template, frame)
+ self.write(', %r)' % self.name)
+ self.writeline('for name, parent_block in parent_template.'
+ 'blocks.%s():' % dict_item_iter)
+ self.indent()
+ self.writeline('context.blocks.setdefault(name, []).'
+ 'append(parent_block)')
+ self.outdent()
+
+ # if this extends statement was in the root level we can take
+ # advantage of that information and simplify the generated code
+ # in the top level from this point onwards
+ if frame.rootlevel:
+ self.has_known_extends = True
+
+ # and now we have one more
+ self.extends_so_far += 1
+
+ def visit_Include(self, node, frame):
+ """Handles includes."""
+ if node.with_context:
+ self.unoptimize_scope(frame)
+ if node.ignore_missing:
+ self.writeline('try:')
+ self.indent()
+
+ func_name = 'get_or_select_template'
+ if isinstance(node.template, nodes.Const):
+ if isinstance(node.template.value, basestring):
+ func_name = 'get_template'
+ elif isinstance(node.template.value, (tuple, list)):
+ func_name = 'select_template'
+ elif isinstance(node.template, (nodes.Tuple, nodes.List)):
+ func_name = 'select_template'
+
+ self.writeline('template = environment.%s(' % func_name, node)
+ self.visit(node.template, frame)
+ self.write(', %r)' % self.name)
+ if node.ignore_missing:
+ self.outdent()
+ self.writeline('except TemplateNotFound:')
+ self.indent()
+ self.writeline('pass')
+ self.outdent()
+ self.writeline('else:')
+ self.indent()
+
+ if node.with_context:
+ self.writeline('for event in template.root_render_func('
+ 'template.new_context(context.parent, True, '
+ 'locals())):')
+ else:
+ self.writeline('for event in template.module._body_stream:')
+
+ self.indent()
+ self.simple_write('event', frame)
+ self.outdent()
+
+ if node.ignore_missing:
+ self.outdent()
+
+ def visit_Import(self, node, frame):
+ """Visit regular imports."""
+ if node.with_context:
+ self.unoptimize_scope(frame)
+ self.writeline('l_%s = ' % node.target, node)
+ if frame.toplevel:
+ self.write('context.vars[%r] = ' % node.target)
+ self.write('environment.get_template(')
+ self.visit(node.template, frame)
+ self.write(', %r).' % self.name)
+ if node.with_context:
+ self.write('make_module(context.parent, True, locals())')
+ else:
+ self.write('module')
+ if frame.toplevel and not node.target.startswith('_'):
+ self.writeline('context.exported_vars.discard(%r)' % node.target)
+ frame.assigned_names.add(node.target)
+
+ def visit_FromImport(self, node, frame):
+ """Visit named imports."""
+ self.newline(node)
+ self.write('included_template = environment.get_template(')
+ self.visit(node.template, frame)
+ self.write(', %r).' % self.name)
+ if node.with_context:
+ self.write('make_module(context.parent, True)')
+ else:
+ self.write('module')
+
+ var_names = []
+ discarded_names = []
+ for name in node.names:
+ if isinstance(name, tuple):
+ name, alias = name
+ else:
+ alias = name
+ self.writeline('l_%s = getattr(included_template, '
+ '%r, missing)' % (alias, name))
+ self.writeline('if l_%s is missing:' % alias)
+ self.indent()
+ self.writeline('l_%s = environment.undefined(%r %% '
+ 'included_template.__name__, '
+ 'name=%r)' %
+ (alias, 'the template %%r (imported on %s) does '
+ 'not export the requested name %s' % (
+ self.position(node),
+ repr(name)
+ ), name))
+ self.outdent()
+ if frame.toplevel:
+ var_names.append(alias)
+ if not alias.startswith('_'):
+ discarded_names.append(alias)
+ frame.assigned_names.add(alias)
+
+ if var_names:
+ if len(var_names) == 1:
+ name = var_names[0]
+ self.writeline('context.vars[%r] = l_%s' % (name, name))
+ else:
+ self.writeline('context.vars.update({%s})' % ', '.join(
+ '%r: l_%s' % (name, name) for name in var_names
+ ))
+ if discarded_names:
+ if len(discarded_names) == 1:
+ self.writeline('context.exported_vars.discard(%r)' %
+ discarded_names[0])
+ else:
+ self.writeline('context.exported_vars.difference_'
+ 'update((%s))' % ', '.join(map(repr, discarded_names)))
+
+ def visit_For(self, node, frame):
+ # when calculating the nodes for the inner frame we have to exclude
+ # the iterator contents from it
+ children = node.iter_child_nodes(exclude=('iter',))
+ if node.recursive:
+ loop_frame = self.function_scoping(node, frame, children,
+ find_special=False)
+ else:
+ loop_frame = frame.inner()
+ loop_frame.inspect(children)
+
+ # try to figure out if we have an extended loop. An extended loop
+ # is necessary if the loop is in recursive mode if the special loop
+ # variable is accessed in the body.
+ extended_loop = node.recursive or 'loop' in \
+ find_undeclared(node.iter_child_nodes(
+ only=('body',)), ('loop',))
+
+ # if we don't have an recursive loop we have to find the shadowed
+ # variables at that point. Because loops can be nested but the loop
+ # variable is a special one we have to enforce aliasing for it.
+ if not node.recursive:
+ aliases = self.push_scope(loop_frame, ('loop',))
+
+ # otherwise we set up a buffer and add a function def
+ else:
+ self.writeline('def loop(reciter, loop_render_func):', node)
+ self.indent()
+ self.buffer(loop_frame)
+ aliases = {}
+
+ # make sure the loop variable is a special one and raise a template
+ # assertion error if a loop tries to write to loop
+ if extended_loop:
+ loop_frame.identifiers.add_special('loop')
+ for name in node.find_all(nodes.Name):
+ if name.ctx == 'store' and name.name == 'loop':
+ self.fail('Can\'t assign to special loop variable '
+ 'in for-loop target', name.lineno)
+
+ self.pull_locals(loop_frame)
+ if node.else_:
+ iteration_indicator = self.temporary_identifier()
+ self.writeline('%s = 1' % iteration_indicator)
+
+ # Create a fake parent loop if the else or test section of a
+ # loop is accessing the special loop variable and no parent loop
+ # exists.
+ if 'loop' not in aliases and 'loop' in find_undeclared(
+ node.iter_child_nodes(only=('else_', 'test')), ('loop',)):
+ self.writeline("l_loop = environment.undefined(%r, name='loop')" %
+ ("'loop' is undefined. the filter section of a loop as well "
+ "as the else block doesn't have access to the special 'loop'"
+ " variable of the current loop. Because there is no parent "
+ "loop it's undefined. Happened in loop on %s" %
+ self.position(node)))
+
+ self.writeline('for ', node)
+ self.visit(node.target, loop_frame)
+ self.write(extended_loop and ', l_loop in LoopContext(' or ' in ')
+
+ # if we have an extened loop and a node test, we filter in the
+ # "outer frame".
+ if extended_loop and node.test is not None:
+ self.write('(')
+ self.visit(node.target, loop_frame)
+ self.write(' for ')
+ self.visit(node.target, loop_frame)
+ self.write(' in ')
+ if node.recursive:
+ self.write('reciter')
+ else:
+ self.visit(node.iter, loop_frame)
+ self.write(' if (')
+ test_frame = loop_frame.copy()
+ self.visit(node.test, test_frame)
+ self.write('))')
+
+ elif node.recursive:
+ self.write('reciter')
+ else:
+ self.visit(node.iter, loop_frame)
+
+ if node.recursive:
+ self.write(', recurse=loop_render_func):')
+ else:
+ self.write(extended_loop and '):' or ':')
+
+ # tests in not extended loops become a continue
+ if not extended_loop and node.test is not None:
+ self.indent()
+ self.writeline('if not ')
+ self.visit(node.test, loop_frame)
+ self.write(':')
+ self.indent()
+ self.writeline('continue')
+ self.outdent(2)
+
+ self.indent()
+ self.blockvisit(node.body, loop_frame)
+ if node.else_:
+ self.writeline('%s = 0' % iteration_indicator)
+ self.outdent()
+
+ if node.else_:
+ self.writeline('if %s:' % iteration_indicator)
+ self.indent()
+ self.blockvisit(node.else_, loop_frame)
+ self.outdent()
+
+ # reset the aliases if there are any.
+ if not node.recursive:
+ self.pop_scope(aliases, loop_frame)
+
+ # if the node was recursive we have to return the buffer contents
+ # and start the iteration code
+ if node.recursive:
+ self.return_buffer_contents(loop_frame)
+ self.outdent()
+ self.start_write(frame, node)
+ self.write('loop(')
+ self.visit(node.iter, frame)
+ self.write(', loop)')
+ self.end_write(frame)
+
+ def visit_If(self, node, frame):
+ if_frame = frame.soft()
+ self.writeline('if ', node)
+ self.visit(node.test, if_frame)
+ self.write(':')
+ self.indent()
+ self.blockvisit(node.body, if_frame)
+ self.outdent()
+ if node.else_:
+ self.writeline('else:')
+ self.indent()
+ self.blockvisit(node.else_, if_frame)
+ self.outdent()
+
+ def visit_Macro(self, node, frame):
+ macro_frame = self.macro_body(node, frame)
+ self.newline()
+ if frame.toplevel:
+ if not node.name.startswith('_'):
+ self.write('context.exported_vars.add(%r)' % node.name)
+ self.writeline('context.vars[%r] = ' % node.name)
+ self.write('l_%s = ' % node.name)
+ self.macro_def(node, macro_frame)
+ frame.assigned_names.add(node.name)
+
+ def visit_CallBlock(self, node, frame):
+ children = node.iter_child_nodes(exclude=('call',))
+ call_frame = self.macro_body(node, frame, children)
+ self.writeline('caller = ')
+ self.macro_def(node, call_frame)
+ self.start_write(frame, node)
+ self.visit_Call(node.call, call_frame, forward_caller=True)
+ self.end_write(frame)
+
+ def visit_FilterBlock(self, node, frame):
+ filter_frame = frame.inner()
+ filter_frame.inspect(node.iter_child_nodes())
+ aliases = self.push_scope(filter_frame)
+ self.pull_locals(filter_frame)
+ self.buffer(filter_frame)
+ self.blockvisit(node.body, filter_frame)
+ self.start_write(frame, node)
+ self.visit_Filter(node.filter, filter_frame)
+ self.end_write(frame)
+ self.pop_scope(aliases, filter_frame)
+
+ def visit_ExprStmt(self, node, frame):
+ self.newline(node)
+ self.visit(node.node, frame)
+
+ def visit_Output(self, node, frame):
+ # if we have a known extends statement, we don't output anything
+ # if we are in a require_output_check section
+ if self.has_known_extends and frame.require_output_check:
+ return
+
+ if self.environment.finalize:
+ finalize = lambda x: unicode(self.environment.finalize(x))
+ else:
+ finalize = unicode
+
+ # if we are inside a frame that requires output checking, we do so
+ outdent_later = False
+ if frame.require_output_check:
+ self.writeline('if parent_template is None:')
+ self.indent()
+ outdent_later = True
+
+ # try to evaluate as many chunks as possible into a static
+ # string at compile time.
+ body = []
+ for child in node.nodes:
+ try:
+ const = child.as_const(frame.eval_ctx)
+ except nodes.Impossible:
+ body.append(child)
+ continue
+ # the frame can't be volatile here, becaus otherwise the
+ # as_const() function would raise an Impossible exception
+ # at that point.
+ try:
+ if frame.eval_ctx.autoescape:
+ if hasattr(const, '__html__'):
+ const = const.__html__()
+ else:
+ const = escape(const)
+ const = finalize(const)
+ except:
+ # if something goes wrong here we evaluate the node
+ # at runtime for easier debugging
+ body.append(child)
+ continue
+ if body and isinstance(body[-1], list):
+ body[-1].append(const)
+ else:
+ body.append([const])
+
+ # if we have less than 3 nodes or a buffer we yield or extend/append
+ if len(body) < 3 or frame.buffer is not None:
+ if frame.buffer is not None:
+ # for one item we append, for more we extend
+ if len(body) == 1:
+ self.writeline('%s.append(' % frame.buffer)
+ else:
+ self.writeline('%s.extend((' % frame.buffer)
+ self.indent()
+ for item in body:
+ if isinstance(item, list):
+ val = repr(concat(item))
+ if frame.buffer is None:
+ self.writeline('yield ' + val)
+ else:
+ self.writeline(val + ', ')
+ else:
+ if frame.buffer is None:
+ self.writeline('yield ', item)
+ else:
+ self.newline(item)
+ close = 1
+ if frame.eval_ctx.volatile:
+ self.write('(context.eval_ctx.autoescape and'
+ ' escape or to_string)(')
+ elif frame.eval_ctx.autoescape:
+ self.write('escape(')
+ else:
+ self.write('to_string(')
+ if self.environment.finalize is not None:
+ self.write('environment.finalize(')
+ close += 1
+ self.visit(item, frame)
+ self.write(')' * close)
+ if frame.buffer is not None:
+ self.write(', ')
+ if frame.buffer is not None:
+ # close the open parentheses
+ self.outdent()
+ self.writeline(len(body) == 1 and ')' or '))')
+
+ # otherwise we create a format string as this is faster in that case
+ else:
+ format = []
+ arguments = []
+ for item in body:
+ if isinstance(item, list):
+ format.append(concat(item).replace('%', '%%'))
+ else:
+ format.append('%s')
+ arguments.append(item)
+ self.writeline('yield ')
+ self.write(repr(concat(format)) + ' % (')
+ idx = -1
+ self.indent()
+ for argument in arguments:
+ self.newline(argument)
+ close = 0
+ if frame.eval_ctx.volatile:
+ self.write('(context.eval_ctx.autoescape and'
+ ' escape or to_string)(')
+ close += 1
+ elif frame.eval_ctx.autoescape:
+ self.write('escape(')
+ close += 1
+ if self.environment.finalize is not None:
+ self.write('environment.finalize(')
+ close += 1
+ self.visit(argument, frame)
+ self.write(')' * close + ', ')
+ self.outdent()
+ self.writeline(')')
+
+ if outdent_later:
+ self.outdent()
+
+ def visit_Assign(self, node, frame):
+ self.newline(node)
+ # toplevel assignments however go into the local namespace and
+ # the current template's context. We create a copy of the frame
+ # here and add a set so that the Name visitor can add the assigned
+ # names here.
+ if frame.toplevel:
+ assignment_frame = frame.copy()
+ assignment_frame.toplevel_assignments = set()
+ else:
+ assignment_frame = frame
+ self.visit(node.target, assignment_frame)
+ self.write(' = ')
+ self.visit(node.node, frame)
+
+ # make sure toplevel assignments are added to the context.
+ if frame.toplevel:
+ public_names = [x for x in assignment_frame.toplevel_assignments
+ if not x.startswith('_')]
+ if len(assignment_frame.toplevel_assignments) == 1:
+ name = next(iter(assignment_frame.toplevel_assignments))
+ self.writeline('context.vars[%r] = l_%s' % (name, name))
+ else:
+ self.writeline('context.vars.update({')
+ for idx, name in enumerate(assignment_frame.toplevel_assignments):
+ if idx:
+ self.write(', ')
+ self.write('%r: l_%s' % (name, name))
+ self.write('})')
+ if public_names:
+ if len(public_names) == 1:
+ self.writeline('context.exported_vars.add(%r)' %
+ public_names[0])
+ else:
+ self.writeline('context.exported_vars.update((%s))' %
+ ', '.join(map(repr, public_names)))
+
+ # -- Expression Visitors
+
+ def visit_Name(self, node, frame):
+ if node.ctx == 'store' and frame.toplevel:
+ frame.toplevel_assignments.add(node.name)
+ self.write('l_' + node.name)
+ frame.assigned_names.add(node.name)
+
+ def visit_Const(self, node, frame):
+ val = node.value
+ if isinstance(val, float):
+ self.write(str(val))
+ else:
+ self.write(repr(val))
+
+ def visit_TemplateData(self, node, frame):
+ try:
+ self.write(repr(node.as_const(frame.eval_ctx)))
+ except nodes.Impossible:
+ self.write('(context.eval_ctx.autoescape and Markup or identity)(%r)'
+ % node.data)
+
+ def visit_Tuple(self, node, frame):
+ self.write('(')
+ idx = -1
+ for idx, item in enumerate(node.items):
+ if idx:
+ self.write(', ')
+ self.visit(item, frame)
+ self.write(idx == 0 and ',)' or ')')
+
+ def visit_List(self, node, frame):
+ self.write('[')
+ for idx, item in enumerate(node.items):
+ if idx:
+ self.write(', ')
+ self.visit(item, frame)
+ self.write(']')
+
+ def visit_Dict(self, node, frame):
+ self.write('{')
+ for idx, item in enumerate(node.items):
+ if idx:
+ self.write(', ')
+ self.visit(item.key, frame)
+ self.write(': ')
+ self.visit(item.value, frame)
+ self.write('}')
+
+ def binop(operator):
+ def visitor(self, node, frame):
+ self.write('(')
+ self.visit(node.left, frame)
+ self.write(' %s ' % operator)
+ self.visit(node.right, frame)
+ self.write(')')
+ return visitor
+
+ def uaop(operator):
+ def visitor(self, node, frame):
+ self.write('(' + operator)
+ self.visit(node.node, frame)
+ self.write(')')
+ return visitor
+
+ visit_Add = binop('+')
+ visit_Sub = binop('-')
+ visit_Mul = binop('*')
+ visit_Div = binop('/')
+ visit_FloorDiv = binop('//')
+ visit_Pow = binop('**')
+ visit_Mod = binop('%')
+ visit_And = binop('and')
+ visit_Or = binop('or')
+ visit_Pos = uaop('+')
+ visit_Neg = uaop('-')
+ visit_Not = uaop('not ')
+ del binop, uaop
+
+ def visit_Concat(self, node, frame):
+ if frame.eval_ctx.volatile:
+ func_name = '(context.eval_ctx.volatile and' \
+ ' markup_join or unicode_join)'
+ elif frame.eval_ctx.autoescape:
+ func_name = 'markup_join'
+ else:
+ func_name = 'unicode_join'
+ self.write('%s((' % func_name)
+ for arg in node.nodes:
+ self.visit(arg, frame)
+ self.write(', ')
+ self.write('))')
+
+ def visit_Compare(self, node, frame):
+ self.visit(node.expr, frame)
+ for op in node.ops:
+ self.visit(op, frame)
+
+ def visit_Operand(self, node, frame):
+ self.write(' %s ' % operators[node.op])
+ self.visit(node.expr, frame)
+
+ def visit_Getattr(self, node, frame):
+ self.write('environment.getattr(')
+ self.visit(node.node, frame)
+ self.write(', %r)' % node.attr)
+
+ def visit_Getitem(self, node, frame):
+ # slices bypass the environment getitem method.
+ if isinstance(node.arg, nodes.Slice):
+ self.visit(node.node, frame)
+ self.write('[')
+ self.visit(node.arg, frame)
+ self.write(']')
+ else:
+ self.write('environment.getitem(')
+ self.visit(node.node, frame)
+ self.write(', ')
+ self.visit(node.arg, frame)
+ self.write(')')
+
+ def visit_Slice(self, node, frame):
+ if node.start is not None:
+ self.visit(node.start, frame)
+ self.write(':')
+ if node.stop is not None:
+ self.visit(node.stop, frame)
+ if node.step is not None:
+ self.write(':')
+ self.visit(node.step, frame)
+
+ def visit_Filter(self, node, frame):
+ self.write(self.filters[node.name] + '(')
+ func = self.environment.filters.get(node.name)
+ if func is None:
+ self.fail('no filter named %r' % node.name, node.lineno)
+ if getattr(func, 'contextfilter', False):
+ self.write('context, ')
+ elif getattr(func, 'evalcontextfilter', False):
+ self.write('context.eval_ctx, ')
+ elif getattr(func, 'environmentfilter', False):
+ self.write('environment, ')
+
+ # if the filter node is None we are inside a filter block
+ # and want to write to the current buffer
+ if node.node is not None:
+ self.visit(node.node, frame)
+ elif frame.eval_ctx.volatile:
+ self.write('(context.eval_ctx.autoescape and'
+ ' Markup(concat(%s)) or concat(%s))' %
+ (frame.buffer, frame.buffer))
+ elif frame.eval_ctx.autoescape:
+ self.write('Markup(concat(%s))' % frame.buffer)
+ else:
+ self.write('concat(%s)' % frame.buffer)
+ self.signature(node, frame)
+ self.write(')')
+
+ def visit_Test(self, node, frame):
+ self.write(self.tests[node.name] + '(')
+ if node.name not in self.environment.tests:
+ self.fail('no test named %r' % node.name, node.lineno)
+ self.visit(node.node, frame)
+ self.signature(node, frame)
+ self.write(')')
+
+ def visit_CondExpr(self, node, frame):
+ def write_expr2():
+ if node.expr2 is not None:
+ return self.visit(node.expr2, frame)
+ self.write('environment.undefined(%r)' % ('the inline if-'
+ 'expression on %s evaluated to false and '
+ 'no else section was defined.' % self.position(node)))
+
+ if not have_condexpr:
+ self.write('((')
+ self.visit(node.test, frame)
+ self.write(') and (')
+ self.visit(node.expr1, frame)
+ self.write(',) or (')
+ write_expr2()
+ self.write(',))[0]')
+ else:
+ self.write('(')
+ self.visit(node.expr1, frame)
+ self.write(' if ')
+ self.visit(node.test, frame)
+ self.write(' else ')
+ write_expr2()
+ self.write(')')
+
+ def visit_Call(self, node, frame, forward_caller=False):
+ if self.environment.sandboxed:
+ self.write('environment.call(context, ')
+ else:
+ self.write('context.call(')
+ self.visit(node.node, frame)
+ extra_kwargs = forward_caller and {'caller': 'caller'} or None
+ self.signature(node, frame, extra_kwargs)
+ self.write(')')
+
+ def visit_Keyword(self, node, frame):
+ self.write(node.key + '=')
+ self.visit(node.value, frame)
+
+ # -- Unused nodes for extensions
+
+ def visit_MarkSafe(self, node, frame):
+ self.write('Markup(')
+ self.visit(node.expr, frame)
+ self.write(')')
+
+ def visit_MarkSafeIfAutoescape(self, node, frame):
+ self.write('(context.eval_ctx.autoescape and Markup or identity)(')
+ self.visit(node.expr, frame)
+ self.write(')')
+
+ def visit_EnvironmentAttribute(self, node, frame):
+ self.write('environment.' + node.name)
+
+ def visit_ExtensionAttribute(self, node, frame):
+ self.write('environment.extensions[%r].%s' % (node.identifier, node.name))
+
+ def visit_ImportedName(self, node, frame):
+ self.write(self.import_aliases[node.importname])
+
+ def visit_InternalName(self, node, frame):
+ self.write(node.name)
+
+ def visit_ContextReference(self, node, frame):
+ self.write('context')
+
+ def visit_Continue(self, node, frame):
+ self.writeline('continue', node)
+
+ def visit_Break(self, node, frame):
+ self.writeline('break', node)
+
+ def visit_Scope(self, node, frame):
+ scope_frame = frame.inner()
+ scope_frame.inspect(node.iter_child_nodes())
+ aliases = self.push_scope(scope_frame)
+ self.pull_locals(scope_frame)
+ self.blockvisit(node.body, scope_frame)
+ self.pop_scope(aliases, scope_frame)
+
+ def visit_EvalContextModifier(self, node, frame):
+ for keyword in node.options:
+ self.writeline('context.eval_ctx.%s = ' % keyword.key)
+ self.visit(keyword.value, frame)
+ try:
+ val = keyword.value.as_const(frame.eval_ctx)
+ except nodes.Impossible:
+ frame.eval_ctx.volatile = True
+ else:
+ setattr(frame.eval_ctx, keyword.key, val)
+
+ def visit_ScopedEvalContextModifier(self, node, frame):
+ old_ctx_name = self.temporary_identifier()
+ safed_ctx = frame.eval_ctx.save()
+ self.writeline('%s = context.eval_ctx.save()' % old_ctx_name)
+ self.visit_EvalContextModifier(node, frame)
+ for child in node.body:
+ self.visit(child, frame)
+ frame.eval_ctx.revert(safed_ctx)
+ self.writeline('context.eval_ctx.revert(%s)' % old_ctx_name)
diff --git a/slider-agent/src/main/python/jinja2/jinja2/constants.py b/slider-agent/src/main/python/jinja2/jinja2/constants.py
new file mode 100644
index 0000000..cab203c
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/constants.py
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja.constants
+ ~~~~~~~~~~~~~~~
+
+ Various constants.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+
+
+#: list of lorem ipsum words used by the lipsum() helper function
+LOREM_IPSUM_WORDS = u'''\
+a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at
+auctor augue bibendum blandit class commodo condimentum congue consectetuer
+consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus
+diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend
+elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames
+faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac
+hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum
+justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem
+luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie
+mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non
+nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque
+penatibus per pharetra phasellus placerat platea porta porttitor posuere
+potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus
+ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit
+sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor
+tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices
+ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus
+viverra volutpat vulputate'''
diff --git a/slider-agent/src/main/python/jinja2/jinja2/debug.py b/slider-agent/src/main/python/jinja2/jinja2/debug.py
new file mode 100644
index 0000000..eb15456
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/debug.py
@@ -0,0 +1,308 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.debug
+ ~~~~~~~~~~~~
+
+ Implements the debug interface for Jinja. This module does some pretty
+ ugly stuff with the Python traceback system in order to achieve tracebacks
+ with correct line numbers, locals and contents.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import sys
+import traceback
+from jinja2.utils import CodeType, missing, internal_code
+from jinja2.exceptions import TemplateSyntaxError
+
+
+# how does the raise helper look like?
+try:
+ exec "raise TypeError, 'foo'"
+except SyntaxError:
+ raise_helper = 'raise __jinja_exception__[1]'
+except TypeError:
+ raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]'
+
+
+class TracebackFrameProxy(object):
+ """Proxies a traceback frame."""
+
+ def __init__(self, tb):
+ self.tb = tb
+
+ def _set_tb_next(self, next):
+ if tb_set_next is not None:
+ tb_set_next(self.tb, next and next.tb or None)
+ self._tb_next = next
+
+ def _get_tb_next(self):
+ return self._tb_next
+
+ tb_next = property(_get_tb_next, _set_tb_next)
+ del _get_tb_next, _set_tb_next
+
+ @property
+ def is_jinja_frame(self):
+ return '__jinja_template__' in self.tb.tb_frame.f_globals
+
+ def __getattr__(self, name):
+ return getattr(self.tb, name)
+
+
+class ProcessedTraceback(object):
+ """Holds a Jinja preprocessed traceback for priting or reraising."""
+
+ def __init__(self, exc_type, exc_value, frames):
+ assert frames, 'no frames for this traceback?'
+ self.exc_type = exc_type
+ self.exc_value = exc_value
+ self.frames = frames
+
+ def chain_frames(self):
+ """Chains the frames. Requires ctypes or the debugsupport extension."""
+ prev_tb = None
+ for tb in self.frames:
+ if prev_tb is not None:
+ prev_tb.tb_next = tb
+ prev_tb = tb
+ prev_tb.tb_next = None
+
+ def render_as_text(self, limit=None):
+ """Return a string with the traceback."""
+ lines = traceback.format_exception(self.exc_type, self.exc_value,
+ self.frames[0], limit=limit)
+ return ''.join(lines).rstrip()
+
+ def render_as_html(self, full=False):
+ """Return a unicode string with the traceback as rendered HTML."""
+ from jinja2.debugrenderer import render_traceback
+ return u'%s\n\n<!--\n%s\n-->' % (
+ render_traceback(self, full=full),
+ self.render_as_text().decode('utf-8', 'replace')
+ )
+
+ @property
+ def is_template_syntax_error(self):
+ """`True` if this is a template syntax error."""
+ return isinstance(self.exc_value, TemplateSyntaxError)
+
+ @property
+ def exc_info(self):
+ """Exception info tuple with a proxy around the frame objects."""
+ return self.exc_type, self.exc_value, self.frames[0]
+
+ @property
+ def standard_exc_info(self):
+ """Standard python exc_info for re-raising"""
+ return self.exc_type, self.exc_value, self.frames[0].tb
+
+
+def make_traceback(exc_info, source_hint=None):
+ """Creates a processed traceback object from the exc_info."""
+ exc_type, exc_value, tb = exc_info
+ if isinstance(exc_value, TemplateSyntaxError):
+ exc_info = translate_syntax_error(exc_value, source_hint)
+ initial_skip = 0
+ else:
+ initial_skip = 1
+ return translate_exception(exc_info, initial_skip)
+
+
+def translate_syntax_error(error, source=None):
+ """Rewrites a syntax error to please traceback systems."""
+ error.source = source
+ error.translated = True
+ exc_info = (error.__class__, error, None)
+ filename = error.filename
+ if filename is None:
+ filename = '<unknown>'
+ return fake_exc_info(exc_info, filename, error.lineno)
+
+
+def translate_exception(exc_info, initial_skip=0):
+ """If passed an exc_info it will automatically rewrite the exceptions
+ all the way down to the correct line numbers and frames.
+ """
+ tb = exc_info[2]
+ frames = []
+
+ # skip some internal frames if wanted
+ for x in xrange(initial_skip):
+ if tb is not None:
+ tb = tb.tb_next
+ initial_tb = tb
+
+ while tb is not None:
+ # skip frames decorated with @internalcode. These are internal
+ # calls we can't avoid and that are useless in template debugging
+ # output.
+ if tb.tb_frame.f_code in internal_code:
+ tb = tb.tb_next
+ continue
+
+ # save a reference to the next frame if we override the current
+ # one with a faked one.
+ next = tb.tb_next
+
+ # fake template exceptions
+ template = tb.tb_frame.f_globals.get('__jinja_template__')
+ if template is not None:
+ lineno = template.get_corresponding_lineno(tb.tb_lineno)
+ tb = fake_exc_info(exc_info[:2] + (tb,), template.filename,
+ lineno)[2]
+
+ frames.append(TracebackFrameProxy(tb))
+ tb = next
+
+ # if we don't have any exceptions in the frames left, we have to
+ # reraise it unchanged.
+ # XXX: can we backup here? when could this happen?
+ if not frames:
+ raise exc_info[0], exc_info[1], exc_info[2]
+
+ traceback = ProcessedTraceback(exc_info[0], exc_info[1], frames)
+ if tb_set_next is not None:
+ traceback.chain_frames()
+ return traceback
+
+
+def fake_exc_info(exc_info, filename, lineno):
+ """Helper for `translate_exception`."""
+ exc_type, exc_value, tb = exc_info
+
+ # figure the real context out
+ if tb is not None:
+ real_locals = tb.tb_frame.f_locals.copy()
+ ctx = real_locals.get('context')
+ if ctx:
+ locals = ctx.get_all()
+ else:
+ locals = {}
+ for name, value in real_locals.iteritems():
+ if name.startswith('l_') and value is not missing:
+ locals[name[2:]] = value
+
+ # if there is a local called __jinja_exception__, we get
+ # rid of it to not break the debug functionality.
+ locals.pop('__jinja_exception__', None)
+ else:
+ locals = {}
+
+ # assamble fake globals we need
+ globals = {
+ '__name__': filename,
+ '__file__': filename,
+ '__jinja_exception__': exc_info[:2],
+
+ # we don't want to keep the reference to the template around
+ # to not cause circular dependencies, but we mark it as Jinja
+ # frame for the ProcessedTraceback
+ '__jinja_template__': None
+ }
+
+ # and fake the exception
+ code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec')
+
+ # if it's possible, change the name of the code. This won't work
+ # on some python environments such as google appengine
+ try:
+ if tb is None:
+ location = 'template'
+ else:
+ function = tb.tb_frame.f_code.co_name
+ if function == 'root':
+ location = 'top-level template code'
+ elif function.startswith('block_'):
+ location = 'block "%s"' % function[6:]
+ else:
+ location = 'template'
+ code = CodeType(0, code.co_nlocals, code.co_stacksize,
+ code.co_flags, code.co_code, code.co_consts,
+ code.co_names, code.co_varnames, filename,
+ location, code.co_firstlineno,
+ code.co_lnotab, (), ())
+ except:
+ pass
+
+ # execute the code and catch the new traceback
+ try:
+ exec code in globals, locals
+ except:
+ exc_info = sys.exc_info()
+ new_tb = exc_info[2].tb_next
+
+ # return without this frame
+ return exc_info[:2] + (new_tb,)
+
+
+def _init_ugly_crap():
+ """This function implements a few ugly things so that we can patch the
+ traceback objects. The function returned allows resetting `tb_next` on
+ any python traceback object.
+ """
+ import ctypes
+ from types import TracebackType
+
+ # figure out side of _Py_ssize_t
+ if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'):
+ _Py_ssize_t = ctypes.c_int64
+ else:
+ _Py_ssize_t = ctypes.c_int
+
+ # regular python
+ class _PyObject(ctypes.Structure):
+ pass
+ _PyObject._fields_ = [
+ ('ob_refcnt', _Py_ssize_t),
+ ('ob_type', ctypes.POINTER(_PyObject))
+ ]
+
+ # python with trace
+ if hasattr(sys, 'getobjects'):
+ class _PyObject(ctypes.Structure):
+ pass
+ _PyObject._fields_ = [
+ ('_ob_next', ctypes.POINTER(_PyObject)),
+ ('_ob_prev', ctypes.POINTER(_PyObject)),
+ ('ob_refcnt', _Py_ssize_t),
+ ('ob_type', ctypes.POINTER(_PyObject))
+ ]
+
+ class _Traceback(_PyObject):
+ pass
+ _Traceback._fields_ = [
+ ('tb_next', ctypes.POINTER(_Traceback)),
+ ('tb_frame', ctypes.POINTER(_PyObject)),
+ ('tb_lasti', ctypes.c_int),
+ ('tb_lineno', ctypes.c_int)
+ ]
+
+ def tb_set_next(tb, next):
+ """Set the tb_next attribute of a traceback object."""
+ if not (isinstance(tb, TracebackType) and
+ (next is None or isinstance(next, TracebackType))):
+ raise TypeError('tb_set_next arguments must be traceback objects')
+ obj = _Traceback.from_address(id(tb))
+ if tb.tb_next is not None:
+ old = _Traceback.from_address(id(tb.tb_next))
+ old.ob_refcnt -= 1
+ if next is None:
+ obj.tb_next = ctypes.POINTER(_Traceback)()
+ else:
+ next = _Traceback.from_address(id(next))
+ next.ob_refcnt += 1
+ obj.tb_next = ctypes.pointer(next)
+
+ return tb_set_next
+
+
+# try to get a tb_set_next implementation
+try:
+ from jinja2._debugsupport import tb_set_next
+except ImportError:
+ try:
+ tb_set_next = _init_ugly_crap()
+ except:
+ tb_set_next = None
+del _init_ugly_crap
diff --git a/slider-agent/src/main/python/jinja2/jinja2/defaults.py b/slider-agent/src/main/python/jinja2/jinja2/defaults.py
new file mode 100644
index 0000000..d2d4544
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/defaults.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.defaults
+ ~~~~~~~~~~~~~~~
+
+ Jinja default filters and tags.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner
+
+
+# defaults for the parser / lexer
+BLOCK_START_STRING = '{%'
+BLOCK_END_STRING = '%}'
+VARIABLE_START_STRING = '{{'
+VARIABLE_END_STRING = '}}'
+COMMENT_START_STRING = '{#'
+COMMENT_END_STRING = '#}'
+LINE_STATEMENT_PREFIX = None
+LINE_COMMENT_PREFIX = None
+TRIM_BLOCKS = False
+NEWLINE_SEQUENCE = '\n'
+
+
+# default filters, tests and namespace
+from jinja2.filters import FILTERS as DEFAULT_FILTERS
+from jinja2.tests import TESTS as DEFAULT_TESTS
+DEFAULT_NAMESPACE = {
+ 'range': xrange,
+ 'dict': lambda **kw: kw,
+ 'lipsum': generate_lorem_ipsum,
+ 'cycler': Cycler,
+ 'joiner': Joiner
+}
+
+
+# export all constants
+__all__ = tuple(x for x in locals().keys() if x.isupper())
diff --git a/slider-agent/src/main/python/jinja2/jinja2/environment.py b/slider-agent/src/main/python/jinja2/jinja2/environment.py
new file mode 100644
index 0000000..ac74a5c
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/environment.py
@@ -0,0 +1,1118 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.environment
+ ~~~~~~~~~~~~~~~~~~
+
+ Provides a class that holds runtime and parsing time options.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import sys
+from jinja2 import nodes
+from jinja2.defaults import *
+from jinja2.lexer import get_lexer, TokenStream
+from jinja2.parser import Parser
+from jinja2.optimizer import optimize
+from jinja2.compiler import generate
+from jinja2.runtime import Undefined, new_context
+from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \
+ TemplatesNotFound
+from jinja2.utils import import_string, LRUCache, Markup, missing, \
+ concat, consume, internalcode, _encode_filename
+
+
+# for direct template usage we have up to ten living environments
+_spontaneous_environments = LRUCache(10)
+
+# the function to create jinja traceback objects. This is dynamically
+# imported on the first exception in the exception handler.
+_make_traceback = None
+
+
+def get_spontaneous_environment(*args):
+ """Return a new spontaneous environment. A spontaneous environment is an
+ unnamed and unaccessible (in theory) environment that is used for
+ templates generated from a string and not from the file system.
+ """
+ try:
+ env = _spontaneous_environments.get(args)
+ except TypeError:
+ return Environment(*args)
+ if env is not None:
+ return env
+ _spontaneous_environments[args] = env = Environment(*args)
+ env.shared = True
+ return env
+
+
+def create_cache(size):
+ """Return the cache class for the given size."""
+ if size == 0:
+ return None
+ if size < 0:
+ return {}
+ return LRUCache(size)
+
+
+def copy_cache(cache):
+ """Create an empty copy of the given cache."""
+ if cache is None:
+ return None
+ elif type(cache) is dict:
+ return {}
+ return LRUCache(cache.capacity)
+
+
+def load_extensions(environment, extensions):
+ """Load the extensions from the list and bind it to the environment.
+ Returns a dict of instanciated environments.
+ """
+ result = {}
+ for extension in extensions:
+ if isinstance(extension, basestring):
+ extension = import_string(extension)
+ result[extension.identifier] = extension(environment)
+ return result
+
+
+def _environment_sanity_check(environment):
+ """Perform a sanity check on the environment."""
+ assert issubclass(environment.undefined, Undefined), 'undefined must ' \
+ 'be a subclass of undefined because filters depend on it.'
+ assert environment.block_start_string != \
+ environment.variable_start_string != \
+ environment.comment_start_string, 'block, variable and comment ' \
+ 'start strings must be different'
+ assert environment.newline_sequence in ('\r', '\r\n', '\n'), \
+ 'newline_sequence set to unknown line ending string.'
+ return environment
+
+
+class Environment(object):
+ r"""The core component of Jinja is the `Environment`. It contains
+ important shared variables like configuration, filters, tests,
+ globals and others. Instances of this class may be modified if
+ they are not shared and if no template was loaded so far.
+ Modifications on environments after the first template was loaded
+ will lead to surprising effects and undefined behavior.
+
+ Here the possible initialization parameters:
+
+ `block_start_string`
+ The string marking the begin of a block. Defaults to ``'{%'``.
+
+ `block_end_string`
+ The string marking the end of a block. Defaults to ``'%}'``.
+
+ `variable_start_string`
+ The string marking the begin of a print statement.
+ Defaults to ``'{{'``.
+
+ `variable_end_string`
+ The string marking the end of a print statement. Defaults to
+ ``'}}'``.
+
+ `comment_start_string`
+ The string marking the begin of a comment. Defaults to ``'{#'``.
+
+ `comment_end_string`
+ The string marking the end of a comment. Defaults to ``'#}'``.
+
+ `line_statement_prefix`
+ If given and a string, this will be used as prefix for line based
+ statements. See also :ref:`line-statements`.
+
+ `line_comment_prefix`
+ If given and a string, this will be used as prefix for line based
+ based comments. See also :ref:`line-statements`.
+
+ .. versionadded:: 2.2
+
+ `trim_blocks`
+ If this is set to ``True`` the first newline after a block is
+ removed (block, not variable tag!). Defaults to `False`.
+
+ `newline_sequence`
+ The sequence that starts a newline. Must be one of ``'\r'``,
+ ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a
+ useful default for Linux and OS X systems as well as web
+ applications.
+
+ `extensions`
+ List of Jinja extensions to use. This can either be import paths
+ as strings or extension classes. For more information have a
+ look at :ref:`the extensions documentation <jinja-extensions>`.
+
+ `optimized`
+ should the optimizer be enabled? Default is `True`.
+
+ `undefined`
+ :class:`Undefined` or a subclass of it that is used to represent
+ undefined values in the template.
+
+ `finalize`
+ A callable that can be used to process the result of a variable
+ expression before it is output. For example one can convert
+ `None` implicitly into an empty string here.
+
+ `autoescape`
+ If set to true the XML/HTML autoescaping feature is enabled by
+ default. For more details about auto escaping see
+ :class:`~jinja2.utils.Markup`. As of Jinja 2.4 this can also
+ be a callable that is passed the template name and has to
+ return `True` or `False` depending on autoescape should be
+ enabled by default.
+
+ .. versionchanged:: 2.4
+ `autoescape` can now be a function
+
+ `loader`
+ The template loader for this environment.
+
+ `cache_size`
+ The size of the cache. Per default this is ``50`` which means
+ that if more than 50 templates are loaded the loader will clean
+ out the least recently used template. If the cache size is set to
+ ``0`` templates are recompiled all the time, if the cache size is
+ ``-1`` the cache will not be cleaned.
+
+ `auto_reload`
+ Some loaders load templates from locations where the template
+ sources may change (ie: file system or database). If
+ `auto_reload` is set to `True` (default) every time a template is
+ requested the loader checks if the source changed and if yes, it
+ will reload the template. For higher performance it's possible to
+ disable that.
+
+ `bytecode_cache`
+ If set to a bytecode cache object, this object will provide a
+ cache for the internal Jinja bytecode so that templates don't
+ have to be parsed if they were not changed.
+
+ See :ref:`bytecode-cache` for more information.
+ """
+
+ #: if this environment is sandboxed. Modifying this variable won't make
+ #: the environment sandboxed though. For a real sandboxed environment
+ #: have a look at jinja2.sandbox
+ sandboxed = False
+
+ #: True if the environment is just an overlay
+ overlayed = False
+
+ #: the environment this environment is linked to if it is an overlay
+ linked_to = None
+
+ #: shared environments have this set to `True`. A shared environment
+ #: must not be modified
+ shared = False
+
+ #: these are currently EXPERIMENTAL undocumented features.
+ exception_handler = None
+ exception_formatter = None
+
+ def __init__(self,
+ block_start_string=BLOCK_START_STRING,
+ block_end_string=BLOCK_END_STRING,
+ variable_start_string=VARIABLE_START_STRING,
+ variable_end_string=VARIABLE_END_STRING,
+ comment_start_string=COMMENT_START_STRING,
+ comment_end_string=COMMENT_END_STRING,
+ line_statement_prefix=LINE_STATEMENT_PREFIX,
+ line_comment_prefix=LINE_COMMENT_PREFIX,
+ trim_blocks=TRIM_BLOCKS,
+ newline_sequence=NEWLINE_SEQUENCE,
+ extensions=(),
+ optimized=True,
+ undefined=Undefined,
+ finalize=None,
+ autoescape=False,
+ loader=None,
+ cache_size=50,
+ auto_reload=True,
+ bytecode_cache=None):
+ # !!Important notice!!
+ # The constructor accepts quite a few arguments that should be
+ # passed by keyword rather than position. However it's important to
+ # not change the order of arguments because it's used at least
+ # internally in those cases:
+ # - spontaneus environments (i18n extension and Template)
+ # - unittests
+ # If parameter changes are required only add parameters at the end
+ # and don't change the arguments (or the defaults!) of the arguments
+ # existing already.
+
+ # lexer / parser information
+ self.block_start_string = block_start_string
+ self.block_end_string = block_end_string
+ self.variable_start_string = variable_start_string
+ self.variable_end_string = variable_end_string
+ self.comment_start_string = comment_start_string
+ self.comment_end_string = comment_end_string
+ self.line_statement_prefix = line_statement_prefix
+ self.line_comment_prefix = line_comment_prefix
+ self.trim_blocks = trim_blocks
+ self.newline_sequence = newline_sequence
+
+ # runtime information
+ self.undefined = undefined
+ self.optimized = optimized
+ self.finalize = finalize
+ self.autoescape = autoescape
+
+ # defaults
+ self.filters = DEFAULT_FILTERS.copy()
+ self.tests = DEFAULT_TESTS.copy()
+ self.globals = DEFAULT_NAMESPACE.copy()
+
+ # set the loader provided
+ self.loader = loader
+ self.bytecode_cache = None
+ self.cache = create_cache(cache_size)
+ self.bytecode_cache = bytecode_cache
+ self.auto_reload = auto_reload
+
+ # load extensions
+ self.extensions = load_extensions(self, extensions)
+
+ _environment_sanity_check(self)
+
+ def add_extension(self, extension):
+ """Adds an extension after the environment was created.
+
+ .. versionadded:: 2.5
+ """
+ self.extensions.update(load_extensions(self, [extension]))
+
+ def extend(self, **attributes):
+ """Add the items to the instance of the environment if they do not exist
+ yet. This is used by :ref:`extensions <writing-extensions>` to register
+ callbacks and configuration values without breaking inheritance.
+ """
+ for key, value in attributes.iteritems():
+ if not hasattr(self, key):
+ setattr(self, key, value)
+
+ def overlay(self, block_start_string=missing, block_end_string=missing,
+ variable_start_string=missing, variable_end_string=missing,
+ comment_start_string=missing, comment_end_string=missing,
+ line_statement_prefix=missing, line_comment_prefix=missing,
+ trim_blocks=missing, extensions=missing, optimized=missing,
+ undefined=missing, finalize=missing, autoescape=missing,
+ loader=missing, cache_size=missing, auto_reload=missing,
+ bytecode_cache=missing):
+ """Create a new overlay environment that shares all the data with the
+ current environment except of cache and the overridden attributes.
+ Extensions cannot be removed for an overlayed environment. An overlayed
+ environment automatically gets all the extensions of the environment it
+ is linked to plus optional extra extensions.
+
+ Creating overlays should happen after the initial environment was set
+ up completely. Not all attributes are truly linked, some are just
+ copied over so modifications on the original environment may not shine
+ through.
+ """
+ args = dict(locals())
+ del args['self'], args['cache_size'], args['extensions']
+
+ rv = object.__new__(self.__class__)
+ rv.__dict__.update(self.__dict__)
+ rv.overlayed = True
+ rv.linked_to = self
+
+ for key, value in args.iteritems():
+ if value is not missing:
+ setattr(rv, key, value)
+
+ if cache_size is not missing:
+ rv.cache = create_cache(cache_size)
+ else:
+ rv.cache = copy_cache(self.cache)
+
+ rv.extensions = {}
+ for key, value in self.extensions.iteritems():
+ rv.extensions[key] = value.bind(rv)
+ if extensions is not missing:
+ rv.extensions.update(load_extensions(rv, extensions))
+
+ return _environment_sanity_check(rv)
+
+ lexer = property(get_lexer, doc="The lexer for this environment.")
+
+ def iter_extensions(self):
+ """Iterates over the extensions by priority."""
+ return iter(sorted(self.extensions.values(),
+ key=lambda x: x.priority))
+
+ def getitem(self, obj, argument):
+ """Get an item or attribute of an object but prefer the item."""
+ try:
+ return obj[argument]
+ except (TypeError, LookupError):
+ if isinstance(argument, basestring):
+ try:
+ attr = str(argument)
+ except:
+ pass
+ else:
+ try:
+ return getattr(obj, attr)
+ except AttributeError:
+ pass
+ return self.undefined(obj=obj, name=argument)
+
+ def getattr(self, obj, attribute):
+ """Get an item or attribute of an object but prefer the attribute.
+ Unlike :meth:`getitem` the attribute *must* be a bytestring.
+ """
+ try:
+ return getattr(obj, attribute)
+ except AttributeError:
+ pass
+ try:
+ return obj[attribute]
+ except (TypeError, LookupError, AttributeError):
+ return self.undefined(obj=obj, name=attribute)
+
+ @internalcode
+ def parse(self, source, name=None, filename=None):
+ """Parse the sourcecode and return the abstract syntax tree. This
+ tree of nodes is used by the compiler to convert the template into
+ executable source- or bytecode. This is useful for debugging or to
+ extract information from templates.
+
+ If you are :ref:`developing Jinja2 extensions <writing-extensions>`
+ this gives you a good overview of the node tree generated.
+ """
+ try:
+ return self._parse(source, name, filename)
+ except TemplateSyntaxError:
+ exc_info = sys.exc_info()
+ self.handle_exception(exc_info, source_hint=source)
+
+ def _parse(self, source, name, filename):
+ """Internal parsing function used by `parse` and `compile`."""
+ return Parser(self, source, name, _encode_filename(filename)).parse()
+
+ def lex(self, source, name=None, filename=None):
+ """Lex the given sourcecode and return a generator that yields
+ tokens as tuples in the form ``(lineno, token_type, value)``.
+ This can be useful for :ref:`extension development <writing-extensions>`
+ and debugging templates.
+
+ This does not perform preprocessing. If you want the preprocessing
+ of the extensions to be applied you have to filter source through
+ the :meth:`preprocess` method.
+ """
+ source = unicode(source)
+ try:
+ return self.lexer.tokeniter(source, name, filename)
+ except TemplateSyntaxError:
+ exc_info = sys.exc_info()
+ self.handle_exception(exc_info, source_hint=source)
+
+ def preprocess(self, source, name=None, filename=None):
+ """Preprocesses the source with all extensions. This is automatically
+ called for all parsing and compiling methods but *not* for :meth:`lex`
+ because there you usually only want the actual source tokenized.
+ """
+ return reduce(lambda s, e: e.preprocess(s, name, filename),
+ self.iter_extensions(), unicode(source))
+
+ def _tokenize(self, source, name, filename=None, state=None):
+ """Called by the parser to do the preprocessing and filtering
+ for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`.
+ """
+ source = self.preprocess(source, name, filename)
+ stream = self.lexer.tokenize(source, name, filename, state)
+ for ext in self.iter_extensions():
+ stream = ext.filter_stream(stream)
+ if not isinstance(stream, TokenStream):
+ stream = TokenStream(stream, name, filename)
+ return stream
+
+ def _generate(self, source, name, filename, defer_init=False):
+ """Internal hook that can be overriden to hook a different generate
+ method in.
+
+ .. versionadded:: 2.5
+ """
+ return generate(source, self, name, filename, defer_init=defer_init)
+
+ def _compile(self, source, filename):
+ """Internal hook that can be overriden to hook a different compile
+ method in.
+
+ .. versionadded:: 2.5
+ """
+ return compile(source, filename, 'exec')
+
+ @internalcode
+ def compile(self, source, name=None, filename=None, raw=False,
+ defer_init=False):
+ """Compile a node or template source code. The `name` parameter is
+ the load name of the template after it was joined using
+ :meth:`join_path` if necessary, not the filename on the file system.
+ the `filename` parameter is the estimated filename of the template on
+ the file system. If the template came from a database or memory this
+ can be omitted.
+
+ The return value of this method is a python code object. If the `raw`
+ parameter is `True` the return value will be a string with python
+ code equivalent to the bytecode returned otherwise. This method is
+ mainly used internally.
+
+ `defer_init` is use internally to aid the module code generator. This
+ causes the generated code to be able to import without the global
+ environment variable to be set.
+
+ .. versionadded:: 2.4
+ `defer_init` parameter added.
+ """
+ source_hint = None
+ try:
+ if isinstance(source, basestring):
+ source_hint = source
+ source = self._parse(source, name, filename)
+ if self.optimized:
+ source = optimize(source, self)
+ source = self._generate(source, name, filename,
+ defer_init=defer_init)
+ if raw:
+ return source
+ if filename is None:
+ filename = '<template>'
+ else:
+ filename = _encode_filename(filename)
+ return self._compile(source, filename)
+ except TemplateSyntaxError:
+ exc_info = sys.exc_info()
+ self.handle_exception(exc_info, source_hint=source)
+
+ def compile_expression(self, source, undefined_to_none=True):
+ """A handy helper method that returns a callable that accepts keyword
+ arguments that appear as variables in the expression. If called it
+ returns the result of the expression.
+
+ This is useful if applications want to use the same rules as Jinja
+ in template "configuration files" or similar situations.
+
+ Example usage:
+
+ >>> env = Environment()
+ >>> expr = env.compile_expression('foo == 42')
+ >>> expr(foo=23)
+ False
+ >>> expr(foo=42)
+ True
+
+ Per default the return value is converted to `None` if the
+ expression returns an undefined value. This can be changed
+ by setting `undefined_to_none` to `False`.
+
+ >>> env.compile_expression('var')() is None
+ True
+ >>> env.compile_expression('var', undefined_to_none=False)()
+ Undefined
+
+ .. versionadded:: 2.1
+ """
+ parser = Parser(self, source, state='variable')
+ exc_info = None
+ try:
+ expr = parser.parse_expression()
+ if not parser.stream.eos:
+ raise TemplateSyntaxError('chunk after expression',
+ parser.stream.current.lineno,
+ None, None)
+ expr.set_environment(self)
+ except TemplateSyntaxError:
+ exc_info = sys.exc_info()
+ if exc_info is not None:
+ self.handle_exception(exc_info, source_hint=source)
+ body = [nodes.Assign(nodes.Name('result', 'store'), expr, lineno=1)]
+ template = self.from_string(nodes.Template(body, lineno=1))
+ return TemplateExpression(template, undefined_to_none)
+
+ def compile_templates(self, target, extensions=None, filter_func=None,
+ zip='deflated', log_function=None,
+ ignore_errors=True, py_compile=False):
+ """Compiles all the templates the loader can find, compiles them
+ and stores them in `target`. If `zip` is `None`, instead of in a
+ zipfile, the templates will be will be stored in a directory.
+ By default a deflate zip algorithm is used, to switch to
+ the stored algorithm, `zip` can be set to ``'stored'``.
+
+ `extensions` and `filter_func` are passed to :meth:`list_templates`.
+ Each template returned will be compiled to the target folder or
+ zipfile.
+
+ By default template compilation errors are ignored. In case a
+ log function is provided, errors are logged. If you want template
+ syntax errors to abort the compilation you can set `ignore_errors`
+ to `False` and you will get an exception on syntax errors.
+
+ If `py_compile` is set to `True` .pyc files will be written to the
+ target instead of standard .py files.
+
+ .. versionadded:: 2.4
+ """
+ from jinja2.loaders import ModuleLoader
+
+ if log_function is None:
+ log_function = lambda x: None
+
+ if py_compile:
+ import imp, struct, marshal
+ py_header = imp.get_magic() + \
+ u'\xff\xff\xff\xff'.encode('iso-8859-15')
+
+ def write_file(filename, data, mode):
+ if zip:
+ info = ZipInfo(filename)
+ info.external_attr = 0755 << 16L
+ zip_file.writestr(info, data)
+ else:
+ f = open(os.path.join(target, filename), mode)
+ try:
+ f.write(data)
+ finally:
+ f.close()
+
+ if zip is not None:
+ from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED, ZIP_STORED
+ zip_file = ZipFile(target, 'w', dict(deflated=ZIP_DEFLATED,
+ stored=ZIP_STORED)[zip])
+ log_function('Compiling into Zip archive "%s"' % target)
+ else:
+ if not os.path.isdir(target):
+ os.makedirs(target)
+ log_function('Compiling into folder "%s"' % target)
+
+ try:
+ for name in self.list_templates(extensions, filter_func):
+ source, filename, _ = self.loader.get_source(self, name)
+ try:
+ code = self.compile(source, name, filename, True, True)
+ except TemplateSyntaxError, e:
+ if not ignore_errors:
+ raise
+ log_function('Could not compile "%s": %s' % (name, e))
+ continue
+
+ filename = ModuleLoader.get_module_filename(name)
+
+ if py_compile:
+ c = self._compile(code, _encode_filename(filename))
+ write_file(filename + 'c', py_header +
+ marshal.dumps(c), 'wb')
+ log_function('Byte-compiled "%s" as %s' %
+ (name, filename + 'c'))
+ else:
+ write_file(filename, code, 'w')
+ log_function('Compiled "%s" as %s' % (name, filename))
+ finally:
+ if zip:
+ zip_file.close()
+
+ log_function('Finished compiling templates')
+
+ def list_templates(self, extensions=None, filter_func=None):
+ """Returns a list of templates for this environment. This requires
+ that the loader supports the loader's
+ :meth:`~BaseLoader.list_templates` method.
+
+ If there are other files in the template folder besides the
+ actual templates, the returned list can be filtered. There are two
+ ways: either `extensions` is set to a list of file extensions for
+ templates, or a `filter_func` can be provided which is a callable that
+ is passed a template name and should return `True` if it should end up
+ in the result list.
+
+ If the loader does not support that, a :exc:`TypeError` is raised.
+ """
+ x = self.loader.list_templates()
+ if extensions is not None:
+ if filter_func is not None:
+ raise TypeError('either extensions or filter_func '
+ 'can be passed, but not both')
+ filter_func = lambda x: '.' in x and \
+ x.rsplit('.', 1)[1] in extensions
+ if filter_func is not None:
+ x = filter(filter_func, x)
+ return x
+
+ def handle_exception(self, exc_info=None, rendered=False, source_hint=None):
+ """Exception handling helper. This is used internally to either raise
+ rewritten exceptions or return a rendered traceback for the template.
+ """
+ global _make_traceback
+ if exc_info is None:
+ exc_info = sys.exc_info()
+
+ # the debugging module is imported when it's used for the first time.
+ # we're doing a lot of stuff there and for applications that do not
+ # get any exceptions in template rendering there is no need to load
+ # all of that.
+ if _make_traceback is None:
+ from jinja2.debug import make_traceback as _make_traceback
+ traceback = _make_traceback(exc_info, source_hint)
+ if rendered and self.exception_formatter is not None:
+ return self.exception_formatter(traceback)
+ if self.exception_handler is not None:
+ self.exception_handler(traceback)
+ exc_type, exc_value, tb = traceback.standard_exc_info
+ raise exc_type, exc_value, tb
+
+ def join_path(self, template, parent):
+ """Join a template with the parent. By default all the lookups are
+ relative to the loader root so this method returns the `template`
+ parameter unchanged, but if the paths should be relative to the
+ parent template, this function can be used to calculate the real
+ template name.
+
+ Subclasses may override this method and implement template path
+ joining here.
+ """
+ return template
+
+ @internalcode
+ def _load_template(self, name, globals):
+ if self.loader is None:
+ raise TypeError('no loader for this environment specified')
+ if self.cache is not None:
+ template = self.cache.get(name)
+ if template is not None and (not self.auto_reload or \
+ template.is_up_to_date):
+ return template
+ template = self.loader.load(self, name, globals)
+ if self.cache is not None:
+ self.cache[name] = template
+ return template
+
+ @internalcode
+ def get_template(self, name, parent=None, globals=None):
+ """Load a template from the loader. If a loader is configured this
+ method ask the loader for the template and returns a :class:`Template`.
+ If the `parent` parameter is not `None`, :meth:`join_path` is called
+ to get the real template name before loading.
+
+ The `globals` parameter can be used to provide template wide globals.
+ These variables are available in the context at render time.
+
+ If the template does not exist a :exc:`TemplateNotFound` exception is
+ raised.
+
+ .. versionchanged:: 2.4
+ If `name` is a :class:`Template` object it is returned from the
+ function unchanged.
+ """
+ if isinstance(name, Template):
+ return name
+ if parent is not None:
+ name = self.join_path(name, parent)
+ return self._load_template(name, self.make_globals(globals))
+
+ @internalcode
+ def select_template(self, names, parent=None, globals=None):
+ """Works like :meth:`get_template` but tries a number of templates
+ before it fails. If it cannot find any of the templates, it will
+ raise a :exc:`TemplatesNotFound` exception.
+
+ .. versionadded:: 2.3
+
+ .. versionchanged:: 2.4
+ If `names` contains a :class:`Template` object it is returned
+ from the function unchanged.
+ """
+ if not names:
+ raise TemplatesNotFound(message=u'Tried to select from an empty list '
+ u'of templates.')
+ globals = self.make_globals(globals)
+ for name in names:
+ if isinstance(name, Template):
+ return name
+ if parent is not None:
+ name = self.join_path(name, parent)
+ try:
+ return self._load_template(name, globals)
+ except TemplateNotFound:
+ pass
+ raise TemplatesNotFound(names)
+
+ @internalcode
+ def get_or_select_template(self, template_name_or_list,
+ parent=None, globals=None):
+ """Does a typecheck and dispatches to :meth:`select_template`
+ if an iterable of template names is given, otherwise to
+ :meth:`get_template`.
+
+ .. versionadded:: 2.3
+ """
+ if isinstance(template_name_or_list, basestring):
+ return self.get_template(template_name_or_list, parent, globals)
+ elif isinstance(template_name_or_list, Template):
+ return template_name_or_list
+ return self.select_template(template_name_or_list, parent, globals)
+
+ def from_string(self, source, globals=None, template_class=None):
+ """Load a template from a string. This parses the source given and
+ returns a :class:`Template` object.
+ """
+ globals = self.make_globals(globals)
+ cls = template_class or self.template_class
+ return cls.from_code(self, self.compile(source), globals, None)
+
+ def make_globals(self, d):
+ """Return a dict for the globals."""
+ if not d:
+ return self.globals
+ return dict(self.globals, **d)
+
+
+class Template(object):
+ """The central template object. This class represents a compiled template
+ and is used to evaluate it.
+
+ Normally the template object is generated from an :class:`Environment` but
+ it also has a constructor that makes it possible to create a template
+ instance directly using the constructor. It takes the same arguments as
+ the environment constructor but it's not possible to specify a loader.
+
+ Every template object has a few methods and members that are guaranteed
+ to exist. However it's important that a template object should be
+ considered immutable. Modifications on the object are not supported.
+
+ Template objects created from the constructor rather than an environment
+ do have an `environment` attribute that points to a temporary environment
+ that is probably shared with other templates created with the constructor
+ and compatible settings.
+
+ >>> template = Template('Hello {{ name }}!')
+ >>> template.render(name='John Doe')
+ u'Hello John Doe!'
+
+ >>> stream = template.stream(name='John Doe')
+ >>> stream.next()
+ u'Hello John Doe!'
+ >>> stream.next()
+ Traceback (most recent call last):
+ ...
+ StopIteration
+ """
+
+ def __new__(cls, source,
+ block_start_string=BLOCK_START_STRING,
+ block_end_string=BLOCK_END_STRING,
+ variable_start_string=VARIABLE_START_STRING,
+ variable_end_string=VARIABLE_END_STRING,
+ comment_start_string=COMMENT_START_STRING,
+ comment_end_string=COMMENT_END_STRING,
+ line_statement_prefix=LINE_STATEMENT_PREFIX,
+ line_comment_prefix=LINE_COMMENT_PREFIX,
+ trim_blocks=TRIM_BLOCKS,
+ newline_sequence=NEWLINE_SEQUENCE,
+ extensions=(),
+ optimized=True,
+ undefined=Undefined,
+ finalize=None,
+ autoescape=False):
+ env = get_spontaneous_environment(
+ block_start_string, block_end_string, variable_start_string,
+ variable_end_string, comment_start_string, comment_end_string,
+ line_statement_prefix, line_comment_prefix, trim_blocks,
+ newline_sequence, frozenset(extensions), optimized, undefined,
+ finalize, autoescape, None, 0, False, None)
+ return env.from_string(source, template_class=cls)
+
+ @classmethod
+ def from_code(cls, environment, code, globals, uptodate=None):
+ """Creates a template object from compiled code and the globals. This
+ is used by the loaders and environment to create a template object.
+ """
+ namespace = {
+ 'environment': environment,
+ '__file__': code.co_filename
+ }
+ exec code in namespace
+ rv = cls._from_namespace(environment, namespace, globals)
+ rv._uptodate = uptodate
+ return rv
+
+ @classmethod
+ def from_module_dict(cls, environment, module_dict, globals):
+ """Creates a template object from a module. This is used by the
+ module loader to create a template object.
+
+ .. versionadded:: 2.4
+ """
+ return cls._from_namespace(environment, module_dict, globals)
+
+ @classmethod
+ def _from_namespace(cls, environment, namespace, globals):
+ t = object.__new__(cls)
+ t.environment = environment
+ t.globals = globals
+ t.name = namespace['name']
+ t.filename = namespace['__file__']
+ t.blocks = namespace['blocks']
+
+ # render function and module
+ t.root_render_func = namespace['root']
+ t._module = None
+
+ # debug and loader helpers
+ t._debug_info = namespace['debug_info']
+ t._uptodate = None
+
+ # store the reference
+ namespace['environment'] = environment
+ namespace['__jinja_template__'] = t
+
+ return t
+
+ def render(self, *args, **kwargs):
+ """This method accepts the same arguments as the `dict` constructor:
+ A dict, a dict subclass or some keyword arguments. If no arguments
+ are given the context will be empty. These two calls do the same::
+
+ template.render(knights='that say nih')
+ template.render({'knights': 'that say nih'})
+
+ This will return the rendered template as unicode string.
+ """
+ vars = dict(*args, **kwargs)
+ try:
+ return concat(self.root_render_func(self.new_context(vars)))
+ except:
+ exc_info = sys.exc_info()
+ return self.environment.handle_exception(exc_info, True)
+
+ def stream(self, *args, **kwargs):
+ """Works exactly like :meth:`generate` but returns a
+ :class:`TemplateStream`.
+ """
+ return TemplateStream(self.generate(*args, **kwargs))
+
+ def generate(self, *args, **kwargs):
+ """For very large templates it can be useful to not render the whole
+ template at once but evaluate each statement after another and yield
+ piece for piece. This method basically does exactly that and returns
+ a generator that yields one item after another as unicode strings.
+
+ It accepts the same arguments as :meth:`render`.
+ """
+ vars = dict(*args, **kwargs)
+ try:
+ for event in self.root_render_func(self.new_context(vars)):
+ yield event
+ except:
+ exc_info = sys.exc_info()
+ else:
+ return
+ yield self.environment.handle_exception(exc_info, True)
+
+ def new_context(self, vars=None, shared=False, locals=None):
+ """Create a new :class:`Context` for this template. The vars
+ provided will be passed to the template. Per default the globals
+ are added to the context. If shared is set to `True` the data
+ is passed as it to the context without adding the globals.
+
+ `locals` can be a dict of local variables for internal usage.
+ """
+ return new_context(self.environment, self.name, self.blocks,
+ vars, shared, self.globals, locals)
+
+ def make_module(self, vars=None, shared=False, locals=None):
+ """This method works like the :attr:`module` attribute when called
+ without arguments but it will evaluate the template on every call
+ rather than caching it. It's also possible to provide
+ a dict which is then used as context. The arguments are the same
+ as for the :meth:`new_context` method.
+ """
+ return TemplateModule(self, self.new_context(vars, shared, locals))
+
+ @property
+ def module(self):
+ """The template as module. This is used for imports in the
+ template runtime but is also useful if one wants to access
+ exported template variables from the Python layer:
+
+ >>> t = Template('{% macro foo() %}42{% endmacro %}23')
+ >>> unicode(t.module)
+ u'23'
+ >>> t.module.foo()
+ u'42'
+ """
+ if self._module is not None:
+ return self._module
+ self._module = rv = self.make_module()
+ return rv
+
+ def get_corresponding_lineno(self, lineno):
+ """Return the source line number of a line number in the
+ generated bytecode as they are not in sync.
+ """
+ for template_line, code_line in reversed(self.debug_info):
+ if code_line <= lineno:
+ return template_line
+ return 1
+
+ @property
+ def is_up_to_date(self):
+ """If this variable is `False` there is a newer version available."""
+ if self._uptodate is None:
+ return True
+ return self._uptodate()
+
+ @property
+ def debug_info(self):
+ """The debug info mapping."""
+ return [tuple(map(int, x.split('='))) for x in
+ self._debug_info.split('&')]
+
+ def __repr__(self):
+ if self.name is None:
+ name = 'memory:%x' % id(self)
+ else:
+ name = repr(self.name)
+ return '<%s %s>' % (self.__class__.__name__, name)
+
+
+class TemplateModule(object):
+ """Represents an imported template. All the exported names of the
+ template are available as attributes on this object. Additionally
+ converting it into an unicode- or bytestrings renders the contents.
+ """
+
+ def __init__(self, template, context):
+ self._body_stream = list(template.root_render_func(context))
+ self.__dict__.update(context.get_exported())
+ self.__name__ = template.name
+
+ def __html__(self):
+ return Markup(concat(self._body_stream))
+
+ def __str__(self):
+ return unicode(self).encode('utf-8')
+
+ # unicode goes after __str__ because we configured 2to3 to rename
+ # __unicode__ to __str__. because the 2to3 tree is not designed to
+ # remove nodes from it, we leave the above __str__ around and let
+ # it override at runtime.
+ def __unicode__(self):
+ return concat(self._body_stream)
+
+ def __repr__(self):
+ if self.__name__ is None:
+ name = 'memory:%x' % id(self)
+ else:
+ name = repr(self.__name__)
+ return '<%s %s>' % (self.__class__.__name__, name)
+
+
+class TemplateExpression(object):
+ """The :meth:`jinja2.Environment.compile_expression` method returns an
+ instance of this object. It encapsulates the expression-like access
+ to the template with an expression it wraps.
+ """
+
+ def __init__(self, template, undefined_to_none):
+ self._template = template
+ self._undefined_to_none = undefined_to_none
+
+ def __call__(self, *args, **kwargs):
+ context = self._template.new_context(dict(*args, **kwargs))
+ consume(self._template.root_render_func(context))
+ rv = context.vars['result']
+ if self._undefined_to_none and isinstance(rv, Undefined):
+ rv = None
+ return rv
+
+
+class TemplateStream(object):
+ """A template stream works pretty much like an ordinary python generator
+ but it can buffer multiple items to reduce the number of total iterations.
+ Per default the output is unbuffered which means that for every unbuffered
+ instruction in the template one unicode string is yielded.
+
+ If buffering is enabled with a buffer size of 5, five items are combined
+ into a new unicode string. This is mainly useful if you are streaming
+ big templates to a client via WSGI which flushes after each iteration.
+ """
+
+ def __init__(self, gen):
+ self._gen = gen
+ self.disable_buffering()
+
+ def dump(self, fp, encoding=None, errors='strict'):
+ """Dump the complete stream into a file or file-like object.
+ Per default unicode strings are written, if you want to encode
+ before writing specifiy an `encoding`.
+
+ Example usage::
+
+ Template('Hello {{ name }}!').stream(name='foo').dump('hello.html')
+ """
+ close = False
+ if isinstance(fp, basestring):
+ fp = file(fp, 'w')
+ close = True
+ try:
+ if encoding is not None:
+ iterable = (x.encode(encoding, errors) for x in self)
+ else:
+ iterable = self
+ if hasattr(fp, 'writelines'):
+ fp.writelines(iterable)
+ else:
+ for item in iterable:
+ fp.write(item)
+ finally:
+ if close:
+ fp.close()
+
+ def disable_buffering(self):
+ """Disable the output buffering."""
+ self._next = self._gen.next
+ self.buffered = False
+
+ def enable_buffering(self, size=5):
+ """Enable buffering. Buffer `size` items before yielding them."""
+ if size <= 1:
+ raise ValueError('buffer size too small')
+
+ def generator(next):
+ buf = []
+ c_size = 0
+ push = buf.append
+
+ while 1:
+ try:
+ while c_size < size:
+ c = next()
+ push(c)
+ if c:
+ c_size += 1
+ except StopIteration:
+ if not c_size:
+ return
+ yield concat(buf)
+ del buf[:]
+ c_size = 0
+
+ self.buffered = True
+ self._next = generator(self._gen.next).next
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ return self._next()
+
+
+# hook in default template class. if anyone reads this comment: ignore that
+# it's possible to use custom templates ;-)
+Environment.template_class = Template
diff --git a/slider-agent/src/main/python/jinja2/jinja2/exceptions.py b/slider-agent/src/main/python/jinja2/jinja2/exceptions.py
new file mode 100644
index 0000000..771f6a8
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/exceptions.py
@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.exceptions
+ ~~~~~~~~~~~~~~~~~
+
+ Jinja exceptions.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+
+
+class TemplateError(Exception):
+ """Baseclass for all template errors."""
+
+ def __init__(self, message=None):
+ if message is not None:
+ message = unicode(message).encode('utf-8')
+ Exception.__init__(self, message)
+
+ @property
+ def message(self):
+ if self.args:
+ message = self.args[0]
+ if message is not None:
+ return message.decode('utf-8', 'replace')
+
+
+class TemplateNotFound(IOError, LookupError, TemplateError):
+ """Raised if a template does not exist."""
+
+ # looks weird, but removes the warning descriptor that just
+ # bogusly warns us about message being deprecated
+ message = None
+
+ def __init__(self, name, message=None):
+ IOError.__init__(self)
+ if message is None:
+ message = name
+ self.message = message
+ self.name = name
+ self.templates = [name]
+
+ def __str__(self):
+ return self.message.encode('utf-8')
+
+ # unicode goes after __str__ because we configured 2to3 to rename
+ # __unicode__ to __str__. because the 2to3 tree is not designed to
+ # remove nodes from it, we leave the above __str__ around and let
+ # it override at runtime.
+ def __unicode__(self):
+ return self.message
+
+
+class TemplatesNotFound(TemplateNotFound):
+ """Like :class:`TemplateNotFound` but raised if multiple templates
+ are selected. This is a subclass of :class:`TemplateNotFound`
+ exception, so just catching the base exception will catch both.
+
+ .. versionadded:: 2.2
+ """
+
+ def __init__(self, names=(), message=None):
+ if message is None:
+ message = u'non of the templates given were found: ' + \
+ u', '.join(map(unicode, names))
+ TemplateNotFound.__init__(self, names and names[-1] or None, message)
+ self.templates = list(names)
+
+
+class TemplateSyntaxError(TemplateError):
+ """Raised to tell the user that there is a problem with the template."""
+
+ def __init__(self, message, lineno, name=None, filename=None):
+ TemplateError.__init__(self, message)
+ self.lineno = lineno
+ self.name = name
+ self.filename = filename
+ self.source = None
+
+ # this is set to True if the debug.translate_syntax_error
+ # function translated the syntax error into a new traceback
+ self.translated = False
+
+ def __str__(self):
+ return unicode(self).encode('utf-8')
+
+ # unicode goes after __str__ because we configured 2to3 to rename
+ # __unicode__ to __str__. because the 2to3 tree is not designed to
+ # remove nodes from it, we leave the above __str__ around and let
+ # it override at runtime.
+ def __unicode__(self):
+ # for translated errors we only return the message
+ if self.translated:
+ return self.message
+
+ # otherwise attach some stuff
+ location = 'line %d' % self.lineno
+ name = self.filename or self.name
+ if name:
+ location = 'File "%s", %s' % (name, location)
+ lines = [self.message, ' ' + location]
+
+ # if the source is set, add the line to the output
+ if self.source is not None:
+ try:
+ line = self.source.splitlines()[self.lineno - 1]
+ except IndexError:
+ line = None
+ if line:
+ lines.append(' ' + line.strip())
+
+ return u'\n'.join(lines)
+
+
+class TemplateAssertionError(TemplateSyntaxError):
+ """Like a template syntax error, but covers cases where something in the
+ template caused an error at compile time that wasn't necessarily caused
+ by a syntax error. However it's a direct subclass of
+ :exc:`TemplateSyntaxError` and has the same attributes.
+ """
+
+
+class TemplateRuntimeError(TemplateError):
+ """A generic runtime error in the template engine. Under some situations
+ Jinja may raise this exception.
+ """
+
+
+class UndefinedError(TemplateRuntimeError):
+ """Raised if a template tries to operate on :class:`Undefined`."""
+
+
+class SecurityError(TemplateRuntimeError):
+ """Raised if a template tries to do something insecure if the
+ sandbox is enabled.
+ """
+
+
+class FilterArgumentError(TemplateRuntimeError):
+ """This error is raised if a filter was called with inappropriate
+ arguments
+ """
diff --git a/slider-agent/src/main/python/jinja2/jinja2/ext.py b/slider-agent/src/main/python/jinja2/jinja2/ext.py
new file mode 100644
index 0000000..ceb3895
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/ext.py
@@ -0,0 +1,610 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.ext
+ ~~~~~~~~~~
+
+ Jinja extensions allow to add custom tags similar to the way django custom
+ tags work. By default two example extensions exist: an i18n and a cache
+ extension.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD.
+"""
+from collections import deque
+from jinja2 import nodes
+from jinja2.defaults import *
+from jinja2.environment import Environment
+from jinja2.runtime import Undefined, concat
+from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError
+from jinja2.utils import contextfunction, import_string, Markup, next
+
+
+# the only real useful gettext functions for a Jinja template. Note
+# that ugettext must be assigned to gettext as Jinja doesn't support
+# non unicode strings.
+GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext')
+
+
+class ExtensionRegistry(type):
+ """Gives the extension an unique identifier."""
+
+ def __new__(cls, name, bases, d):
+ rv = type.__new__(cls, name, bases, d)
+ rv.identifier = rv.__module__ + '.' + rv.__name__
+ return rv
+
+
+class Extension(object):
+ """Extensions can be used to add extra functionality to the Jinja template
+ system at the parser level. Custom extensions are bound to an environment
+ but may not store environment specific data on `self`. The reason for
+ this is that an extension can be bound to another environment (for
+ overlays) by creating a copy and reassigning the `environment` attribute.
+
+ As extensions are created by the environment they cannot accept any
+ arguments for configuration. One may want to work around that by using
+ a factory function, but that is not possible as extensions are identified
+ by their import name. The correct way to configure the extension is
+ storing the configuration values on the environment. Because this way the
+ environment ends up acting as central configuration storage the
+ attributes may clash which is why extensions have to ensure that the names
+ they choose for configuration are not too generic. ``prefix`` for example
+ is a terrible name, ``fragment_cache_prefix`` on the other hand is a good
+ name as includes the name of the extension (fragment cache).
+ """
+ __metaclass__ = ExtensionRegistry
+
+ #: if this extension parses this is the list of tags it's listening to.
+ tags = set()
+
+ #: the priority of that extension. This is especially useful for
+ #: extensions that preprocess values. A lower value means higher
+ #: priority.
+ #:
+ #: .. versionadded:: 2.4
+ priority = 100
+
+ def __init__(self, environment):
+ self.environment = environment
+
+ def bind(self, environment):
+ """Create a copy of this extension bound to another environment."""
+ rv = object.__new__(self.__class__)
+ rv.__dict__.update(self.__dict__)
+ rv.environment = environment
+ return rv
+
+ def preprocess(self, source, name, filename=None):
+ """This method is called before the actual lexing and can be used to
+ preprocess the source. The `filename` is optional. The return value
+ must be the preprocessed source.
+ """
+ return source
+
+ def filter_stream(self, stream):
+ """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used
+ to filter tokens returned. This method has to return an iterable of
+ :class:`~jinja2.lexer.Token`\s, but it doesn't have to return a
+ :class:`~jinja2.lexer.TokenStream`.
+
+ In the `ext` folder of the Jinja2 source distribution there is a file
+ called `inlinegettext.py` which implements a filter that utilizes this
+ method.
+ """
+ return stream
+
+ def parse(self, parser):
+ """If any of the :attr:`tags` matched this method is called with the
+ parser as first argument. The token the parser stream is pointing at
+ is the name token that matched. This method has to return one or a
+ list of multiple nodes.
+ """
+ raise NotImplementedError()
+
+ def attr(self, name, lineno=None):
+ """Return an attribute node for the current extension. This is useful
+ to pass constants on extensions to generated template code::
+
+ self.attr('_my_attribute', lineno=lineno)
+ """
+ return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno)
+
+ def call_method(self, name, args=None, kwargs=None, dyn_args=None,
+ dyn_kwargs=None, lineno=None):
+ """Call a method of the extension. This is a shortcut for
+ :meth:`attr` + :class:`jinja2.nodes.Call`.
+ """
+ if args is None:
+ args = []
+ if kwargs is None:
+ kwargs = []
+ return nodes.Call(self.attr(name, lineno=lineno), args, kwargs,
+ dyn_args, dyn_kwargs, lineno=lineno)
+
+
+@contextfunction
+def _gettext_alias(__context, *args, **kwargs):
+ return __context.call(__context.resolve('gettext'), *args, **kwargs)
+
+
+def _make_new_gettext(func):
+ @contextfunction
+ def gettext(__context, __string, **variables):
+ rv = __context.call(func, __string)
+ if __context.eval_ctx.autoescape:
+ rv = Markup(rv)
+ return rv % variables
+ return gettext
+
+
+def _make_new_ngettext(func):
+ @contextfunction
+ def ngettext(__context, __singular, __plural, __num, **variables):
+ variables.setdefault('num', __num)
+ rv = __context.call(func, __singular, __plural, __num)
+ if __context.eval_ctx.autoescape:
+ rv = Markup(rv)
+ return rv % variables
+ return ngettext
+
+
+class InternationalizationExtension(Extension):
+ """This extension adds gettext support to Jinja2."""
+ tags = set(['trans'])
+
+ # TODO: the i18n extension is currently reevaluating values in a few
+ # situations. Take this example:
+ # {% trans count=something() %}{{ count }} foo{% pluralize
+ # %}{{ count }} fooss{% endtrans %}
+ # something is called twice here. One time for the gettext value and
+ # the other time for the n-parameter of the ngettext function.
+
+ def __init__(self, environment):
+ Extension.__init__(self, environment)
+ environment.globals['_'] = _gettext_alias
+ environment.extend(
+ install_gettext_translations=self._install,
+ install_null_translations=self._install_null,
+ install_gettext_callables=self._install_callables,
+ uninstall_gettext_translations=self._uninstall,
+ extract_translations=self._extract,
+ newstyle_gettext=False
+ )
+
+ def _install(self, translations, newstyle=None):
+ gettext = getattr(translations, 'ugettext', None)
+ if gettext is None:
+ gettext = translations.gettext
+ ngettext = getattr(translations, 'ungettext', None)
+ if ngettext is None:
+ ngettext = translations.ngettext
+ self._install_callables(gettext, ngettext, newstyle)
+
+ def _install_null(self, newstyle=None):
+ self._install_callables(
+ lambda x: x,
+ lambda s, p, n: (n != 1 and (p,) or (s,))[0],
+ newstyle
+ )
+
+ def _install_callables(self, gettext, ngettext, newstyle=None):
+ if newstyle is not None:
+ self.environment.newstyle_gettext = newstyle
+ if self.environment.newstyle_gettext:
+ gettext = _make_new_gettext(gettext)
+ ngettext = _make_new_ngettext(ngettext)
+ self.environment.globals.update(
+ gettext=gettext,
+ ngettext=ngettext
+ )
+
+ def _uninstall(self, translations):
+ for key in 'gettext', 'ngettext':
+ self.environment.globals.pop(key, None)
+
+ def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS):
+ if isinstance(source, basestring):
+ source = self.environment.parse(source)
+ return extract_from_ast(source, gettext_functions)
+
+ def parse(self, parser):
+ """Parse a translatable tag."""
+ lineno = next(parser.stream).lineno
+ num_called_num = False
+
+ # find all the variables referenced. Additionally a variable can be
+ # defined in the body of the trans block too, but this is checked at
+ # a later state.
+ plural_expr = None
+ variables = {}
+ while parser.stream.current.type != 'block_end':
+ if variables:
+ parser.stream.expect('comma')
+
+ # skip colon for python compatibility
+ if parser.stream.skip_if('colon'):
+ break
+
+ name = parser.stream.expect('name')
+ if name.value in variables:
+ parser.fail('translatable variable %r defined twice.' %
+ name.value, name.lineno,
+ exc=TemplateAssertionError)
+
+ # expressions
+ if parser.stream.current.type == 'assign':
+ next(parser.stream)
+ variables[name.value] = var = parser.parse_expression()
+ else:
+ variables[name.value] = var = nodes.Name(name.value, 'load')
+
+ if plural_expr is None:
+ plural_expr = var
+ num_called_num = name.value == 'num'
+
+ parser.stream.expect('block_end')
+
+ plural = plural_names = None
+ have_plural = False
+ referenced = set()
+
+ # now parse until endtrans or pluralize
+ singular_names, singular = self._parse_block(parser, True)
+ if singular_names:
+ referenced.update(singular_names)
+ if plural_expr is None:
+ plural_expr = nodes.Name(singular_names[0], 'load')
+ num_called_num = singular_names[0] == 'num'
+
+ # if we have a pluralize block, we parse that too
+ if parser.stream.current.test('name:pluralize'):
+ have_plural = True
+ next(parser.stream)
+ if parser.stream.current.type != 'block_end':
+ name = parser.stream.expect('name')
+ if name.value not in variables:
+ parser.fail('unknown variable %r for pluralization' %
+ name.value, name.lineno,
+ exc=TemplateAssertionError)
+ plural_expr = variables[name.value]
+ num_called_num = name.value == 'num'
+ parser.stream.expect('block_end')
+ plural_names, plural = self._parse_block(parser, False)
+ next(parser.stream)
+ referenced.update(plural_names)
+ else:
+ next(parser.stream)
+
+ # register free names as simple name expressions
+ for var in referenced:
+ if var not in variables:
+ variables[var] = nodes.Name(var, 'load')
+
+ if not have_plural:
+ plural_expr = None
+ elif plural_expr is None:
+ parser.fail('pluralize without variables', lineno)
+
+ node = self._make_node(singular, plural, variables, plural_expr,
+ bool(referenced),
+ num_called_num and have_plural)
+ node.set_lineno(lineno)
+ return node
+
+ def _parse_block(self, parser, allow_pluralize):
+ """Parse until the next block tag with a given name."""
+ referenced = []
+ buf = []
+ while 1:
+ if parser.stream.current.type == 'data':
+ buf.append(parser.stream.current.value.replace('%', '%%'))
+ next(parser.stream)
+ elif parser.stream.current.type == 'variable_begin':
+ next(parser.stream)
+ name = parser.stream.expect('name').value
+ referenced.append(name)
+ buf.append('%%(%s)s' % name)
+ parser.stream.expect('variable_end')
+ elif parser.stream.current.type == 'block_begin':
+ next(parser.stream)
+ if parser.stream.current.test('name:endtrans'):
+ break
+ elif parser.stream.current.test('name:pluralize'):
+ if allow_pluralize:
+ break
+ parser.fail('a translatable section can have only one '
+ 'pluralize section')
+ parser.fail('control structures in translatable sections are '
+ 'not allowed')
+ elif parser.stream.eos:
+ parser.fail('unclosed translation block')
+ else:
+ assert False, 'internal parser error'
+
+ return referenced, concat(buf)
+
+ def _make_node(self, singular, plural, variables, plural_expr,
+ vars_referenced, num_called_num):
+ """Generates a useful node from the data provided."""
+ # no variables referenced? no need to escape for old style
+ # gettext invocations only if there are vars.
+ if not vars_referenced and not self.environment.newstyle_gettext:
+ singular = singular.replace('%%', '%')
+ if plural:
+ plural = plural.replace('%%', '%')
+
+ # singular only:
+ if plural_expr is None:
+ gettext = nodes.Name('gettext', 'load')
+ node = nodes.Call(gettext, [nodes.Const(singular)],
+ [], None, None)
+
+ # singular and plural
+ else:
+ ngettext = nodes.Name('ngettext', 'load')
+ node = nodes.Call(ngettext, [
+ nodes.Const(singular),
+ nodes.Const(plural),
+ plural_expr
+ ], [], None, None)
+
+ # in case newstyle gettext is used, the method is powerful
+ # enough to handle the variable expansion and autoescape
+ # handling itself
+ if self.environment.newstyle_gettext:
+ for key, value in variables.iteritems():
+ # the function adds that later anyways in case num was
+ # called num, so just skip it.
+ if num_called_num and key == 'num':
+ continue
+ node.kwargs.append(nodes.Keyword(key, value))
+
+ # otherwise do that here
+ else:
+ # mark the return value as safe if we are in an
+ # environment with autoescaping turned on
+ node = nodes.MarkSafeIfAutoescape(node)
+ if variables:
+ node = nodes.Mod(node, nodes.Dict([
+ nodes.Pair(nodes.Const(key), value)
+ for key, value in variables.items()
+ ]))
+ return nodes.Output([node])
+
+
+class ExprStmtExtension(Extension):
+ """Adds a `do` tag to Jinja2 that works like the print statement just
+ that it doesn't print the return value.
+ """
+ tags = set(['do'])
+
+ def parse(self, parser):
+ node = nodes.ExprStmt(lineno=next(parser.stream).lineno)
+ node.node = parser.parse_tuple()
+ return node
+
+
+class LoopControlExtension(Extension):
+ """Adds break and continue to the template engine."""
+ tags = set(['break', 'continue'])
+
+ def parse(self, parser):
+ token = next(parser.stream)
+ if token.value == 'break':
+ return nodes.Break(lineno=token.lineno)
+ return nodes.Continue(lineno=token.lineno)
+
+
+class WithExtension(Extension):
+ """Adds support for a django-like with block."""
+ tags = set(['with'])
+
+ def parse(self, parser):
+ node = nodes.Scope(lineno=next(parser.stream).lineno)
+ assignments = []
+ while parser.stream.current.type != 'block_end':
+ lineno = parser.stream.current.lineno
+ if assignments:
+ parser.stream.expect('comma')
+ target = parser.parse_assign_target()
+ parser.stream.expect('assign')
+ expr = parser.parse_expression()
+ assignments.append(nodes.Assign(target, expr, lineno=lineno))
+ node.body = assignments + \
+ list(parser.parse_statements(('name:endwith',),
+ drop_needle=True))
+ return node
+
+
+class AutoEscapeExtension(Extension):
+ """Changes auto escape rules for a scope."""
+ tags = set(['autoescape'])
+
+ def parse(self, parser):
+ node = nodes.ScopedEvalContextModifier(lineno=next(parser.stream).lineno)
+ node.options = [
+ nodes.Keyword('autoescape', parser.parse_expression())
+ ]
+ node.body = parser.parse_statements(('name:endautoescape',),
+ drop_needle=True)
+ return nodes.Scope([node])
+
+
+def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS,
+ babel_style=True):
+ """Extract localizable strings from the given template node. Per
+ default this function returns matches in babel style that means non string
+ parameters as well as keyword arguments are returned as `None`. This
+ allows Babel to figure out what you really meant if you are using
+ gettext functions that allow keyword arguments for placeholder expansion.
+ If you don't want that behavior set the `babel_style` parameter to `False`
+ which causes only strings to be returned and parameters are always stored
+ in tuples. As a consequence invalid gettext calls (calls without a single
+ string parameter or string parameters after non-string parameters) are
+ skipped.
+
+ This example explains the behavior:
+
+ >>> from jinja2 import Environment
+ >>> env = Environment()
+ >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}')
+ >>> list(extract_from_ast(node))
+ [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))]
+ >>> list(extract_from_ast(node, babel_style=False))
+ [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))]
+
+ For every string found this function yields a ``(lineno, function,
+ message)`` tuple, where:
+
+ * ``lineno`` is the number of the line on which the string was found,
+ * ``function`` is the name of the ``gettext`` function used (if the
+ string was extracted from embedded Python code), and
+ * ``message`` is the string itself (a ``unicode`` object, or a tuple
+ of ``unicode`` objects for functions with multiple string arguments).
+
+ This extraction function operates on the AST and is because of that unable
+ to extract any comments. For comment support you have to use the babel
+ extraction interface or extract comments yourself.
+ """
+ for node in node.find_all(nodes.Call):
+ if not isinstance(node.node, nodes.Name) or \
+ node.node.name not in gettext_functions:
+ continue
+
+ strings = []
+ for arg in node.args:
+ if isinstance(arg, nodes.Const) and \
+ isinstance(arg.value, basestring):
+ strings.append(arg.value)
+ else:
+ strings.append(None)
+
+ for arg in node.kwargs:
+ strings.append(None)
+ if node.dyn_args is not None:
+ strings.append(None)
+ if node.dyn_kwargs is not None:
+ strings.append(None)
+
+ if not babel_style:
+ strings = tuple(x for x in strings if x is not None)
+ if not strings:
+ continue
+ else:
+ if len(strings) == 1:
+ strings = strings[0]
+ else:
+ strings = tuple(strings)
+ yield node.lineno, node.node.name, strings
+
+
+class _CommentFinder(object):
+ """Helper class to find comments in a token stream. Can only
+ find comments for gettext calls forwards. Once the comment
+ from line 4 is found, a comment for line 1 will not return a
+ usable value.
+ """
+
+ def __init__(self, tokens, comment_tags):
+ self.tokens = tokens
+ self.comment_tags = comment_tags
+ self.offset = 0
+ self.last_lineno = 0
+
+ def find_backwards(self, offset):
+ try:
+ for _, token_type, token_value in \
+ reversed(self.tokens[self.offset:offset]):
+ if token_type in ('comment', 'linecomment'):
+ try:
+ prefix, comment = token_value.split(None, 1)
+ except ValueError:
+ continue
+ if prefix in self.comment_tags:
+ return [comment.rstrip()]
+ return []
+ finally:
+ self.offset = offset
+
+ def find_comments(self, lineno):
+ if not self.comment_tags or self.last_lineno > lineno:
+ return []
+ for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]):
+ if token_lineno > lineno:
+ return self.find_backwards(self.offset + idx)
+ return self.find_backwards(len(self.tokens))
+
+
+def babel_extract(fileobj, keywords, comment_tags, options):
+ """Babel extraction method for Jinja templates.
+
+ .. versionchanged:: 2.3
+ Basic support for translation comments was added. If `comment_tags`
+ is now set to a list of keywords for extraction, the extractor will
+ try to find the best preceeding comment that begins with one of the
+ keywords. For best results, make sure to not have more than one
+ gettext call in one line of code and the matching comment in the
+ same line or the line before.
+
+ .. versionchanged:: 2.5.1
+ The `newstyle_gettext` flag can be set to `True` to enable newstyle
+ gettext calls.
+
+ :param fileobj: the file-like object the messages should be extracted from
+ :param keywords: a list of keywords (i.e. function names) that should be
+ recognized as translation functions
+ :param comment_tags: a list of translator tags to search for and include
+ in the results.
+ :param options: a dictionary of additional options (optional)
+ :return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
+ (comments will be empty currently)
+ """
+ extensions = set()
+ for extension in options.get('extensions', '').split(','):
+ extension = extension.strip()
+ if not extension:
+ continue
+ extensions.add(import_string(extension))
+ if InternationalizationExtension not in extensions:
+ extensions.add(InternationalizationExtension)
+
+ def getbool(options, key, default=False):
+ options.get(key, str(default)).lower() in ('1', 'on', 'yes', 'true')
+
+ environment = Environment(
+ options.get('block_start_string', BLOCK_START_STRING),
+ options.get('block_end_string', BLOCK_END_STRING),
+ options.get('variable_start_string', VARIABLE_START_STRING),
+ options.get('variable_end_string', VARIABLE_END_STRING),
+ options.get('comment_start_string', COMMENT_START_STRING),
+ options.get('comment_end_string', COMMENT_END_STRING),
+ options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
+ options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
+ getbool(options, 'trim_blocks', TRIM_BLOCKS),
+ NEWLINE_SEQUENCE, frozenset(extensions),
+ cache_size=0,
+ auto_reload=False
+ )
+
+ if getbool(options, 'newstyle_gettext'):
+ environment.newstyle_gettext = True
+
+ source = fileobj.read().decode(options.get('encoding', 'utf-8'))
+ try:
+ node = environment.parse(source)
+ tokens = list(environment.lex(environment.preprocess(source)))
+ except TemplateSyntaxError, e:
+ # skip templates with syntax errors
+ return
+
+ finder = _CommentFinder(tokens, comment_tags)
+ for lineno, func, message in extract_from_ast(node, keywords):
+ yield lineno, func, message, finder.find_comments(lineno)
+
+
+#: nicer import names
+i18n = InternationalizationExtension
+do = ExprStmtExtension
+loopcontrols = LoopControlExtension
+with_ = WithExtension
+autoescape = AutoEscapeExtension
diff --git a/slider-agent/src/main/python/jinja2/jinja2/filters.py b/slider-agent/src/main/python/jinja2/jinja2/filters.py
new file mode 100644
index 0000000..d1848e4
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/filters.py
@@ -0,0 +1,719 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.filters
+ ~~~~~~~~~~~~~~
+
+ Bundled jinja filters.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import re
+import math
+from random import choice
+from operator import itemgetter
+from itertools import imap, groupby
+from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode
+from jinja2.runtime import Undefined
+from jinja2.exceptions import FilterArgumentError, SecurityError
+
+
+_word_re = re.compile(r'\w+(?u)')
+
+
+def contextfilter(f):
+ """Decorator for marking context dependent filters. The current
+ :class:`Context` will be passed as first argument.
+ """
+ f.contextfilter = True
+ return f
+
+
+def evalcontextfilter(f):
+ """Decorator for marking eval-context dependent filters. An eval
+ context object is passed as first argument. For more information
+ about the eval context, see :ref:`eval-context`.
+
+ .. versionadded:: 2.4
+ """
+ f.evalcontextfilter = True
+ return f
+
+
+def environmentfilter(f):
+ """Decorator for marking evironment dependent filters. The current
+ :class:`Environment` is passed to the filter as first argument.
+ """
+ f.environmentfilter = True
+ return f
+
+
+def do_forceescape(value):
+ """Enforce HTML escaping. This will probably double escape variables."""
+ if hasattr(value, '__html__'):
+ value = value.__html__()
+ return escape(unicode(value))
+
+
+@evalcontextfilter
+def do_replace(eval_ctx, s, old, new, count=None):
+ """Return a copy of the value with all occurrences of a substring
+ replaced with a new one. The first argument is the substring
+ that should be replaced, the second is the replacement string.
+ If the optional third argument ``count`` is given, only the first
+ ``count`` occurrences are replaced:
+
+ .. sourcecode:: jinja
+
+ {{ "Hello World"|replace("Hello", "Goodbye") }}
+ -> Goodbye World
+
+ {{ "aaaaargh"|replace("a", "d'oh, ", 2) }}
+ -> d'oh, d'oh, aaargh
+ """
+ if count is None:
+ count = -1
+ if not eval_ctx.autoescape:
+ return unicode(s).replace(unicode(old), unicode(new), count)
+ if hasattr(old, '__html__') or hasattr(new, '__html__') and \
+ not hasattr(s, '__html__'):
+ s = escape(s)
+ else:
+ s = soft_unicode(s)
+ return s.replace(soft_unicode(old), soft_unicode(new), count)
+
+
+def do_upper(s):
+ """Convert a value to uppercase."""
+ return soft_unicode(s).upper()
+
+
+def do_lower(s):
+ """Convert a value to lowercase."""
+ return soft_unicode(s).lower()
+
+
+@evalcontextfilter
+def do_xmlattr(_eval_ctx, d, autospace=True):
+ """Create an SGML/XML attribute string based on the items in a dict.
+ All values that are neither `none` nor `undefined` are automatically
+ escaped:
+
+ .. sourcecode:: html+jinja
+
+ <ul{{ {'class': 'my_list', 'missing': none,
+ 'id': 'list-%d'|format(variable)}|xmlattr }}>
+ ...
+ </ul>
+
+ Results in something like this:
+
+ .. sourcecode:: html
+
+ <ul class="my_list" id="list-42">
+ ...
+ </ul>
+
+ As you can see it automatically prepends a space in front of the item
+ if the filter returned something unless the second parameter is false.
+ """
+ rv = u' '.join(
+ u'%s="%s"' % (escape(key), escape(value))
+ for key, value in d.iteritems()
+ if value is not None and not isinstance(value, Undefined)
+ )
+ if autospace and rv:
+ rv = u' ' + rv
+ if _eval_ctx.autoescape:
+ rv = Markup(rv)
+ return rv
+
+
+def do_capitalize(s):
+ """Capitalize a value. The first character will be uppercase, all others
+ lowercase.
+ """
+ return soft_unicode(s).capitalize()
+
+
+def do_title(s):
+ """Return a titlecased version of the value. I.e. words will start with
+ uppercase letters, all remaining characters are lowercase.
+ """
+ return soft_unicode(s).title()
+
+
+def do_dictsort(value, case_sensitive=False, by='key'):
+ """Sort a dict and yield (key, value) pairs. Because python dicts are
+ unsorted you may want to use this function to order them by either
+ key or value:
+
+ .. sourcecode:: jinja
+
+ {% for item in mydict|dictsort %}
+ sort the dict by key, case insensitive
+
+ {% for item in mydict|dicsort(true) %}
+ sort the dict by key, case sensitive
+
+ {% for item in mydict|dictsort(false, 'value') %}
+ sort the dict by key, case insensitive, sorted
+ normally and ordered by value.
+ """
+ if by == 'key':
+ pos = 0
+ elif by == 'value':
+ pos = 1
+ else:
+ raise FilterArgumentError('You can only sort by either '
+ '"key" or "value"')
+ def sort_func(item):
+ value = item[pos]
+ if isinstance(value, basestring) and not case_sensitive:
+ value = value.lower()
+ return value
+
+ return sorted(value.items(), key=sort_func)
+
+
+def do_sort(value, reverse=False, case_sensitive=False):
+ """Sort an iterable. Per default it sorts ascending, if you pass it
+ true as first argument it will reverse the sorting.
+
+ If the iterable is made of strings the third parameter can be used to
+ control the case sensitiveness of the comparison which is disabled by
+ default.
+
+ .. sourcecode:: jinja
+
+ {% for item in iterable|sort %}
+ ...
+ {% endfor %}
+ """
+ if not case_sensitive:
+ def sort_func(item):
+ if isinstance(item, basestring):
+ item = item.lower()
+ return item
+ else:
+ sort_func = None
+ return sorted(value, key=sort_func, reverse=reverse)
+
+
+def do_default(value, default_value=u'', boolean=False):
+ """If the value is undefined it will return the passed default value,
+ otherwise the value of the variable:
+
+ .. sourcecode:: jinja
+
+ {{ my_variable|default('my_variable is not defined') }}
+
+ This will output the value of ``my_variable`` if the variable was
+ defined, otherwise ``'my_variable is not defined'``. If you want
+ to use default with variables that evaluate to false you have to
+ set the second parameter to `true`:
+
+ .. sourcecode:: jinja
+
+ {{ ''|default('the string was empty', true) }}
+ """
+ if (boolean and not value) or isinstance(value, Undefined):
+ return default_value
+ return value
+
+
+@evalcontextfilter
+def do_join(eval_ctx, value, d=u''):
+ """Return a string which is the concatenation of the strings in the
+ sequence. The separator between elements is an empty string per
+ default, you can define it with the optional parameter:
+
+ .. sourcecode:: jinja
+
+ {{ [1, 2, 3]|join('|') }}
+ -> 1|2|3
+
+ {{ [1, 2, 3]|join }}
+ -> 123
+ """
+ # no automatic escaping? joining is a lot eaiser then
+ if not eval_ctx.autoescape:
+ return unicode(d).join(imap(unicode, value))
+
+ # if the delimiter doesn't have an html representation we check
+ # if any of the items has. If yes we do a coercion to Markup
+ if not hasattr(d, '__html__'):
+ value = list(value)
+ do_escape = False
+ for idx, item in enumerate(value):
+ if hasattr(item, '__html__'):
+ do_escape = True
+ else:
+ value[idx] = unicode(item)
+ if do_escape:
+ d = escape(d)
+ else:
+ d = unicode(d)
+ return d.join(value)
+
+ # no html involved, to normal joining
+ return soft_unicode(d).join(imap(soft_unicode, value))
+
+
+def do_center(value, width=80):
+ """Centers the value in a field of a given width."""
+ return unicode(value).center(width)
+
+
+@environmentfilter
+def do_first(environment, seq):
+ """Return the first item of a sequence."""
+ try:
+ return iter(seq).next()
+ except StopIteration:
+ return environment.undefined('No first item, sequence was empty.')
+
+
+@environmentfilter
+def do_last(environment, seq):
+ """Return the last item of a sequence."""
+ try:
+ return iter(reversed(seq)).next()
+ except StopIteration:
+ return environment.undefined('No last item, sequence was empty.')
+
+
+@environmentfilter
+def do_random(environment, seq):
+ """Return a random item from the sequence."""
+ try:
+ return choice(seq)
+ except IndexError:
+ return environment.undefined('No random item, sequence was empty.')
+
+
+def do_filesizeformat(value, binary=False):
+ """Format the value like a 'human-readable' file size (i.e. 13 KB,
+ 4.1 MB, 102 bytes, etc). Per default decimal prefixes are used (mega,
+ giga, etc.), if the second parameter is set to `True` the binary
+ prefixes are used (mebi, gibi).
+ """
+ bytes = float(value)
+ base = binary and 1024 or 1000
+ middle = binary and 'i' or ''
+ if bytes < base:
+ return "%d Byte%s" % (bytes, bytes != 1 and 's' or '')
+ elif bytes < base * base:
+ return "%.1f K%sB" % (bytes / base, middle)
+ elif bytes < base * base * base:
+ return "%.1f M%sB" % (bytes / (base * base), middle)
+ return "%.1f G%sB" % (bytes / (base * base * base), middle)
+
+
+def do_pprint(value, verbose=False):
+ """Pretty print a variable. Useful for debugging.
+
+ With Jinja 1.2 onwards you can pass it a parameter. If this parameter
+ is truthy the output will be more verbose (this requires `pretty`)
+ """
+ return pformat(value, verbose=verbose)
+
+
+@evalcontextfilter
+def do_urlize(eval_ctx, value, trim_url_limit=None, nofollow=False):
+ """Converts URLs in plain text into clickable links.
+
+ If you pass the filter an additional integer it will shorten the urls
+ to that number. Also a third argument exists that makes the urls
+ "nofollow":
+
+ .. sourcecode:: jinja
+
+ {{ mytext|urlize(40, true) }}
+ links are shortened to 40 chars and defined with rel="nofollow"
+ """
+ rv = urlize(value, trim_url_limit, nofollow)
+ if eval_ctx.autoescape:
+ rv = Markup(rv)
+ return rv
+
+
+def do_indent(s, width=4, indentfirst=False):
+ """Return a copy of the passed string, each line indented by
+ 4 spaces. The first line is not indented. If you want to
+ change the number of spaces or indent the first line too
+ you can pass additional parameters to the filter:
+
+ .. sourcecode:: jinja
+
+ {{ mytext|indent(2, true) }}
+ indent by two spaces and indent the first line too.
+ """
+ indention = u' ' * width
+ rv = (u'\n' + indention).join(s.splitlines())
+ if indentfirst:
+ rv = indention + rv
+ return rv
+
+
+def do_truncate(s, length=255, killwords=False, end='...'):
+ """Return a truncated copy of the string. The length is specified
+ with the first parameter which defaults to ``255``. If the second
+ parameter is ``true`` the filter will cut the text at length. Otherwise
+ it will try to save the last word. If the text was in fact
+ truncated it will append an ellipsis sign (``"..."``). If you want a
+ different ellipsis sign than ``"..."`` you can specify it using the
+ third parameter.
+
+ .. sourcecode jinja::
+
+ {{ mytext|truncate(300, false, '»') }}
+ truncate mytext to 300 chars, don't split up words, use a
+ right pointing double arrow as ellipsis sign.
+ """
+ if len(s) <= length:
+ return s
+ elif killwords:
+ return s[:length] + end
+ words = s.split(' ')
+ result = []
+ m = 0
+ for word in words:
+ m += len(word) + 1
+ if m > length:
+ break
+ result.append(word)
+ result.append(end)
+ return u' '.join(result)
+
+
+def do_wordwrap(s, width=79, break_long_words=True):
+ """
+ Return a copy of the string passed to the filter wrapped after
+ ``79`` characters. You can override this default using the first
+ parameter. If you set the second parameter to `false` Jinja will not
+ split words apart if they are longer than `width`.
+ """
+ import textwrap
+ return u'\n'.join(textwrap.wrap(s, width=width, expand_tabs=False,
+ replace_whitespace=False,
+ break_long_words=break_long_words))
+
+
+def do_wordcount(s):
+ """Count the words in that string."""
+ return len(_word_re.findall(s))
+
+
+def do_int(value, default=0):
+ """Convert the value into an integer. If the
+ conversion doesn't work it will return ``0``. You can
+ override this default using the first parameter.
+ """
+ try:
+ return int(value)
+ except (TypeError, ValueError):
+ # this quirk is necessary so that "42.23"|int gives 42.
+ try:
+ return int(float(value))
+ except (TypeError, ValueError):
+ return default
+
+
+def do_float(value, default=0.0):
+ """Convert the value into a floating point number. If the
+ conversion doesn't work it will return ``0.0``. You can
+ override this default using the first parameter.
+ """
+ try:
+ return float(value)
+ except (TypeError, ValueError):
+ return default
+
+
+def do_format(value, *args, **kwargs):
+ """
+ Apply python string formatting on an object:
+
+ .. sourcecode:: jinja
+
+ {{ "%s - %s"|format("Hello?", "Foo!") }}
+ -> Hello? - Foo!
+ """
+ if args and kwargs:
+ raise FilterArgumentError('can\'t handle positional and keyword '
+ 'arguments at the same time')
+ return soft_unicode(value) % (kwargs or args)
+
+
+def do_trim(value):
+ """Strip leading and trailing whitespace."""
+ return soft_unicode(value).strip()
+
+
+def do_striptags(value):
+ """Strip SGML/XML tags and replace adjacent whitespace by one space.
+ """
+ if hasattr(value, '__html__'):
+ value = value.__html__()
+ return Markup(unicode(value)).striptags()
+
+
+def do_slice(value, slices, fill_with=None):
+ """Slice an iterator and return a list of lists containing
+ those items. Useful if you want to create a div containing
+ three ul tags that represent columns:
+
+ .. sourcecode:: html+jinja
+
+ <div class="columwrapper">
+ {%- for column in items|slice(3) %}
+ <ul class="column-{{ loop.index }}">
+ {%- for item in column %}
+ <li>{{ item }}</li>
+ {%- endfor %}
+ </ul>
+ {%- endfor %}
+ </div>
+
+ If you pass it a second argument it's used to fill missing
+ values on the last iteration.
+ """
+ seq = list(value)
+ length = len(seq)
+ items_per_slice = length // slices
+ slices_with_extra = length % slices
+ offset = 0
+ for slice_number in xrange(slices):
+ start = offset + slice_number * items_per_slice
+ if slice_number < slices_with_extra:
+ offset += 1
+ end = offset + (slice_number + 1) * items_per_slice
+ tmp = seq[start:end]
+ if fill_with is not None and slice_number >= slices_with_extra:
+ tmp.append(fill_with)
+ yield tmp
+
+
+def do_batch(value, linecount, fill_with=None):
+ """
+ A filter that batches items. It works pretty much like `slice`
+ just the other way round. It returns a list of lists with the
+ given number of items. If you provide a second parameter this
+ is used to fill missing items. See this example:
+
+ .. sourcecode:: html+jinja
+
+ <table>
+ {%- for row in items|batch(3, ' ') %}
+ <tr>
+ {%- for column in row %}
+ <td>{{ column }}</td>
+ {%- endfor %}
+ </tr>
+ {%- endfor %}
+ </table>
+ """
+ result = []
+ tmp = []
+ for item in value:
+ if len(tmp) == linecount:
+ yield tmp
+ tmp = []
+ tmp.append(item)
+ if tmp:
+ if fill_with is not None and len(tmp) < linecount:
+ tmp += [fill_with] * (linecount - len(tmp))
+ yield tmp
+
+
+def do_round(value, precision=0, method='common'):
+ """Round the number to a given precision. The first
+ parameter specifies the precision (default is ``0``), the
+ second the rounding method:
+
+ - ``'common'`` rounds either up or down
+ - ``'ceil'`` always rounds up
+ - ``'floor'`` always rounds down
+
+ If you don't specify a method ``'common'`` is used.
+
+ .. sourcecode:: jinja
+
+ {{ 42.55|round }}
+ -> 43.0
+ {{ 42.55|round(1, 'floor') }}
+ -> 42.5
+
+ Note that even if rounded to 0 precision, a float is returned. If
+ you need a real integer, pipe it through `int`:
+
+ .. sourcecode:: jinja
+
+ {{ 42.55|round|int }}
+ -> 43
+ """
+ if not method in ('common', 'ceil', 'floor'):
+ raise FilterArgumentError('method must be common, ceil or floor')
+ if method == 'common':
+ return round(value, precision)
+ func = getattr(math, method)
+ return func(value * (10 ** precision)) / (10 ** precision)
+
+
+@environmentfilter
+def do_groupby(environment, value, attribute):
+ """Group a sequence of objects by a common attribute.
+
+ If you for example have a list of dicts or objects that represent persons
+ with `gender`, `first_name` and `last_name` attributes and you want to
+ group all users by genders you can do something like the following
+ snippet:
+
+ .. sourcecode:: html+jinja
+
+ <ul>
+ {% for group in persons|groupby('gender') %}
+ <li>{{ group.grouper }}<ul>
+ {% for person in group.list %}
+ <li>{{ person.first_name }} {{ person.last_name }}</li>
+ {% endfor %}</ul></li>
+ {% endfor %}
+ </ul>
+
+ Additionally it's possible to use tuple unpacking for the grouper and
+ list:
+
+ .. sourcecode:: html+jinja
+
+ <ul>
+ {% for grouper, list in persons|groupby('gender') %}
+ ...
+ {% endfor %}
+ </ul>
+
+ As you can see the item we're grouping by is stored in the `grouper`
+ attribute and the `list` contains all the objects that have this grouper
+ in common.
+ """
+ expr = lambda x: environment.getitem(x, attribute)
+ return sorted(map(_GroupTuple, groupby(sorted(value, key=expr), expr)))
+
+
+class _GroupTuple(tuple):
+ __slots__ = ()
+ grouper = property(itemgetter(0))
+ list = property(itemgetter(1))
+
+ def __new__(cls, (key, value)):
+ return tuple.__new__(cls, (key, list(value)))
+
+
+def do_list(value):
+ """Convert the value into a list. If it was a string the returned list
+ will be a list of characters.
+ """
+ return list(value)
+
+
+def do_mark_safe(value):
+ """Mark the value as safe which means that in an environment with automatic
+ escaping enabled this variable will not be escaped.
+ """
+ return Markup(value)
+
+
+def do_mark_unsafe(value):
+ """Mark a value as unsafe. This is the reverse operation for :func:`safe`."""
+ return unicode(value)
+
+
+def do_reverse(value):
+ """Reverse the object or return an iterator the iterates over it the other
+ way round.
+ """
+ if isinstance(value, basestring):
+ return value[::-1]
+ try:
+ return reversed(value)
+ except TypeError:
+ try:
+ rv = list(value)
+ rv.reverse()
+ return rv
+ except TypeError:
+ raise FilterArgumentError('argument must be iterable')
+
+
+@environmentfilter
+def do_attr(environment, obj, name):
+ """Get an attribute of an object. ``foo|attr("bar")`` works like
+ ``foo["bar"]`` just that always an attribute is returned and items are not
+ looked up.
+
+ See :ref:`Notes on subscriptions <notes-on-subscriptions>` for more details.
+ """
+ try:
+ name = str(name)
+ except UnicodeError:
+ pass
+ else:
+ try:
+ value = getattr(obj, name)
+ except AttributeError:
+ pass
+ else:
+ if environment.sandboxed and not \
+ environment.is_safe_attribute(obj, name, value):
+ return environment.unsafe_undefined(obj, name)
+ return value
+ return environment.undefined(obj=obj, name=name)
+
+
+FILTERS = {
+ 'attr': do_attr,
+ 'replace': do_replace,
+ 'upper': do_upper,
+ 'lower': do_lower,
+ 'escape': escape,
+ 'e': escape,
+ 'forceescape': do_forceescape,
+ 'capitalize': do_capitalize,
+ 'title': do_title,
+ 'default': do_default,
+ 'd': do_default,
+ 'join': do_join,
+ 'count': len,
+ 'dictsort': do_dictsort,
+ 'sort': do_sort,
+ 'length': len,
+ 'reverse': do_reverse,
+ 'center': do_center,
+ 'indent': do_indent,
+ 'title': do_title,
+ 'capitalize': do_capitalize,
+ 'first': do_first,
+ 'last': do_last,
+ 'random': do_random,
+ 'filesizeformat': do_filesizeformat,
+ 'pprint': do_pprint,
+ 'truncate': do_truncate,
+ 'wordwrap': do_wordwrap,
+ 'wordcount': do_wordcount,
+ 'int': do_int,
+ 'float': do_float,
+ 'string': soft_unicode,
+ 'list': do_list,
+ 'urlize': do_urlize,
+ 'format': do_format,
+ 'trim': do_trim,
+ 'striptags': do_striptags,
+ 'slice': do_slice,
+ 'batch': do_batch,
+ 'sum': sum,
+ 'abs': abs,
+ 'round': do_round,
+ 'groupby': do_groupby,
+ 'safe': do_mark_safe,
+ 'xmlattr': do_xmlattr
+}
diff --git a/slider-agent/src/main/python/jinja2/jinja2/lexer.py b/slider-agent/src/main/python/jinja2/jinja2/lexer.py
new file mode 100644
index 0000000..0d3f696
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/lexer.py
@@ -0,0 +1,681 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.lexer
+ ~~~~~~~~~~~~
+
+ This module implements a Jinja / Python combination lexer. The
+ `Lexer` class provided by this module is used to do some preprocessing
+ for Jinja.
+
+ On the one hand it filters out invalid operators like the bitshift
+ operators we don't allow in templates. On the other hand it separates
+ template code and python code in expressions.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import re
+from operator import itemgetter
+from collections import deque
+from jinja2.exceptions import TemplateSyntaxError
+from jinja2.utils import LRUCache, next
+
+
+# cache for the lexers. Exists in order to be able to have multiple
+# environments with the same lexer
+_lexer_cache = LRUCache(50)
+
+# static regular expressions
+whitespace_re = re.compile(r'\s+', re.U)
+string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
+ r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)
+integer_re = re.compile(r'\d+')
+
+# we use the unicode identifier rule if this python version is able
+# to handle unicode identifiers, otherwise the standard ASCII one.
+try:
+ compile('föö', '<unknown>', 'eval')
+except SyntaxError:
+ name_re = re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b')
+else:
+ from jinja2 import _stringdefs
+ name_re = re.compile(r'[%s][%s]*' % (_stringdefs.xid_start,
+ _stringdefs.xid_continue))
+
+float_re = re.compile(r'(?<!\.)\d+\.\d+')
+newline_re = re.compile(r'(\r\n|\r|\n)')
+
+# internal the tokens and keep references to them
+TOKEN_ADD = intern('add')
+TOKEN_ASSIGN = intern('assign')
+TOKEN_COLON = intern('colon')
+TOKEN_COMMA = intern('comma')
+TOKEN_DIV = intern('div')
+TOKEN_DOT = intern('dot')
+TOKEN_EQ = intern('eq')
+TOKEN_FLOORDIV = intern('floordiv')
+TOKEN_GT = intern('gt')
+TOKEN_GTEQ = intern('gteq')
+TOKEN_LBRACE = intern('lbrace')
+TOKEN_LBRACKET = intern('lbracket')
+TOKEN_LPAREN = intern('lparen')
+TOKEN_LT = intern('lt')
+TOKEN_LTEQ = intern('lteq')
+TOKEN_MOD = intern('mod')
+TOKEN_MUL = intern('mul')
+TOKEN_NE = intern('ne')
+TOKEN_PIPE = intern('pipe')
+TOKEN_POW = intern('pow')
+TOKEN_RBRACE = intern('rbrace')
+TOKEN_RBRACKET = intern('rbracket')
+TOKEN_RPAREN = intern('rparen')
+TOKEN_SEMICOLON = intern('semicolon')
+TOKEN_SUB = intern('sub')
+TOKEN_TILDE = intern('tilde')
+TOKEN_WHITESPACE = intern('whitespace')
+TOKEN_FLOAT = intern('float')
+TOKEN_INTEGER = intern('integer')
+TOKEN_NAME = intern('name')
+TOKEN_STRING = intern('string')
+TOKEN_OPERATOR = intern('operator')
+TOKEN_BLOCK_BEGIN = intern('block_begin')
+TOKEN_BLOCK_END = intern('block_end')
+TOKEN_VARIABLE_BEGIN = intern('variable_begin')
+TOKEN_VARIABLE_END = intern('variable_end')
+TOKEN_RAW_BEGIN = intern('raw_begin')
+TOKEN_RAW_END = intern('raw_end')
+TOKEN_COMMENT_BEGIN = intern('comment_begin')
+TOKEN_COMMENT_END = intern('comment_end')
+TOKEN_COMMENT = intern('comment')
+TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin')
+TOKEN_LINESTATEMENT_END = intern('linestatement_end')
+TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin')
+TOKEN_LINECOMMENT_END = intern('linecomment_end')
+TOKEN_LINECOMMENT = intern('linecomment')
+TOKEN_DATA = intern('data')
+TOKEN_INITIAL = intern('initial')
+TOKEN_EOF = intern('eof')
+
+# bind operators to token types
+operators = {
+ '+': TOKEN_ADD,
+ '-': TOKEN_SUB,
+ '/': TOKEN_DIV,
+ '//': TOKEN_FLOORDIV,
+ '*': TOKEN_MUL,
+ '%': TOKEN_MOD,
+ '**': TOKEN_POW,
+ '~': TOKEN_TILDE,
+ '[': TOKEN_LBRACKET,
+ ']': TOKEN_RBRACKET,
+ '(': TOKEN_LPAREN,
+ ')': TOKEN_RPAREN,
+ '{': TOKEN_LBRACE,
+ '}': TOKEN_RBRACE,
+ '==': TOKEN_EQ,
+ '!=': TOKEN_NE,
+ '>': TOKEN_GT,
+ '>=': TOKEN_GTEQ,
+ '<': TOKEN_LT,
+ '<=': TOKEN_LTEQ,
+ '=': TOKEN_ASSIGN,
+ '.': TOKEN_DOT,
+ ':': TOKEN_COLON,
+ '|': TOKEN_PIPE,
+ ',': TOKEN_COMMA,
+ ';': TOKEN_SEMICOLON
+}
+
+reverse_operators = dict([(v, k) for k, v in operators.iteritems()])
+assert len(operators) == len(reverse_operators), 'operators dropped'
+operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
+ sorted(operators, key=lambda x: -len(x))))
+
+ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT,
+ TOKEN_COMMENT_END, TOKEN_WHITESPACE,
+ TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN,
+ TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT])
+ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA,
+ TOKEN_COMMENT, TOKEN_LINECOMMENT])
+
+
+def _describe_token_type(token_type):
+ if token_type in reverse_operators:
+ return reverse_operators[token_type]
+ return {
+ TOKEN_COMMENT_BEGIN: 'begin of comment',
+ TOKEN_COMMENT_END: 'end of comment',
+ TOKEN_COMMENT: 'comment',
+ TOKEN_LINECOMMENT: 'comment',
+ TOKEN_BLOCK_BEGIN: 'begin of statement block',
+ TOKEN_BLOCK_END: 'end of statement block',
+ TOKEN_VARIABLE_BEGIN: 'begin of print statement',
+ TOKEN_VARIABLE_END: 'end of print statement',
+ TOKEN_LINESTATEMENT_BEGIN: 'begin of line statement',
+ TOKEN_LINESTATEMENT_END: 'end of line statement',
+ TOKEN_DATA: 'template data / text',
+ TOKEN_EOF: 'end of template'
+ }.get(token_type, token_type)
+
+
+def describe_token(token):
+ """Returns a description of the token."""
+ if token.type == 'name':
+ return token.value
+ return _describe_token_type(token.type)
+
+
+def describe_token_expr(expr):
+ """Like `describe_token` but for token expressions."""
+ if ':' in expr:
+ type, value = expr.split(':', 1)
+ if type == 'name':
+ return value
+ else:
+ type = expr
+ return _describe_token_type(type)
+
+
+def count_newlines(value):
+ """Count the number of newline characters in the string. This is
+ useful for extensions that filter a stream.
+ """
+ return len(newline_re.findall(value))
+
+
+def compile_rules(environment):
+ """Compiles all the rules from the environment into a list of rules."""
+ e = re.escape
+ rules = [
+ (len(environment.comment_start_string), 'comment',
+ e(environment.comment_start_string)),
+ (len(environment.block_start_string), 'block',
+ e(environment.block_start_string)),
+ (len(environment.variable_start_string), 'variable',
+ e(environment.variable_start_string))
+ ]
+
+ if environment.line_statement_prefix is not None:
+ rules.append((len(environment.line_statement_prefix), 'linestatement',
+ r'^\s*' + e(environment.line_statement_prefix)))
+ if environment.line_comment_prefix is not None:
+ rules.append((len(environment.line_comment_prefix), 'linecomment',
+ r'(?:^|(?<=\S))[^\S\r\n]*' +
+ e(environment.line_comment_prefix)))
+
+ return [x[1:] for x in sorted(rules, reverse=True)]
+
+
+class Failure(object):
+ """Class that raises a `TemplateSyntaxError` if called.
+ Used by the `Lexer` to specify known errors.
+ """
+
+ def __init__(self, message, cls=TemplateSyntaxError):
+ self.message = message
+ self.error_class = cls
+
+ def __call__(self, lineno, filename):
+ raise self.error_class(self.message, lineno, filename)
+
+
+class Token(tuple):
+ """Token class."""
+ __slots__ = ()
+ lineno, type, value = (property(itemgetter(x)) for x in range(3))
+
+ def __new__(cls, lineno, type, value):
+ return tuple.__new__(cls, (lineno, intern(str(type)), value))
+
+ def __str__(self):
+ if self.type in reverse_operators:
+ return reverse_operators[self.type]
+ elif self.type == 'name':
+ return self.value
+ return self.type
+
+ def test(self, expr):
+ """Test a token against a token expression. This can either be a
+ token type or ``'token_type:token_value'``. This can only test
+ against string values and types.
+ """
+ # here we do a regular string equality check as test_any is usually
+ # passed an iterable of not interned strings.
+ if self.type == expr:
+ return True
+ elif ':' in expr:
+ return expr.split(':', 1) == [self.type, self.value]
+ return False
+
+ def test_any(self, *iterable):
+ """Test against multiple token expressions."""
+ for expr in iterable:
+ if self.test(expr):
+ return True
+ return False
+
+ def __repr__(self):
+ return 'Token(%r, %r, %r)' % (
+ self.lineno,
+ self.type,
+ self.value
+ )
+
+
+class TokenStreamIterator(object):
+ """The iterator for tokenstreams. Iterate over the stream
+ until the eof token is reached.
+ """
+
+ def __init__(self, stream):
+ self.stream = stream
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ token = self.stream.current
+ if token.type is TOKEN_EOF:
+ self.stream.close()
+ raise StopIteration()
+ next(self.stream)
+ return token
+
+
+class TokenStream(object):
+ """A token stream is an iterable that yields :class:`Token`\s. The
+ parser however does not iterate over it but calls :meth:`next` to go
+ one token ahead. The current active token is stored as :attr:`current`.
+ """
+
+ def __init__(self, generator, name, filename):
+ self._next = iter(generator).next
+ self._pushed = deque()
+ self.name = name
+ self.filename = filename
+ self.closed = False
+ self.current = Token(1, TOKEN_INITIAL, '')
+ next(self)
+
+ def __iter__(self):
+ return TokenStreamIterator(self)
+
+ def __nonzero__(self):
+ return bool(self._pushed) or self.current.type is not TOKEN_EOF
+
+ eos = property(lambda x: not x, doc="Are we at the end of the stream?")
+
+ def push(self, token):
+ """Push a token back to the stream."""
+ self._pushed.append(token)
+
+ def look(self):
+ """Look at the next token."""
+ old_token = next(self)
+ result = self.current
+ self.push(result)
+ self.current = old_token
+ return result
+
+ def skip(self, n=1):
+ """Got n tokens ahead."""
+ for x in xrange(n):
+ next(self)
+
+ def next_if(self, expr):
+ """Perform the token test and return the token if it matched.
+ Otherwise the return value is `None`.
+ """
+ if self.current.test(expr):
+ return next(self)
+
+ def skip_if(self, expr):
+ """Like :meth:`next_if` but only returns `True` or `False`."""
+ return self.next_if(expr) is not None
+
+ def next(self):
+ """Go one token ahead and return the old one"""
+ rv = self.current
+ if self._pushed:
+ self.current = self._pushed.popleft()
+ elif self.current.type is not TOKEN_EOF:
+ try:
+ self.current = self._next()
+ except StopIteration:
+ self.close()
+ return rv
+
+ def close(self):
+ """Close the stream."""
+ self.current = Token(self.current.lineno, TOKEN_EOF, '')
+ self._next = None
+ self.closed = True
+
+ def expect(self, expr):
+ """Expect a given token type and return it. This accepts the same
+ argument as :meth:`jinja2.lexer.Token.test`.
+ """
+ if not self.current.test(expr):
+ expr = describe_token_expr(expr)
+ if self.current.type is TOKEN_EOF:
+ raise TemplateSyntaxError('unexpected end of template, '
+ 'expected %r.' % expr,
+ self.current.lineno,
+ self.name, self.filename)
+ raise TemplateSyntaxError("expected token %r, got %r" %
+ (expr, describe_token(self.current)),
+ self.current.lineno,
+ self.name, self.filename)
+ try:
+ return self.current
+ finally:
+ next(self)
+
+
+def get_lexer(environment):
+ """Return a lexer which is probably cached."""
+ key = (environment.block_start_string,
+ environment.block_end_string,
+ environment.variable_start_string,
+ environment.variable_end_string,
+ environment.comment_start_string,
+ environment.comment_end_string,
+ environment.line_statement_prefix,
+ environment.line_comment_prefix,
+ environment.trim_blocks,
+ environment.newline_sequence)
+ lexer = _lexer_cache.get(key)
+ if lexer is None:
+ lexer = Lexer(environment)
+ _lexer_cache[key] = lexer
+ return lexer
+
+
+class Lexer(object):
+ """Class that implements a lexer for a given environment. Automatically
+ created by the environment class, usually you don't have to do that.
+
+ Note that the lexer is not automatically bound to an environment.
+ Multiple environments can share the same lexer.
+ """
+
+ def __init__(self, environment):
+ # shortcuts
+ c = lambda x: re.compile(x, re.M | re.S)
+ e = re.escape
+
+ # lexing rules for tags
+ tag_rules = [
+ (whitespace_re, TOKEN_WHITESPACE, None),
+ (float_re, TOKEN_FLOAT, None),
+ (integer_re, TOKEN_INTEGER, None),
+ (name_re, TOKEN_NAME, None),
+ (string_re, TOKEN_STRING, None),
+ (operator_re, TOKEN_OPERATOR, None)
+ ]
+
+ # assamble the root lexing rule. because "|" is ungreedy
+ # we have to sort by length so that the lexer continues working
+ # as expected when we have parsing rules like <% for block and
+ # <%= for variables. (if someone wants asp like syntax)
+ # variables are just part of the rules if variable processing
+ # is required.
+ root_tag_rules = compile_rules(environment)
+
+ # block suffix if trimming is enabled
+ block_suffix_re = environment.trim_blocks and '\\n?' or ''
+
+ self.newline_sequence = environment.newline_sequence
+
+ # global lexing rules
+ self.rules = {
+ 'root': [
+ # directives
+ (c('(.*?)(?:%s)' % '|'.join(
+ [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % (
+ e(environment.block_start_string),
+ e(environment.block_start_string),
+ e(environment.block_end_string),
+ e(environment.block_end_string)
+ )] + [
+ r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, r)
+ for n, r in root_tag_rules
+ ])), (TOKEN_DATA, '#bygroup'), '#bygroup'),
+ # data
+ (c('.+'), TOKEN_DATA, None)
+ ],
+ # comments
+ TOKEN_COMMENT_BEGIN: [
+ (c(r'(.*?)((?:\-%s\s*|%s)%s)' % (
+ e(environment.comment_end_string),
+ e(environment.comment_end_string),
+ block_suffix_re
+ )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'),
+ (c('(.)'), (Failure('Missing end of comment tag'),), None)
+ ],
+ # blocks
+ TOKEN_BLOCK_BEGIN: [
+ (c('(?:\-%s\s*|%s)%s' % (
+ e(environment.block_end_string),
+ e(environment.block_end_string),
+ block_suffix_re
+ )), TOKEN_BLOCK_END, '#pop'),
+ ] + tag_rules,
+ # variables
+ TOKEN_VARIABLE_BEGIN: [
+ (c('\-%s\s*|%s' % (
+ e(environment.variable_end_string),
+ e(environment.variable_end_string)
+ )), TOKEN_VARIABLE_END, '#pop')
+ ] + tag_rules,
+ # raw block
+ TOKEN_RAW_BEGIN: [
+ (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % (
+ e(environment.block_start_string),
+ e(environment.block_start_string),
+ e(environment.block_end_string),
+ e(environment.block_end_string),
+ block_suffix_re
+ )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'),
+ (c('(.)'), (Failure('Missing end of raw directive'),), None)
+ ],
+ # line statements
+ TOKEN_LINESTATEMENT_BEGIN: [
+ (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop')
+ ] + tag_rules,
+ # line comments
+ TOKEN_LINECOMMENT_BEGIN: [
+ (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT,
+ TOKEN_LINECOMMENT_END), '#pop')
+ ]
+ }
+
+ def _normalize_newlines(self, value):
+ """Called for strings and template data to normlize it to unicode."""
+ return newline_re.sub(self.newline_sequence, value)
+
+ def tokenize(self, source, name=None, filename=None, state=None):
+ """Calls tokeniter + tokenize and wraps it in a token stream.
+ """
+ stream = self.tokeniter(source, name, filename, state)
+ return TokenStream(self.wrap(stream, name, filename), name, filename)
+
+ def wrap(self, stream, name=None, filename=None):
+ """This is called with the stream as returned by `tokenize` and wraps
+ every token in a :class:`Token` and converts the value.
+ """
+ for lineno, token, value in stream:
+ if token in ignored_tokens:
+ continue
+ elif token == 'linestatement_begin':
+ token = 'block_begin'
+ elif token == 'linestatement_end':
+ token = 'block_end'
+ # we are not interested in those tokens in the parser
+ elif token in ('raw_begin', 'raw_end'):
+ continue
+ elif token == 'data':
+ value = self._normalize_newlines(value)
+ elif token == 'keyword':
+ token = value
+ elif token == 'name':
+ value = str(value)
+ elif token == 'string':
+ # try to unescape string
+ try:
+ value = self._normalize_newlines(value[1:-1]) \
+ .encode('ascii', 'backslashreplace') \
+ .decode('unicode-escape')
+ except Exception, e:
+ msg = str(e).split(':')[-1].strip()
+ raise TemplateSyntaxError(msg, lineno, name, filename)
+ # if we can express it as bytestring (ascii only)
+ # we do that for support of semi broken APIs
+ # as datetime.datetime.strftime. On python 3 this
+ # call becomes a noop thanks to 2to3
+ try:
+ value = str(value)
+ except UnicodeError:
+ pass
+ elif token == 'integer':
+ value = int(value)
+ elif token == 'float':
+ value = float(value)
+ elif token == 'operator':
+ token = operators[value]
+ yield Token(lineno, token, value)
+
+ def tokeniter(self, source, name, filename=None, state=None):
+ """This method tokenizes the text and returns the tokens in a
+ generator. Use this method if you just want to tokenize a template.
+ """
+ source = '\n'.join(unicode(source).splitlines())
+ pos = 0
+ lineno = 1
+ stack = ['root']
+ if state is not None and state != 'root':
+ assert state in ('variable', 'block'), 'invalid state'
+ stack.append(state + '_begin')
+ else:
+ state = 'root'
+ statetokens = self.rules[stack[-1]]
+ source_length = len(source)
+
+ balancing_stack = []
+
+ while 1:
+ # tokenizer loop
+ for regex, tokens, new_state in statetokens:
+ m = regex.match(source, pos)
+ # if no match we try again with the next rule
+ if m is None:
+ continue
+
+ # we only match blocks and variables if brances / parentheses
+ # are balanced. continue parsing with the lower rule which
+ # is the operator rule. do this only if the end tags look
+ # like operators
+ if balancing_stack and \
+ tokens in ('variable_end', 'block_end',
+ 'linestatement_end'):
+ continue
+
+ # tuples support more options
+ if isinstance(tokens, tuple):
+ for idx, token in enumerate(tokens):
+ # failure group
+ if token.__class__ is Failure:
+ raise token(lineno, filename)
+ # bygroup is a bit more complex, in that case we
+ # yield for the current token the first named
+ # group that matched
+ elif token == '#bygroup':
+ for key, value in m.groupdict().iteritems():
+ if value is not None:
+ yield lineno, key, value
+ lineno += value.count('\n')
+ break
+ else:
+ raise RuntimeError('%r wanted to resolve '
+ 'the token dynamically'
+ ' but no group matched'
+ % regex)
+ # normal group
+ else:
+ data = m.group(idx + 1)
+ if data or token not in ignore_if_empty:
+ yield lineno, token, data
+ lineno += data.count('\n')
+
+ # strings as token just are yielded as it.
+ else:
+ data = m.group()
+ # update brace/parentheses balance
+ if tokens == 'operator':
+ if data == '{':
+ balancing_stack.append('}')
+ elif data == '(':
+ balancing_stack.append(')')
+ elif data == '[':
+ balancing_stack.append(']')
+ elif data in ('}', ')', ']'):
+ if not balancing_stack:
+ raise TemplateSyntaxError('unexpected \'%s\'' %
+ data, lineno, name,
+ filename)
+ expected_op = balancing_stack.pop()
+ if expected_op != data:
+ raise TemplateSyntaxError('unexpected \'%s\', '
+ 'expected \'%s\'' %
+ (data, expected_op),
+ lineno, name,
+ filename)
+ # yield items
+ if data or tokens not in ignore_if_empty:
+ yield lineno, tokens, data
+ lineno += data.count('\n')
+
+ # fetch new position into new variable so that we can check
+ # if there is a internal parsing error which would result
+ # in an infinite loop
+ pos2 = m.end()
+
+ # handle state changes
+ if new_state is not None:
+ # remove the uppermost state
+ if new_state == '#pop':
+ stack.pop()
+ # resolve the new state by group checking
+ elif new_state == '#bygroup':
+ for key, value in m.groupdict().iteritems():
+ if value is not None:
+ stack.append(key)
+ break
+ else:
+ raise RuntimeError('%r wanted to resolve the '
+ 'new state dynamically but'
+ ' no group matched' %
+ regex)
+ # direct state name given
+ else:
+ stack.append(new_state)
+ statetokens = self.rules[stack[-1]]
+ # we are still at the same position and no stack change.
+ # this means a loop without break condition, avoid that and
+ # raise error
+ elif pos2 == pos:
+ raise RuntimeError('%r yielded empty string without '
+ 'stack change' % regex)
+ # publish new function and start again
+ pos = pos2
+ break
+ # if loop terminated without break we havn't found a single match
+ # either we are at the end of the file or we have a problem
+ else:
+ # end of text
+ if pos >= source_length:
+ return
+ # something went wrong
+ raise TemplateSyntaxError('unexpected char %r at %d' %
+ (source[pos], pos), lineno,
+ name, filename)
diff --git a/slider-agent/src/main/python/jinja2/jinja2/loaders.py b/slider-agent/src/main/python/jinja2/jinja2/loaders.py
new file mode 100644
index 0000000..bd435e8
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/loaders.py
@@ -0,0 +1,449 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.loaders
+ ~~~~~~~~~~~~~~
+
+ Jinja loader classes.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import sys
+import weakref
+from types import ModuleType
+from os import path
+try:
+ from hashlib import sha1
+except ImportError:
+ from sha import new as sha1
+from jinja2.exceptions import TemplateNotFound
+from jinja2.utils import LRUCache, open_if_exists, internalcode
+
+
+def split_template_path(template):
+ """Split a path into segments and perform a sanity check. If it detects
+ '..' in the path it will raise a `TemplateNotFound` error.
+ """
+ pieces = []
+ for piece in template.split('/'):
+ if path.sep in piece \
+ or (path.altsep and path.altsep in piece) or \
+ piece == path.pardir:
+ raise TemplateNotFound(template)
+ elif piece and piece != '.':
+ pieces.append(piece)
+ return pieces
+
+
+class BaseLoader(object):
+ """Baseclass for all loaders. Subclass this and override `get_source` to
+ implement a custom loading mechanism. The environment provides a
+ `get_template` method that calls the loader's `load` method to get the
+ :class:`Template` object.
+
+ A very basic example for a loader that looks up templates on the file
+ system could look like this::
+
+ from jinja2 import BaseLoader, TemplateNotFound
+ from os.path import join, exists, getmtime
+
+ class MyLoader(BaseLoader):
+
+ def __init__(self, path):
+ self.path = path
+
+ def get_source(self, environment, template):
+ path = join(self.path, template)
+ if not exists(path):
+ raise TemplateNotFound(template)
+ mtime = getmtime(path)
+ with file(path) as f:
+ source = f.read().decode('utf-8')
+ return source, path, lambda: mtime == getmtime(path)
+ """
+
+ #: if set to `False` it indicates that the loader cannot provide access
+ #: to the source of templates.
+ #:
+ #: .. versionadded:: 2.4
+ has_source_access = True
+
+ def get_source(self, environment, template):
+ """Get the template source, filename and reload helper for a template.
+ It's passed the environment and template name and has to return a
+ tuple in the form ``(source, filename, uptodate)`` or raise a
+ `TemplateNotFound` error if it can't locate the template.
+
+ The source part of the returned tuple must be the source of the
+ template as unicode string or a ASCII bytestring. The filename should
+ be the name of the file on the filesystem if it was loaded from there,
+ otherwise `None`. The filename is used by python for the tracebacks
+ if no loader extension is used.
+
+ The last item in the tuple is the `uptodate` function. If auto
+ reloading is enabled it's always called to check if the template
+ changed. No arguments are passed so the function must store the
+ old state somewhere (for example in a closure). If it returns `False`
+ the template will be reloaded.
+ """
+ if not self.has_source_access:
+ raise RuntimeError('%s cannot provide access to the source' %
+ self.__class__.__name__)
+ raise TemplateNotFound(template)
+
+ def list_templates(self):
+ """Iterates over all templates. If the loader does not support that
+ it should raise a :exc:`TypeError` which is the default behavior.
+ """
+ raise TypeError('this loader cannot iterate over all templates')
+
+ @internalcode
+ def load(self, environment, name, globals=None):
+ """Loads a template. This method looks up the template in the cache
+ or loads one by calling :meth:`get_source`. Subclasses should not
+ override this method as loaders working on collections of other
+ loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`)
+ will not call this method but `get_source` directly.
+ """
+ code = None
+ if globals is None:
+ globals = {}
+
+ # first we try to get the source for this template together
+ # with the filename and the uptodate function.
+ source, filename, uptodate = self.get_source(environment, name)
+
+ # try to load the code from the bytecode cache if there is a
+ # bytecode cache configured.
+ bcc = environment.bytecode_cache
+ if bcc is not None:
+ bucket = bcc.get_bucket(environment, name, filename, source)
+ code = bucket.code
+
+ # if we don't have code so far (not cached, no longer up to
+ # date) etc. we compile the template
+ if code is None:
+ code = environment.compile(source, name, filename)
+
+ # if the bytecode cache is available and the bucket doesn't
+ # have a code so far, we give the bucket the new code and put
+ # it back to the bytecode cache.
+ if bcc is not None and bucket.code is None:
+ bucket.code = code
+ bcc.set_bucket(bucket)
+
+ return environment.template_class.from_code(environment, code,
+ globals, uptodate)
+
+
+class FileSystemLoader(BaseLoader):
+ """Loads templates from the file system. This loader can find templates
+ in folders on the file system and is the preferred way to load them.
+
+ The loader takes the path to the templates as string, or if multiple
+ locations are wanted a list of them which is then looked up in the
+ given order:
+
+ >>> loader = FileSystemLoader('/path/to/templates')
+ >>> loader = FileSystemLoader(['/path/to/templates', '/other/path'])
+
+ Per default the template encoding is ``'utf-8'`` which can be changed
+ by setting the `encoding` parameter to something else.
+ """
+
+ def __init__(self, searchpath, encoding='utf-8'):
+ if isinstance(searchpath, basestring):
+ searchpath = [searchpath]
+ self.searchpath = list(searchpath)
+ self.encoding = encoding
+
+ def get_source(self, environment, template):
+ pieces = split_template_path(template)
+ for searchpath in self.searchpath:
+ filename = path.join(searchpath, *pieces)
+ f = open_if_exists(filename)
+ if f is None:
+ continue
+ try:
+ contents = f.read().decode(self.encoding)
+ finally:
+ f.close()
+
+ mtime = path.getmtime(filename)
+ def uptodate():
+ try:
+ return path.getmtime(filename) == mtime
+ except OSError:
+ return False
+ return contents, filename, uptodate
+ raise TemplateNotFound(template)
+
+ def list_templates(self):
+ found = set()
+ for searchpath in self.searchpath:
+ for dirpath, dirnames, filenames in os.walk(searchpath):
+ for filename in filenames:
+ template = os.path.join(dirpath, filename) \
+ [len(searchpath):].strip(os.path.sep) \
+ .replace(os.path.sep, '/')
+ if template[:2] == './':
+ template = template[2:]
+ if template not in found:
+ found.add(template)
+ return sorted(found)
+
+
+class PackageLoader(BaseLoader):
+ """Load templates from python eggs or packages. It is constructed with
+ the name of the python package and the path to the templates in that
+ package::
+
+ loader = PackageLoader('mypackage', 'views')
+
+ If the package path is not given, ``'templates'`` is assumed.
+
+ Per default the template encoding is ``'utf-8'`` which can be changed
+ by setting the `encoding` parameter to something else. Due to the nature
+ of eggs it's only possible to reload templates if the package was loaded
+ from the file system and not a zip file.
+ """
+
+ def __init__(self, package_name, package_path='templates',
+ encoding='utf-8'):
+ from pkg_resources import DefaultProvider, ResourceManager, \
+ get_provider
+ provider = get_provider(package_name)
+ self.encoding = encoding
+ self.manager = ResourceManager()
+ self.filesystem_bound = isinstance(provider, DefaultProvider)
+ self.provider = provider
+ self.package_path = package_path
+
+ def get_source(self, environment, template):
+ pieces = split_template_path(template)
+ p = '/'.join((self.package_path,) + tuple(pieces))
+ if not self.provider.has_resource(p):
+ raise TemplateNotFound(template)
+
+ filename = uptodate = None
+ if self.filesystem_bound:
+ filename = self.provider.get_resource_filename(self.manager, p)
+ mtime = path.getmtime(filename)
+ def uptodate():
+ try:
+ return path.getmtime(filename) == mtime
+ except OSError:
+ return False
+
+ source = self.provider.get_resource_string(self.manager, p)
+ return source.decode(self.encoding), filename, uptodate
+
+ def list_templates(self):
+ path = self.package_path
+ if path[:2] == './':
+ path = path[2:]
+ elif path == '.':
+ path = ''
+ offset = len(path)
+ results = []
+ def _walk(path):
+ for filename in self.provider.resource_listdir(path):
+ fullname = path + '/' + filename
+ if self.provider.resource_isdir(fullname):
+ for item in _walk(fullname):
+ results.append(item)
+ else:
+ results.append(fullname[offset:].lstrip('/'))
+ _walk(path)
+ results.sort()
+ return results
+
+
+class DictLoader(BaseLoader):
+ """Loads a template from a python dict. It's passed a dict of unicode
+ strings bound to template names. This loader is useful for unittesting:
+
+ >>> loader = DictLoader({'index.html': 'source here'})
+
+ Because auto reloading is rarely useful this is disabled per default.
+ """
+
+ def __init__(self, mapping):
+ self.mapping = mapping
+
+ def get_source(self, environment, template):
+ if template in self.mapping:
+ source = self.mapping[template]
+ return source, None, lambda: source != self.mapping.get(template)
+ raise TemplateNotFound(template)
+
+ def list_templates(self):
+ return sorted(self.mapping)
+
+
+class FunctionLoader(BaseLoader):
+ """A loader that is passed a function which does the loading. The
+ function becomes the name of the template passed and has to return either
+ an unicode string with the template source, a tuple in the form ``(source,
+ filename, uptodatefunc)`` or `None` if the template does not exist.
+
+ >>> def load_template(name):
+ ... if name == 'index.html':
+ ... return '...'
+ ...
+ >>> loader = FunctionLoader(load_template)
+
+ The `uptodatefunc` is a function that is called if autoreload is enabled
+ and has to return `True` if the template is still up to date. For more
+ details have a look at :meth:`BaseLoader.get_source` which has the same
+ return value.
+ """
+
+ def __init__(self, load_func):
+ self.load_func = load_func
+
+ def get_source(self, environment, template):
+ rv = self.load_func(template)
+ if rv is None:
+ raise TemplateNotFound(template)
+ elif isinstance(rv, basestring):
+ return rv, None, None
+ return rv
+
+
+class PrefixLoader(BaseLoader):
+ """A loader that is passed a dict of loaders where each loader is bound
+ to a prefix. The prefix is delimited from the template by a slash per
+ default, which can be changed by setting the `delimiter` argument to
+ something else::
+
+ loader = PrefixLoader({
+ 'app1': PackageLoader('mypackage.app1'),
+ 'app2': PackageLoader('mypackage.app2')
+ })
+
+ By loading ``'app1/index.html'`` the file from the app1 package is loaded,
+ by loading ``'app2/index.html'`` the file from the second.
+ """
+
+ def __init__(self, mapping, delimiter='/'):
+ self.mapping = mapping
+ self.delimiter = delimiter
+
+ def get_source(self, environment, template):
+ try:
+ prefix, name = template.split(self.delimiter, 1)
+ loader = self.mapping[prefix]
+ except (ValueError, KeyError):
+ raise TemplateNotFound(template)
+ try:
+ return loader.get_source(environment, name)
+ except TemplateNotFound:
+ # re-raise the exception with the correct fileame here.
+ # (the one that includes the prefix)
+ raise TemplateNotFound(template)
+
+ def list_templates(self):
+ result = []
+ for prefix, loader in self.mapping.iteritems():
+ for template in loader.list_templates():
+ result.append(prefix + self.delimiter + template)
+ return result
+
+
+class ChoiceLoader(BaseLoader):
+ """This loader works like the `PrefixLoader` just that no prefix is
+ specified. If a template could not be found by one loader the next one
+ is tried.
+
+ >>> loader = ChoiceLoader([
+ ... FileSystemLoader('/path/to/user/templates'),
+ ... FileSystemLoader('/path/to/system/templates')
+ ... ])
+
+ This is useful if you want to allow users to override builtin templates
+ from a different location.
+ """
+
+ def __init__(self, loaders):
+ self.loaders = loaders
+
+ def get_source(self, environment, template):
+ for loader in self.loaders:
+ try:
+ return loader.get_source(environment, template)
+ except TemplateNotFound:
+ pass
+ raise TemplateNotFound(template)
+
+ def list_templates(self):
+ found = set()
+ for loader in self.loaders:
+ found.update(loader.list_templates())
+ return sorted(found)
+
+
+class _TemplateModule(ModuleType):
+ """Like a normal module but with support for weak references"""
+
+
+class ModuleLoader(BaseLoader):
+ """This loader loads templates from precompiled templates.
+
+ Example usage:
+
+ >>> loader = ChoiceLoader([
+ ... ModuleLoader('/path/to/compiled/templates'),
+ ... FileSystemLoader('/path/to/templates')
+ ... ])
+ """
+
+ has_source_access = False
+
+ def __init__(self, path):
+ package_name = '_jinja2_module_templates_%x' % id(self)
+
+ # create a fake module that looks for the templates in the
+ # path given.
+ mod = _TemplateModule(package_name)
+ if isinstance(path, basestring):
+ path = [path]
+ else:
+ path = list(path)
+ mod.__path__ = path
+
+ sys.modules[package_name] = weakref.proxy(mod,
+ lambda x: sys.modules.pop(package_name, None))
+
+ # the only strong reference, the sys.modules entry is weak
+ # so that the garbage collector can remove it once the
+ # loader that created it goes out of business.
+ self.module = mod
+ self.package_name = package_name
+
+ @staticmethod
+ def get_template_key(name):
+ return 'tmpl_' + sha1(name.encode('utf-8')).hexdigest()
+
+ @staticmethod
+ def get_module_filename(name):
+ return ModuleLoader.get_template_key(name) + '.py'
+
+ @internalcode
+ def load(self, environment, name, globals=None):
+ key = self.get_template_key(name)
+ module = '%s.%s' % (self.package_name, key)
+ mod = getattr(self.module, module, None)
+ if mod is None:
+ try:
+ mod = __import__(module, None, None, ['root'])
+ except ImportError:
+ raise TemplateNotFound(name)
+
+ # remove the entry from sys.modules, we only want the attribute
+ # on the module object we have stored on the loader.
+ sys.modules.pop(module, None)
+
+ return environment.template_class.from_module_dict(
+ environment, mod.__dict__, globals)
diff --git a/slider-agent/src/main/python/jinja2/jinja2/meta.py b/slider-agent/src/main/python/jinja2/jinja2/meta.py
new file mode 100644
index 0000000..3a779a5
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/meta.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.meta
+ ~~~~~~~~~~~
+
+ This module implements various functions that exposes information about
+ templates that might be interesting for various kinds of applications.
+
+ :copyright: (c) 2010 by the Jinja Team, see AUTHORS for more details.
+ :license: BSD, see LICENSE for more details.
+"""
+from jinja2 import nodes
+from jinja2.compiler import CodeGenerator
+
+
+class TrackingCodeGenerator(CodeGenerator):
+ """We abuse the code generator for introspection."""
+
+ def __init__(self, environment):
+ CodeGenerator.__init__(self, environment, '<introspection>',
+ '<introspection>')
+ self.undeclared_identifiers = set()
+
+ def write(self, x):
+ """Don't write."""
+
+ def pull_locals(self, frame):
+ """Remember all undeclared identifiers."""
+ self.undeclared_identifiers.update(frame.identifiers.undeclared)
+
+
+def find_undeclared_variables(ast):
+ """Returns a set of all variables in the AST that will be looked up from
+ the context at runtime. Because at compile time it's not known which
+ variables will be used depending on the path the execution takes at
+ runtime, all variables are returned.
+
+ >>> from jinja2 import Environment, meta
+ >>> env = Environment()
+ >>> ast = env.parse('{% set foo = 42 %}{{ bar + foo }}')
+ >>> meta.find_undeclared_variables(ast)
+ set(['bar'])
+
+ .. admonition:: Implementation
+
+ Internally the code generator is used for finding undeclared variables.
+ This is good to know because the code generator might raise a
+ :exc:`TemplateAssertionError` during compilation and as a matter of
+ fact this function can currently raise that exception as well.
+ """
+ codegen = TrackingCodeGenerator(ast.environment)
+ codegen.visit(ast)
+ return codegen.undeclared_identifiers
+
+
+def find_referenced_templates(ast):
+ """Finds all the referenced templates from the AST. This will return an
+ iterator over all the hardcoded template extensions, inclusions and
+ imports. If dynamic inheritance or inclusion is used, `None` will be
+ yielded.
+
+ >>> from jinja2 import Environment, meta
+ >>> env = Environment()
+ >>> ast = env.parse('{% extends "layout.html" %}{% include helper %}')
+ >>> list(meta.find_referenced_templates(ast))
+ ['layout.html', None]
+
+ This function is useful for dependency tracking. For example if you want
+ to rebuild parts of the website after a layout template has changed.
+ """
+ for node in ast.find_all((nodes.Extends, nodes.FromImport, nodes.Import,
+ nodes.Include)):
+ if not isinstance(node.template, nodes.Const):
+ # a tuple with some non consts in there
+ if isinstance(node.template, (nodes.Tuple, nodes.List)):
+ for template_name in node.template.items:
+ # something const, only yield the strings and ignore
+ # non-string consts that really just make no sense
+ if isinstance(template_name, nodes.Const):
+ if isinstance(template_name.value, basestring):
+ yield template_name.value
+ # something dynamic in there
+ else:
+ yield None
+ # something dynamic we don't know about here
+ else:
+ yield None
+ continue
+ # constant is a basestring, direct template name
+ if isinstance(node.template.value, basestring):
+ yield node.template.value
+ # a tuple or list (latter *should* not happen) made of consts,
+ # yield the consts that are strings. We could warn here for
+ # non string values
+ elif isinstance(node, nodes.Include) and \
+ isinstance(node.template.value, (tuple, list)):
+ for template_name in node.template.value:
+ if isinstance(template_name, basestring):
+ yield template_name
+ # something else we don't care about, we could warn here
+ else:
+ yield None
diff --git a/slider-agent/src/main/python/jinja2/jinja2/nodes.py b/slider-agent/src/main/python/jinja2/jinja2/nodes.py
new file mode 100644
index 0000000..6446c70
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/nodes.py
@@ -0,0 +1,901 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.nodes
+ ~~~~~~~~~~~~
+
+ This module implements additional nodes derived from the ast base node.
+
+ It also provides some node tree helper functions like `in_lineno` and
+ `get_nodes` used by the parser and translator in order to normalize
+ python and jinja nodes.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import operator
+from itertools import chain, izip
+from collections import deque
+from jinja2.utils import Markup, MethodType, FunctionType
+
+
+#: the types we support for context functions
+_context_function_types = (FunctionType, MethodType)
+
+
+_binop_to_func = {
+ '*': operator.mul,
+ '/': operator.truediv,
+ '//': operator.floordiv,
+ '**': operator.pow,
+ '%': operator.mod,
+ '+': operator.add,
+ '-': operator.sub
+}
+
+_uaop_to_func = {
+ 'not': operator.not_,
+ '+': operator.pos,
+ '-': operator.neg
+}
+
+_cmpop_to_func = {
+ 'eq': operator.eq,
+ 'ne': operator.ne,
+ 'gt': operator.gt,
+ 'gteq': operator.ge,
+ 'lt': operator.lt,
+ 'lteq': operator.le,
+ 'in': lambda a, b: a in b,
+ 'notin': lambda a, b: a not in b
+}
+
+
+class Impossible(Exception):
+ """Raised if the node could not perform a requested action."""
+
+
+class NodeType(type):
+ """A metaclass for nodes that handles the field and attribute
+ inheritance. fields and attributes from the parent class are
+ automatically forwarded to the child."""
+
+ def __new__(cls, name, bases, d):
+ for attr in 'fields', 'attributes':
+ storage = []
+ storage.extend(getattr(bases[0], attr, ()))
+ storage.extend(d.get(attr, ()))
+ assert len(bases) == 1, 'multiple inheritance not allowed'
+ assert len(storage) == len(set(storage)), 'layout conflict'
+ d[attr] = tuple(storage)
+ d.setdefault('abstract', False)
+ return type.__new__(cls, name, bases, d)
+
+
+class EvalContext(object):
+ """Holds evaluation time information. Custom attributes can be attached
+ to it in extensions.
+ """
+
+ def __init__(self, environment, template_name=None):
+ if callable(environment.autoescape):
+ self.autoescape = environment.autoescape(template_name)
+ else:
+ self.autoescape = environment.autoescape
+ self.volatile = False
+
+ def save(self):
+ return self.__dict__.copy()
+
+ def revert(self, old):
+ self.__dict__.clear()
+ self.__dict__.update(old)
+
+
+def get_eval_context(node, ctx):
+ if ctx is None:
+ if node.environment is None:
+ raise RuntimeError('if no eval context is passed, the '
+ 'node must have an attached '
+ 'environment.')
+ return EvalContext(node.environment)
+ return ctx
+
+
+class Node(object):
+ """Baseclass for all Jinja2 nodes. There are a number of nodes available
+ of different types. There are three major types:
+
+ - :class:`Stmt`: statements
+ - :class:`Expr`: expressions
+ - :class:`Helper`: helper nodes
+ - :class:`Template`: the outermost wrapper node
+
+ All nodes have fields and attributes. Fields may be other nodes, lists,
+ or arbitrary values. Fields are passed to the constructor as regular
+ positional arguments, attributes as keyword arguments. Each node has
+ two attributes: `lineno` (the line number of the node) and `environment`.
+ The `environment` attribute is set at the end of the parsing process for
+ all nodes automatically.
+ """
+ __metaclass__ = NodeType
+ fields = ()
+ attributes = ('lineno', 'environment')
+ abstract = True
+
+ def __init__(self, *fields, **attributes):
+ if self.abstract:
+ raise TypeError('abstract nodes are not instanciable')
+ if fields:
+ if len(fields) != len(self.fields):
+ if not self.fields:
+ raise TypeError('%r takes 0 arguments' %
+ self.__class__.__name__)
+ raise TypeError('%r takes 0 or %d argument%s' % (
+ self.__class__.__name__,
+ len(self.fields),
+ len(self.fields) != 1 and 's' or ''
+ ))
+ for name, arg in izip(self.fields, fields):
+ setattr(self, name, arg)
+ for attr in self.attributes:
+ setattr(self, attr, attributes.pop(attr, None))
+ if attributes:
+ raise TypeError('unknown attribute %r' %
+ iter(attributes).next())
+
+ def iter_fields(self, exclude=None, only=None):
+ """This method iterates over all fields that are defined and yields
+ ``(key, value)`` tuples. Per default all fields are returned, but
+ it's possible to limit that to some fields by providing the `only`
+ parameter or to exclude some using the `exclude` parameter. Both
+ should be sets or tuples of field names.
+ """
+ for name in self.fields:
+ if (exclude is only is None) or \
+ (exclude is not None and name not in exclude) or \
+ (only is not None and name in only):
+ try:
+ yield name, getattr(self, name)
+ except AttributeError:
+ pass
+
+ def iter_child_nodes(self, exclude=None, only=None):
+ """Iterates over all direct child nodes of the node. This iterates
+ over all fields and yields the values of they are nodes. If the value
+ of a field is a list all the nodes in that list are returned.
+ """
+ for field, item in self.iter_fields(exclude, only):
+ if isinstance(item, list):
+ for n in item:
+ if isinstance(n, Node):
+ yield n
+ elif isinstance(item, Node):
+ yield item
+
+ def find(self, node_type):
+ """Find the first node of a given type. If no such node exists the
+ return value is `None`.
+ """
+ for result in self.find_all(node_type):
+ return result
+
+ def find_all(self, node_type):
+ """Find all the nodes of a given type. If the type is a tuple,
+ the check is performed for any of the tuple items.
+ """
+ for child in self.iter_child_nodes():
+ if isinstance(child, node_type):
+ yield child
+ for result in child.find_all(node_type):
+ yield result
+
+ def set_ctx(self, ctx):
+ """Reset the context of a node and all child nodes. Per default the
+ parser will all generate nodes that have a 'load' context as it's the
+ most common one. This method is used in the parser to set assignment
+ targets and other nodes to a store context.
+ """
+ todo = deque([self])
+ while todo:
+ node = todo.popleft()
+ if 'ctx' in node.fields:
+ node.ctx = ctx
+ todo.extend(node.iter_child_nodes())
+ return self
+
+ def set_lineno(self, lineno, override=False):
+ """Set the line numbers of the node and children."""
+ todo = deque([self])
+ while todo:
+ node = todo.popleft()
+ if 'lineno' in node.attributes:
+ if node.lineno is None or override:
+ node.lineno = lineno
+ todo.extend(node.iter_child_nodes())
+ return self
+
+ def set_environment(self, environment):
+ """Set the environment for all nodes."""
+ todo = deque([self])
+ while todo:
+ node = todo.popleft()
+ node.environment = environment
+ todo.extend(node.iter_child_nodes())
+ return self
+
+ def __eq__(self, other):
+ return type(self) is type(other) and \
+ tuple(self.iter_fields()) == tuple(other.iter_fields())
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __repr__(self):
+ return '%s(%s)' % (
+ self.__class__.__name__,
+ ', '.join('%s=%r' % (arg, getattr(self, arg, None)) for
+ arg in self.fields)
+ )
+
+
+class Stmt(Node):
+ """Base node for all statements."""
+ abstract = True
+
+
+class Helper(Node):
+ """Nodes that exist in a specific context only."""
+ abstract = True
+
+
+class Template(Node):
+ """Node that represents a template. This must be the outermost node that
+ is passed to the compiler.
+ """
+ fields = ('body',)
+
+
+class Output(Stmt):
+ """A node that holds multiple expressions which are then printed out.
+ This is used both for the `print` statement and the regular template data.
+ """
+ fields = ('nodes',)
+
+
+class Extends(Stmt):
+ """Represents an extends statement."""
+ fields = ('template',)
+
+
+class For(Stmt):
+ """The for loop. `target` is the target for the iteration (usually a
+ :class:`Name` or :class:`Tuple`), `iter` the iterable. `body` is a list
+ of nodes that are used as loop-body, and `else_` a list of nodes for the
+ `else` block. If no else node exists it has to be an empty list.
+
+ For filtered nodes an expression can be stored as `test`, otherwise `None`.
+ """
+ fields = ('target', 'iter', 'body', 'else_', 'test', 'recursive')
+
+
+class If(Stmt):
+ """If `test` is true, `body` is rendered, else `else_`."""
+ fields = ('test', 'body', 'else_')
+
+
+class Macro(Stmt):
+ """A macro definition. `name` is the name of the macro, `args` a list of
+ arguments and `defaults` a list of defaults if there are any. `body` is
+ a list of nodes for the macro body.
+ """
+ fields = ('name', 'args', 'defaults', 'body')
+
+
+class CallBlock(Stmt):
+ """Like a macro without a name but a call instead. `call` is called with
+ the unnamed macro as `caller` argument this node holds.
+ """
+ fields = ('call', 'args', 'defaults', 'body')
+
+
+class FilterBlock(Stmt):
+ """Node for filter sections."""
+ fields = ('body', 'filter')
+
+
+class Block(Stmt):
+ """A node that represents a block."""
+ fields = ('name', 'body', 'scoped')
+
+
+class Include(Stmt):
+ """A node that represents the include tag."""
+ fields = ('template', 'with_context', 'ignore_missing')
+
+
+class Import(Stmt):
+ """A node that represents the import tag."""
+ fields = ('template', 'target', 'with_context')
+
+
+class FromImport(Stmt):
+ """A node that represents the from import tag. It's important to not
+ pass unsafe names to the name attribute. The compiler translates the
+ attribute lookups directly into getattr calls and does *not* use the
+ subscript callback of the interface. As exported variables may not
+ start with double underscores (which the parser asserts) this is not a
+ problem for regular Jinja code, but if this node is used in an extension
+ extra care must be taken.
+
+ The list of names may contain tuples if aliases are wanted.
+ """
+ fields = ('template', 'names', 'with_context')
+
+
+class ExprStmt(Stmt):
+ """A statement that evaluates an expression and discards the result."""
+ fields = ('node',)
+
+
+class Assign(Stmt):
+ """Assigns an expression to a target."""
+ fields = ('target', 'node')
+
+
+class Expr(Node):
+ """Baseclass for all expressions."""
+ abstract = True
+
+ def as_const(self, eval_ctx=None):
+ """Return the value of the expression as constant or raise
+ :exc:`Impossible` if this was not possible.
+
+ An :class:`EvalContext` can be provided, if none is given
+ a default context is created which requires the nodes to have
+ an attached environment.
+
+ .. versionchanged:: 2.4
+ the `eval_ctx` parameter was added.
+ """
+ raise Impossible()
+
+ def can_assign(self):
+ """Check if it's possible to assign something to this node."""
+ return False
+
+
+class BinExpr(Expr):
+ """Baseclass for all binary expressions."""
+ fields = ('left', 'right')
+ operator = None
+ abstract = True
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ f = _binop_to_func[self.operator]
+ try:
+ return f(self.left.as_const(eval_ctx), self.right.as_const(eval_ctx))
+ except:
+ raise Impossible()
+
+
+class UnaryExpr(Expr):
+ """Baseclass for all unary expressions."""
+ fields = ('node',)
+ operator = None
+ abstract = True
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ f = _uaop_to_func[self.operator]
+ try:
+ return f(self.node.as_const(eval_ctx))
+ except:
+ raise Impossible()
+
+
+class Name(Expr):
+ """Looks up a name or stores a value in a name.
+ The `ctx` of the node can be one of the following values:
+
+ - `store`: store a value in the name
+ - `load`: load that name
+ - `param`: like `store` but if the name was defined as function parameter.
+ """
+ fields = ('name', 'ctx')
+
+ def can_assign(self):
+ return self.name not in ('true', 'false', 'none',
+ 'True', 'False', 'None')
+
+
+class Literal(Expr):
+ """Baseclass for literals."""
+ abstract = True
+
+
+class Const(Literal):
+ """All constant values. The parser will return this node for simple
+ constants such as ``42`` or ``"foo"`` but it can be used to store more
+ complex values such as lists too. Only constants with a safe
+ representation (objects where ``eval(repr(x)) == x`` is true).
+ """
+ fields = ('value',)
+
+ def as_const(self, eval_ctx=None):
+ return self.value
+
+ @classmethod
+ def from_untrusted(cls, value, lineno=None, environment=None):
+ """Return a const object if the value is representable as
+ constant value in the generated code, otherwise it will raise
+ an `Impossible` exception.
+ """
+ from compiler import has_safe_repr
+ if not has_safe_repr(value):
+ raise Impossible()
+ return cls(value, lineno=lineno, environment=environment)
+
+
+class TemplateData(Literal):
+ """A constant template string."""
+ fields = ('data',)
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ if eval_ctx.volatile:
+ raise Impossible()
+ if eval_ctx.autoescape:
+ return Markup(self.data)
+ return self.data
+
+
+class Tuple(Literal):
+ """For loop unpacking and some other things like multiple arguments
+ for subscripts. Like for :class:`Name` `ctx` specifies if the tuple
+ is used for loading the names or storing.
+ """
+ fields = ('items', 'ctx')
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ return tuple(x.as_const(eval_ctx) for x in self.items)
+
+ def can_assign(self):
+ for item in self.items:
+ if not item.can_assign():
+ return False
+ return True
+
+
+class List(Literal):
+ """Any list literal such as ``[1, 2, 3]``"""
+ fields = ('items',)
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ return [x.as_const(eval_ctx) for x in self.items]
+
+
+class Dict(Literal):
+ """Any dict literal such as ``{1: 2, 3: 4}``. The items must be a list of
+ :class:`Pair` nodes.
+ """
+ fields = ('items',)
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ return dict(x.as_const(eval_ctx) for x in self.items)
+
+
+class Pair(Helper):
+ """A key, value pair for dicts."""
+ fields = ('key', 'value')
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ return self.key.as_const(eval_ctx), self.value.as_const(eval_ctx)
+
+
+class Keyword(Helper):
+ """A key, value pair for keyword arguments where key is a string."""
+ fields = ('key', 'value')
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ return self.key, self.value.as_const(eval_ctx)
+
+
+class CondExpr(Expr):
+ """A conditional expression (inline if expression). (``{{
+ foo if bar else baz }}``)
+ """
+ fields = ('test', 'expr1', 'expr2')
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ if self.test.as_const(eval_ctx):
+ return self.expr1.as_const(eval_ctx)
+
+ # if we evaluate to an undefined object, we better do that at runtime
+ if self.expr2 is None:
+ raise Impossible()
+
+ return self.expr2.as_const(eval_ctx)
+
+
+class Filter(Expr):
+ """This node applies a filter on an expression. `name` is the name of
+ the filter, the rest of the fields are the same as for :class:`Call`.
+
+ If the `node` of a filter is `None` the contents of the last buffer are
+ filtered. Buffers are created by macros and filter blocks.
+ """
+ fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ if eval_ctx.volatile or self.node is None:
+ raise Impossible()
+ # we have to be careful here because we call filter_ below.
+ # if this variable would be called filter, 2to3 would wrap the
+ # call in a list beause it is assuming we are talking about the
+ # builtin filter function here which no longer returns a list in
+ # python 3. because of that, do not rename filter_ to filter!
+ filter_ = self.environment.filters.get(self.name)
+ if filter_ is None or getattr(filter_, 'contextfilter', False):
+ raise Impossible()
+ obj = self.node.as_const(eval_ctx)
+ args = [x.as_const(eval_ctx) for x in self.args]
+ if getattr(filter_, 'evalcontextfilter', False):
+ args.insert(0, eval_ctx)
+ elif getattr(filter_, 'environmentfilter', False):
+ args.insert(0, self.environment)
+ kwargs = dict(x.as_const(eval_ctx) for x in self.kwargs)
+ if self.dyn_args is not None:
+ try:
+ args.extend(self.dyn_args.as_const(eval_ctx))
+ except:
+ raise Impossible()
+ if self.dyn_kwargs is not None:
+ try:
+ kwargs.update(self.dyn_kwargs.as_const(eval_ctx))
+ except:
+ raise Impossible()
+ try:
+ return filter_(obj, *args, **kwargs)
+ except:
+ raise Impossible()
+
+
+class Test(Expr):
+ """Applies a test on an expression. `name` is the name of the test, the
+ rest of the fields are the same as for :class:`Call`.
+ """
+ fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+
+
+class Call(Expr):
+ """Calls an expression. `args` is a list of arguments, `kwargs` a list
+ of keyword arguments (list of :class:`Keyword` nodes), and `dyn_args`
+ and `dyn_kwargs` has to be either `None` or a node that is used as
+ node for dynamic positional (``*args``) or keyword (``**kwargs``)
+ arguments.
+ """
+ fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ if eval_ctx.volatile:
+ raise Impossible()
+ obj = self.node.as_const(eval_ctx)
+
+ # don't evaluate context functions
+ args = [x.as_const(eval_ctx) for x in self.args]
+ if isinstance(obj, _context_function_types):
+ if getattr(obj, 'contextfunction', False):
+ raise Impossible()
+ elif getattr(obj, 'evalcontextfunction', False):
+ args.insert(0, eval_ctx)
+ elif getattr(obj, 'environmentfunction', False):
+ args.insert(0, self.environment)
+
+ kwargs = dict(x.as_const(eval_ctx) for x in self.kwargs)
+ if self.dyn_args is not None:
+ try:
+ args.extend(self.dyn_args.as_const(eval_ctx))
+ except:
+ raise Impossible()
+ if self.dyn_kwargs is not None:
+ try:
+ kwargs.update(self.dyn_kwargs.as_const(eval_ctx))
+ except:
+ raise Impossible()
+ try:
+ return obj(*args, **kwargs)
+ except:
+ raise Impossible()
+
+
+class Getitem(Expr):
+ """Get an attribute or item from an expression and prefer the item."""
+ fields = ('node', 'arg', 'ctx')
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ if self.ctx != 'load':
+ raise Impossible()
+ try:
+ return self.environment.getitem(self.node.as_const(eval_ctx),
+ self.arg.as_const(eval_ctx))
+ except:
+ raise Impossible()
+
+ def can_assign(self):
+ return False
+
+
+class Getattr(Expr):
+ """Get an attribute or item from an expression that is a ascii-only
+ bytestring and prefer the attribute.
+ """
+ fields = ('node', 'attr', 'ctx')
+
+ def as_const(self, eval_ctx=None):
+ if self.ctx != 'load':
+ raise Impossible()
+ try:
+ eval_ctx = get_eval_context(self, eval_ctx)
+ return self.environment.getattr(self.node.as_const(eval_ctx),
+ self.attr)
+ except:
+ raise Impossible()
+
+ def can_assign(self):
+ return False
+
+
+class Slice(Expr):
+ """Represents a slice object. This must only be used as argument for
+ :class:`Subscript`.
+ """
+ fields = ('start', 'stop', 'step')
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ def const(obj):
+ if obj is None:
+ return None
+ return obj.as_const(eval_ctx)
+ return slice(const(self.start), const(self.stop), const(self.step))
+
+
+class Concat(Expr):
+ """Concatenates the list of expressions provided after converting them to
+ unicode.
+ """
+ fields = ('nodes',)
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ return ''.join(unicode(x.as_const(eval_ctx)) for x in self.nodes)
+
+
+class Compare(Expr):
+ """Compares an expression with some other expressions. `ops` must be a
+ list of :class:`Operand`\s.
+ """
+ fields = ('expr', 'ops')
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ result = value = self.expr.as_const(eval_ctx)
+ try:
+ for op in self.ops:
+ new_value = op.expr.as_const(eval_ctx)
+ result = _cmpop_to_func[op.op](value, new_value)
+ value = new_value
+ except:
+ raise Impossible()
+ return result
+
+
+class Operand(Helper):
+ """Holds an operator and an expression."""
+ fields = ('op', 'expr')
+
+if __debug__:
+ Operand.__doc__ += '\nThe following operators are available: ' + \
+ ', '.join(sorted('``%s``' % x for x in set(_binop_to_func) |
+ set(_uaop_to_func) | set(_cmpop_to_func)))
+
+
+class Mul(BinExpr):
+ """Multiplies the left with the right node."""
+ operator = '*'
+
+
+class Div(BinExpr):
+ """Divides the left by the right node."""
+ operator = '/'
+
+
+class FloorDiv(BinExpr):
+ """Divides the left by the right node and truncates conver the
+ result into an integer by truncating.
+ """
+ operator = '//'
+
+
+class Add(BinExpr):
+ """Add the left to the right node."""
+ operator = '+'
+
+
+class Sub(BinExpr):
+ """Substract the right from the left node."""
+ operator = '-'
+
+
+class Mod(BinExpr):
+ """Left modulo right."""
+ operator = '%'
+
+
+class Pow(BinExpr):
+ """Left to the power of right."""
+ operator = '**'
+
+
+class And(BinExpr):
+ """Short circuited AND."""
+ operator = 'and'
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ return self.left.as_const(eval_ctx) and self.right.as_const(eval_ctx)
+
+
+class Or(BinExpr):
+ """Short circuited OR."""
+ operator = 'or'
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ return self.left.as_const(eval_ctx) or self.right.as_const(eval_ctx)
+
+
+class Not(UnaryExpr):
+ """Negate the expression."""
+ operator = 'not'
+
+
+class Neg(UnaryExpr):
+ """Make the expression negative."""
+ operator = '-'
+
+
+class Pos(UnaryExpr):
+ """Make the expression positive (noop for most expressions)"""
+ operator = '+'
+
+
+# Helpers for extensions
+
+
+class EnvironmentAttribute(Expr):
+ """Loads an attribute from the environment object. This is useful for
+ extensions that want to call a callback stored on the environment.
+ """
+ fields = ('name',)
+
+
+class ExtensionAttribute(Expr):
+ """Returns the attribute of an extension bound to the environment.
+ The identifier is the identifier of the :class:`Extension`.
+
+ This node is usually constructed by calling the
+ :meth:`~jinja2.ext.Extension.attr` method on an extension.
+ """
+ fields = ('identifier', 'name')
+
+
+class ImportedName(Expr):
+ """If created with an import name the import name is returned on node
+ access. For example ``ImportedName('cgi.escape')`` returns the `escape`
+ function from the cgi module on evaluation. Imports are optimized by the
+ compiler so there is no need to assign them to local variables.
+ """
+ fields = ('importname',)
+
+
+class InternalName(Expr):
+ """An internal name in the compiler. You cannot create these nodes
+ yourself but the parser provides a
+ :meth:`~jinja2.parser.Parser.free_identifier` method that creates
+ a new identifier for you. This identifier is not available from the
+ template and is not threated specially by the compiler.
+ """
+ fields = ('name',)
+
+ def __init__(self):
+ raise TypeError('Can\'t create internal names. Use the '
+ '`free_identifier` method on a parser.')
+
+
+class MarkSafe(Expr):
+ """Mark the wrapped expression as safe (wrap it as `Markup`)."""
+ fields = ('expr',)
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ return Markup(self.expr.as_const(eval_ctx))
+
+
+class MarkSafeIfAutoescape(Expr):
+ """Mark the wrapped expression as safe (wrap it as `Markup`) but
+ only if autoescaping is active.
+
+ .. versionadded:: 2.5
+ """
+ fields = ('expr',)
+
+ def as_const(self, eval_ctx=None):
+ eval_ctx = get_eval_context(self, eval_ctx)
+ if eval_ctx.volatile:
+ raise Impossible()
+ expr = self.expr.as_const(eval_ctx)
+ if eval_ctx.autoescape:
+ return Markup(expr)
+ return expr
+
+
+class ContextReference(Expr):
+ """Returns the current template context. It can be used like a
+ :class:`Name` node, with a ``'load'`` ctx and will return the
+ current :class:`~jinja2.runtime.Context` object.
+
+ Here an example that assigns the current template name to a
+ variable named `foo`::
+
+ Assign(Name('foo', ctx='store'),
+ Getattr(ContextReference(), 'name'))
+ """
+
+
+class Continue(Stmt):
+ """Continue a loop."""
+
+
+class Break(Stmt):
+ """Break a loop."""
+
+
+class Scope(Stmt):
+ """An artificial scope."""
+ fields = ('body',)
+
+
+class EvalContextModifier(Stmt):
+ """Modifies the eval context. For each option that should be modified,
+ a :class:`Keyword` has to be added to the :attr:`options` list.
+
+ Example to change the `autoescape` setting::
+
+ EvalContextModifier(options=[Keyword('autoescape', Const(True))])
+ """
+ fields = ('options',)
+
+
+class ScopedEvalContextModifier(EvalContextModifier):
+ """Modifies the eval context and reverts it later. Works exactly like
+ :class:`EvalContextModifier` but will only modify the
+ :class:`~jinja2.nodes.EvalContext` for nodes in the :attr:`body`.
+ """
+ fields = ('body',)
+
+
+# make sure nobody creates custom nodes
+def _failing_new(*args, **kwargs):
+ raise TypeError('can\'t create custom node types')
+NodeType.__new__ = staticmethod(_failing_new); del _failing_new
diff --git a/slider-agent/src/main/python/jinja2/jinja2/optimizer.py b/slider-agent/src/main/python/jinja2/jinja2/optimizer.py
new file mode 100644
index 0000000..00eab11
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/optimizer.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.optimizer
+ ~~~~~~~~~~~~~~~~
+
+ The jinja optimizer is currently trying to constant fold a few expressions
+ and modify the AST in place so that it should be easier to evaluate it.
+
+ Because the AST does not contain all the scoping information and the
+ compiler has to find that out, we cannot do all the optimizations we
+ want. For example loop unrolling doesn't work because unrolled loops would
+ have a different scoping.
+
+ The solution would be a second syntax tree that has the scoping rules stored.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD.
+"""
+from jinja2 import nodes
+from jinja2.visitor import NodeTransformer
+
+
+def optimize(node, environment):
+ """The context hint can be used to perform an static optimization
+ based on the context given."""
+ optimizer = Optimizer(environment)
+ return optimizer.visit(node)
+
+
+class Optimizer(NodeTransformer):
+
+ def __init__(self, environment):
+ self.environment = environment
+
+ def visit_If(self, node):
+ """Eliminate dead code."""
+ # do not optimize ifs that have a block inside so that it doesn't
+ # break super().
+ if node.find(nodes.Block) is not None:
+ return self.generic_visit(node)
+ try:
+ val = self.visit(node.test).as_const()
+ except nodes.Impossible:
+ return self.generic_visit(node)
+ if val:
+ body = node.body
+ else:
+ body = node.else_
+ result = []
+ for node in body:
+ result.extend(self.visit_list(node))
+ return result
+
+ def fold(self, node):
+ """Do constant folding."""
+ node = self.generic_visit(node)
+ try:
+ return nodes.Const.from_untrusted(node.as_const(),
+ lineno=node.lineno,
+ environment=self.environment)
+ except nodes.Impossible:
+ return node
+
+ visit_Add = visit_Sub = visit_Mul = visit_Div = visit_FloorDiv = \
+ visit_Pow = visit_Mod = visit_And = visit_Or = visit_Pos = visit_Neg = \
+ visit_Not = visit_Compare = visit_Getitem = visit_Getattr = visit_Call = \
+ visit_Filter = visit_Test = visit_CondExpr = fold
+ del fold
diff --git a/slider-agent/src/main/python/jinja2/jinja2/parser.py b/slider-agent/src/main/python/jinja2/jinja2/parser.py
new file mode 100644
index 0000000..d44229a
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/parser.py
@@ -0,0 +1,896 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.parser
+ ~~~~~~~~~~~~~
+
+ Implements the template parser.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+from jinja2 import nodes
+from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
+from jinja2.utils import next
+from jinja2.lexer import describe_token, describe_token_expr
+
+
+#: statements that callinto
+_statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
+ 'macro', 'include', 'from', 'import',
+ 'set'])
+_compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq'])
+
+
+class Parser(object):
+ """This is the central parsing class Jinja2 uses. It's passed to
+ extensions and can be used to parse expressions or statements.
+ """
+
+ def __init__(self, environment, source, name=None, filename=None,
+ state=None):
+ self.environment = environment
+ self.stream = environment._tokenize(source, name, filename, state)
+ self.name = name
+ self.filename = filename
+ self.closed = False
+ self.extensions = {}
+ for extension in environment.iter_extensions():
+ for tag in extension.tags:
+ self.extensions[tag] = extension.parse
+ self._last_identifier = 0
+ self._tag_stack = []
+ self._end_token_stack = []
+
+ def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
+ """Convenience method that raises `exc` with the message, passed
+ line number or last line number as well as the current name and
+ filename.
+ """
+ if lineno is None:
+ lineno = self.stream.current.lineno
+ raise exc(msg, lineno, self.name, self.filename)
+
+ def _fail_ut_eof(self, name, end_token_stack, lineno):
+ expected = []
+ for exprs in end_token_stack:
+ expected.extend(map(describe_token_expr, exprs))
+ if end_token_stack:
+ currently_looking = ' or '.join(
+ "'%s'" % describe_token_expr(expr)
+ for expr in end_token_stack[-1])
+ else:
+ currently_looking = None
+
+ if name is None:
+ message = ['Unexpected end of template.']
+ else:
+ message = ['Encountered unknown tag \'%s\'.' % name]
+
+ if currently_looking:
+ if name is not None and name in expected:
+ message.append('You probably made a nesting mistake. Jinja '
+ 'is expecting this tag, but currently looking '
+ 'for %s.' % currently_looking)
+ else:
+ message.append('Jinja was looking for the following tags: '
+ '%s.' % currently_looking)
+
+ if self._tag_stack:
+ message.append('The innermost block that needs to be '
+ 'closed is \'%s\'.' % self._tag_stack[-1])
+
+ self.fail(' '.join(message), lineno)
+
+ def fail_unknown_tag(self, name, lineno=None):
+ """Called if the parser encounters an unknown tag. Tries to fail
+ with a human readable error message that could help to identify
+ the problem.
+ """
+ return self._fail_ut_eof(name, self._end_token_stack, lineno)
+
+ def fail_eof(self, end_tokens=None, lineno=None):
+ """Like fail_unknown_tag but for end of template situations."""
+ stack = list(self._end_token_stack)
+ if end_tokens is not None:
+ stack.append(end_tokens)
+ return self._fail_ut_eof(None, stack, lineno)
+
+ def is_tuple_end(self, extra_end_rules=None):
+ """Are we at the end of a tuple?"""
+ if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
+ return True
+ elif extra_end_rules is not None:
+ return self.stream.current.test_any(extra_end_rules)
+ return False
+
+ def free_identifier(self, lineno=None):
+ """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
+ self._last_identifier += 1
+ rv = object.__new__(nodes.InternalName)
+ nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
+ return rv
+
+ def parse_statement(self):
+ """Parse a single statement."""
+ token = self.stream.current
+ if token.type != 'name':
+ self.fail('tag name expected', token.lineno)
+ self._tag_stack.append(token.value)
+ pop_tag = True
+ try:
+ if token.value in _statement_keywords:
+ return getattr(self, 'parse_' + self.stream.current.value)()
+ if token.value == 'call':
+ return self.parse_call_block()
+ if token.value == 'filter':
+ return self.parse_filter_block()
+ ext = self.extensions.get(token.value)
+ if ext is not None:
+ return ext(self)
+
+ # did not work out, remove the token we pushed by accident
+ # from the stack so that the unknown tag fail function can
+ # produce a proper error message.
+ self._tag_stack.pop()
+ pop_tag = False
+ self.fail_unknown_tag(token.value, token.lineno)
+ finally:
+ if pop_tag:
+ self._tag_stack.pop()
+
+ def parse_statements(self, end_tokens, drop_needle=False):
+ """Parse multiple statements into a list until one of the end tokens
+ is reached. This is used to parse the body of statements as it also
+ parses template data if appropriate. The parser checks first if the
+ current token is a colon and skips it if there is one. Then it checks
+ for the block end and parses until if one of the `end_tokens` is
+ reached. Per default the active token in the stream at the end of
+ the call is the matched end token. If this is not wanted `drop_needle`
+ can be set to `True` and the end token is removed.
+ """
+ # the first token may be a colon for python compatibility
+ self.stream.skip_if('colon')
+
+ # in the future it would be possible to add whole code sections
+ # by adding some sort of end of statement token and parsing those here.
+ self.stream.expect('block_end')
+ result = self.subparse(end_tokens)
+
+ # we reached the end of the template too early, the subparser
+ # does not check for this, so we do that now
+ if self.stream.current.type == 'eof':
+ self.fail_eof(end_tokens)
+
+ if drop_needle:
+ next(self.stream)
+ return result
+
+ def parse_set(self):
+ """Parse an assign statement."""
+ lineno = next(self.stream).lineno
+ target = self.parse_assign_target()
+ self.stream.expect('assign')
+ expr = self.parse_tuple()
+ return nodes.Assign(target, expr, lineno=lineno)
+
+ def parse_for(self):
+ """Parse a for loop."""
+ lineno = self.stream.expect('name:for').lineno
+ target = self.parse_assign_target(extra_end_rules=('name:in',))
+ self.stream.expect('name:in')
+ iter = self.parse_tuple(with_condexpr=False,
+ extra_end_rules=('name:recursive',))
+ test = None
+ if self.stream.skip_if('name:if'):
+ test = self.parse_expression()
+ recursive = self.stream.skip_if('name:recursive')
+ body = self.parse_statements(('name:endfor', 'name:else'))
+ if next(self.stream).value == 'endfor':
+ else_ = []
+ else:
+ else_ = self.parse_statements(('name:endfor',), drop_needle=True)
+ return nodes.For(target, iter, body, else_, test,
+ recursive, lineno=lineno)
+
+ def parse_if(self):
+ """Parse an if construct."""
+ node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
+ while 1:
+ node.test = self.parse_tuple(with_condexpr=False)
+ node.body = self.parse_statements(('name:elif', 'name:else',
+ 'name:endif'))
+ token = next(self.stream)
+ if token.test('name:elif'):
+ new_node = nodes.If(lineno=self.stream.current.lineno)
+ node.else_ = [new_node]
+ node = new_node
+ continue
+ elif token.test('name:else'):
+ node.else_ = self.parse_statements(('name:endif',),
+ drop_needle=True)
+ else:
+ node.else_ = []
+ break
+ return result
+
+ def parse_block(self):
+ node = nodes.Block(lineno=next(self.stream).lineno)
+ node.name = self.stream.expect('name').value
+ node.scoped = self.stream.skip_if('name:scoped')
+
+ # common problem people encounter when switching from django
+ # to jinja. we do not support hyphens in block names, so let's
+ # raise a nicer error message in that case.
+ if self.stream.current.type == 'sub':
+ self.fail('Block names in Jinja have to be valid Python '
+ 'identifiers and may not contain hypens, use an '
+ 'underscore instead.')
+
+ node.body = self.parse_statements(('name:endblock',), drop_needle=True)
+ self.stream.skip_if('name:' + node.name)
+ return node
+
+ def parse_extends(self):
+ node = nodes.Extends(lineno=next(self.stream).lineno)
+ node.template = self.parse_expression()
+ return node
+
+ def parse_import_context(self, node, default):
+ if self.stream.current.test_any('name:with', 'name:without') and \
+ self.stream.look().test('name:context'):
+ node.with_context = next(self.stream).value == 'with'
+ self.stream.skip()
+ else:
+ node.with_context = default
+ return node
+
+ def parse_include(self):
+ node = nodes.Include(lineno=next(self.stream).lineno)
+ node.template = self.parse_expression()
+ if self.stream.current.test('name:ignore') and \
+ self.stream.look().test('name:missing'):
+ node.ignore_missing = True
+ self.stream.skip(2)
+ else:
+ node.ignore_missing = False
+ return self.parse_import_context(node, True)
+
+ def parse_import(self):
+ node = nodes.Import(lineno=next(self.stream).lineno)
+ node.template = self.parse_expression()
+ self.stream.expect('name:as')
+ node.target = self.parse_assign_target(name_only=True).name
+ return self.parse_import_context(node, False)
+
+ def parse_from(self):
+ node = nodes.FromImport(lineno=next(self.stream).lineno)
+ node.template = self.parse_expression()
+ self.stream.expect('name:import')
+ node.names = []
+
+ def parse_context():
+ if self.stream.current.value in ('with', 'without') and \
+ self.stream.look().test('name:context'):
+ node.with_context = next(self.stream).value == 'with'
+ self.stream.skip()
+ return True
+ return False
+
+ while 1:
+ if node.names:
+ self.stream.expect('comma')
+ if self.stream.current.type == 'name':
+ if parse_context():
+ break
+ target = self.parse_assign_target(name_only=True)
+ if target.name.startswith('_'):
+ self.fail('names starting with an underline can not '
+ 'be imported', target.lineno,
+ exc=TemplateAssertionError)
+ if self.stream.skip_if('name:as'):
+ alias = self.parse_assign_target(name_only=True)
+ node.names.append((target.name, alias.name))
+ else:
+ node.names.append(target.name)
+ if parse_context() or self.stream.current.type != 'comma':
+ break
+ else:
+ break
+ if not hasattr(node, 'with_context'):
+ node.with_context = False
+ self.stream.skip_if('comma')
+ return node
+
+ def parse_signature(self, node):
+ node.args = args = []
+ node.defaults = defaults = []
+ self.stream.expect('lparen')
+ while self.stream.current.type != 'rparen':
+ if args:
+ self.stream.expect('comma')
+ arg = self.parse_assign_target(name_only=True)
+ arg.set_ctx('param')
+ if self.stream.skip_if('assign'):
+ defaults.append(self.parse_expression())
+ args.append(arg)
+ self.stream.expect('rparen')
+
+ def parse_call_block(self):
+ node = nodes.CallBlock(lineno=next(self.stream).lineno)
+ if self.stream.current.type == 'lparen':
+ self.parse_signature(node)
+ else:
+ node.args = []
+ node.defaults = []
+
+ node.call = self.parse_expression()
+ if not isinstance(node.call, nodes.Call):
+ self.fail('expected call', node.lineno)
+ node.body = self.parse_statements(('name:endcall',), drop_needle=True)
+ return node
+
+ def parse_filter_block(self):
+ node = nodes.FilterBlock(lineno=next(self.stream).lineno)
+ node.filter = self.parse_filter(None, start_inline=True)
+ node.body = self.parse_statements(('name:endfilter',),
+ drop_needle=True)
+ return node
+
+ def parse_macro(self):
+ node = nodes.Macro(lineno=next(self.stream).lineno)
+ node.name = self.parse_assign_target(name_only=True).name
+ self.parse_signature(node)
+ node.body = self.parse_statements(('name:endmacro',),
+ drop_needle=True)
+ return node
+
+ def parse_print(self):
+ node = nodes.Output(lineno=next(self.stream).lineno)
+ node.nodes = []
+ while self.stream.current.type != 'block_end':
+ if node.nodes:
+ self.stream.expect('comma')
+ node.nodes.append(self.parse_expression())
+ return node
+
+ def parse_assign_target(self, with_tuple=True, name_only=False,
+ extra_end_rules=None):
+ """Parse an assignment target. As Jinja2 allows assignments to
+ tuples, this function can parse all allowed assignment targets. Per
+ default assignments to tuples are parsed, that can be disable however
+ by setting `with_tuple` to `False`. If only assignments to names are
+ wanted `name_only` can be set to `True`. The `extra_end_rules`
+ parameter is forwarded to the tuple parsing function.
+ """
+ if name_only:
+ token = self.stream.expect('name')
+ target = nodes.Name(token.value, 'store', lineno=token.lineno)
+ else:
+ if with_tuple:
+ target = self.parse_tuple(simplified=True,
+ extra_end_rules=extra_end_rules)
+ else:
+ target = self.parse_primary()
+ target.set_ctx('store')
+ if not target.can_assign():
+ self.fail('can\'t assign to %r' % target.__class__.
+ __name__.lower(), target.lineno)
+ return target
+
+ def parse_expression(self, with_condexpr=True):
+ """Parse an expression. Per default all expressions are parsed, if
+ the optional `with_condexpr` parameter is set to `False` conditional
+ expressions are not parsed.
+ """
+ if with_condexpr:
+ return self.parse_condexpr()
+ return self.parse_or()
+
+ def parse_condexpr(self):
+ lineno = self.stream.current.lineno
+ expr1 = self.parse_or()
+ while self.stream.skip_if('name:if'):
+ expr2 = self.parse_or()
+ if self.stream.skip_if('name:else'):
+ expr3 = self.parse_condexpr()
+ else:
+ expr3 = None
+ expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
+ lineno = self.stream.current.lineno
+ return expr1
+
+ def parse_or(self):
+ lineno = self.stream.current.lineno
+ left = self.parse_and()
+ while self.stream.skip_if('name:or'):
+ right = self.parse_and()
+ left = nodes.Or(left, right, lineno=lineno)
+ lineno = self.stream.current.lineno
+ return left
+
+ def parse_and(self):
+ lineno = self.stream.current.lineno
+ left = self.parse_not()
+ while self.stream.skip_if('name:and'):
+ right = self.parse_not()
+ left = nodes.And(left, right, lineno=lineno)
+ lineno = self.stream.current.lineno
+ return left
+
+ def parse_not(self):
+ if self.stream.current.test('name:not'):
+ lineno = next(self.stream).lineno
+ return nodes.Not(self.parse_not(), lineno=lineno)
+ return self.parse_compare()
+
+ def parse_compare(self):
+ lineno = self.stream.current.lineno
+ expr = self.parse_add()
+ ops = []
+ while 1:
+ token_type = self.stream.current.type
+ if token_type in _compare_operators:
+ next(self.stream)
+ ops.append(nodes.Operand(token_type, self.parse_add()))
+ elif self.stream.skip_if('name:in'):
+ ops.append(nodes.Operand('in', self.parse_add()))
+ elif self.stream.current.test('name:not') and \
+ self.stream.look().test('name:in'):
+ self.stream.skip(2)
+ ops.append(nodes.Operand('notin', self.parse_add()))
+ else:
+ break
+ lineno = self.stream.current.lineno
+ if not ops:
+ return expr
+ return nodes.Compare(expr, ops, lineno=lineno)
+
+ def parse_add(self):
+ lineno = self.stream.current.lineno
+ left = self.parse_sub()
+ while self.stream.current.type == 'add':
+ next(self.stream)
+ right = self.parse_sub()
+ left = nodes.Add(left, right, lineno=lineno)
+ lineno = self.stream.current.lineno
+ return left
+
+ def parse_sub(self):
+ lineno = self.stream.current.lineno
+ left = self.parse_concat()
+ while self.stream.current.type == 'sub':
+ next(self.stream)
+ right = self.parse_concat()
+ left = nodes.Sub(left, right, lineno=lineno)
+ lineno = self.stream.current.lineno
+ return left
+
+ def parse_concat(self):
+ lineno = self.stream.current.lineno
+ args = [self.parse_mul()]
+ while self.stream.current.type == 'tilde':
+ next(self.stream)
+ args.append(self.parse_mul())
+ if len(args) == 1:
+ return args[0]
+ return nodes.Concat(args, lineno=lineno)
+
+ def parse_mul(self):
+ lineno = self.stream.current.lineno
+ left = self.parse_div()
+ while self.stream.current.type == 'mul':
+ next(self.stream)
+ right = self.parse_div()
+ left = nodes.Mul(left, right, lineno=lineno)
+ lineno = self.stream.current.lineno
+ return left
+
+ def parse_div(self):
+ lineno = self.stream.current.lineno
+ left = self.parse_floordiv()
+ while self.stream.current.type == 'div':
+ next(self.stream)
+ right = self.parse_floordiv()
+ left = nodes.Div(left, right, lineno=lineno)
+ lineno = self.stream.current.lineno
+ return left
+
+ def parse_floordiv(self):
+ lineno = self.stream.current.lineno
+ left = self.parse_mod()
+ while self.stream.current.type == 'floordiv':
+ next(self.stream)
+ right = self.parse_mod()
+ left = nodes.FloorDiv(left, right, lineno=lineno)
+ lineno = self.stream.current.lineno
+ return left
+
+ def parse_mod(self):
+ lineno = self.stream.current.lineno
+ left = self.parse_pow()
+ while self.stream.current.type == 'mod':
+ next(self.stream)
+ right = self.parse_pow()
+ left = nodes.Mod(left, right, lineno=lineno)
+ lineno = self.stream.current.lineno
+ return left
+
+ def parse_pow(self):
+ lineno = self.stream.current.lineno
+ left = self.parse_unary()
+ while self.stream.current.type == 'pow':
+ next(self.stream)
+ right = self.parse_unary()
+ left = nodes.Pow(left, right, lineno=lineno)
+ lineno = self.stream.current.lineno
+ return left
+
+ def parse_unary(self, with_filter=True):
+ token_type = self.stream.current.type
+ lineno = self.stream.current.lineno
+ if token_type == 'sub':
+ next(self.stream)
+ node = nodes.Neg(self.parse_unary(False), lineno=lineno)
+ elif token_type == 'add':
+ next(self.stream)
+ node = nodes.Pos(self.parse_unary(False), lineno=lineno)
+ else:
+ node = self.parse_primary()
+ node = self.parse_postfix(node)
+ if with_filter:
+ node = self.parse_filter_expr(node)
+ return node
+
+ def parse_primary(self):
+ token = self.stream.current
+ if token.type == 'name':
+ if token.value in ('true', 'false', 'True', 'False'):
+ node = nodes.Const(token.value in ('true', 'True'),
+ lineno=token.lineno)
+ elif token.value in ('none', 'None'):
+ node = nodes.Const(None, lineno=token.lineno)
+ else:
+ node = nodes.Name(token.value, 'load', lineno=token.lineno)
+ next(self.stream)
+ elif token.type == 'string':
+ next(self.stream)
+ buf = [token.value]
+ lineno = token.lineno
+ while self.stream.current.type == 'string':
+ buf.append(self.stream.current.value)
+ next(self.stream)
+ node = nodes.Const(''.join(buf), lineno=lineno)
+ elif token.type in ('integer', 'float'):
+ next(self.stream)
+ node = nodes.Const(token.value, lineno=token.lineno)
+ elif token.type == 'lparen':
+ next(self.stream)
+ node = self.parse_tuple(explicit_parentheses=True)
+ self.stream.expect('rparen')
+ elif token.type == 'lbracket':
+ node = self.parse_list()
+ elif token.type == 'lbrace':
+ node = self.parse_dict()
+ else:
+ self.fail("unexpected '%s'" % describe_token(token), token.lineno)
+ return node
+
+ def parse_tuple(self, simplified=False, with_condexpr=True,
+ extra_end_rules=None, explicit_parentheses=False):
+ """Works like `parse_expression` but if multiple expressions are
+ delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
+ This method could also return a regular expression instead of a tuple
+ if no commas where found.
+
+ The default parsing mode is a full tuple. If `simplified` is `True`
+ only names and literals are parsed. The `no_condexpr` parameter is
+ forwarded to :meth:`parse_expression`.
+
+ Because tuples do not require delimiters and may end in a bogus comma
+ an extra hint is needed that marks the end of a tuple. For example
+ for loops support tuples between `for` and `in`. In that case the
+ `extra_end_rules` is set to ``['name:in']``.
+
+ `explicit_parentheses` is true if the parsing was triggered by an
+ expression in parentheses. This is used to figure out if an empty
+ tuple is a valid expression or not.
+ """
+ lineno = self.stream.current.lineno
+ if simplified:
+ parse = self.parse_primary
+ elif with_condexpr:
+ parse = self.parse_expression
+ else:
+ parse = lambda: self.parse_expression(with_condexpr=False)
+ args = []
+ is_tuple = False
+ while 1:
+ if args:
+ self.stream.expect('comma')
+ if self.is_tuple_end(extra_end_rules):
+ break
+ args.append(parse())
+ if self.stream.current.type == 'comma':
+ is_tuple = True
+ else:
+ break
+ lineno = self.stream.current.lineno
+
+ if not is_tuple:
+ if args:
+ return args[0]
+
+ # if we don't have explicit parentheses, an empty tuple is
+ # not a valid expression. This would mean nothing (literally
+ # nothing) in the spot of an expression would be an empty
+ # tuple.
+ if not explicit_parentheses:
+ self.fail('Expected an expression, got \'%s\'' %
+ describe_token(self.stream.current))
+
+ return nodes.Tuple(args, 'load', lineno=lineno)
+
+ def parse_list(self):
+ token = self.stream.expect('lbracket')
+ items = []
+ while self.stream.current.type != 'rbracket':
+ if items:
+ self.stream.expect('comma')
+ if self.stream.current.type == 'rbracket':
+ break
+ items.append(self.parse_expression())
+ self.stream.expect('rbracket')
+ return nodes.List(items, lineno=token.lineno)
+
+ def parse_dict(self):
+ token = self.stream.expect('lbrace')
+ items = []
+ while self.stream.current.type != 'rbrace':
+ if items:
+ self.stream.expect('comma')
+ if self.stream.current.type == 'rbrace':
+ break
+ key = self.parse_expression()
+ self.stream.expect('colon')
+ value = self.parse_expression()
+ items.append(nodes.Pair(key, value, lineno=key.lineno))
+ self.stream.expect('rbrace')
+ return nodes.Dict(items, lineno=token.lineno)
+
+ def parse_postfix(self, node):
+ while 1:
+ token_type = self.stream.current.type
+ if token_type == 'dot' or token_type == 'lbracket':
+ node = self.parse_subscript(node)
+ # calls are valid both after postfix expressions (getattr
+ # and getitem) as well as filters and tests
+ elif token_type == 'lparen':
+ node = self.parse_call(node)
+ else:
+ break
+ return node
+
+ def parse_filter_expr(self, node):
+ while 1:
+ token_type = self.stream.current.type
+ if token_type == 'pipe':
+ node = self.parse_filter(node)
+ elif token_type == 'name' and self.stream.current.value == 'is':
+ node = self.parse_test(node)
+ # calls are valid both after postfix expressions (getattr
+ # and getitem) as well as filters and tests
+ elif token_type == 'lparen':
+ node = self.parse_call(node)
+ else:
+ break
+ return node
+
+ def parse_subscript(self, node):
+ token = next(self.stream)
+ if token.type == 'dot':
+ attr_token = self.stream.current
+ next(self.stream)
+ if attr_token.type == 'name':
+ return nodes.Getattr(node, attr_token.value, 'load',
+ lineno=token.lineno)
+ elif attr_token.type != 'integer':
+ self.fail('expected name or number', attr_token.lineno)
+ arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
+ return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
+ if token.type == 'lbracket':
+ priority_on_attribute = False
+ args = []
+ while self.stream.current.type != 'rbracket':
+ if args:
+ self.stream.expect('comma')
+ args.append(self.parse_subscribed())
+ self.stream.expect('rbracket')
+ if len(args) == 1:
+ arg = args[0]
+ else:
+ arg = nodes.Tuple(args, 'load', lineno=token.lineno)
+ return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
+ self.fail('expected subscript expression', self.lineno)
+
+ def parse_subscribed(self):
+ lineno = self.stream.current.lineno
+
+ if self.stream.current.type == 'colon':
+ next(self.stream)
+ args = [None]
+ else:
+ node = self.parse_expression()
+ if self.stream.current.type != 'colon':
+ return node
+ next(self.stream)
+ args = [node]
+
+ if self.stream.current.type == 'colon':
+ args.append(None)
+ elif self.stream.current.type not in ('rbracket', 'comma'):
+ args.append(self.parse_expression())
+ else:
+ args.append(None)
+
+ if self.stream.current.type == 'colon':
+ next(self.stream)
+ if self.stream.current.type not in ('rbracket', 'comma'):
+ args.append(self.parse_expression())
+ else:
+ args.append(None)
+ else:
+ args.append(None)
+
+ return nodes.Slice(lineno=lineno, *args)
+
+ def parse_call(self, node):
+ token = self.stream.expect('lparen')
+ args = []
+ kwargs = []
+ dyn_args = dyn_kwargs = None
+ require_comma = False
+
+ def ensure(expr):
+ if not expr:
+ self.fail('invalid syntax for function call expression',
+ token.lineno)
+
+ while self.stream.current.type != 'rparen':
+ if require_comma:
+ self.stream.expect('comma')
+ # support for trailing comma
+ if self.stream.current.type == 'rparen':
+ break
+ if self.stream.current.type == 'mul':
+ ensure(dyn_args is None and dyn_kwargs is None)
+ next(self.stream)
+ dyn_args = self.parse_expression()
+ elif self.stream.current.type == 'pow':
+ ensure(dyn_kwargs is None)
+ next(self.stream)
+ dyn_kwargs = self.parse_expression()
+ else:
+ ensure(dyn_args is None and dyn_kwargs is None)
+ if self.stream.current.type == 'name' and \
+ self.stream.look().type == 'assign':
+ key = self.stream.current.value
+ self.stream.skip(2)
+ value = self.parse_expression()
+ kwargs.append(nodes.Keyword(key, value,
+ lineno=value.lineno))
+ else:
+ ensure(not kwargs)
+ args.append(self.parse_expression())
+
+ require_comma = True
+ self.stream.expect('rparen')
+
+ if node is None:
+ return args, kwargs, dyn_args, dyn_kwargs
+ return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs,
+ lineno=token.lineno)
+
+ def parse_filter(self, node, start_inline=False):
+ while self.stream.current.type == 'pipe' or start_inline:
+ if not start_inline:
+ next(self.stream)
+ token = self.stream.expect('name')
+ name = token.value
+ while self.stream.current.type == 'dot':
+ next(self.stream)
+ name += '.' + self.stream.expect('name').value
+ if self.stream.current.type == 'lparen':
+ args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
+ else:
+ args = []
+ kwargs = []
+ dyn_args = dyn_kwargs = None
+ node = nodes.Filter(node, name, args, kwargs, dyn_args,
+ dyn_kwargs, lineno=token.lineno)
+ start_inline = False
+ return node
+
+ def parse_test(self, node):
+ token = next(self.stream)
+ if self.stream.current.test('name:not'):
+ next(self.stream)
+ negated = True
+ else:
+ negated = False
+ name = self.stream.expect('name').value
+ while self.stream.current.type == 'dot':
+ next(self.stream)
+ name += '.' + self.stream.expect('name').value
+ dyn_args = dyn_kwargs = None
+ kwargs = []
+ if self.stream.current.type == 'lparen':
+ args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
+ elif self.stream.current.type in ('name', 'string', 'integer',
+ 'float', 'lparen', 'lbracket',
+ 'lbrace') and not \
+ self.stream.current.test_any('name:else', 'name:or',
+ 'name:and'):
+ if self.stream.current.test('name:is'):
+ self.fail('You cannot chain multiple tests with is')
+ args = [self.parse_expression()]
+ else:
+ args = []
+ node = nodes.Test(node, name, args, kwargs, dyn_args,
+ dyn_kwargs, lineno=token.lineno)
+ if negated:
+ node = nodes.Not(node, lineno=token.lineno)
+ return node
+
+ def subparse(self, end_tokens=None):
+ body = []
+ data_buffer = []
+ add_data = data_buffer.append
+
+ if end_tokens is not None:
+ self._end_token_stack.append(end_tokens)
+
+ def flush_data():
+ if data_buffer:
+ lineno = data_buffer[0].lineno
+ body.append(nodes.Output(data_buffer[:], lineno=lineno))
+ del data_buffer[:]
+
+ try:
+ while self.stream:
+ token = self.stream.current
+ if token.type == 'data':
+ if token.value:
+ add_data(nodes.TemplateData(token.value,
+ lineno=token.lineno))
+ next(self.stream)
+ elif token.type == 'variable_begin':
+ next(self.stream)
+ add_data(self.parse_tuple(with_condexpr=True))
+ self.stream.expect('variable_end')
+ elif token.type == 'block_begin':
+ flush_data()
+ next(self.stream)
+ if end_tokens is not None and \
+ self.stream.current.test_any(*end_tokens):
+ return body
+ rv = self.parse_statement()
+ if isinstance(rv, list):
+ body.extend(rv)
+ else:
+ body.append(rv)
+ self.stream.expect('block_end')
+ else:
+ raise AssertionError('internal parsing error')
+
+ flush_data()
+ finally:
+ if end_tokens is not None:
+ self._end_token_stack.pop()
+
+ return body
+
+ def parse(self):
+ """Parse the whole template into a `Template` node."""
+ result = nodes.Template(self.subparse(), lineno=1)
+ result.set_environment(self.environment)
+ return result
diff --git a/slider-agent/src/main/python/jinja2/jinja2/runtime.py b/slider-agent/src/main/python/jinja2/jinja2/runtime.py
new file mode 100644
index 0000000..6fea3aa
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/runtime.py
@@ -0,0 +1,544 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.runtime
+ ~~~~~~~~~~~~~~
+
+ Runtime helpers.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD.
+"""
+import sys
+from itertools import chain, imap
+from jinja2.nodes import EvalContext, _context_function_types
+from jinja2.utils import Markup, partial, soft_unicode, escape, missing, \
+ concat, internalcode, next, object_type_repr
+from jinja2.exceptions import UndefinedError, TemplateRuntimeError, \
+ TemplateNotFound
+
+
+# these variables are exported to the template runtime
+__all__ = ['LoopContext', 'TemplateReference', 'Macro', 'Markup',
+ 'TemplateRuntimeError', 'missing', 'concat', 'escape',
+ 'markup_join', 'unicode_join', 'to_string', 'identity',
+ 'TemplateNotFound']
+
+#: the name of the function that is used to convert something into
+#: a string. 2to3 will adopt that automatically and the generated
+#: code can take advantage of it.
+to_string = unicode
+
+#: the identity function. Useful for certain things in the environment
+identity = lambda x: x
+
+
+def markup_join(seq):
+ """Concatenation that escapes if necessary and converts to unicode."""
+ buf = []
+ iterator = imap(soft_unicode, seq)
+ for arg in iterator:
+ buf.append(arg)
+ if hasattr(arg, '__html__'):
+ return Markup(u'').join(chain(buf, iterator))
+ return concat(buf)
+
+
+def unicode_join(seq):
+ """Simple args to unicode conversion and concatenation."""
+ return concat(imap(unicode, seq))
+
+
+def new_context(environment, template_name, blocks, vars=None,
+ shared=None, globals=None, locals=None):
+ """Internal helper to for context creation."""
+ if vars is None:
+ vars = {}
+ if shared:
+ parent = vars
+ else:
+ parent = dict(globals or (), **vars)
+ if locals:
+ # if the parent is shared a copy should be created because
+ # we don't want to modify the dict passed
+ if shared:
+ parent = dict(parent)
+ for key, value in locals.iteritems():
+ if key[:2] == 'l_' and value is not missing:
+ parent[key[2:]] = value
+ return Context(environment, parent, template_name, blocks)
+
+
+class TemplateReference(object):
+ """The `self` in templates."""
+
+ def __init__(self, context):
+ self.__context = context
+
+ def __getitem__(self, name):
+ blocks = self.__context.blocks[name]
+ wrap = self.__context.eval_ctx.autoescape and \
+ Markup or (lambda x: x)
+ return BlockReference(name, self.__context, blocks, 0)
+
+ def __repr__(self):
+ return '<%s %r>' % (
+ self.__class__.__name__,
+ self.__context.name
+ )
+
+
+class Context(object):
+ """The template context holds the variables of a template. It stores the
+ values passed to the template and also the names the template exports.
+ Creating instances is neither supported nor useful as it's created
+ automatically at various stages of the template evaluation and should not
+ be created by hand.
+
+ The context is immutable. Modifications on :attr:`parent` **must not**
+ happen and modifications on :attr:`vars` are allowed from generated
+ template code only. Template filters and global functions marked as
+ :func:`contextfunction`\s get the active context passed as first argument
+ and are allowed to access the context read-only.
+
+ The template context supports read only dict operations (`get`,
+ `keys`, `values`, `items`, `iterkeys`, `itervalues`, `iteritems`,
+ `__getitem__`, `__contains__`). Additionally there is a :meth:`resolve`
+ method that doesn't fail with a `KeyError` but returns an
+ :class:`Undefined` object for missing variables.
+ """
+ __slots__ = ('parent', 'vars', 'environment', 'eval_ctx', 'exported_vars',
+ 'name', 'blocks', '__weakref__')
+
+ def __init__(self, environment, parent, name, blocks):
+ self.parent = parent
+ self.vars = {}
+ self.environment = environment
+ self.eval_ctx = EvalContext(self.environment, name)
+ self.exported_vars = set()
+ self.name = name
+
+ # create the initial mapping of blocks. Whenever template inheritance
+ # takes place the runtime will update this mapping with the new blocks
+ # from the template.
+ self.blocks = dict((k, [v]) for k, v in blocks.iteritems())
+
+ def super(self, name, current):
+ """Render a parent block."""
+ try:
+ blocks = self.blocks[name]
+ index = blocks.index(current) + 1
+ blocks[index]
+ except LookupError:
+ return self.environment.undefined('there is no parent block '
+ 'called %r.' % name,
+ name='super')
+ return BlockReference(name, self, blocks, index)
+
+ def get(self, key, default=None):
+ """Returns an item from the template context, if it doesn't exist
+ `default` is returned.
+ """
+ try:
+ return self[key]
+ except KeyError:
+ return default
+
+ def resolve(self, key):
+ """Looks up a variable like `__getitem__` or `get` but returns an
+ :class:`Undefined` object with the name of the name looked up.
+ """
+ if key in self.vars:
+ return self.vars[key]
+ if key in self.parent:
+ return self.parent[key]
+ return self.environment.undefined(name=key)
+
+ def get_exported(self):
+ """Get a new dict with the exported variables."""
+ return dict((k, self.vars[k]) for k in self.exported_vars)
+
+ def get_all(self):
+ """Return a copy of the complete context as dict including the
+ exported variables.
+ """
+ return dict(self.parent, **self.vars)
+
+ @internalcode
+ def call(__self, __obj, *args, **kwargs):
+ """Call the callable with the arguments and keyword arguments
+ provided but inject the active context or environment as first
+ argument if the callable is a :func:`contextfunction` or
+ :func:`environmentfunction`.
+ """
+ if __debug__:
+ __traceback_hide__ = True
+ if isinstance(__obj, _context_function_types):
+ if getattr(__obj, 'contextfunction', 0):
+ args = (__self,) + args
+ elif getattr(__obj, 'evalcontextfunction', 0):
+ args = (__self.eval_ctx,) + args
+ elif getattr(__obj, 'environmentfunction', 0):
+ args = (__self.environment,) + args
+ try:
+ return __obj(*args, **kwargs)
+ except StopIteration:
+ return __self.environment.undefined('value was undefined because '
+ 'a callable raised a '
+ 'StopIteration exception')
+
+ def derived(self, locals=None):
+ """Internal helper function to create a derived context."""
+ context = new_context(self.environment, self.name, {},
+ self.parent, True, None, locals)
+ context.eval_ctx = self.eval_ctx
+ context.blocks.update((k, list(v)) for k, v in self.blocks.iteritems())
+ return context
+
+ def _all(meth):
+ proxy = lambda self: getattr(self.get_all(), meth)()
+ proxy.__doc__ = getattr(dict, meth).__doc__
+ proxy.__name__ = meth
+ return proxy
+
+ keys = _all('keys')
+ values = _all('values')
+ items = _all('items')
+
+ # not available on python 3
+ if hasattr(dict, 'iterkeys'):
+ iterkeys = _all('iterkeys')
+ itervalues = _all('itervalues')
+ iteritems = _all('iteritems')
+ del _all
+
+ def __contains__(self, name):
+ return name in self.vars or name in self.parent
+
+ def __getitem__(self, key):
+ """Lookup a variable or raise `KeyError` if the variable is
+ undefined.
+ """
+ item = self.resolve(key)
+ if isinstance(item, Undefined):
+ raise KeyError(key)
+ return item
+
+ def __repr__(self):
+ return '<%s %s of %r>' % (
+ self.__class__.__name__,
+ repr(self.get_all()),
+ self.name
+ )
+
+
+# register the context as mapping if possible
+try:
+ from collections import Mapping
+ Mapping.register(Context)
+except ImportError:
+ pass
+
+
+class BlockReference(object):
+ """One block on a template reference."""
+
+ def __init__(self, name, context, stack, depth):
+ self.name = name
+ self._context = context
+ self._stack = stack
+ self._depth = depth
+
+ @property
+ def super(self):
+ """Super the block."""
+ if self._depth + 1 >= len(self._stack):
+ return self._context.environment. \
+ undefined('there is no parent block called %r.' %
+ self.name, name='super')
+ return BlockReference(self.name, self._context, self._stack,
+ self._depth + 1)
+
+ @internalcode
+ def __call__(self):
+ rv = concat(self._stack[self._depth](self._context))
+ if self._context.eval_ctx.autoescape:
+ rv = Markup(rv)
+ return rv
+
+
+class LoopContext(object):
+ """A loop context for dynamic iteration."""
+
+ def __init__(self, iterable, recurse=None):
+ self._iterator = iter(iterable)
+ self._recurse = recurse
+ self.index0 = -1
+
+ # try to get the length of the iterable early. This must be done
+ # here because there are some broken iterators around where there
+ # __len__ is the number of iterations left (i'm looking at your
+ # listreverseiterator!).
+ try:
+ self._length = len(iterable)
+ except (TypeError, AttributeError):
+ self._length = None
+
+ def cycle(self, *args):
+ """Cycles among the arguments with the current loop index."""
+ if not args:
+ raise TypeError('no items for cycling given')
+ return args[self.index0 % len(args)]
+
+ first = property(lambda x: x.index0 == 0)
+ last = property(lambda x: x.index0 + 1 == x.length)
+ index = property(lambda x: x.index0 + 1)
+ revindex = property(lambda x: x.length - x.index0)
+ revindex0 = property(lambda x: x.length - x.index)
+
+ def __len__(self):
+ return self.length
+
+ def __iter__(self):
+ return LoopContextIterator(self)
+
+ @internalcode
+ def loop(self, iterable):
+ if self._recurse is None:
+ raise TypeError('Tried to call non recursive loop. Maybe you '
+ "forgot the 'recursive' modifier.")
+ return self._recurse(iterable, self._recurse)
+
+ # a nifty trick to enhance the error message if someone tried to call
+ # the the loop without or with too many arguments.
+ __call__ = loop
+ del loop
+
+ @property
+ def length(self):
+ if self._length is None:
+ # if was not possible to get the length of the iterator when
+ # the loop context was created (ie: iterating over a generator)
+ # we have to convert the iterable into a sequence and use the
+ # length of that.
+ iterable = tuple(self._iterator)
+ self._iterator = iter(iterable)
+ self._length = len(iterable) + self.index0 + 1
+ return self._length
+
+ def __repr__(self):
+ return '<%s %r/%r>' % (
+ self.__class__.__name__,
+ self.index,
+ self.length
+ )
+
+
+class LoopContextIterator(object):
+ """The iterator for a loop context."""
+ __slots__ = ('context',)
+
+ def __init__(self, context):
+ self.context = context
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ ctx = self.context
+ ctx.index0 += 1
+ return next(ctx._iterator), ctx
+
+
+class Macro(object):
+ """Wraps a macro function."""
+
+ def __init__(self, environment, func, name, arguments, defaults,
+ catch_kwargs, catch_varargs, caller):
+ self._environment = environment
+ self._func = func
+ self._argument_count = len(arguments)
+ self.name = name
+ self.arguments = arguments
+ self.defaults = defaults
+ self.catch_kwargs = catch_kwargs
+ self.catch_varargs = catch_varargs
+ self.caller = caller
+
+ @internalcode
+ def __call__(self, *args, **kwargs):
+ # try to consume the positional arguments
+ arguments = list(args[:self._argument_count])
+ off = len(arguments)
+
+ # if the number of arguments consumed is not the number of
+ # arguments expected we start filling in keyword arguments
+ # and defaults.
+ if off != self._argument_count:
+ for idx, name in enumerate(self.arguments[len(arguments):]):
+ try:
+ value = kwargs.pop(name)
+ except KeyError:
+ try:
+ value = self.defaults[idx - self._argument_count + off]
+ except IndexError:
+ value = self._environment.undefined(
+ 'parameter %r was not provided' % name, name=name)
+ arguments.append(value)
+
+ # it's important that the order of these arguments does not change
+ # if not also changed in the compiler's `function_scoping` method.
+ # the order is caller, keyword arguments, positional arguments!
+ if self.caller:
+ caller = kwargs.pop('caller', None)
+ if caller is None:
+ caller = self._environment.undefined('No caller defined',
+ name='caller')
+ arguments.append(caller)
+ if self.catch_kwargs:
+ arguments.append(kwargs)
+ elif kwargs:
+ raise TypeError('macro %r takes no keyword argument %r' %
+ (self.name, next(iter(kwargs))))
+ if self.catch_varargs:
+ arguments.append(args[self._argument_count:])
+ elif len(args) > self._argument_count:
+ raise TypeError('macro %r takes not more than %d argument(s)' %
+ (self.name, len(self.arguments)))
+ return self._func(*arguments)
+
+ def __repr__(self):
+ return '<%s %s>' % (
+ self.__class__.__name__,
+ self.name is None and 'anonymous' or repr(self.name)
+ )
+
+
+class Undefined(object):
+ """The default undefined type. This undefined type can be printed and
+ iterated over, but every other access will raise an :exc:`UndefinedError`:
+
+ >>> foo = Undefined(name='foo')
+ >>> str(foo)
+ ''
+ >>> not foo
+ True
+ >>> foo + 42
+ Traceback (most recent call last):
+ ...
+ UndefinedError: 'foo' is undefined
+ """
+ __slots__ = ('_undefined_hint', '_undefined_obj', '_undefined_name',
+ '_undefined_exception')
+
+ def __init__(self, hint=None, obj=missing, name=None, exc=UndefinedError):
+ self._undefined_hint = hint
+ self._undefined_obj = obj
+ self._undefined_name = name
+ self._undefined_exception = exc
+
+ @internalcode
+ def _fail_with_undefined_error(self, *args, **kwargs):
+ """Regular callback function for undefined objects that raises an
+ `UndefinedError` on call.
+ """
+ if self._undefined_hint is None:
+ if self._undefined_obj is missing:
+ hint = '%r is undefined' % self._undefined_name
+ elif not isinstance(self._undefined_name, basestring):
+ hint = '%s has no element %r' % (
+ object_type_repr(self._undefined_obj),
+ self._undefined_name
+ )
+ else:
+ hint = '%r has no attribute %r' % (
+ object_type_repr(self._undefined_obj),
+ self._undefined_name
+ )
+ else:
+ hint = self._undefined_hint
+ raise self._undefined_exception(hint)
+
+ __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \
+ __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \
+ __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \
+ __getattr__ = __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = \
+ __int__ = __float__ = __complex__ = __pow__ = __rpow__ = \
+ _fail_with_undefined_error
+
+ def __str__(self):
+ return unicode(self).encode('utf-8')
+
+ # unicode goes after __str__ because we configured 2to3 to rename
+ # __unicode__ to __str__. because the 2to3 tree is not designed to
+ # remove nodes from it, we leave the above __str__ around and let
+ # it override at runtime.
+ def __unicode__(self):
+ return u''
+
+ def __len__(self):
+ return 0
+
+ def __iter__(self):
+ if 0:
+ yield None
+
+ def __nonzero__(self):
+ return False
+
+ def __repr__(self):
+ return 'Undefined'
+
+
+class DebugUndefined(Undefined):
+ """An undefined that returns the debug info when printed.
+
+ >>> foo = DebugUndefined(name='foo')
+ >>> str(foo)
+ '{{ foo }}'
+ >>> not foo
+ True
+ >>> foo + 42
+ Traceback (most recent call last):
+ ...
+ UndefinedError: 'foo' is undefined
+ """
+ __slots__ = ()
+
+ def __unicode__(self):
+ if self._undefined_hint is None:
+ if self._undefined_obj is missing:
+ return u'{{ %s }}' % self._undefined_name
+ return '{{ no such element: %s[%r] }}' % (
+ object_type_repr(self._undefined_obj),
+ self._undefined_name
+ )
+ return u'{{ undefined value printed: %s }}' % self._undefined_hint
+
+
+class StrictUndefined(Undefined):
+ """An undefined that barks on print and iteration as well as boolean
+ tests and all kinds of comparisons. In other words: you can do nothing
+ with it except checking if it's defined using the `defined` test.
+
+ >>> foo = StrictUndefined(name='foo')
+ >>> str(foo)
+ Traceback (most recent call last):
+ ...
+ UndefinedError: 'foo' is undefined
+ >>> not foo
+ Traceback (most recent call last):
+ ...
+ UndefinedError: 'foo' is undefined
+ >>> foo + 42
+ Traceback (most recent call last):
+ ...
+ UndefinedError: 'foo' is undefined
+ """
+ __slots__ = ()
+ __iter__ = __unicode__ = __str__ = __len__ = __nonzero__ = __eq__ = \
+ __ne__ = __bool__ = Undefined._fail_with_undefined_error
+
+
+# remove remaining slots attributes, after the metaclass did the magic they
+# are unneeded and irritating as they contain wrong data for the subclasses.
+del Undefined.__slots__, DebugUndefined.__slots__, StrictUndefined.__slots__
diff --git a/slider-agent/src/main/python/jinja2/jinja2/sandbox.py b/slider-agent/src/main/python/jinja2/jinja2/sandbox.py
new file mode 100644
index 0000000..7497195
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/sandbox.py
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.sandbox
+ ~~~~~~~~~~~~~~
+
+ Adds a sandbox layer to Jinja as it was the default behavior in the old
+ Jinja 1 releases. This sandbox is slightly different from Jinja 1 as the
+ default behavior is easier to use.
+
+ The behavior can be changed by subclassing the environment.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD.
+"""
+import operator
+from jinja2.runtime import Undefined
+from jinja2.environment import Environment
+from jinja2.exceptions import SecurityError
+from jinja2.utils import FunctionType, MethodType, TracebackType, CodeType, \
+ FrameType, GeneratorType
+
+
+#: maximum number of items a range may produce
+MAX_RANGE = 100000
+
+#: attributes of function objects that are considered unsafe.
+UNSAFE_FUNCTION_ATTRIBUTES = set(['func_closure', 'func_code', 'func_dict',
+ 'func_defaults', 'func_globals'])
+
+#: unsafe method attributes. function attributes are unsafe for methods too
+UNSAFE_METHOD_ATTRIBUTES = set(['im_class', 'im_func', 'im_self'])
+
+
+import warnings
+
+# make sure we don't warn in python 2.6 about stuff we don't care about
+warnings.filterwarnings('ignore', 'the sets module', DeprecationWarning,
+ module='jinja2.sandbox')
+
+from collections import deque
+
+_mutable_set_types = (set,)
+_mutable_mapping_types = (dict,)
+_mutable_sequence_types = (list,)
+
+
+# on python 2.x we can register the user collection types
+try:
+ from UserDict import UserDict, DictMixin
+ from UserList import UserList
+ _mutable_mapping_types += (UserDict, DictMixin)
+ _mutable_set_types += (UserList,)
+except ImportError:
+ pass
+
+# if sets is still available, register the mutable set from there as well
+try:
+ from sets import Set
+ _mutable_set_types += (Set,)
+except ImportError:
+ pass
+
+#: register Python 2.6 abstract base classes
+try:
+ from collections import MutableSet, MutableMapping, MutableSequence
+ _mutable_set_types += (MutableSet,)
+ _mutable_mapping_types += (MutableMapping,)
+ _mutable_sequence_types += (MutableSequence,)
+except ImportError:
+ pass
+
+_mutable_spec = (
+ (_mutable_set_types, frozenset([
+ 'add', 'clear', 'difference_update', 'discard', 'pop', 'remove',
+ 'symmetric_difference_update', 'update'
+ ])),
+ (_mutable_mapping_types, frozenset([
+ 'clear', 'pop', 'popitem', 'setdefault', 'update'
+ ])),
+ (_mutable_sequence_types, frozenset([
+ 'append', 'reverse', 'insert', 'sort', 'extend', 'remove'
+ ])),
+ (deque, frozenset([
+ 'append', 'appendleft', 'clear', 'extend', 'extendleft', 'pop',
+ 'popleft', 'remove', 'rotate'
+ ]))
+)
+
+
+def safe_range(*args):
+ """A range that can't generate ranges with a length of more than
+ MAX_RANGE items.
+ """
+ rng = xrange(*args)
+ if len(rng) > MAX_RANGE:
+ raise OverflowError('range too big, maximum size for range is %d' %
+ MAX_RANGE)
+ return rng
+
+
+def unsafe(f):
+ """
+ Mark a function or method as unsafe::
+
+ @unsafe
+ def delete(self):
+ pass
+ """
+ f.unsafe_callable = True
+ return f
+
+
+def is_internal_attribute(obj, attr):
+ """Test if the attribute given is an internal python attribute. For
+ example this function returns `True` for the `func_code` attribute of
+ python objects. This is useful if the environment method
+ :meth:`~SandboxedEnvironment.is_safe_attribute` is overriden.
+
+ >>> from jinja2.sandbox import is_internal_attribute
+ >>> is_internal_attribute(lambda: None, "func_code")
+ True
+ >>> is_internal_attribute((lambda x:x).func_code, 'co_code')
+ True
+ >>> is_internal_attribute(str, "upper")
+ False
+ """
+ if isinstance(obj, FunctionType):
+ if attr in UNSAFE_FUNCTION_ATTRIBUTES:
+ return True
+ elif isinstance(obj, MethodType):
+ if attr in UNSAFE_FUNCTION_ATTRIBUTES or \
+ attr in UNSAFE_METHOD_ATTRIBUTES:
+ return True
+ elif isinstance(obj, type):
+ if attr == 'mro':
+ return True
+ elif isinstance(obj, (CodeType, TracebackType, FrameType)):
+ return True
+ elif isinstance(obj, GeneratorType):
+ if attr == 'gi_frame':
+ return True
+ return attr.startswith('__')
+
+
+def modifies_known_mutable(obj, attr):
+ """This function checks if an attribute on a builtin mutable object
+ (list, dict, set or deque) would modify it if called. It also supports
+ the "user"-versions of the objects (`sets.Set`, `UserDict.*` etc.) and
+ with Python 2.6 onwards the abstract base classes `MutableSet`,
+ `MutableMapping`, and `MutableSequence`.
+
+ >>> modifies_known_mutable({}, "clear")
+ True
+ >>> modifies_known_mutable({}, "keys")
+ False
+ >>> modifies_known_mutable([], "append")
+ True
+ >>> modifies_known_mutable([], "index")
+ False
+
+ If called with an unsupported object (such as unicode) `False` is
+ returned.
+
+ >>> modifies_known_mutable("foo", "upper")
+ False
+ """
+ for typespec, unsafe in _mutable_spec:
+ if isinstance(obj, typespec):
+ return attr in unsafe
+ return False
+
+
+class SandboxedEnvironment(Environment):
+ """The sandboxed environment. It works like the regular environment but
+ tells the compiler to generate sandboxed code. Additionally subclasses of
+ this environment may override the methods that tell the runtime what
+ attributes or functions are safe to access.
+
+ If the template tries to access insecure code a :exc:`SecurityError` is
+ raised. However also other exceptions may occour during the rendering so
+ the caller has to ensure that all exceptions are catched.
+ """
+ sandboxed = True
+
+ def __init__(self, *args, **kwargs):
+ Environment.__init__(self, *args, **kwargs)
+ self.globals['range'] = safe_range
+
+ def is_safe_attribute(self, obj, attr, value):
+ """The sandboxed environment will call this method to check if the
+ attribute of an object is safe to access. Per default all attributes
+ starting with an underscore are considered private as well as the
+ special attributes of internal python objects as returned by the
+ :func:`is_internal_attribute` function.
+ """
+ return not (attr.startswith('_') or is_internal_attribute(obj, attr))
+
+ def is_safe_callable(self, obj):
+ """Check if an object is safely callable. Per default a function is
+ considered safe unless the `unsafe_callable` attribute exists and is
+ True. Override this method to alter the behavior, but this won't
+ affect the `unsafe` decorator from this module.
+ """
+ return not (getattr(obj, 'unsafe_callable', False) or \
+ getattr(obj, 'alters_data', False))
+
+ def getitem(self, obj, argument):
+ """Subscribe an object from sandboxed code."""
+ try:
+ return obj[argument]
+ except (TypeError, LookupError):
+ if isinstance(argument, basestring):
+ try:
+ attr = str(argument)
+ except:
+ pass
+ else:
+ try:
+ value = getattr(obj, attr)
+ except AttributeError:
+ pass
+ else:
+ if self.is_safe_attribute(obj, argument, value):
+ return value
+ return self.unsafe_undefined(obj, argument)
+ return self.undefined(obj=obj, name=argument)
+
+ def getattr(self, obj, attribute):
+ """Subscribe an object from sandboxed code and prefer the
+ attribute. The attribute passed *must* be a bytestring.
+ """
+ try:
+ value = getattr(obj, attribute)
+ except AttributeError:
+ try:
+ return obj[attribute]
+ except (TypeError, LookupError):
+ pass
+ else:
+ if self.is_safe_attribute(obj, attribute, value):
+ return value
+ return self.unsafe_undefined(obj, attribute)
+ return self.undefined(obj=obj, name=attribute)
+
+ def unsafe_undefined(self, obj, attribute):
+ """Return an undefined object for unsafe attributes."""
+ return self.undefined('access to attribute %r of %r '
+ 'object is unsafe.' % (
+ attribute,
+ obj.__class__.__name__
+ ), name=attribute, obj=obj, exc=SecurityError)
+
+ def call(__self, __context, __obj, *args, **kwargs):
+ """Call an object from sandboxed code."""
+ # the double prefixes are to avoid double keyword argument
+ # errors when proxying the call.
+ if not __self.is_safe_callable(__obj):
+ raise SecurityError('%r is not safely callable' % (__obj,))
+ return __context.call(__obj, *args, **kwargs)
+
+
+class ImmutableSandboxedEnvironment(SandboxedEnvironment):
+ """Works exactly like the regular `SandboxedEnvironment` but does not
+ permit modifications on the builtin mutable objects `list`, `set`, and
+ `dict` by using the :func:`modifies_known_mutable` function.
+ """
+
+ def is_safe_attribute(self, obj, attr, value):
+ if not SandboxedEnvironment.is_safe_attribute(self, obj, attr, value):
+ return False
+ return not modifies_known_mutable(obj, attr)
diff --git a/slider-agent/src/main/python/jinja2/jinja2/tests.py b/slider-agent/src/main/python/jinja2/jinja2/tests.py
new file mode 100644
index 0000000..d257eca
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/tests.py
@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.tests
+ ~~~~~~~~~~~~
+
+ Jinja test functions. Used with the "is" operator.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import re
+from jinja2.runtime import Undefined
+
+# nose, nothing here to test
+__test__ = False
+
+
+number_re = re.compile(r'^-?\d+(\.\d+)?$')
+regex_type = type(number_re)
+
+
+try:
+ test_callable = callable
+except NameError:
+ def test_callable(x):
+ return hasattr(x, '__call__')
+
+
+def test_odd(value):
+ """Return true if the variable is odd."""
+ return value % 2 == 1
+
+
+def test_even(value):
+ """Return true if the variable is even."""
+ return value % 2 == 0
+
+
+def test_divisibleby(value, num):
+ """Check if a variable is divisible by a number."""
+ return value % num == 0
+
+
+def test_defined(value):
+ """Return true if the variable is defined:
+
+ .. sourcecode:: jinja
+
+ {% if variable is defined %}
+ value of variable: {{ variable }}
+ {% else %}
+ variable is not defined
+ {% endif %}
+
+ See the :func:`default` filter for a simple way to set undefined
+ variables.
+ """
+ return not isinstance(value, Undefined)
+
+
+def test_undefined(value):
+ """Like :func:`defined` but the other way round."""
+ return isinstance(value, Undefined)
+
+
+def test_none(value):
+ """Return true if the variable is none."""
+ return value is None
+
+
+def test_lower(value):
+ """Return true if the variable is lowercased."""
+ return unicode(value).islower()
+
+
+def test_upper(value):
+ """Return true if the variable is uppercased."""
+ return unicode(value).isupper()
+
+
+def test_string(value):
+ """Return true if the object is a string."""
+ return isinstance(value, basestring)
+
+
+def test_number(value):
+ """Return true if the variable is a number."""
+ return isinstance(value, (int, long, float, complex))
+
+
+def test_sequence(value):
+ """Return true if the variable is a sequence. Sequences are variables
+ that are iterable.
+ """
+ try:
+ len(value)
+ value.__getitem__
+ except:
+ return False
+ return True
+
+
+def test_sameas(value, other):
+ """Check if an object points to the same memory address than another
+ object:
+
+ .. sourcecode:: jinja
+
+ {% if foo.attribute is sameas false %}
+ the foo attribute really is the `False` singleton
+ {% endif %}
+ """
+ return value is other
+
+
+def test_iterable(value):
+ """Check if it's possible to iterate over an object."""
+ try:
+ iter(value)
+ except TypeError:
+ return False
+ return True
+
+
+def test_escaped(value):
+ """Check if the value is escaped."""
+ return hasattr(value, '__html__')
+
+
+TESTS = {
+ 'odd': test_odd,
+ 'even': test_even,
+ 'divisibleby': test_divisibleby,
+ 'defined': test_defined,
+ 'undefined': test_undefined,
+ 'none': test_none,
+ 'lower': test_lower,
+ 'upper': test_upper,
+ 'string': test_string,
+ 'number': test_number,
+ 'sequence': test_sequence,
+ 'iterable': test_iterable,
+ 'callable': test_callable,
+ 'sameas': test_sameas,
+ 'escaped': test_escaped
+}
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/__init__.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/__init__.py
new file mode 100644
index 0000000..1f10ef6
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/__init__.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite
+ ~~~~~~~~~~~~~~~~
+
+ All the unittests of Jinja2. These tests can be executed by
+ either running run-tests.py using multiple Python versions at
+ the same time.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import re
+import sys
+import unittest
+from traceback import format_exception
+from jinja2 import loaders
+
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+dict_loader = loaders.DictLoader({
+ 'justdict.html': 'FOO'
+})
+package_loader = loaders.PackageLoader('jinja2.testsuite.res', 'templates')
+filesystem_loader = loaders.FileSystemLoader(here + '/res/templates')
+function_loader = loaders.FunctionLoader({'justfunction.html': 'FOO'}.get)
+choice_loader = loaders.ChoiceLoader([dict_loader, package_loader])
+prefix_loader = loaders.PrefixLoader({
+ 'a': filesystem_loader,
+ 'b': dict_loader
+})
+
+
+class JinjaTestCase(unittest.TestCase):
+
+ ### use only these methods for testing. If you need standard
+ ### unittest method, wrap them!
+
+ def setup(self):
+ pass
+
+ def teardown(self):
+ pass
+
+ def setUp(self):
+ self.setup()
+
+ def tearDown(self):
+ self.teardown()
+
+ def assert_equal(self, a, b):
+ return self.assertEqual(a, b)
+
+ def assert_raises(self, *args, **kwargs):
+ return self.assertRaises(*args, **kwargs)
+
+ def assert_traceback_matches(self, callback, expected_tb):
+ try:
+ callback()
+ except Exception, e:
+ tb = format_exception(*sys.exc_info())
+ if re.search(expected_tb.strip(), ''.join(tb)) is None:
+ raise self.fail('Traceback did not match:\n\n%s\nexpected:\n%s'
+ % (''.join(tb), expected_tb))
+ else:
+ self.fail('Expected exception')
+
+
+def suite():
+ from jinja2.testsuite import ext, filters, tests, core_tags, \
+ loader, inheritance, imports, lexnparse, security, api, \
+ regression, debug, utils, doctests
+ suite = unittest.TestSuite()
+ suite.addTest(ext.suite())
+ suite.addTest(filters.suite())
+ suite.addTest(tests.suite())
+ suite.addTest(core_tags.suite())
+ suite.addTest(loader.suite())
+ suite.addTest(inheritance.suite())
+ suite.addTest(imports.suite())
+ suite.addTest(lexnparse.suite())
+ suite.addTest(security.suite())
+ suite.addTest(api.suite())
+ suite.addTest(regression.suite())
+ suite.addTest(debug.suite())
+ suite.addTest(utils.suite())
+
+ # doctests will not run on python 3 currently. Too many issues
+ # with that, do not test that on that platform.
+ if sys.version_info < (3, 0):
+ suite.addTest(doctests.suite())
+
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/api.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/api.py
new file mode 100644
index 0000000..7463c7f
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/api.py
@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.api
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Tests the public API and related stuff.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import time
+import tempfile
+import unittest
+
+from jinja2.testsuite import JinjaTestCase
+
+from jinja2 import Environment, Undefined, DebugUndefined, \
+ StrictUndefined, UndefinedError, Template, meta, \
+ is_undefined, Template, DictLoader
+from jinja2.utils import Cycler
+
+env = Environment()
+
+
+class ExtendedAPITestCase(JinjaTestCase):
+
+ def test_item_and_attribute(self):
+ from jinja2.sandbox import SandboxedEnvironment
+
+ for env in Environment(), SandboxedEnvironment():
+ # the |list is necessary for python3
+ tmpl = env.from_string('{{ foo.items()|list }}')
+ assert tmpl.render(foo={'items': 42}) == "[('items', 42)]"
+ tmpl = env.from_string('{{ foo|attr("items")()|list }}')
+ assert tmpl.render(foo={'items': 42}) == "[('items', 42)]"
+ tmpl = env.from_string('{{ foo["items"] }}')
+ assert tmpl.render(foo={'items': 42}) == '42'
+
+ def test_finalizer(self):
+ def finalize_none_empty(value):
+ if value is None:
+ value = u''
+ return value
+ env = Environment(finalize=finalize_none_empty)
+ tmpl = env.from_string('{% for item in seq %}|{{ item }}{% endfor %}')
+ assert tmpl.render(seq=(None, 1, "foo")) == '||1|foo'
+ tmpl = env.from_string('<{{ none }}>')
+ assert tmpl.render() == '<>'
+
+ def test_cycler(self):
+ items = 1, 2, 3
+ c = Cycler(*items)
+ for item in items + items:
+ assert c.current == item
+ assert c.next() == item
+ c.next()
+ assert c.current == 2
+ c.reset()
+ assert c.current == 1
+
+ def test_expressions(self):
+ expr = env.compile_expression("foo")
+ assert expr() is None
+ assert expr(foo=42) == 42
+ expr2 = env.compile_expression("foo", undefined_to_none=False)
+ assert is_undefined(expr2())
+
+ expr = env.compile_expression("42 + foo")
+ assert expr(foo=42) == 84
+
+ def test_template_passthrough(self):
+ t = Template('Content')
+ assert env.get_template(t) is t
+ assert env.select_template([t]) is t
+ assert env.get_or_select_template([t]) is t
+ assert env.get_or_select_template(t) is t
+
+ def test_autoescape_autoselect(self):
+ def select_autoescape(name):
+ if name is None or '.' not in name:
+ return False
+ return name.endswith('.html')
+ env = Environment(autoescape=select_autoescape,
+ loader=DictLoader({
+ 'test.txt': '{{ foo }}',
+ 'test.html': '{{ foo }}'
+ }))
+ t = env.get_template('test.txt')
+ assert t.render(foo='<foo>') == '<foo>'
+ t = env.get_template('test.html')
+ assert t.render(foo='<foo>') == '<foo>'
+ t = env.from_string('{{ foo }}')
+ assert t.render(foo='<foo>') == '<foo>'
+
+
+class MetaTestCase(JinjaTestCase):
+
+ def test_find_undeclared_variables(self):
+ ast = env.parse('{% set foo = 42 %}{{ bar + foo }}')
+ x = meta.find_undeclared_variables(ast)
+ assert x == set(['bar'])
+
+ ast = env.parse('{% set foo = 42 %}{{ bar + foo }}'
+ '{% macro meh(x) %}{{ x }}{% endmacro %}'
+ '{% for item in seq %}{{ muh(item) + meh(seq) }}{% endfor %}')
+ x = meta.find_undeclared_variables(ast)
+ assert x == set(['bar', 'seq', 'muh'])
+
+ def test_find_refererenced_templates(self):
+ ast = env.parse('{% extends "layout.html" %}{% include helper %}')
+ i = meta.find_referenced_templates(ast)
+ assert i.next() == 'layout.html'
+ assert i.next() is None
+ assert list(i) == []
+
+ ast = env.parse('{% extends "layout.html" %}'
+ '{% from "test.html" import a, b as c %}'
+ '{% import "meh.html" as meh %}'
+ '{% include "muh.html" %}')
+ i = meta.find_referenced_templates(ast)
+ assert list(i) == ['layout.html', 'test.html', 'meh.html', 'muh.html']
+
+ def test_find_included_templates(self):
+ ast = env.parse('{% include ["foo.html", "bar.html"] %}')
+ i = meta.find_referenced_templates(ast)
+ assert list(i) == ['foo.html', 'bar.html']
+
+ ast = env.parse('{% include ("foo.html", "bar.html") %}')
+ i = meta.find_referenced_templates(ast)
+ assert list(i) == ['foo.html', 'bar.html']
+
+ ast = env.parse('{% include ["foo.html", "bar.html", foo] %}')
+ i = meta.find_referenced_templates(ast)
+ assert list(i) == ['foo.html', 'bar.html', None]
+
+ ast = env.parse('{% include ("foo.html", "bar.html", foo) %}')
+ i = meta.find_referenced_templates(ast)
+ assert list(i) == ['foo.html', 'bar.html', None]
+
+
+class StreamingTestCase(JinjaTestCase):
+
+ def test_basic_streaming(self):
+ tmpl = env.from_string("<ul>{% for item in seq %}<li>{{ loop.index "
+ "}} - {{ item }}</li>{%- endfor %}</ul>")
+ stream = tmpl.stream(seq=range(4))
+ self.assert_equal(stream.next(), '<ul>')
+ self.assert_equal(stream.next(), '<li>1 - 0</li>')
+ self.assert_equal(stream.next(), '<li>2 - 1</li>')
+ self.assert_equal(stream.next(), '<li>3 - 2</li>')
+ self.assert_equal(stream.next(), '<li>4 - 3</li>')
+ self.assert_equal(stream.next(), '</ul>')
+
+ def test_buffered_streaming(self):
+ tmpl = env.from_string("<ul>{% for item in seq %}<li>{{ loop.index "
+ "}} - {{ item }}</li>{%- endfor %}</ul>")
+ stream = tmpl.stream(seq=range(4))
+ stream.enable_buffering(size=3)
+ self.assert_equal(stream.next(), u'<ul><li>1 - 0</li><li>2 - 1</li>')
+ self.assert_equal(stream.next(), u'<li>3 - 2</li><li>4 - 3</li></ul>')
+
+ def test_streaming_behavior(self):
+ tmpl = env.from_string("")
+ stream = tmpl.stream()
+ assert not stream.buffered
+ stream.enable_buffering(20)
+ assert stream.buffered
+ stream.disable_buffering()
+ assert not stream.buffered
+
+
+class UndefinedTestCase(JinjaTestCase):
+
+ def test_stopiteration_is_undefined(self):
+ def test():
+ raise StopIteration()
+ t = Template('A{{ test() }}B')
+ assert t.render(test=test) == 'AB'
+ t = Template('A{{ test().missingattribute }}B')
+ self.assert_raises(UndefinedError, t.render, test=test)
+
+ def test_default_undefined(self):
+ env = Environment(undefined=Undefined)
+ self.assert_equal(env.from_string('{{ missing }}').render(), u'')
+ self.assert_raises(UndefinedError,
+ env.from_string('{{ missing.attribute }}').render)
+ self.assert_equal(env.from_string('{{ missing|list }}').render(), '[]')
+ self.assert_equal(env.from_string('{{ missing is not defined }}').render(), 'True')
+ self.assert_equal(env.from_string('{{ foo.missing }}').render(foo=42), '')
+ self.assert_equal(env.from_string('{{ not missing }}').render(), 'True')
+
+ def test_debug_undefined(self):
+ env = Environment(undefined=DebugUndefined)
+ self.assert_equal(env.from_string('{{ missing }}').render(), '{{ missing }}')
+ self.assert_raises(UndefinedError,
+ env.from_string('{{ missing.attribute }}').render)
+ self.assert_equal(env.from_string('{{ missing|list }}').render(), '[]')
+ self.assert_equal(env.from_string('{{ missing is not defined }}').render(), 'True')
+ self.assert_equal(env.from_string('{{ foo.missing }}').render(foo=42),
+ u"{{ no such element: int object['missing'] }}")
+ self.assert_equal(env.from_string('{{ not missing }}').render(), 'True')
+
+ def test_strict_undefined(self):
+ env = Environment(undefined=StrictUndefined)
+ self.assert_raises(UndefinedError, env.from_string('{{ missing }}').render)
+ self.assert_raises(UndefinedError, env.from_string('{{ missing.attribute }}').render)
+ self.assert_raises(UndefinedError, env.from_string('{{ missing|list }}').render)
+ self.assert_equal(env.from_string('{{ missing is not defined }}').render(), 'True')
+ self.assert_raises(UndefinedError, env.from_string('{{ foo.missing }}').render, foo=42)
+ self.assert_raises(UndefinedError, env.from_string('{{ not missing }}').render)
+
+ def test_indexing_gives_undefined(self):
+ t = Template("{{ var[42].foo }}")
+ self.assert_raises(UndefinedError, t.render, var=0)
+
+ def test_none_gives_proper_error(self):
+ try:
+ Environment().getattr(None, 'split')()
+ except UndefinedError, e:
+ assert e.message == "'None' has no attribute 'split'"
+ else:
+ assert False, 'expected exception'
+
+ def test_object_repr(self):
+ try:
+ Undefined(obj=42, name='upper')()
+ except UndefinedError, e:
+ assert e.message == "'int object' has no attribute 'upper'"
+ else:
+ assert False, 'expected exception'
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(ExtendedAPITestCase))
+ suite.addTest(unittest.makeSuite(MetaTestCase))
+ suite.addTest(unittest.makeSuite(StreamingTestCase))
+ suite.addTest(unittest.makeSuite(UndefinedTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/core_tags.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/core_tags.py
new file mode 100644
index 0000000..d456b4f
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/core_tags.py
@@ -0,0 +1,286 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.core_tags
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Test the core tags like for and if.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import re
+import unittest
+
+from jinja2.testsuite import JinjaTestCase
+
+from jinja2 import Environment, TemplateSyntaxError, UndefinedError, \
+ DictLoader
+
+env = Environment()
+
+
+class ForLoopTestCase(JinjaTestCase):
+
+ def test_simple(self):
+ tmpl = env.from_string('{% for item in seq %}{{ item }}{% endfor %}')
+ assert tmpl.render(seq=range(10)) == '0123456789'
+
+ def test_else(self):
+ tmpl = env.from_string('{% for item in seq %}XXX{% else %}...{% endfor %}')
+ assert tmpl.render() == '...'
+
+ def test_empty_blocks(self):
+ tmpl = env.from_string('<{% for item in seq %}{% else %}{% endfor %}>')
+ assert tmpl.render() == '<>'
+
+ def test_context_vars(self):
+ tmpl = env.from_string('''{% for item in seq -%}
+ {{ loop.index }}|{{ loop.index0 }}|{{ loop.revindex }}|{{
+ loop.revindex0 }}|{{ loop.first }}|{{ loop.last }}|{{
+ loop.length }}###{% endfor %}''')
+ one, two, _ = tmpl.render(seq=[0, 1]).split('###')
+ (one_index, one_index0, one_revindex, one_revindex0, one_first,
+ one_last, one_length) = one.split('|')
+ (two_index, two_index0, two_revindex, two_revindex0, two_first,
+ two_last, two_length) = two.split('|')
+
+ assert int(one_index) == 1 and int(two_index) == 2
+ assert int(one_index0) == 0 and int(two_index0) == 1
+ assert int(one_revindex) == 2 and int(two_revindex) == 1
+ assert int(one_revindex0) == 1 and int(two_revindex0) == 0
+ assert one_first == 'True' and two_first == 'False'
+ assert one_last == 'False' and two_last == 'True'
+ assert one_length == two_length == '2'
+
+ def test_cycling(self):
+ tmpl = env.from_string('''{% for item in seq %}{{
+ loop.cycle('<1>', '<2>') }}{% endfor %}{%
+ for item in seq %}{{ loop.cycle(*through) }}{% endfor %}''')
+ output = tmpl.render(seq=range(4), through=('<1>', '<2>'))
+ assert output == '<1><2>' * 4
+
+ def test_scope(self):
+ tmpl = env.from_string('{% for item in seq %}{% endfor %}{{ item }}')
+ output = tmpl.render(seq=range(10))
+ assert not output
+
+ def test_varlen(self):
+ def inner():
+ for item in range(5):
+ yield item
+ tmpl = env.from_string('{% for item in iter %}{{ item }}{% endfor %}')
+ output = tmpl.render(iter=inner())
+ assert output == '01234'
+
+ def test_noniter(self):
+ tmpl = env.from_string('{% for item in none %}...{% endfor %}')
+ self.assert_raises(TypeError, tmpl.render)
+
+ def test_recursive(self):
+ tmpl = env.from_string('''{% for item in seq recursive -%}
+ [{{ item.a }}{% if item.b %}<{{ loop(item.b) }}>{% endif %}]
+ {%- endfor %}''')
+ assert tmpl.render(seq=[
+ dict(a=1, b=[dict(a=1), dict(a=2)]),
+ dict(a=2, b=[dict(a=1), dict(a=2)]),
+ dict(a=3, b=[dict(a='a')])
+ ]) == '[1<[1][2]>][2<[1][2]>][3<[a]>]'
+
+ def test_looploop(self):
+ tmpl = env.from_string('''{% for row in table %}
+ {%- set rowloop = loop -%}
+ {% for cell in row -%}
+ [{{ rowloop.index }}|{{ loop.index }}]
+ {%- endfor %}
+ {%- endfor %}''')
+ assert tmpl.render(table=['ab', 'cd']) == '[1|1][1|2][2|1][2|2]'
+
+ def test_reversed_bug(self):
+ tmpl = env.from_string('{% for i in items %}{{ i }}'
+ '{% if not loop.last %}'
+ ',{% endif %}{% endfor %}')
+ assert tmpl.render(items=reversed([3, 2, 1])) == '1,2,3'
+
+ def test_loop_errors(self):
+ tmpl = env.from_string('''{% for item in [1] if loop.index
+ == 0 %}...{% endfor %}''')
+ self.assert_raises(UndefinedError, tmpl.render)
+ tmpl = env.from_string('''{% for item in [] %}...{% else
+ %}{{ loop }}{% endfor %}''')
+ assert tmpl.render() == ''
+
+ def test_loop_filter(self):
+ tmpl = env.from_string('{% for item in range(10) if item '
+ 'is even %}[{{ item }}]{% endfor %}')
+ assert tmpl.render() == '[0][2][4][6][8]'
+ tmpl = env.from_string('''
+ {%- for item in range(10) if item is even %}[{{
+ loop.index }}:{{ item }}]{% endfor %}''')
+ assert tmpl.render() == '[1:0][2:2][3:4][4:6][5:8]'
+
+ def test_loop_unassignable(self):
+ self.assert_raises(TemplateSyntaxError, env.from_string,
+ '{% for loop in seq %}...{% endfor %}')
+
+ def test_scoped_special_var(self):
+ t = env.from_string('{% for s in seq %}[{{ loop.first }}{% for c in s %}'
+ '|{{ loop.first }}{% endfor %}]{% endfor %}')
+ assert t.render(seq=('ab', 'cd')) == '[True|True|False][False|True|False]'
+
+ def test_scoped_loop_var(self):
+ t = env.from_string('{% for x in seq %}{{ loop.first }}'
+ '{% for y in seq %}{% endfor %}{% endfor %}')
+ assert t.render(seq='ab') == 'TrueFalse'
+ t = env.from_string('{% for x in seq %}{% for y in seq %}'
+ '{{ loop.first }}{% endfor %}{% endfor %}')
+ assert t.render(seq='ab') == 'TrueFalseTrueFalse'
+
+ def test_recursive_empty_loop_iter(self):
+ t = env.from_string('''
+ {%- for item in foo recursive -%}{%- endfor -%}
+ ''')
+ assert t.render(dict(foo=[])) == ''
+
+ def test_call_in_loop(self):
+ t = env.from_string('''
+ {%- macro do_something() -%}
+ [{{ caller() }}]
+ {%- endmacro %}
+
+ {%- for i in [1, 2, 3] %}
+ {%- call do_something() -%}
+ {{ i }}
+ {%- endcall %}
+ {%- endfor -%}
+ ''')
+ assert t.render() == '[1][2][3]'
+
+ def test_scoping_bug(self):
+ t = env.from_string('''
+ {%- for item in foo %}...{{ item }}...{% endfor %}
+ {%- macro item(a) %}...{{ a }}...{% endmacro %}
+ {{- item(2) -}}
+ ''')
+ assert t.render(foo=(1,)) == '...1......2...'
+
+ def test_unpacking(self):
+ tmpl = env.from_string('{% for a, b, c in [[1, 2, 3]] %}'
+ '{{ a }}|{{ b }}|{{ c }}{% endfor %}')
+ assert tmpl.render() == '1|2|3'
+
+
+class IfConditionTestCase(JinjaTestCase):
+
+ def test_simple(self):
+ tmpl = env.from_string('''{% if true %}...{% endif %}''')
+ assert tmpl.render() == '...'
+
+ def test_elif(self):
+ tmpl = env.from_string('''{% if false %}XXX{% elif true
+ %}...{% else %}XXX{% endif %}''')
+ assert tmpl.render() == '...'
+
+ def test_else(self):
+ tmpl = env.from_string('{% if false %}XXX{% else %}...{% endif %}')
+ assert tmpl.render() == '...'
+
+ def test_empty(self):
+ tmpl = env.from_string('[{% if true %}{% else %}{% endif %}]')
+ assert tmpl.render() == '[]'
+
+ def test_complete(self):
+ tmpl = env.from_string('{% if a %}A{% elif b %}B{% elif c == d %}'
+ 'C{% else %}D{% endif %}')
+ assert tmpl.render(a=0, b=False, c=42, d=42.0) == 'C'
+
+ def test_no_scope(self):
+ tmpl = env.from_string('{% if a %}{% set foo = 1 %}{% endif %}{{ foo }}')
+ assert tmpl.render(a=True) == '1'
+ tmpl = env.from_string('{% if true %}{% set foo = 1 %}{% endif %}{{ foo }}')
+ assert tmpl.render() == '1'
+
+
+class MacrosTestCase(JinjaTestCase):
+ env = Environment(trim_blocks=True)
+
+ def test_simple(self):
+ tmpl = self.env.from_string('''\
+{% macro say_hello(name) %}Hello {{ name }}!{% endmacro %}
+{{ say_hello('Peter') }}''')
+ assert tmpl.render() == 'Hello Peter!'
+
+ def test_scoping(self):
+ tmpl = self.env.from_string('''\
+{% macro level1(data1) %}
+{% macro level2(data2) %}{{ data1 }}|{{ data2 }}{% endmacro %}
+{{ level2('bar') }}{% endmacro %}
+{{ level1('foo') }}''')
+ assert tmpl.render() == 'foo|bar'
+
+ def test_arguments(self):
+ tmpl = self.env.from_string('''\
+{% macro m(a, b, c='c', d='d') %}{{ a }}|{{ b }}|{{ c }}|{{ d }}{% endmacro %}
+{{ m() }}|{{ m('a') }}|{{ m('a', 'b') }}|{{ m(1, 2, 3) }}''')
+ assert tmpl.render() == '||c|d|a||c|d|a|b|c|d|1|2|3|d'
+
+ def test_varargs(self):
+ tmpl = self.env.from_string('''\
+{% macro test() %}{{ varargs|join('|') }}{% endmacro %}\
+{{ test(1, 2, 3) }}''')
+ assert tmpl.render() == '1|2|3'
+
+ def test_simple_call(self):
+ tmpl = self.env.from_string('''\
+{% macro test() %}[[{{ caller() }}]]{% endmacro %}\
+{% call test() %}data{% endcall %}''')
+ assert tmpl.render() == '[[data]]'
+
+ def test_complex_call(self):
+ tmpl = self.env.from_string('''\
+{% macro test() %}[[{{ caller('data') }}]]{% endmacro %}\
+{% call(data) test() %}{{ data }}{% endcall %}''')
+ assert tmpl.render() == '[[data]]'
+
+ def test_caller_undefined(self):
+ tmpl = self.env.from_string('''\
+{% set caller = 42 %}\
+{% macro test() %}{{ caller is not defined }}{% endmacro %}\
+{{ test() }}''')
+ assert tmpl.render() == 'True'
+
+ def test_include(self):
+ self.env = Environment(loader=DictLoader({'include':
+ '{% macro test(foo) %}[{{ foo }}]{% endmacro %}'}))
+ tmpl = self.env.from_string('{% from "include" import test %}{{ test("foo") }}')
+ assert tmpl.render() == '[foo]'
+
+ def test_macro_api(self):
+ tmpl = self.env.from_string('{% macro foo(a, b) %}{% endmacro %}'
+ '{% macro bar() %}{{ varargs }}{{ kwargs }}{% endmacro %}'
+ '{% macro baz() %}{{ caller() }}{% endmacro %}')
+ assert tmpl.module.foo.arguments == ('a', 'b')
+ assert tmpl.module.foo.defaults == ()
+ assert tmpl.module.foo.name == 'foo'
+ assert not tmpl.module.foo.caller
+ assert not tmpl.module.foo.catch_kwargs
+ assert not tmpl.module.foo.catch_varargs
+ assert tmpl.module.bar.arguments == ()
+ assert tmpl.module.bar.defaults == ()
+ assert not tmpl.module.bar.caller
+ assert tmpl.module.bar.catch_kwargs
+ assert tmpl.module.bar.catch_varargs
+ assert tmpl.module.baz.caller
+
+ def test_callself(self):
+ tmpl = self.env.from_string('{% macro foo(x) %}{{ x }}{% if x > 1 %}|'
+ '{{ foo(x - 1) }}{% endif %}{% endmacro %}'
+ '{{ foo(5) }}')
+ assert tmpl.render() == '5|4|3|2|1'
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(ForLoopTestCase))
+ suite.addTest(unittest.makeSuite(IfConditionTestCase))
+ suite.addTest(unittest.makeSuite(MacrosTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/debug.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/debug.py
new file mode 100644
index 0000000..7552dec
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/debug.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.debug
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Tests the debug system.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import sys
+import unittest
+
+from jinja2.testsuite import JinjaTestCase, filesystem_loader
+
+from jinja2 import Environment, TemplateSyntaxError
+
+env = Environment(loader=filesystem_loader)
+
+
+class DebugTestCase(JinjaTestCase):
+
+ if sys.version_info[:2] != (2, 4):
+ def test_runtime_error(self):
+ def test():
+ tmpl.render(fail=lambda: 1 / 0)
+ tmpl = env.get_template('broken.html')
+ self.assert_traceback_matches(test, r'''
+ File ".*?broken.html", line 2, in (top-level template code|<module>)
+ \{\{ fail\(\) \}\}
+ File ".*?debug.pyc?", line \d+, in <lambda>
+ tmpl\.render\(fail=lambda: 1 / 0\)
+ZeroDivisionError: (int(eger)? )?division (or modulo )?by zero
+''')
+
+ def test_syntax_error(self):
+ # XXX: the .*? is necessary for python3 which does not hide
+ # some of the stack frames we don't want to show. Not sure
+ # what's up with that, but that is not that critical. Should
+ # be fixed though.
+ self.assert_traceback_matches(lambda: env.get_template('syntaxerror.html'), r'''(?sm)
+ File ".*?syntaxerror.html", line 4, in (template|<module>)
+ \{% endif %\}.*?
+(jinja2\.exceptions\.)?TemplateSyntaxError: Encountered unknown tag 'endif'. Jinja was looking for the following tags: 'endfor' or 'else'. The innermost block that needs to be closed is 'for'.
+ ''')
+
+ def test_regular_syntax_error(self):
+ def test():
+ raise TemplateSyntaxError('wtf', 42)
+ self.assert_traceback_matches(test, r'''
+ File ".*debug.pyc?", line \d+, in test
+ raise TemplateSyntaxError\('wtf', 42\)
+(jinja2\.exceptions\.)?TemplateSyntaxError: wtf
+ line 42''')
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(DebugTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/doctests.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/doctests.py
new file mode 100644
index 0000000..616d3b6
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/doctests.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.doctests
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ The doctests. Collects all tests we want to test from
+ the Jinja modules.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import unittest
+import doctest
+
+
+def suite():
+ from jinja2 import utils, sandbox, runtime, meta, loaders, \
+ ext, environment, bccache, nodes
+ suite = unittest.TestSuite()
+ suite.addTest(doctest.DocTestSuite(utils))
+ suite.addTest(doctest.DocTestSuite(sandbox))
+ suite.addTest(doctest.DocTestSuite(runtime))
+ suite.addTest(doctest.DocTestSuite(meta))
+ suite.addTest(doctest.DocTestSuite(loaders))
+ suite.addTest(doctest.DocTestSuite(ext))
+ suite.addTest(doctest.DocTestSuite(environment))
+ suite.addTest(doctest.DocTestSuite(bccache))
+ suite.addTest(doctest.DocTestSuite(nodes))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/ext.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/ext.py
new file mode 100644
index 0000000..89b8579
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/ext.py
@@ -0,0 +1,455 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.ext
+ ~~~~~~~~~~~~~~~~~~~~
+
+ Tests for the extensions.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import re
+import unittest
+
+from jinja2.testsuite import JinjaTestCase, filesystem_loader
+
+from jinja2 import Environment, DictLoader, contextfunction, nodes
+from jinja2.exceptions import TemplateAssertionError
+from jinja2.ext import Extension
+from jinja2.lexer import Token, count_newlines
+from jinja2.utils import next
+
+# 2.x / 3.x
+try:
+ from io import BytesIO
+except ImportError:
+ from StringIO import StringIO as BytesIO
+
+
+importable_object = 23
+
+_gettext_re = re.compile(r'_\((.*?)\)(?s)')
+
+
+i18n_templates = {
+ 'master.html': '<title>{{ page_title|default(_("missing")) }}</title>'
+ '{% block body %}{% endblock %}',
+ 'child.html': '{% extends "master.html" %}{% block body %}'
+ '{% trans %}watch out{% endtrans %}{% endblock %}',
+ 'plural.html': '{% trans user_count %}One user online{% pluralize %}'
+ '{{ user_count }} users online{% endtrans %}',
+ 'stringformat.html': '{{ _("User: %(num)s")|format(num=user_count) }}'
+}
+
+newstyle_i18n_templates = {
+ 'master.html': '<title>{{ page_title|default(_("missing")) }}</title>'
+ '{% block body %}{% endblock %}',
+ 'child.html': '{% extends "master.html" %}{% block body %}'
+ '{% trans %}watch out{% endtrans %}{% endblock %}',
+ 'plural.html': '{% trans user_count %}One user online{% pluralize %}'
+ '{{ user_count }} users online{% endtrans %}',
+ 'stringformat.html': '{{ _("User: %(num)s", num=user_count) }}',
+ 'ngettext.html': '{{ ngettext("%(num)s apple", "%(num)s apples", apples) }}',
+ 'ngettext_long.html': '{% trans num=apples %}{{ num }} apple{% pluralize %}'
+ '{{ num }} apples{% endtrans %}',
+ 'transvars1.html': '{% trans %}User: {{ num }}{% endtrans %}',
+ 'transvars2.html': '{% trans num=count %}User: {{ num }}{% endtrans %}',
+ 'transvars3.html': '{% trans count=num %}User: {{ count }}{% endtrans %}',
+ 'novars.html': '{% trans %}%(hello)s{% endtrans %}',
+ 'vars.html': '{% trans %}{{ foo }}%(foo)s{% endtrans %}',
+ 'explicitvars.html': '{% trans foo="42" %}%(foo)s{% endtrans %}'
+}
+
+
+languages = {
+ 'de': {
+ 'missing': u'fehlend',
+ 'watch out': u'pass auf',
+ 'One user online': u'Ein Benutzer online',
+ '%(user_count)s users online': u'%(user_count)s Benutzer online',
+ 'User: %(num)s': u'Benutzer: %(num)s',
+ 'User: %(count)s': u'Benutzer: %(count)s',
+ '%(num)s apple': u'%(num)s Apfel',
+ '%(num)s apples': u'%(num)s Äpfel'
+ }
+}
+
+
+@contextfunction
+def gettext(context, string):
+ language = context.get('LANGUAGE', 'en')
+ return languages.get(language, {}).get(string, string)
+
+
+@contextfunction
+def ngettext(context, s, p, n):
+ language = context.get('LANGUAGE', 'en')
+ if n != 1:
+ return languages.get(language, {}).get(p, p)
+ return languages.get(language, {}).get(s, s)
+
+
+i18n_env = Environment(
+ loader=DictLoader(i18n_templates),
+ extensions=['jinja2.ext.i18n']
+)
+i18n_env.globals.update({
+ '_': gettext,
+ 'gettext': gettext,
+ 'ngettext': ngettext
+})
+
+newstyle_i18n_env = Environment(
+ loader=DictLoader(newstyle_i18n_templates),
+ extensions=['jinja2.ext.i18n']
+)
+newstyle_i18n_env.install_gettext_callables(gettext, ngettext, newstyle=True)
+
+class TestExtension(Extension):
+ tags = set(['test'])
+ ext_attr = 42
+
+ def parse(self, parser):
+ return nodes.Output([self.call_method('_dump', [
+ nodes.EnvironmentAttribute('sandboxed'),
+ self.attr('ext_attr'),
+ nodes.ImportedName(__name__ + '.importable_object'),
+ nodes.ContextReference()
+ ])]).set_lineno(next(parser.stream).lineno)
+
+ def _dump(self, sandboxed, ext_attr, imported_object, context):
+ return '%s|%s|%s|%s' % (
+ sandboxed,
+ ext_attr,
+ imported_object,
+ context.blocks
+ )
+
+
+class PreprocessorExtension(Extension):
+
+ def preprocess(self, source, name, filename=None):
+ return source.replace('[[TEST]]', '({{ foo }})')
+
+
+class StreamFilterExtension(Extension):
+
+ def filter_stream(self, stream):
+ for token in stream:
+ if token.type == 'data':
+ for t in self.interpolate(token):
+ yield t
+ else:
+ yield token
+
+ def interpolate(self, token):
+ pos = 0
+ end = len(token.value)
+ lineno = token.lineno
+ while 1:
+ match = _gettext_re.search(token.value, pos)
+ if match is None:
+ break
+ value = token.value[pos:match.start()]
+ if value:
+ yield Token(lineno, 'data', value)
+ lineno += count_newlines(token.value)
+ yield Token(lineno, 'variable_begin', None)
+ yield Token(lineno, 'name', 'gettext')
+ yield Token(lineno, 'lparen', None)
+ yield Token(lineno, 'string', match.group(1))
+ yield Token(lineno, 'rparen', None)
+ yield Token(lineno, 'variable_end', None)
+ pos = match.end()
+ if pos < end:
+ yield Token(lineno, 'data', token.value[pos:])
+
+
+class ExtensionsTestCase(JinjaTestCase):
+
+ def test_extend_late(self):
+ env = Environment()
+ env.add_extension('jinja2.ext.autoescape')
+ t = env.from_string('{% autoescape true %}{{ "<test>" }}{% endautoescape %}')
+ assert t.render() == '<test>'
+
+ def test_loop_controls(self):
+ env = Environment(extensions=['jinja2.ext.loopcontrols'])
+
+ tmpl = env.from_string('''
+ {%- for item in [1, 2, 3, 4] %}
+ {%- if item % 2 == 0 %}{% continue %}{% endif -%}
+ {{ item }}
+ {%- endfor %}''')
+ assert tmpl.render() == '13'
+
+ tmpl = env.from_string('''
+ {%- for item in [1, 2, 3, 4] %}
+ {%- if item > 2 %}{% break %}{% endif -%}
+ {{ item }}
+ {%- endfor %}''')
+ assert tmpl.render() == '12'
+
+ def test_do(self):
+ env = Environment(extensions=['jinja2.ext.do'])
+ tmpl = env.from_string('''
+ {%- set items = [] %}
+ {%- for char in "foo" %}
+ {%- do items.append(loop.index0 ~ char) %}
+ {%- endfor %}{{ items|join(', ') }}''')
+ assert tmpl.render() == '0f, 1o, 2o'
+
+ def test_with(self):
+ env = Environment(extensions=['jinja2.ext.with_'])
+ tmpl = env.from_string('''\
+ {% with a=42, b=23 -%}
+ {{ a }} = {{ b }}
+ {% endwith -%}
+ {{ a }} = {{ b }}\
+ ''')
+ assert [x.strip() for x in tmpl.render(a=1, b=2).splitlines()] \
+ == ['42 = 23', '1 = 2']
+
+ def test_extension_nodes(self):
+ env = Environment(extensions=[TestExtension])
+ tmpl = env.from_string('{% test %}')
+ assert tmpl.render() == 'False|42|23|{}'
+
+ def test_identifier(self):
+ assert TestExtension.identifier == __name__ + '.TestExtension'
+
+ def test_rebinding(self):
+ original = Environment(extensions=[TestExtension])
+ overlay = original.overlay()
+ for env in original, overlay:
+ for ext in env.extensions.itervalues():
+ assert ext.environment is env
+
+ def test_preprocessor_extension(self):
+ env = Environment(extensions=[PreprocessorExtension])
+ tmpl = env.from_string('{[[TEST]]}')
+ assert tmpl.render(foo=42) == '{(42)}'
+
+ def test_streamfilter_extension(self):
+ env = Environment(extensions=[StreamFilterExtension])
+ env.globals['gettext'] = lambda x: x.upper()
+ tmpl = env.from_string('Foo _(bar) Baz')
+ out = tmpl.render()
+ assert out == 'Foo BAR Baz'
+
+ def test_extension_ordering(self):
+ class T1(Extension):
+ priority = 1
+ class T2(Extension):
+ priority = 2
+ env = Environment(extensions=[T1, T2])
+ ext = list(env.iter_extensions())
+ assert ext[0].__class__ is T1
+ assert ext[1].__class__ is T2
+
+
+class InternationalizationTestCase(JinjaTestCase):
+
+ def test_trans(self):
+ tmpl = i18n_env.get_template('child.html')
+ assert tmpl.render(LANGUAGE='de') == '<title>fehlend</title>pass auf'
+
+ def test_trans_plural(self):
+ tmpl = i18n_env.get_template('plural.html')
+ assert tmpl.render(LANGUAGE='de', user_count=1) == 'Ein Benutzer online'
+ assert tmpl.render(LANGUAGE='de', user_count=2) == '2 Benutzer online'
+
+ def test_complex_plural(self):
+ tmpl = i18n_env.from_string('{% trans foo=42, count=2 %}{{ count }} item{% '
+ 'pluralize count %}{{ count }} items{% endtrans %}')
+ assert tmpl.render() == '2 items'
+ self.assert_raises(TemplateAssertionError, i18n_env.from_string,
+ '{% trans foo %}...{% pluralize bar %}...{% endtrans %}')
+
+ def test_trans_stringformatting(self):
+ tmpl = i18n_env.get_template('stringformat.html')
+ assert tmpl.render(LANGUAGE='de', user_count=5) == 'Benutzer: 5'
+
+ def test_extract(self):
+ from jinja2.ext import babel_extract
+ source = BytesIO('''
+ {{ gettext('Hello World') }}
+ {% trans %}Hello World{% endtrans %}
+ {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
+ '''.encode('ascii')) # make python 3 happy
+ assert list(babel_extract(source, ('gettext', 'ngettext', '_'), [], {})) == [
+ (2, 'gettext', u'Hello World', []),
+ (3, 'gettext', u'Hello World', []),
+ (4, 'ngettext', (u'%(users)s user', u'%(users)s users', None), [])
+ ]
+
+ def test_comment_extract(self):
+ from jinja2.ext import babel_extract
+ source = BytesIO('''
+ {# trans first #}
+ {{ gettext('Hello World') }}
+ {% trans %}Hello World{% endtrans %}{# trans second #}
+ {#: third #}
+ {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
+ '''.encode('utf-8')) # make python 3 happy
+ assert list(babel_extract(source, ('gettext', 'ngettext', '_'), ['trans', ':'], {})) == [
+ (3, 'gettext', u'Hello World', ['first']),
+ (4, 'gettext', u'Hello World', ['second']),
+ (6, 'ngettext', (u'%(users)s user', u'%(users)s users', None), ['third'])
+ ]
+
+
+class NewstyleInternationalizationTestCase(JinjaTestCase):
+
+ def test_trans(self):
+ tmpl = newstyle_i18n_env.get_template('child.html')
+ assert tmpl.render(LANGUAGE='de') == '<title>fehlend</title>pass auf'
+
+ def test_trans_plural(self):
+ tmpl = newstyle_i18n_env.get_template('plural.html')
+ assert tmpl.render(LANGUAGE='de', user_count=1) == 'Ein Benutzer online'
+ assert tmpl.render(LANGUAGE='de', user_count=2) == '2 Benutzer online'
+
+ def test_complex_plural(self):
+ tmpl = newstyle_i18n_env.from_string('{% trans foo=42, count=2 %}{{ count }} item{% '
+ 'pluralize count %}{{ count }} items{% endtrans %}')
+ assert tmpl.render() == '2 items'
+ self.assert_raises(TemplateAssertionError, i18n_env.from_string,
+ '{% trans foo %}...{% pluralize bar %}...{% endtrans %}')
+
+ def test_trans_stringformatting(self):
+ tmpl = newstyle_i18n_env.get_template('stringformat.html')
+ assert tmpl.render(LANGUAGE='de', user_count=5) == 'Benutzer: 5'
+
+ def test_newstyle_plural(self):
+ tmpl = newstyle_i18n_env.get_template('ngettext.html')
+ assert tmpl.render(LANGUAGE='de', apples=1) == '1 Apfel'
+ assert tmpl.render(LANGUAGE='de', apples=5) == u'5 Äpfel'
+
+ def test_autoescape_support(self):
+ env = Environment(extensions=['jinja2.ext.autoescape',
+ 'jinja2.ext.i18n'])
+ env.install_gettext_callables(lambda x: u'<strong>Wert: %(name)s</strong>',
+ lambda s, p, n: s, newstyle=True)
+ t = env.from_string('{% autoescape ae %}{{ gettext("foo", name='
+ '"<test>") }}{% endautoescape %}')
+ assert t.render(ae=True) == '<strong>Wert: <test></strong>'
+ assert t.render(ae=False) == '<strong>Wert: <test></strong>'
+
+ def test_num_used_twice(self):
+ tmpl = newstyle_i18n_env.get_template('ngettext_long.html')
+ assert tmpl.render(apples=5, LANGUAGE='de') == u'5 Äpfel'
+
+ def test_num_called_num(self):
+ source = newstyle_i18n_env.compile('''
+ {% trans num=3 %}{{ num }} apple{% pluralize
+ %}{{ num }} apples{% endtrans %}
+ ''', raw=True)
+ # quite hacky, but the only way to properly test that. The idea is
+ # that the generated code does not pass num twice (although that
+ # would work) for better performance. This only works on the
+ # newstyle gettext of course
+ assert re.search(r"l_ngettext, u?'\%\(num\)s apple', u?'\%\(num\)s "
+ r"apples', 3", source) is not None
+
+ def test_trans_vars(self):
+ t1 = newstyle_i18n_env.get_template('transvars1.html')
+ t2 = newstyle_i18n_env.get_template('transvars2.html')
+ t3 = newstyle_i18n_env.get_template('transvars3.html')
+ assert t1.render(num=1, LANGUAGE='de') == 'Benutzer: 1'
+ assert t2.render(count=23, LANGUAGE='de') == 'Benutzer: 23'
+ assert t3.render(num=42, LANGUAGE='de') == 'Benutzer: 42'
+
+ def test_novars_vars_escaping(self):
+ t = newstyle_i18n_env.get_template('novars.html')
+ assert t.render() == '%(hello)s'
+ t = newstyle_i18n_env.get_template('vars.html')
+ assert t.render(foo='42') == '42%(foo)s'
+ t = newstyle_i18n_env.get_template('explicitvars.html')
+ assert t.render() == '%(foo)s'
+
+
+class AutoEscapeTestCase(JinjaTestCase):
+
+ def test_scoped_setting(self):
+ env = Environment(extensions=['jinja2.ext.autoescape'],
+ autoescape=True)
+ tmpl = env.from_string('''
+ {{ "<HelloWorld>" }}
+ {% autoescape false %}
+ {{ "<HelloWorld>" }}
+ {% endautoescape %}
+ {{ "<HelloWorld>" }}
+ ''')
+ assert tmpl.render().split() == \
+ [u'<HelloWorld>', u'<HelloWorld>', u'<HelloWorld>']
+
+ env = Environment(extensions=['jinja2.ext.autoescape'],
+ autoescape=False)
+ tmpl = env.from_string('''
+ {{ "<HelloWorld>" }}
+ {% autoescape true %}
+ {{ "<HelloWorld>" }}
+ {% endautoescape %}
+ {{ "<HelloWorld>" }}
+ ''')
+ assert tmpl.render().split() == \
+ [u'<HelloWorld>', u'<HelloWorld>', u'<HelloWorld>']
+
+ def test_nonvolatile(self):
+ env = Environment(extensions=['jinja2.ext.autoescape'],
+ autoescape=True)
+ tmpl = env.from_string('{{ {"foo": "<test>"}|xmlattr|escape }}')
+ assert tmpl.render() == ' foo="<test>"'
+ tmpl = env.from_string('{% autoescape false %}{{ {"foo": "<test>"}'
+ '|xmlattr|escape }}{% endautoescape %}')
+ assert tmpl.render() == ' foo="&lt;test&gt;"'
+
+ def test_volatile(self):
+ env = Environment(extensions=['jinja2.ext.autoescape'],
+ autoescape=True)
+ tmpl = env.from_string('{% autoescape foo %}{{ {"foo": "<test>"}'
+ '|xmlattr|escape }}{% endautoescape %}')
+ assert tmpl.render(foo=False) == ' foo="&lt;test&gt;"'
+ assert tmpl.render(foo=True) == ' foo="<test>"'
+
+ def test_scoping(self):
+ env = Environment(extensions=['jinja2.ext.autoescape'])
+ tmpl = env.from_string('{% autoescape true %}{% set x = "<x>" %}{{ x }}'
+ '{% endautoescape %}{{ x }}{{ "<y>" }}')
+ assert tmpl.render(x=1) == '<x>1<y>'
+
+ def test_volatile_scoping(self):
+ env = Environment(extensions=['jinja2.ext.autoescape'])
+ tmplsource = '''
+ {% autoescape val %}
+ {% macro foo(x) %}
+ [{{ x }}]
+ {% endmacro %}
+ {{ foo().__class__.__name__ }}
+ {% endautoescape %}
+ {{ '<testing>' }}
+ '''
+ tmpl = env.from_string(tmplsource)
+ assert tmpl.render(val=True).split()[0] == 'Markup'
+ assert tmpl.render(val=False).split()[0] == unicode.__name__
+
+ # looking at the source we should see <testing> there in raw
+ # (and then escaped as well)
+ env = Environment(extensions=['jinja2.ext.autoescape'])
+ pysource = env.compile(tmplsource, raw=True)
+ assert '<testing>\\n' in pysource
+
+ env = Environment(extensions=['jinja2.ext.autoescape'],
+ autoescape=True)
+ pysource = env.compile(tmplsource, raw=True)
+ assert '<testing>\\n' in pysource
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(ExtensionsTestCase))
+ suite.addTest(unittest.makeSuite(InternationalizationTestCase))
+ suite.addTest(unittest.makeSuite(NewstyleInternationalizationTestCase))
+ suite.addTest(unittest.makeSuite(AutoEscapeTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/filters.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/filters.py
new file mode 100644
index 0000000..b59c9e3
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/filters.py
@@ -0,0 +1,291 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.filters
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Tests for the jinja filters.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import unittest
+from jinja2.testsuite import JinjaTestCase
+
+from jinja2 import Markup, Environment
+
+env = Environment()
+
+
+class FilterTestCase(JinjaTestCase):
+
+ def test_capitalize(self):
+ tmpl = env.from_string('{{ "foo bar"|capitalize }}')
+ assert tmpl.render() == 'Foo bar'
+
+ def test_center(self):
+ tmpl = env.from_string('{{ "foo"|center(9) }}')
+ assert tmpl.render() == ' foo '
+
+ def test_default(self):
+ tmpl = env.from_string(
+ "{{ missing|default('no') }}|{{ false|default('no') }}|"
+ "{{ false|default('no', true) }}|{{ given|default('no') }}"
+ )
+ assert tmpl.render(given='yes') == 'no|False|no|yes'
+
+ def test_dictsort(self):
+ tmpl = env.from_string(
+ '{{ foo|dictsort }}|'
+ '{{ foo|dictsort(true) }}|'
+ '{{ foo|dictsort(false, "value") }}'
+ )
+ out = tmpl.render(foo={"aa": 0, "b": 1, "c": 2, "AB": 3})
+ assert out == ("[('aa', 0), ('AB', 3), ('b', 1), ('c', 2)]|"
+ "[('AB', 3), ('aa', 0), ('b', 1), ('c', 2)]|"
+ "[('aa', 0), ('b', 1), ('c', 2), ('AB', 3)]")
+
+ def test_batch(self):
+ tmpl = env.from_string("{{ foo|batch(3)|list }}|"
+ "{{ foo|batch(3, 'X')|list }}")
+ out = tmpl.render(foo=range(10))
+ assert out == ("[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]|"
+ "[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 'X', 'X']]")
+
+ def test_slice(self):
+ tmpl = env.from_string('{{ foo|slice(3)|list }}|'
+ '{{ foo|slice(3, "X")|list }}')
+ out = tmpl.render(foo=range(10))
+ assert out == ("[[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]]|"
+ "[[0, 1, 2, 3], [4, 5, 6, 'X'], [7, 8, 9, 'X']]")
+
+ def test_escape(self):
+ tmpl = env.from_string('''{{ '<">&'|escape }}''')
+ out = tmpl.render()
+ assert out == '<">&'
+
+ def test_striptags(self):
+ tmpl = env.from_string('''{{ foo|striptags }}''')
+ out = tmpl.render(foo=' <p>just a small \n <a href="#">'
+ 'example</a> link</p>\n<p>to a webpage</p> '
+ '<!-- <p>and some commented stuff</p> -->')
+ assert out == 'just a small example link to a webpage'
+
+ def test_filesizeformat(self):
+ tmpl = env.from_string(
+ '{{ 100|filesizeformat }}|'
+ '{{ 1000|filesizeformat }}|'
+ '{{ 1000000|filesizeformat }}|'
+ '{{ 1000000000|filesizeformat }}|'
+ '{{ 1000000000000|filesizeformat }}|'
+ '{{ 100|filesizeformat(true) }}|'
+ '{{ 1000|filesizeformat(true) }}|'
+ '{{ 1000000|filesizeformat(true) }}|'
+ '{{ 1000000000|filesizeformat(true) }}|'
+ '{{ 1000000000000|filesizeformat(true) }}'
+ )
+ out = tmpl.render()
+ assert out == (
+ '100 Bytes|1.0 KB|1.0 MB|1.0 GB|1000.0 GB|'
+ '100 Bytes|1000 Bytes|976.6 KiB|953.7 MiB|931.3 GiB'
+ )
+
+ def test_first(self):
+ tmpl = env.from_string('{{ foo|first }}')
+ out = tmpl.render(foo=range(10))
+ assert out == '0'
+
+ def test_float(self):
+ tmpl = env.from_string('{{ "42"|float }}|'
+ '{{ "ajsghasjgd"|float }}|'
+ '{{ "32.32"|float }}')
+ out = tmpl.render()
+ assert out == '42.0|0.0|32.32'
+
+ def test_format(self):
+ tmpl = env.from_string('''{{ "%s|%s"|format("a", "b") }}''')
+ out = tmpl.render()
+ assert out == 'a|b'
+
+ def test_indent(self):
+ tmpl = env.from_string('{{ foo|indent(2) }}|{{ foo|indent(2, true) }}')
+ text = '\n'.join([' '.join(['foo', 'bar'] * 2)] * 2)
+ out = tmpl.render(foo=text)
+ assert out == ('foo bar foo bar\n foo bar foo bar| '
+ 'foo bar foo bar\n foo bar foo bar')
+
+ def test_int(self):
+ tmpl = env.from_string('{{ "42"|int }}|{{ "ajsghasjgd"|int }}|'
+ '{{ "32.32"|int }}')
+ out = tmpl.render()
+ assert out == '42|0|32'
+
+ def test_join(self):
+ tmpl = env.from_string('{{ [1, 2, 3]|join("|") }}')
+ out = tmpl.render()
+ assert out == '1|2|3'
+
+ env2 = Environment(autoescape=True)
+ tmpl = env2.from_string('{{ ["<foo>", "<span>foo</span>"|safe]|join }}')
+ assert tmpl.render() == '<foo><span>foo</span>'
+
+ def test_last(self):
+ tmpl = env.from_string('''{{ foo|last }}''')
+ out = tmpl.render(foo=range(10))
+ assert out == '9'
+
+ def test_length(self):
+ tmpl = env.from_string('''{{ "hello world"|length }}''')
+ out = tmpl.render()
+ assert out == '11'
+
+ def test_lower(self):
+ tmpl = env.from_string('''{{ "FOO"|lower }}''')
+ out = tmpl.render()
+ assert out == 'foo'
+
+ def test_pprint(self):
+ from pprint import pformat
+ tmpl = env.from_string('''{{ data|pprint }}''')
+ data = range(1000)
+ assert tmpl.render(data=data) == pformat(data)
+
+ def test_random(self):
+ tmpl = env.from_string('''{{ seq|random }}''')
+ seq = range(100)
+ for _ in range(10):
+ assert int(tmpl.render(seq=seq)) in seq
+
+ def test_reverse(self):
+ tmpl = env.from_string('{{ "foobar"|reverse|join }}|'
+ '{{ [1, 2, 3]|reverse|list }}')
+ assert tmpl.render() == 'raboof|[3, 2, 1]'
+
+ def test_string(self):
+ x = [1, 2, 3, 4, 5]
+ tmpl = env.from_string('''{{ obj|string }}''')
+ assert tmpl.render(obj=x) == unicode(x)
+
+ def test_title(self):
+ tmpl = env.from_string('''{{ "foo bar"|title }}''')
+ assert tmpl.render() == "Foo Bar"
+
+ def test_truncate(self):
+ tmpl = env.from_string(
+ '{{ data|truncate(15, true, ">>>") }}|'
+ '{{ data|truncate(15, false, ">>>") }}|'
+ '{{ smalldata|truncate(15) }}'
+ )
+ out = tmpl.render(data='foobar baz bar' * 1000,
+ smalldata='foobar baz bar')
+ assert out == 'foobar baz barf>>>|foobar baz >>>|foobar baz bar'
+
+ def test_upper(self):
+ tmpl = env.from_string('{{ "foo"|upper }}')
+ assert tmpl.render() == 'FOO'
+
+ def test_urlize(self):
+ tmpl = env.from_string('{{ "foo http://www.example.com/ bar"|urlize }}')
+ assert tmpl.render() == 'foo <a href="http://www.example.com/">'\
+ 'http://www.example.com/</a> bar'
+
+ def test_wordcount(self):
+ tmpl = env.from_string('{{ "foo bar baz"|wordcount }}')
+ assert tmpl.render() == '3'
+
+ def test_block(self):
+ tmpl = env.from_string('{% filter lower|escape %}<HEHE>{% endfilter %}')
+ assert tmpl.render() == '<hehe>'
+
+ def test_chaining(self):
+ tmpl = env.from_string('''{{ ['<foo>', '<bar>']|first|upper|escape }}''')
+ assert tmpl.render() == '<FOO>'
+
+ def test_sum(self):
+ tmpl = env.from_string('''{{ [1, 2, 3, 4, 5, 6]|sum }}''')
+ assert tmpl.render() == '21'
+
+ def test_abs(self):
+ tmpl = env.from_string('''{{ -1|abs }}|{{ 1|abs }}''')
+ assert tmpl.render() == '1|1', tmpl.render()
+
+ def test_round_positive(self):
+ tmpl = env.from_string('{{ 2.7|round }}|{{ 2.1|round }}|'
+ "{{ 2.1234|round(3, 'floor') }}|"
+ "{{ 2.1|round(0, 'ceil') }}")
+ assert tmpl.render() == '3.0|2.0|2.123|3.0', tmpl.render()
+
+ def test_round_negative(self):
+ tmpl = env.from_string('{{ 21.3|round(-1)}}|'
+ "{{ 21.3|round(-1, 'ceil')}}|"
+ "{{ 21.3|round(-1, 'floor')}}")
+ assert tmpl.render() == '20.0|30.0|20.0',tmpl.render()
+
+ def test_xmlattr(self):
+ tmpl = env.from_string("{{ {'foo': 42, 'bar': 23, 'fish': none, "
+ "'spam': missing, 'blub:blub': '<?>'}|xmlattr }}")
+ out = tmpl.render().split()
+ assert len(out) == 3
+ assert 'foo="42"' in out
+ assert 'bar="23"' in out
+ assert 'blub:blub="<?>"' in out
+
+ def test_sort1(self):
+ tmpl = env.from_string('{{ [2, 3, 1]|sort }}|{{ [2, 3, 1]|sort(true) }}')
+ assert tmpl.render() == '[1, 2, 3]|[3, 2, 1]'
+
+ def test_sort2(self):
+ tmpl = env.from_string('{{ "".join(["c", "A", "b", "D"]|sort(false, true)) }}')
+ assert tmpl.render() == 'AbcD'
+
+ def test_groupby(self):
+ tmpl = env.from_string('''
+ {%- for grouper, list in [{'foo': 1, 'bar': 2},
+ {'foo': 2, 'bar': 3},
+ {'foo': 1, 'bar': 1},
+ {'foo': 3, 'bar': 4}]|groupby('foo') -%}
+ {{ grouper }}{% for x in list %}: {{ x.foo }}, {{ x.bar }}{% endfor %}|
+ {%- endfor %}''')
+ assert tmpl.render().split('|') == [
+ "1: 1, 2: 1, 1",
+ "2: 2, 3",
+ "3: 3, 4",
+ ""
+ ]
+
+ def test_filtertag(self):
+ tmpl = env.from_string("{% filter upper|replace('FOO', 'foo') %}"
+ "foobar{% endfilter %}")
+ assert tmpl.render() == 'fooBAR'
+
+ def test_replace(self):
+ env = Environment()
+ tmpl = env.from_string('{{ string|replace("o", 42) }}')
+ assert tmpl.render(string='<foo>') == '<f4242>'
+ env = Environment(autoescape=True)
+ tmpl = env.from_string('{{ string|replace("o", 42) }}')
+ assert tmpl.render(string='<foo>') == '<f4242>'
+ tmpl = env.from_string('{{ string|replace("<", 42) }}')
+ assert tmpl.render(string='<foo>') == '42foo>'
+ tmpl = env.from_string('{{ string|replace("o", ">x<") }}')
+ assert tmpl.render(string=Markup('foo')) == 'f>x<>x<'
+
+ def test_forceescape(self):
+ tmpl = env.from_string('{{ x|forceescape }}')
+ assert tmpl.render(x=Markup('<div />')) == u'<div />'
+
+ def test_safe(self):
+ env = Environment(autoescape=True)
+ tmpl = env.from_string('{{ "<div>foo</div>"|safe }}')
+ assert tmpl.render() == '<div>foo</div>'
+ tmpl = env.from_string('{{ "<div>foo</div>" }}')
+ assert tmpl.render() == '<div>foo</div>'
+
+ def test_sort2(self):
+ tmpl = env.from_string('''{{ ['foo', 'Bar', 'blah']|sort }}''')
+ assert tmpl.render() == "['Bar', 'blah', 'foo']"
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(FilterTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/imports.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/imports.py
new file mode 100644
index 0000000..c785606
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/imports.py
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.imports
+ ~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Tests the import features (with includes).
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import time
+import tempfile
+import unittest
+
+from jinja2.testsuite import JinjaTestCase
+
+from jinja2 import Environment, DictLoader
+from jinja2.exceptions import TemplateNotFound, TemplatesNotFound
+
+
+test_env = Environment(loader=DictLoader(dict(
+ module='{% macro test() %}[{{ foo }}|{{ bar }}]{% endmacro %}',
+ header='[{{ foo }}|{{ 23 }}]',
+ o_printer='({{ o }})'
+)))
+test_env.globals['bar'] = 23
+
+
+class ImportsTestCase(JinjaTestCase):
+
+ def test_context_imports(self):
+ t = test_env.from_string('{% import "module" as m %}{{ m.test() }}')
+ assert t.render(foo=42) == '[|23]'
+ t = test_env.from_string('{% import "module" as m without context %}{{ m.test() }}')
+ assert t.render(foo=42) == '[|23]'
+ t = test_env.from_string('{% import "module" as m with context %}{{ m.test() }}')
+ assert t.render(foo=42) == '[42|23]'
+ t = test_env.from_string('{% from "module" import test %}{{ test() }}')
+ assert t.render(foo=42) == '[|23]'
+ t = test_env.from_string('{% from "module" import test without context %}{{ test() }}')
+ assert t.render(foo=42) == '[|23]'
+ t = test_env.from_string('{% from "module" import test with context %}{{ test() }}')
+ assert t.render(foo=42) == '[42|23]'
+
+ def test_trailing_comma(self):
+ test_env.from_string('{% from "foo" import bar, baz with context %}')
+ test_env.from_string('{% from "foo" import bar, baz, with context %}')
+ test_env.from_string('{% from "foo" import bar, with context %}')
+ test_env.from_string('{% from "foo" import bar, with, context %}')
+ test_env.from_string('{% from "foo" import bar, with with context %}')
+
+ def test_exports(self):
+ m = test_env.from_string('''
+ {% macro toplevel() %}...{% endmacro %}
+ {% macro __private() %}...{% endmacro %}
+ {% set variable = 42 %}
+ {% for item in [1] %}
+ {% macro notthere() %}{% endmacro %}
+ {% endfor %}
+ ''').module
+ assert m.toplevel() == '...'
+ assert not hasattr(m, '__missing')
+ assert m.variable == 42
+ assert not hasattr(m, 'notthere')
+
+
+class IncludesTestCase(JinjaTestCase):
+
+ def test_context_include(self):
+ t = test_env.from_string('{% include "header" %}')
+ assert t.render(foo=42) == '[42|23]'
+ t = test_env.from_string('{% include "header" with context %}')
+ assert t.render(foo=42) == '[42|23]'
+ t = test_env.from_string('{% include "header" without context %}')
+ assert t.render(foo=42) == '[|23]'
+
+ def test_choice_includes(self):
+ t = test_env.from_string('{% include ["missing", "header"] %}')
+ assert t.render(foo=42) == '[42|23]'
+
+ t = test_env.from_string('{% include ["missing", "missing2"] ignore missing %}')
+ assert t.render(foo=42) == ''
+
+ t = test_env.from_string('{% include ["missing", "missing2"] %}')
+ self.assert_raises(TemplateNotFound, t.render)
+ try:
+ t.render()
+ except TemplatesNotFound, e:
+ assert e.templates == ['missing', 'missing2']
+ assert e.name == 'missing2'
+ else:
+ assert False, 'thou shalt raise'
+
+ def test_includes(t, **ctx):
+ ctx['foo'] = 42
+ assert t.render(ctx) == '[42|23]'
+
+ t = test_env.from_string('{% include ["missing", "header"] %}')
+ test_includes(t)
+ t = test_env.from_string('{% include x %}')
+ test_includes(t, x=['missing', 'header'])
+ t = test_env.from_string('{% include [x, "header"] %}')
+ test_includes(t, x='missing')
+ t = test_env.from_string('{% include x %}')
+ test_includes(t, x='header')
+ t = test_env.from_string('{% include x %}')
+ test_includes(t, x='header')
+ t = test_env.from_string('{% include [x] %}')
+ test_includes(t, x='header')
+
+ def test_include_ignoring_missing(self):
+ t = test_env.from_string('{% include "missing" %}')
+ self.assert_raises(TemplateNotFound, t.render)
+ for extra in '', 'with context', 'without context':
+ t = test_env.from_string('{% include "missing" ignore missing ' +
+ extra + ' %}')
+ assert t.render() == ''
+
+ def test_context_include_with_overrides(self):
+ env = Environment(loader=DictLoader(dict(
+ main="{% for item in [1, 2, 3] %}{% include 'item' %}{% endfor %}",
+ item="{{ item }}"
+ )))
+ assert env.get_template("main").render() == "123"
+
+ def test_unoptimized_scopes(self):
+ t = test_env.from_string("""
+ {% macro outer(o) %}
+ {% macro inner() %}
+ {% include "o_printer" %}
+ {% endmacro %}
+ {{ inner() }}
+ {% endmacro %}
+ {{ outer("FOO") }}
+ """)
+ assert t.render().strip() == '(FOO)'
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(ImportsTestCase))
+ suite.addTest(unittest.makeSuite(IncludesTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/inheritance.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/inheritance.py
new file mode 100644
index 0000000..87b4a59
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/inheritance.py
@@ -0,0 +1,208 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.inheritance
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Tests the template inheritance feature.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import time
+import tempfile
+import unittest
+
+from jinja2.testsuite import JinjaTestCase
+
+from jinja2 import Environment, DictLoader
+from jinja2.exceptions import TemplateSyntaxError
+
+
+LAYOUTTEMPLATE = '''\
+|{% block block1 %}block 1 from layout{% endblock %}
+|{% block block2 %}block 2 from layout{% endblock %}
+|{% block block3 %}
+{% block block4 %}nested block 4 from layout{% endblock %}
+{% endblock %}|'''
+
+LEVEL1TEMPLATE = '''\
+{% extends "layout" %}
+{% block block1 %}block 1 from level1{% endblock %}'''
+
+LEVEL2TEMPLATE = '''\
+{% extends "level1" %}
+{% block block2 %}{% block block5 %}nested block 5 from level2{%
+endblock %}{% endblock %}'''
+
+LEVEL3TEMPLATE = '''\
+{% extends "level2" %}
+{% block block5 %}block 5 from level3{% endblock %}
+{% block block4 %}block 4 from level3{% endblock %}
+'''
+
+LEVEL4TEMPLATE = '''\
+{% extends "level3" %}
+{% block block3 %}block 3 from level4{% endblock %}
+'''
+
+WORKINGTEMPLATE = '''\
+{% extends "layout" %}
+{% block block1 %}
+ {% if false %}
+ {% block block2 %}
+ this should workd
+ {% endblock %}
+ {% endif %}
+{% endblock %}
+'''
+
+env = Environment(loader=DictLoader({
+ 'layout': LAYOUTTEMPLATE,
+ 'level1': LEVEL1TEMPLATE,
+ 'level2': LEVEL2TEMPLATE,
+ 'level3': LEVEL3TEMPLATE,
+ 'level4': LEVEL4TEMPLATE,
+ 'working': WORKINGTEMPLATE
+}), trim_blocks=True)
+
+
+class InheritanceTestCase(JinjaTestCase):
+
+ def test_layout(self):
+ tmpl = env.get_template('layout')
+ assert tmpl.render() == ('|block 1 from layout|block 2 from '
+ 'layout|nested block 4 from layout|')
+
+ def test_level1(self):
+ tmpl = env.get_template('level1')
+ assert tmpl.render() == ('|block 1 from level1|block 2 from '
+ 'layout|nested block 4 from layout|')
+
+ def test_level2(self):
+ tmpl = env.get_template('level2')
+ assert tmpl.render() == ('|block 1 from level1|nested block 5 from '
+ 'level2|nested block 4 from layout|')
+
+ def test_level3(self):
+ tmpl = env.get_template('level3')
+ assert tmpl.render() == ('|block 1 from level1|block 5 from level3|'
+ 'block 4 from level3|')
+
+ def test_level4(sel):
+ tmpl = env.get_template('level4')
+ assert tmpl.render() == ('|block 1 from level1|block 5 from '
+ 'level3|block 3 from level4|')
+
+ def test_super(self):
+ env = Environment(loader=DictLoader({
+ 'a': '{% block intro %}INTRO{% endblock %}|'
+ 'BEFORE|{% block data %}INNER{% endblock %}|AFTER',
+ 'b': '{% extends "a" %}{% block data %}({{ '
+ 'super() }}){% endblock %}',
+ 'c': '{% extends "b" %}{% block intro %}--{{ '
+ 'super() }}--{% endblock %}\n{% block data '
+ '%}[{{ super() }}]{% endblock %}'
+ }))
+ tmpl = env.get_template('c')
+ assert tmpl.render() == '--INTRO--|BEFORE|[(INNER)]|AFTER'
+
+ def test_working(self):
+ tmpl = env.get_template('working')
+
+ def test_reuse_blocks(self):
+ tmpl = env.from_string('{{ self.foo() }}|{% block foo %}42'
+ '{% endblock %}|{{ self.foo() }}')
+ assert tmpl.render() == '42|42|42'
+
+ def test_preserve_blocks(self):
+ env = Environment(loader=DictLoader({
+ 'a': '{% if false %}{% block x %}A{% endblock %}{% endif %}{{ self.x() }}',
+ 'b': '{% extends "a" %}{% block x %}B{{ super() }}{% endblock %}'
+ }))
+ tmpl = env.get_template('b')
+ assert tmpl.render() == 'BA'
+
+ def test_dynamic_inheritance(self):
+ env = Environment(loader=DictLoader({
+ 'master1': 'MASTER1{% block x %}{% endblock %}',
+ 'master2': 'MASTER2{% block x %}{% endblock %}',
+ 'child': '{% extends master %}{% block x %}CHILD{% endblock %}'
+ }))
+ tmpl = env.get_template('child')
+ for m in range(1, 3):
+ assert tmpl.render(master='master%d' % m) == 'MASTER%dCHILD' % m
+
+ def test_multi_inheritance(self):
+ env = Environment(loader=DictLoader({
+ 'master1': 'MASTER1{% block x %}{% endblock %}',
+ 'master2': 'MASTER2{% block x %}{% endblock %}',
+ 'child': '''{% if master %}{% extends master %}{% else %}{% extends
+ 'master1' %}{% endif %}{% block x %}CHILD{% endblock %}'''
+ }))
+ tmpl = env.get_template('child')
+ assert tmpl.render(master='master2') == 'MASTER2CHILD'
+ assert tmpl.render(master='master1') == 'MASTER1CHILD'
+ assert tmpl.render() == 'MASTER1CHILD'
+
+ def test_scoped_block(self):
+ env = Environment(loader=DictLoader({
+ 'master.html': '{% for item in seq %}[{% block item scoped %}'
+ '{% endblock %}]{% endfor %}'
+ }))
+ t = env.from_string('{% extends "master.html" %}{% block item %}'
+ '{{ item }}{% endblock %}')
+ assert t.render(seq=range(5)) == '[0][1][2][3][4]'
+
+ def test_super_in_scoped_block(self):
+ env = Environment(loader=DictLoader({
+ 'master.html': '{% for item in seq %}[{% block item scoped %}'
+ '{{ item }}{% endblock %}]{% endfor %}'
+ }))
+ t = env.from_string('{% extends "master.html" %}{% block item %}'
+ '{{ super() }}|{{ item * 2 }}{% endblock %}')
+ assert t.render(seq=range(5)) == '[0|0][1|2][2|4][3|6][4|8]'
+
+
+class BugFixTestCase(JinjaTestCase):
+
+ def test_fixed_macro_scoping_bug(self):
+ assert Environment(loader=DictLoader({
+ 'test.html': '''\
+ {% extends 'details.html' %}
+
+ {% macro my_macro() %}
+ my_macro
+ {% endmacro %}
+
+ {% block inner_box %}
+ {{ my_macro() }}
+ {% endblock %}
+ ''',
+ 'details.html': '''\
+ {% extends 'standard.html' %}
+
+ {% macro my_macro() %}
+ my_macro
+ {% endmacro %}
+
+ {% block content %}
+ {% block outer_box %}
+ outer_box
+ {% block inner_box %}
+ inner_box
+ {% endblock %}
+ {% endblock %}
+ {% endblock %}
+ ''',
+ 'standard.html': '''
+ {% block content %} {% endblock %}
+ '''
+ })).get_template("test.html").render().split() == [u'outer_box', u'my_macro']
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(InheritanceTestCase))
+ suite.addTest(unittest.makeSuite(BugFixTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/lexnparse.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/lexnparse.py
new file mode 100644
index 0000000..008a0a9
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/lexnparse.py
@@ -0,0 +1,390 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.lexnparse
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ All the unittests regarding lexing, parsing and syntax.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import sys
+import time
+import tempfile
+import unittest
+
+from jinja2.testsuite import JinjaTestCase
+
+from jinja2 import Environment, Template, TemplateSyntaxError, \
+ UndefinedError, nodes
+
+env = Environment()
+
+
+# how does a string look like in jinja syntax?
+if sys.version_info < (3, 0):
+ def jinja_string_repr(string):
+ return repr(string)[1:]
+else:
+ jinja_string_repr = repr
+
+
+class LexerTestCase(JinjaTestCase):
+
+ def test_raw1(self):
+ tmpl = env.from_string('{% raw %}foo{% endraw %}|'
+ '{%raw%}{{ bar }}|{% baz %}{% endraw %}')
+ assert tmpl.render() == 'foo|{{ bar }}|{% baz %}'
+
+ def test_raw2(self):
+ tmpl = env.from_string('1 {%- raw -%} 2 {%- endraw -%} 3')
+ assert tmpl.render() == '123'
+
+ def test_balancing(self):
+ env = Environment('{%', '%}', '${', '}')
+ tmpl = env.from_string('''{% for item in seq
+ %}${{'foo': item}|upper}{% endfor %}''')
+ assert tmpl.render(seq=range(3)) == "{'FOO': 0}{'FOO': 1}{'FOO': 2}"
+
+ def test_comments(self):
+ env = Environment('<!--', '-->', '{', '}')
+ tmpl = env.from_string('''\
+<ul>
+<!--- for item in seq -->
+ <li>{item}</li>
+<!--- endfor -->
+</ul>''')
+ assert tmpl.render(seq=range(3)) == ("<ul>\n <li>0</li>\n "
+ "<li>1</li>\n <li>2</li>\n</ul>")
+
+ def test_string_escapes(self):
+ for char in u'\0', u'\u2668', u'\xe4', u'\t', u'\r', u'\n':
+ tmpl = env.from_string('{{ %s }}' % jinja_string_repr(char))
+ assert tmpl.render() == char
+ assert env.from_string('{{ "\N{HOT SPRINGS}" }}').render() == u'\u2668'
+
+ def test_bytefallback(self):
+ from pprint import pformat
+ tmpl = env.from_string(u'''{{ 'foo'|pprint }}|{{ 'bär'|pprint }}''')
+ assert tmpl.render() == pformat('foo') + '|' + pformat(u'bär')
+
+ def test_operators(self):
+ from jinja2.lexer import operators
+ for test, expect in operators.iteritems():
+ if test in '([{}])':
+ continue
+ stream = env.lexer.tokenize('{{ %s }}' % test)
+ stream.next()
+ assert stream.current.type == expect
+
+ def test_normalizing(self):
+ for seq in '\r', '\r\n', '\n':
+ env = Environment(newline_sequence=seq)
+ tmpl = env.from_string('1\n2\r\n3\n4\n')
+ result = tmpl.render()
+ assert result.replace(seq, 'X') == '1X2X3X4'
+
+
+class ParserTestCase(JinjaTestCase):
+
+ def test_php_syntax(self):
+ env = Environment('<?', '?>', '<?=', '?>', '<!--', '-->')
+ tmpl = env.from_string('''\
+<!-- I'm a comment, I'm not interesting -->\
+<? for item in seq -?>
+ <?= item ?>
+<?- endfor ?>''')
+ assert tmpl.render(seq=range(5)) == '01234'
+
+ def test_erb_syntax(self):
+ env = Environment('<%', '%>', '<%=', '%>', '<%#', '%>')
+ tmpl = env.from_string('''\
+<%# I'm a comment, I'm not interesting %>\
+<% for item in seq -%>
+ <%= item %>
+<%- endfor %>''')
+ assert tmpl.render(seq=range(5)) == '01234'
+
+ def test_comment_syntax(self):
+ env = Environment('<!--', '-->', '${', '}', '<!--#', '-->')
+ tmpl = env.from_string('''\
+<!--# I'm a comment, I'm not interesting -->\
+<!-- for item in seq --->
+ ${item}
+<!--- endfor -->''')
+ assert tmpl.render(seq=range(5)) == '01234'
+
+ def test_balancing(self):
+ tmpl = env.from_string('''{{{'foo':'bar'}.foo}}''')
+ assert tmpl.render() == 'bar'
+
+ def test_start_comment(self):
+ tmpl = env.from_string('''{# foo comment
+and bar comment #}
+{% macro blub() %}foo{% endmacro %}
+{{ blub() }}''')
+ assert tmpl.render().strip() == 'foo'
+
+ def test_line_syntax(self):
+ env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%')
+ tmpl = env.from_string('''\
+<%# regular comment %>
+% for item in seq:
+ ${item}
+% endfor''')
+ assert [int(x.strip()) for x in tmpl.render(seq=range(5)).split()] == \
+ range(5)
+
+ env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##')
+ tmpl = env.from_string('''\
+<%# regular comment %>
+% for item in seq:
+ ${item} ## the rest of the stuff
+% endfor''')
+ assert [int(x.strip()) for x in tmpl.render(seq=range(5)).split()] == \
+ range(5)
+
+ def test_line_syntax_priority(self):
+ # XXX: why is the whitespace there in front of the newline?
+ env = Environment('{%', '%}', '${', '}', '/*', '*/', '##', '#')
+ tmpl = env.from_string('''\
+/* ignore me.
+ I'm a multiline comment */
+## for item in seq:
+* ${item} # this is just extra stuff
+## endfor''')
+ assert tmpl.render(seq=[1, 2]).strip() == '* 1\n* 2'
+ env = Environment('{%', '%}', '${', '}', '/*', '*/', '#', '##')
+ tmpl = env.from_string('''\
+/* ignore me.
+ I'm a multiline comment */
+# for item in seq:
+* ${item} ## this is just extra stuff
+ ## extra stuff i just want to ignore
+# endfor''')
+ assert tmpl.render(seq=[1, 2]).strip() == '* 1\n\n* 2'
+
+ def test_error_messages(self):
+ def assert_error(code, expected):
+ try:
+ Template(code)
+ except TemplateSyntaxError, e:
+ assert str(e) == expected, 'unexpected error message'
+ else:
+ assert False, 'that was suposed to be an error'
+
+ assert_error('{% for item in seq %}...{% endif %}',
+ "Encountered unknown tag 'endif'. Jinja was looking "
+ "for the following tags: 'endfor' or 'else'. The "
+ "innermost block that needs to be closed is 'for'.")
+ assert_error('{% if foo %}{% for item in seq %}...{% endfor %}{% endfor %}',
+ "Encountered unknown tag 'endfor'. Jinja was looking for "
+ "the following tags: 'elif' or 'else' or 'endif'. The "
+ "innermost block that needs to be closed is 'if'.")
+ assert_error('{% if foo %}',
+ "Unexpected end of template. Jinja was looking for the "
+ "following tags: 'elif' or 'else' or 'endif'. The "
+ "innermost block that needs to be closed is 'if'.")
+ assert_error('{% for item in seq %}',
+ "Unexpected end of template. Jinja was looking for the "
+ "following tags: 'endfor' or 'else'. The innermost block "
+ "that needs to be closed is 'for'.")
+ assert_error('{% block foo-bar-baz %}',
+ "Block names in Jinja have to be valid Python identifiers "
+ "and may not contain hypens, use an underscore instead.")
+ assert_error('{% unknown_tag %}',
+ "Encountered unknown tag 'unknown_tag'.")
+
+
+class SyntaxTestCase(JinjaTestCase):
+
+ def test_call(self):
+ env = Environment()
+ env.globals['foo'] = lambda a, b, c, e, g: a + b + c + e + g
+ tmpl = env.from_string("{{ foo('a', c='d', e='f', *['b'], **{'g': 'h'}) }}")
+ assert tmpl.render() == 'abdfh'
+
+ def test_slicing(self):
+ tmpl = env.from_string('{{ [1, 2, 3][:] }}|{{ [1, 2, 3][::-1] }}')
+ assert tmpl.render() == '[1, 2, 3]|[3, 2, 1]'
+
+ def test_attr(self):
+ tmpl = env.from_string("{{ foo.bar }}|{{ foo['bar'] }}")
+ assert tmpl.render(foo={'bar': 42}) == '42|42'
+
+ def test_subscript(self):
+ tmpl = env.from_string("{{ foo[0] }}|{{ foo[-1] }}")
+ assert tmpl.render(foo=[0, 1, 2]) == '0|2'
+
+ def test_tuple(self):
+ tmpl = env.from_string('{{ () }}|{{ (1,) }}|{{ (1, 2) }}')
+ assert tmpl.render() == '()|(1,)|(1, 2)'
+
+ def test_math(self):
+ tmpl = env.from_string('{{ (1 + 1 * 2) - 3 / 2 }}|{{ 2**3 }}')
+ assert tmpl.render() == '1.5|8'
+
+ def test_div(self):
+ tmpl = env.from_string('{{ 3 // 2 }}|{{ 3 / 2 }}|{{ 3 % 2 }}')
+ assert tmpl.render() == '1|1.5|1'
+
+ def test_unary(self):
+ tmpl = env.from_string('{{ +3 }}|{{ -3 }}')
+ assert tmpl.render() == '3|-3'
+
+ def test_concat(self):
+ tmpl = env.from_string("{{ [1, 2] ~ 'foo' }}")
+ assert tmpl.render() == '[1, 2]foo'
+
+ def test_compare(self):
+ tmpl = env.from_string('{{ 1 > 0 }}|{{ 1 >= 1 }}|{{ 2 < 3 }}|'
+ '{{ 2 == 2 }}|{{ 1 <= 1 }}')
+ assert tmpl.render() == 'True|True|True|True|True'
+
+ def test_inop(self):
+ tmpl = env.from_string('{{ 1 in [1, 2, 3] }}|{{ 1 not in [1, 2, 3] }}')
+ assert tmpl.render() == 'True|False'
+
+ def test_literals(self):
+ tmpl = env.from_string('{{ [] }}|{{ {} }}|{{ () }}')
+ assert tmpl.render().lower() == '[]|{}|()'
+
+ def test_bool(self):
+ tmpl = env.from_string('{{ true and false }}|{{ false '
+ 'or true }}|{{ not false }}')
+ assert tmpl.render() == 'False|True|True'
+
+ def test_grouping(self):
+ tmpl = env.from_string('{{ (true and false) or (false and true) and not false }}')
+ assert tmpl.render() == 'False'
+
+ def test_django_attr(self):
+ tmpl = env.from_string('{{ [1, 2, 3].0 }}|{{ [[1]].0.0 }}')
+ assert tmpl.render() == '1|1'
+
+ def test_conditional_expression(self):
+ tmpl = env.from_string('''{{ 0 if true else 1 }}''')
+ assert tmpl.render() == '0'
+
+ def test_short_conditional_expression(self):
+ tmpl = env.from_string('<{{ 1 if false }}>')
+ assert tmpl.render() == '<>'
+
+ tmpl = env.from_string('<{{ (1 if false).bar }}>')
+ self.assert_raises(UndefinedError, tmpl.render)
+
+ def test_filter_priority(self):
+ tmpl = env.from_string('{{ "foo"|upper + "bar"|upper }}')
+ assert tmpl.render() == 'FOOBAR'
+
+ def test_function_calls(self):
+ tests = [
+ (True, '*foo, bar'),
+ (True, '*foo, *bar'),
+ (True, '*foo, bar=42'),
+ (True, '**foo, *bar'),
+ (True, '**foo, bar'),
+ (False, 'foo, bar'),
+ (False, 'foo, bar=42'),
+ (False, 'foo, bar=23, *args'),
+ (False, 'a, b=c, *d, **e'),
+ (False, '*foo, **bar')
+ ]
+ for should_fail, sig in tests:
+ if should_fail:
+ self.assert_raises(TemplateSyntaxError,
+ env.from_string, '{{ foo(%s) }}' % sig)
+ else:
+ env.from_string('foo(%s)' % sig)
+
+ def test_tuple_expr(self):
+ for tmpl in [
+ '{{ () }}',
+ '{{ (1, 2) }}',
+ '{{ (1, 2,) }}',
+ '{{ 1, }}',
+ '{{ 1, 2 }}',
+ '{% for foo, bar in seq %}...{% endfor %}',
+ '{% for x in foo, bar %}...{% endfor %}',
+ '{% for x in foo, %}...{% endfor %}'
+ ]:
+ assert env.from_string(tmpl)
+
+ def test_trailing_comma(self):
+ tmpl = env.from_string('{{ (1, 2,) }}|{{ [1, 2,] }}|{{ {1: 2,} }}')
+ assert tmpl.render().lower() == '(1, 2)|[1, 2]|{1: 2}'
+
+ def test_block_end_name(self):
+ env.from_string('{% block foo %}...{% endblock foo %}')
+ self.assert_raises(TemplateSyntaxError, env.from_string,
+ '{% block x %}{% endblock y %}')
+
+ def test_contant_casing(self):
+ for const in True, False, None:
+ tmpl = env.from_string('{{ %s }}|{{ %s }}|{{ %s }}' % (
+ str(const), str(const).lower(), str(const).upper()
+ ))
+ assert tmpl.render() == '%s|%s|' % (const, const)
+
+ def test_test_chaining(self):
+ self.assert_raises(TemplateSyntaxError, env.from_string,
+ '{{ foo is string is sequence }}')
+ env.from_string('{{ 42 is string or 42 is number }}'
+ ).render() == 'True'
+
+ def test_string_concatenation(self):
+ tmpl = env.from_string('{{ "foo" "bar" "baz" }}')
+ assert tmpl.render() == 'foobarbaz'
+
+ def test_notin(self):
+ bar = xrange(100)
+ tmpl = env.from_string('''{{ not 42 in bar }}''')
+ assert tmpl.render(bar=bar) == unicode(not 42 in bar)
+
+ def test_implicit_subscribed_tuple(self):
+ class Foo(object):
+ def __getitem__(self, x):
+ return x
+ t = env.from_string('{{ foo[1, 2] }}')
+ assert t.render(foo=Foo()) == u'(1, 2)'
+
+ def test_raw2(self):
+ tmpl = env.from_string('{% raw %}{{ FOO }} and {% BAR %}{% endraw %}')
+ assert tmpl.render() == '{{ FOO }} and {% BAR %}'
+
+ def test_const(self):
+ tmpl = env.from_string('{{ true }}|{{ false }}|{{ none }}|'
+ '{{ none is defined }}|{{ missing is defined }}')
+ assert tmpl.render() == 'True|False|None|True|False'
+
+ def test_neg_filter_priority(self):
+ node = env.parse('{{ -1|foo }}')
+ assert isinstance(node.body[0].nodes[0], nodes.Filter)
+ assert isinstance(node.body[0].nodes[0].node, nodes.Neg)
+
+ def test_const_assign(self):
+ constass1 = '''{% set true = 42 %}'''
+ constass2 = '''{% for none in seq %}{% endfor %}'''
+ for tmpl in constass1, constass2:
+ self.assert_raises(TemplateSyntaxError, env.from_string, tmpl)
+
+ def test_localset(self):
+ tmpl = env.from_string('''{% set foo = 0 %}\
+{% for item in [1, 2] %}{% set foo = 1 %}{% endfor %}\
+{{ foo }}''')
+ assert tmpl.render() == '0'
+
+ def test_parse_unary(self):
+ tmpl = env.from_string('{{ -foo["bar"] }}')
+ assert tmpl.render(foo={'bar': 42}) == '-42'
+ tmpl = env.from_string('{{ -foo["bar"]|abs }}')
+ assert tmpl.render(foo={'bar': 42}) == '42'
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(LexerTestCase))
+ suite.addTest(unittest.makeSuite(ParserTestCase))
+ suite.addTest(unittest.makeSuite(SyntaxTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/loader.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/loader.py
new file mode 100644
index 0000000..0ff0d04
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/loader.py
@@ -0,0 +1,191 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.loader
+ ~~~~~~~~~~~~~~~~~~~~~~~
+
+ Test the loaders.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import sys
+import time
+import tempfile
+import shutil
+import unittest
+
+from jinja2.testsuite import JinjaTestCase, dict_loader, \
+ package_loader, filesystem_loader, function_loader, \
+ choice_loader, prefix_loader
+
+from jinja2 import Environment, loaders
+from jinja2.loaders import split_template_path
+from jinja2.exceptions import TemplateNotFound
+
+
+class LoaderTestCase(JinjaTestCase):
+
+ def test_dict_loader(self):
+ env = Environment(loader=dict_loader)
+ tmpl = env.get_template('justdict.html')
+ assert tmpl.render().strip() == 'FOO'
+ self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')
+
+ def test_package_loader(self):
+ env = Environment(loader=package_loader)
+ tmpl = env.get_template('test.html')
+ assert tmpl.render().strip() == 'BAR'
+ self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')
+
+ def test_filesystem_loader(self):
+ env = Environment(loader=filesystem_loader)
+ tmpl = env.get_template('test.html')
+ assert tmpl.render().strip() == 'BAR'
+ tmpl = env.get_template('foo/test.html')
+ assert tmpl.render().strip() == 'FOO'
+ self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')
+
+ def test_choice_loader(self):
+ env = Environment(loader=choice_loader)
+ tmpl = env.get_template('justdict.html')
+ assert tmpl.render().strip() == 'FOO'
+ tmpl = env.get_template('test.html')
+ assert tmpl.render().strip() == 'BAR'
+ self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')
+
+ def test_function_loader(self):
+ env = Environment(loader=function_loader)
+ tmpl = env.get_template('justfunction.html')
+ assert tmpl.render().strip() == 'FOO'
+ self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')
+
+ def test_prefix_loader(self):
+ env = Environment(loader=prefix_loader)
+ tmpl = env.get_template('a/test.html')
+ assert tmpl.render().strip() == 'BAR'
+ tmpl = env.get_template('b/justdict.html')
+ assert tmpl.render().strip() == 'FOO'
+ self.assert_raises(TemplateNotFound, env.get_template, 'missing')
+
+ def test_caching(self):
+ changed = False
+ class TestLoader(loaders.BaseLoader):
+ def get_source(self, environment, template):
+ return u'foo', None, lambda: not changed
+ env = Environment(loader=TestLoader(), cache_size=-1)
+ tmpl = env.get_template('template')
+ assert tmpl is env.get_template('template')
+ changed = True
+ assert tmpl is not env.get_template('template')
+ changed = False
+
+ env = Environment(loader=TestLoader(), cache_size=0)
+ assert env.get_template('template') \
+ is not env.get_template('template')
+
+ env = Environment(loader=TestLoader(), cache_size=2)
+ t1 = env.get_template('one')
+ t2 = env.get_template('two')
+ assert t2 is env.get_template('two')
+ assert t1 is env.get_template('one')
+ t3 = env.get_template('three')
+ assert 'one' in env.cache
+ assert 'two' not in env.cache
+ assert 'three' in env.cache
+
+ def test_split_template_path(self):
+ assert split_template_path('foo/bar') == ['foo', 'bar']
+ assert split_template_path('./foo/bar') == ['foo', 'bar']
+ self.assert_raises(TemplateNotFound, split_template_path, '../foo')
+
+
+class ModuleLoaderTestCase(JinjaTestCase):
+ archive = None
+
+ def compile_down(self, zip='deflated', py_compile=False):
+ super(ModuleLoaderTestCase, self).setup()
+ log = []
+ self.reg_env = Environment(loader=prefix_loader)
+ if zip is not None:
+ self.archive = tempfile.mkstemp(suffix='.zip')[1]
+ else:
+ self.archive = tempfile.mkdtemp()
+ self.reg_env.compile_templates(self.archive, zip=zip,
+ log_function=log.append,
+ py_compile=py_compile)
+ self.mod_env = Environment(loader=loaders.ModuleLoader(self.archive))
+ return ''.join(log)
+
+ def teardown(self):
+ super(ModuleLoaderTestCase, self).teardown()
+ if hasattr(self, 'mod_env'):
+ if os.path.isfile(self.archive):
+ os.remove(self.archive)
+ else:
+ shutil.rmtree(self.archive)
+ self.archive = None
+
+ def test_log(self):
+ log = self.compile_down()
+ assert 'Compiled "a/foo/test.html" as ' \
+ 'tmpl_a790caf9d669e39ea4d280d597ec891c4ef0404a' in log
+ assert 'Finished compiling templates' in log
+ assert 'Could not compile "a/syntaxerror.html": ' \
+ 'Encountered unknown tag \'endif\'' in log
+
+ def _test_common(self):
+ tmpl1 = self.reg_env.get_template('a/test.html')
+ tmpl2 = self.mod_env.get_template('a/test.html')
+ assert tmpl1.render() == tmpl2.render()
+
+ tmpl1 = self.reg_env.get_template('b/justdict.html')
+ tmpl2 = self.mod_env.get_template('b/justdict.html')
+ assert tmpl1.render() == tmpl2.render()
+
+ def test_deflated_zip_compile(self):
+ self.compile_down(zip='deflated')
+ self._test_common()
+
+ def test_stored_zip_compile(self):
+ self.compile_down(zip='stored')
+ self._test_common()
+
+ def test_filesystem_compile(self):
+ self.compile_down(zip=None)
+ self._test_common()
+
+ def test_weak_references(self):
+ self.compile_down()
+ tmpl = self.mod_env.get_template('a/test.html')
+ key = loaders.ModuleLoader.get_template_key('a/test.html')
+ name = self.mod_env.loader.module.__name__
+
+ assert hasattr(self.mod_env.loader.module, key)
+ assert name in sys.modules
+
+ # unset all, ensure the module is gone from sys.modules
+ self.mod_env = tmpl = None
+
+ try:
+ import gc
+ gc.collect()
+ except:
+ pass
+
+ assert name not in sys.modules
+
+ def test_byte_compilation(self):
+ log = self.compile_down(py_compile=True)
+ assert 'Byte-compiled "a/test.html"' in log
+ tmpl1 = self.mod_env.get_template('a/test.html')
+ mod = self.mod_env.loader.module. \
+ tmpl_3c4ddf650c1a73df961a6d3d2ce2752f1b8fd490
+ assert mod.__file__.endswith('.pyc')
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(LoaderTestCase))
+ suite.addTest(unittest.makeSuite(ModuleLoaderTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/regression.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/regression.py
new file mode 100644
index 0000000..a39c3cb
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/regression.py
@@ -0,0 +1,258 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.regression
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Tests corner cases and bugs.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import time
+import tempfile
+import unittest
+
+from jinja2.testsuite import JinjaTestCase
+
+from jinja2 import Template, Environment, DictLoader, TemplateSyntaxError, \
+ TemplateNotFound, PrefixLoader
+
+env = Environment()
+
+
+class CornerTestCase(JinjaTestCase):
+
+ def test_assigned_scoping(self):
+ t = env.from_string('''
+ {%- for item in (1, 2, 3, 4) -%}
+ [{{ item }}]
+ {%- endfor %}
+ {{- item -}}
+ ''')
+ assert t.render(item=42) == '[1][2][3][4]42'
+
+ t = env.from_string('''
+ {%- for item in (1, 2, 3, 4) -%}
+ [{{ item }}]
+ {%- endfor %}
+ {%- set item = 42 %}
+ {{- item -}}
+ ''')
+ assert t.render() == '[1][2][3][4]42'
+
+ t = env.from_string('''
+ {%- set item = 42 %}
+ {%- for item in (1, 2, 3, 4) -%}
+ [{{ item }}]
+ {%- endfor %}
+ {{- item -}}
+ ''')
+ assert t.render() == '[1][2][3][4]42'
+
+ def test_closure_scoping(self):
+ t = env.from_string('''
+ {%- set wrapper = "<FOO>" %}
+ {%- for item in (1, 2, 3, 4) %}
+ {%- macro wrapper() %}[{{ item }}]{% endmacro %}
+ {{- wrapper() }}
+ {%- endfor %}
+ {{- wrapper -}}
+ ''')
+ assert t.render() == '[1][2][3][4]<FOO>'
+
+ t = env.from_string('''
+ {%- for item in (1, 2, 3, 4) %}
+ {%- macro wrapper() %}[{{ item }}]{% endmacro %}
+ {{- wrapper() }}
+ {%- endfor %}
+ {%- set wrapper = "<FOO>" %}
+ {{- wrapper -}}
+ ''')
+ assert t.render() == '[1][2][3][4]<FOO>'
+
+ t = env.from_string('''
+ {%- for item in (1, 2, 3, 4) %}
+ {%- macro wrapper() %}[{{ item }}]{% endmacro %}
+ {{- wrapper() }}
+ {%- endfor %}
+ {{- wrapper -}}
+ ''')
+ assert t.render(wrapper=23) == '[1][2][3][4]23'
+
+
+class BugTestCase(JinjaTestCase):
+
+ def test_keyword_folding(self):
+ env = Environment()
+ env.filters['testing'] = lambda value, some: value + some
+ assert env.from_string("{{ 'test'|testing(some='stuff') }}") \
+ .render() == 'teststuff'
+
+ def test_extends_output_bugs(self):
+ env = Environment(loader=DictLoader({
+ 'parent.html': '(({% block title %}{% endblock %}))'
+ }))
+
+ t = env.from_string('{% if expr %}{% extends "parent.html" %}{% endif %}'
+ '[[{% block title %}title{% endblock %}]]'
+ '{% for item in [1, 2, 3] %}({{ item }}){% endfor %}')
+ assert t.render(expr=False) == '[[title]](1)(2)(3)'
+ assert t.render(expr=True) == '((title))'
+
+ def test_urlize_filter_escaping(self):
+ tmpl = env.from_string('{{ "http://www.example.org/<foo"|urlize }}')
+ assert tmpl.render() == '<a href="http://www.example.org/<foo">http://www.example.org/<foo</a>'
+
+ def test_loop_call_loop(self):
+ tmpl = env.from_string('''
+
+ {% macro test() %}
+ {{ caller() }}
+ {% endmacro %}
+
+ {% for num1 in range(5) %}
+ {% call test() %}
+ {% for num2 in range(10) %}
+ {{ loop.index }}
+ {% endfor %}
+ {% endcall %}
+ {% endfor %}
+
+ ''')
+
+ assert tmpl.render().split() == map(unicode, range(1, 11)) * 5
+
+ def test_weird_inline_comment(self):
+ env = Environment(line_statement_prefix='%')
+ self.assert_raises(TemplateSyntaxError, env.from_string,
+ '% for item in seq {# missing #}\n...% endfor')
+
+ def test_old_macro_loop_scoping_bug(self):
+ tmpl = env.from_string('{% for i in (1, 2) %}{{ i }}{% endfor %}'
+ '{% macro i() %}3{% endmacro %}{{ i() }}')
+ assert tmpl.render() == '123'
+
+ def test_partial_conditional_assignments(self):
+ tmpl = env.from_string('{% if b %}{% set a = 42 %}{% endif %}{{ a }}')
+ assert tmpl.render(a=23) == '23'
+ assert tmpl.render(b=True) == '42'
+
+ def test_stacked_locals_scoping_bug(self):
+ env = Environment(line_statement_prefix='#')
+ t = env.from_string('''\
+# for j in [1, 2]:
+# set x = 1
+# for i in [1, 2]:
+# print x
+# if i % 2 == 0:
+# set x = x + 1
+# endif
+# endfor
+# endfor
+# if a
+# print 'A'
+# elif b
+# print 'B'
+# elif c == d
+# print 'C'
+# else
+# print 'D'
+# endif
+ ''')
+ assert t.render(a=0, b=False, c=42, d=42.0) == '1111C'
+
+ def test_stacked_locals_scoping_bug_twoframe(self):
+ t = Template('''
+ {% set x = 1 %}
+ {% for item in foo %}
+ {% if item == 1 %}
+ {% set x = 2 %}
+ {% endif %}
+ {% endfor %}
+ {{ x }}
+ ''')
+ rv = t.render(foo=[1]).strip()
+ assert rv == u'1'
+
+ def test_call_with_args(self):
+ t = Template("""{% macro dump_users(users) -%}
+ <ul>
+ {%- for user in users -%}
+ <li><p>{{ user.username|e }}</p>{{ caller(user) }}</li>
+ {%- endfor -%}
+ </ul>
+ {%- endmacro -%}
+
+ {% call(user) dump_users(list_of_user) -%}
+ <dl>
+ <dl>Realname</dl>
+ <dd>{{ user.realname|e }}</dd>
+ <dl>Description</dl>
+ <dd>{{ user.description }}</dd>
+ </dl>
+ {% endcall %}""")
+
+ assert [x.strip() for x in t.render(list_of_user=[{
+ 'username':'apo',
+ 'realname':'something else',
+ 'description':'test'
+ }]).splitlines()] == [
+ u'<ul><li><p>apo</p><dl>',
+ u'<dl>Realname</dl>',
+ u'<dd>something else</dd>',
+ u'<dl>Description</dl>',
+ u'<dd>test</dd>',
+ u'</dl>',
+ u'</li></ul>'
+ ]
+
+ def test_empty_if_condition_fails(self):
+ self.assert_raises(TemplateSyntaxError, Template, '{% if %}....{% endif %}')
+ self.assert_raises(TemplateSyntaxError, Template, '{% if foo %}...{% elif %}...{% endif %}')
+ self.assert_raises(TemplateSyntaxError, Template, '{% for x in %}..{% endfor %}')
+
+ def test_recursive_loop_bug(self):
+ tpl1 = Template("""
+ {% for p in foo recursive%}
+ {{p.bar}}
+ {% for f in p.fields recursive%}
+ {{f.baz}}
+ {{p.bar}}
+ {% if f.rec %}
+ {{ loop(f.sub) }}
+ {% endif %}
+ {% endfor %}
+ {% endfor %}
+ """)
+
+ tpl2 = Template("""
+ {% for p in foo%}
+ {{p.bar}}
+ {% for f in p.fields recursive%}
+ {{f.baz}}
+ {{p.bar}}
+ {% if f.rec %}
+ {{ loop(f.sub) }}
+ {% endif %}
+ {% endfor %}
+ {% endfor %}
+ """)
+
+ def test_correct_prefix_loader_name(self):
+ env = Environment(loader=PrefixLoader({
+ 'foo': DictLoader({})
+ }))
+ try:
+ env.get_template('foo/bar.html')
+ except TemplateNotFound, e:
+ assert e.name == 'foo/bar.html'
+ else:
+ assert False, 'expected error here'
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(CornerTestCase))
+ suite.addTest(unittest.makeSuite(BugTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/__init__.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/__init__.py
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/broken.html b/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/broken.html
new file mode 100644
index 0000000..77669fa
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/broken.html
@@ -0,0 +1,3 @@
+Before
+{{ fail() }}
+After
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/foo/test.html b/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/foo/test.html
new file mode 100644
index 0000000..b7d6715
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/foo/test.html
@@ -0,0 +1 @@
+FOO
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/syntaxerror.html b/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/syntaxerror.html
new file mode 100644
index 0000000..f21b817
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/syntaxerror.html
@@ -0,0 +1,4 @@
+Foo
+{% for item in broken %}
+ ...
+{% endif %}
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/test.html b/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/test.html
new file mode 100644
index 0000000..ba578e4
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/res/templates/test.html
@@ -0,0 +1 @@
+BAR
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/security.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/security.py
new file mode 100644
index 0000000..b2b4cf1
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/security.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.security
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ Checks the sandbox and other security features.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import time
+import tempfile
+import unittest
+
+from jinja2.testsuite import JinjaTestCase
+
+from jinja2 import Environment
+from jinja2.sandbox import SandboxedEnvironment, \
+ ImmutableSandboxedEnvironment, unsafe
+from jinja2 import Markup, escape
+from jinja2.exceptions import SecurityError, TemplateSyntaxError
+
+
+class PrivateStuff(object):
+
+ def bar(self):
+ return 23
+
+ @unsafe
+ def foo(self):
+ return 42
+
+ def __repr__(self):
+ return 'PrivateStuff'
+
+
+class PublicStuff(object):
+ bar = lambda self: 23
+ _foo = lambda self: 42
+
+ def __repr__(self):
+ return 'PublicStuff'
+
+
+class SandboxTestCase(JinjaTestCase):
+
+ def test_unsafe(self):
+ env = SandboxedEnvironment()
+ self.assert_raises(SecurityError, env.from_string("{{ foo.foo() }}").render,
+ foo=PrivateStuff())
+ self.assert_equal(env.from_string("{{ foo.bar() }}").render(foo=PrivateStuff()), '23')
+
+ self.assert_raises(SecurityError, env.from_string("{{ foo._foo() }}").render,
+ foo=PublicStuff())
+ self.assert_equal(env.from_string("{{ foo.bar() }}").render(foo=PublicStuff()), '23')
+ self.assert_equal(env.from_string("{{ foo.__class__ }}").render(foo=42), '')
+ self.assert_equal(env.from_string("{{ foo.func_code }}").render(foo=lambda:None), '')
+ self.assert_raises(SecurityError, env.from_string(
+ "{{ foo.__class__.__subclasses__() }}").render, foo=42)
+
+ def test_immutable_environment(self):
+ env = ImmutableSandboxedEnvironment()
+ self.assert_raises(SecurityError, env.from_string(
+ '{{ [].append(23) }}').render)
+ self.assert_raises(SecurityError, env.from_string(
+ '{{ {1:2}.clear() }}').render)
+
+ def test_restricted(self):
+ env = SandboxedEnvironment()
+ self.assert_raises(TemplateSyntaxError, env.from_string,
+ "{% for item.attribute in seq %}...{% endfor %}")
+ self.assert_raises(TemplateSyntaxError, env.from_string,
+ "{% for foo, bar.baz in seq %}...{% endfor %}")
+
+ def test_markup_operations(self):
+ # adding two strings should escape the unsafe one
+ unsafe = '<script type="application/x-some-script">alert("foo");</script>'
+ safe = Markup('<em>username</em>')
+ assert unsafe + safe == unicode(escape(unsafe)) + unicode(safe)
+
+ # string interpolations are safe to use too
+ assert Markup('<em>%s</em>') % '<bad user>' == \
+ '<em><bad user></em>'
+ assert Markup('<em>%(username)s</em>') % {
+ 'username': '<bad user>'
+ } == '<em><bad user></em>'
+
+ # an escaped object is markup too
+ assert type(Markup('foo') + 'bar') is Markup
+
+ # and it implements __html__ by returning itself
+ x = Markup("foo")
+ assert x.__html__() is x
+
+ # it also knows how to treat __html__ objects
+ class Foo(object):
+ def __html__(self):
+ return '<em>awesome</em>'
+ def __unicode__(self):
+ return 'awesome'
+ assert Markup(Foo()) == '<em>awesome</em>'
+ assert Markup('<strong>%s</strong>') % Foo() == \
+ '<strong><em>awesome</em></strong>'
+
+ # escaping and unescaping
+ assert escape('"<>&\'') == '"<>&''
+ assert Markup("<em>Foo & Bar</em>").striptags() == "Foo & Bar"
+ assert Markup("<test>").unescape() == "<test>"
+
+
+ def test_template_data(self):
+ env = Environment(autoescape=True)
+ t = env.from_string('{% macro say_hello(name) %}'
+ '<p>Hello {{ name }}!</p>{% endmacro %}'
+ '{{ say_hello("<blink>foo</blink>") }}')
+ escaped_out = '<p>Hello <blink>foo</blink>!</p>'
+ assert t.render() == escaped_out
+ assert unicode(t.module) == escaped_out
+ assert escape(t.module) == escaped_out
+ assert t.module.say_hello('<blink>foo</blink>') == escaped_out
+ assert escape(t.module.say_hello('<blink>foo</blink>')) == escaped_out
+
+
+ def test_attr_filter(self):
+ env = SandboxedEnvironment()
+ tmpl = env.from_string('{{ 42|attr("__class__")|attr("__subclasses__")() }}')
+ self.assert_raises(SecurityError, tmpl.render)
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(SandboxTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/tests.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/tests.py
new file mode 100644
index 0000000..cd5006f
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/tests.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.tests
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Who tests the tests?
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import unittest
+from jinja2.testsuite import JinjaTestCase
+
+from jinja2 import Markup, Environment
+
+env = Environment()
+
+
+class TestsTestCase(JinjaTestCase):
+
+ def test_defined(self):
+ tmpl = env.from_string('{{ missing is defined }}|{{ true is defined }}')
+ assert tmpl.render() == 'False|True'
+
+ def test_even(self):
+ tmpl = env.from_string('''{{ 1 is even }}|{{ 2 is even }}''')
+ assert tmpl.render() == 'False|True'
+
+ def test_odd(self):
+ tmpl = env.from_string('''{{ 1 is odd }}|{{ 2 is odd }}''')
+ assert tmpl.render() == 'True|False'
+
+ def test_lower(self):
+ tmpl = env.from_string('''{{ "foo" is lower }}|{{ "FOO" is lower }}''')
+ assert tmpl.render() == 'True|False'
+
+ def test_typechecks(self):
+ tmpl = env.from_string('''
+ {{ 42 is undefined }}
+ {{ 42 is defined }}
+ {{ 42 is none }}
+ {{ none is none }}
+ {{ 42 is number }}
+ {{ 42 is string }}
+ {{ "foo" is string }}
+ {{ "foo" is sequence }}
+ {{ [1] is sequence }}
+ {{ range is callable }}
+ {{ 42 is callable }}
+ {{ range(5) is iterable }}
+ ''')
+ assert tmpl.render().split() == [
+ 'False', 'True', 'False', 'True', 'True', 'False',
+ 'True', 'True', 'True', 'True', 'False', 'True'
+ ]
+
+ def test_sequence(self):
+ tmpl = env.from_string(
+ '{{ [1, 2, 3] is sequence }}|'
+ '{{ "foo" is sequence }}|'
+ '{{ 42 is sequence }}'
+ )
+ assert tmpl.render() == 'True|True|False'
+
+ def test_upper(self):
+ tmpl = env.from_string('{{ "FOO" is upper }}|{{ "foo" is upper }}')
+ assert tmpl.render() == 'True|False'
+
+ def test_sameas(self):
+ tmpl = env.from_string('{{ foo is sameas false }}|'
+ '{{ 0 is sameas false }}')
+ assert tmpl.render(foo=False) == 'True|False'
+
+ def test_no_paren_for_arg1(self):
+ tmpl = env.from_string('{{ foo is sameas none }}')
+ assert tmpl.render(foo=None) == 'True'
+
+ def test_escaped(self):
+ env = Environment(autoescape=True)
+ tmpl = env.from_string('{{ x is escaped }}|{{ y is escaped }}')
+ assert tmpl.render(x='foo', y=Markup('foo')) == 'False|True'
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(TestsTestCase))
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/testsuite/utils.py b/slider-agent/src/main/python/jinja2/jinja2/testsuite/utils.py
new file mode 100644
index 0000000..a402bbc
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/testsuite/utils.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.testsuite.utils
+ ~~~~~~~~~~~~~~~~~~~~~~
+
+ Tests utilities jinja uses.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import gc
+import unittest
+
+import pickle
+
+from jinja2.testsuite import JinjaTestCase
+
+from jinja2 import Environment, Undefined, DebugUndefined, \
+ StrictUndefined, UndefinedError, Template, meta
+from jinja2.utils import LRUCache, escape, object_type_repr
+
+
+class LRUCacheTestCase(JinjaTestCase):
+
+ def test_simple(self):
+ d = LRUCache(3)
+ d["a"] = 1
+ d["b"] = 2
+ d["c"] = 3
+ d["a"]
+ d["d"] = 4
+ assert len(d) == 3
+ assert 'a' in d and 'c' in d and 'd' in d and 'b' not in d
+
+ def test_pickleable(self):
+ cache = LRUCache(2)
+ cache["foo"] = 42
+ cache["bar"] = 23
+ cache["foo"]
+
+ for protocol in range(3):
+ copy = pickle.loads(pickle.dumps(cache, protocol))
+ assert copy.capacity == cache.capacity
+ assert copy._mapping == cache._mapping
+ assert copy._queue == cache._queue
+
+
+class HelpersTestCase(JinjaTestCase):
+
+ def test_object_type_repr(self):
+ class X(object):
+ pass
+ self.assert_equal(object_type_repr(42), 'int object')
+ self.assert_equal(object_type_repr([]), 'list object')
+ self.assert_equal(object_type_repr(X()),
+ 'jinja2.testsuite.utils.X object')
+ self.assert_equal(object_type_repr(None), 'None')
+ self.assert_equal(object_type_repr(Ellipsis), 'Ellipsis')
+
+
+class MarkupLeakTestCase(JinjaTestCase):
+
+ def test_markup_leaks(self):
+ counts = set()
+ for count in xrange(20):
+ for item in xrange(1000):
+ escape("foo")
+ escape("<foo>")
+ escape(u"foo")
+ escape(u"<foo>")
+ counts.add(len(gc.get_objects()))
+ assert len(counts) == 1, 'ouch, c extension seems to leak objects'
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(LRUCacheTestCase))
+ suite.addTest(unittest.makeSuite(HelpersTestCase))
+
+ # this test only tests the c extension
+ if not hasattr(escape, 'func_code'):
+ suite.addTest(unittest.makeSuite(MarkupLeakTestCase))
+
+ return suite
diff --git a/slider-agent/src/main/python/jinja2/jinja2/utils.py b/slider-agent/src/main/python/jinja2/jinja2/utils.py
new file mode 100644
index 0000000..7b77b8e
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/utils.py
@@ -0,0 +1,601 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.utils
+ ~~~~~~~~~~~~
+
+ Utility functions.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import re
+import sys
+import errno
+try:
+ from thread import allocate_lock
+except ImportError:
+ from dummy_thread import allocate_lock
+from collections import deque
+from itertools import imap
+
+
+_word_split_re = re.compile(r'(\s+)')
+_punctuation_re = re.compile(
+ '^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$' % (
+ '|'.join(imap(re.escape, ('(', '<', '<'))),
+ '|'.join(imap(re.escape, ('.', ',', ')', '>', '\n', '>')))
+ )
+)
+_simple_email_re = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$')
+_striptags_re = re.compile(r'(<!--.*?-->|<[^>]*>)')
+_entity_re = re.compile(r'&([^;]+);')
+_letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+_digits = '0123456789'
+
+# special singleton representing missing values for the runtime
+missing = type('MissingType', (), {'__repr__': lambda x: 'missing'})()
+
+# internal code
+internal_code = set()
+
+
+# concatenate a list of strings and convert them to unicode.
+# unfortunately there is a bug in python 2.4 and lower that causes
+# unicode.join trash the traceback.
+_concat = u''.join
+try:
+ def _test_gen_bug():
+ raise TypeError(_test_gen_bug)
+ yield None
+ _concat(_test_gen_bug())
+except TypeError, _error:
+ if not _error.args or _error.args[0] is not _test_gen_bug:
+ def concat(gen):
+ try:
+ return _concat(list(gen))
+ except:
+ # this hack is needed so that the current frame
+ # does not show up in the traceback.
+ exc_type, exc_value, tb = sys.exc_info()
+ raise exc_type, exc_value, tb.tb_next
+ else:
+ concat = _concat
+ del _test_gen_bug, _error
+
+
+# for python 2.x we create outselves a next() function that does the
+# basics without exception catching.
+try:
+ next = next
+except NameError:
+ def next(x):
+ return x.next()
+
+
+# if this python version is unable to deal with unicode filenames
+# when passed to encode we let this function encode it properly.
+# This is used in a couple of places. As far as Jinja is concerned
+# filenames are unicode *or* bytestrings in 2.x and unicode only in
+# 3.x because compile cannot handle bytes
+if sys.version_info < (3, 0):
+ def _encode_filename(filename):
+ if isinstance(filename, unicode):
+ return filename.encode('utf-8')
+ return filename
+else:
+ def _encode_filename(filename):
+ assert filename is None or isinstance(filename, str), \
+ 'filenames must be strings'
+ return filename
+
+from keyword import iskeyword as is_python_keyword
+
+
+# common types. These do exist in the special types module too which however
+# does not exist in IronPython out of the box. Also that way we don't have
+# to deal with implementation specific stuff here
+class _C(object):
+ def method(self): pass
+def _func():
+ yield None
+FunctionType = type(_func)
+GeneratorType = type(_func())
+MethodType = type(_C.method)
+CodeType = type(_C.method.func_code)
+try:
+ raise TypeError()
+except TypeError:
+ _tb = sys.exc_info()[2]
+ TracebackType = type(_tb)
+ FrameType = type(_tb.tb_frame)
+del _C, _tb, _func
+
+
+def contextfunction(f):
+ """This decorator can be used to mark a function or method context callable.
+ A context callable is passed the active :class:`Context` as first argument when
+ called from the template. This is useful if a function wants to get access
+ to the context or functions provided on the context object. For example
+ a function that returns a sorted list of template variables the current
+ template exports could look like this::
+
+ @contextfunction
+ def get_exported_names(context):
+ return sorted(context.exported_vars)
+ """
+ f.contextfunction = True
+ return f
+
+
+def evalcontextfunction(f):
+ """This decoraotr can be used to mark a function or method as an eval
+ context callable. This is similar to the :func:`contextfunction`
+ but instead of passing the context, an evaluation context object is
+ passed. For more information about the eval context, see
+ :ref:`eval-context`.
+
+ .. versionadded:: 2.4
+ """
+ f.evalcontextfunction = True
+ return f
+
+
+def environmentfunction(f):
+ """This decorator can be used to mark a function or method as environment
+ callable. This decorator works exactly like the :func:`contextfunction`
+ decorator just that the first argument is the active :class:`Environment`
+ and not context.
+ """
+ f.environmentfunction = True
+ return f
+
+
+def internalcode(f):
+ """Marks the function as internally used"""
+ internal_code.add(f.func_code)
+ return f
+
+
+def is_undefined(obj):
+ """Check if the object passed is undefined. This does nothing more than
+ performing an instance check against :class:`Undefined` but looks nicer.
+ This can be used for custom filters or tests that want to react to
+ undefined variables. For example a custom default filter can look like
+ this::
+
+ def default(var, default=''):
+ if is_undefined(var):
+ return default
+ return var
+ """
+ from jinja2.runtime import Undefined
+ return isinstance(obj, Undefined)
+
+
+def consume(iterable):
+ """Consumes an iterable without doing anything with it."""
+ for event in iterable:
+ pass
+
+
+def clear_caches():
+ """Jinja2 keeps internal caches for environments and lexers. These are
+ used so that Jinja2 doesn't have to recreate environments and lexers all
+ the time. Normally you don't have to care about that but if you are
+ messuring memory consumption you may want to clean the caches.
+ """
+ from jinja2.environment import _spontaneous_environments
+ from jinja2.lexer import _lexer_cache
+ _spontaneous_environments.clear()
+ _lexer_cache.clear()
+
+
+def import_string(import_name, silent=False):
+ """Imports an object based on a string. This use useful if you want to
+ use import paths as endpoints or something similar. An import path can
+ be specified either in dotted notation (``xml.sax.saxutils.escape``)
+ or with a colon as object delimiter (``xml.sax.saxutils:escape``).
+
+ If the `silent` is True the return value will be `None` if the import
+ fails.
+
+ :return: imported object
+ """
+ try:
+ if ':' in import_name:
+ module, obj = import_name.split(':', 1)
+ elif '.' in import_name:
+ items = import_name.split('.')
+ module = '.'.join(items[:-1])
+ obj = items[-1]
+ else:
+ return __import__(import_name)
+ return getattr(__import__(module, None, None, [obj]), obj)
+ except (ImportError, AttributeError):
+ if not silent:
+ raise
+
+
+def open_if_exists(filename, mode='rb'):
+ """Returns a file descriptor for the filename if that file exists,
+ otherwise `None`.
+ """
+ try:
+ return open(filename, mode)
+ except IOError, e:
+ if e.errno not in (errno.ENOENT, errno.EISDIR):
+ raise
+
+
+def object_type_repr(obj):
+ """Returns the name of the object's type. For some recognized
+ singletons the name of the object is returned instead. (For
+ example for `None` and `Ellipsis`).
+ """
+ if obj is None:
+ return 'None'
+ elif obj is Ellipsis:
+ return 'Ellipsis'
+ # __builtin__ in 2.x, builtins in 3.x
+ if obj.__class__.__module__ in ('__builtin__', 'builtins'):
+ name = obj.__class__.__name__
+ else:
+ name = obj.__class__.__module__ + '.' + obj.__class__.__name__
+ return '%s object' % name
+
+
+def pformat(obj, verbose=False):
+ """Prettyprint an object. Either use the `pretty` library or the
+ builtin `pprint`.
+ """
+ try:
+ from pretty import pretty
+ return pretty(obj, verbose=verbose)
+ except ImportError:
+ from pprint import pformat
+ return pformat(obj)
+
+
+def urlize(text, trim_url_limit=None, nofollow=False):
+ """Converts any URLs in text into clickable links. Works on http://,
+ https:// and www. links. Links can have trailing punctuation (periods,
+ commas, close-parens) and leading punctuation (opening parens) and
+ it'll still do the right thing.
+
+ If trim_url_limit is not None, the URLs in link text will be limited
+ to trim_url_limit characters.
+
+ If nofollow is True, the URLs in link text will get a rel="nofollow"
+ attribute.
+ """
+ trim_url = lambda x, limit=trim_url_limit: limit is not None \
+ and (x[:limit] + (len(x) >=limit and '...'
+ or '')) or x
+ words = _word_split_re.split(unicode(escape(text)))
+ nofollow_attr = nofollow and ' rel="nofollow"' or ''
+ for i, word in enumerate(words):
+ match = _punctuation_re.match(word)
+ if match:
+ lead, middle, trail = match.groups()
+ if middle.startswith('www.') or (
+ '@' not in middle and
+ not middle.startswith('http://') and
+ len(middle) > 0 and
+ middle[0] in _letters + _digits and (
+ middle.endswith('.org') or
+ middle.endswith('.net') or
+ middle.endswith('.com')
+ )):
+ middle = '<a href="http://%s"%s>%s</a>' % (middle,
+ nofollow_attr, trim_url(middle))
+ if middle.startswith('http://') or \
+ middle.startswith('https://'):
+ middle = '<a href="%s"%s>%s</a>' % (middle,
+ nofollow_attr, trim_url(middle))
+ if '@' in middle and not middle.startswith('www.') and \
+ not ':' in middle and _simple_email_re.match(middle):
+ middle = '<a href="mailto:%s">%s</a>' % (middle, middle)
+ if lead + middle + trail != word:
+ words[i] = lead + middle + trail
+ return u''.join(words)
+
+
+def generate_lorem_ipsum(n=5, html=True, min=20, max=100):
+ """Generate some lorem impsum for the template."""
+ from jinja2.constants import LOREM_IPSUM_WORDS
+ from random import choice, randrange
+ words = LOREM_IPSUM_WORDS.split()
+ result = []
+
+ for _ in xrange(n):
+ next_capitalized = True
+ last_comma = last_fullstop = 0
+ word = None
+ last = None
+ p = []
+
+ # each paragraph contains out of 20 to 100 words.
+ for idx, _ in enumerate(xrange(randrange(min, max))):
+ while True:
+ word = choice(words)
+ if word != last:
+ last = word
+ break
+ if next_capitalized:
+ word = word.capitalize()
+ next_capitalized = False
+ # add commas
+ if idx - randrange(3, 8) > last_comma:
+ last_comma = idx
+ last_fullstop += 2
+ word += ','
+ # add end of sentences
+ if idx - randrange(10, 20) > last_fullstop:
+ last_comma = last_fullstop = idx
+ word += '.'
+ next_capitalized = True
+ p.append(word)
+
+ # ensure that the paragraph ends with a dot.
+ p = u' '.join(p)
+ if p.endswith(','):
+ p = p[:-1] + '.'
+ elif not p.endswith('.'):
+ p += '.'
+ result.append(p)
+
+ if not html:
+ return u'\n\n'.join(result)
+ return Markup(u'\n'.join(u'<p>%s</p>' % escape(x) for x in result))
+
+
+class LRUCache(object):
+ """A simple LRU Cache implementation."""
+
+ # this is fast for small capacities (something below 1000) but doesn't
+ # scale. But as long as it's only used as storage for templates this
+ # won't do any harm.
+
+ def __init__(self, capacity):
+ self.capacity = capacity
+ self._mapping = {}
+ self._queue = deque()
+ self._postinit()
+
+ def _postinit(self):
+ # alias all queue methods for faster lookup
+ self._popleft = self._queue.popleft
+ self._pop = self._queue.pop
+ if hasattr(self._queue, 'remove'):
+ self._remove = self._queue.remove
+ self._wlock = allocate_lock()
+ self._append = self._queue.append
+
+ def _remove(self, obj):
+ """Python 2.4 compatibility."""
+ for idx, item in enumerate(self._queue):
+ if item == obj:
+ del self._queue[idx]
+ break
+
+ def __getstate__(self):
+ return {
+ 'capacity': self.capacity,
+ '_mapping': self._mapping,
+ '_queue': self._queue
+ }
+
+ def __setstate__(self, d):
+ self.__dict__.update(d)
+ self._postinit()
+
+ def __getnewargs__(self):
+ return (self.capacity,)
+
+ def copy(self):
+ """Return an shallow copy of the instance."""
+ rv = self.__class__(self.capacity)
+ rv._mapping.update(self._mapping)
+ rv._queue = deque(self._queue)
+ return rv
+
+ def get(self, key, default=None):
+ """Return an item from the cache dict or `default`"""
+ try:
+ return self[key]
+ except KeyError:
+ return default
+
+ def setdefault(self, key, default=None):
+ """Set `default` if the key is not in the cache otherwise
+ leave unchanged. Return the value of this key.
+ """
+ try:
+ return self[key]
+ except KeyError:
+ self[key] = default
+ return default
+
+ def clear(self):
+ """Clear the cache."""
+ self._wlock.acquire()
+ try:
+ self._mapping.clear()
+ self._queue.clear()
+ finally:
+ self._wlock.release()
+
+ def __contains__(self, key):
+ """Check if a key exists in this cache."""
+ return key in self._mapping
+
+ def __len__(self):
+ """Return the current size of the cache."""
+ return len(self._mapping)
+
+ def __repr__(self):
+ return '<%s %r>' % (
+ self.__class__.__name__,
+ self._mapping
+ )
+
+ def __getitem__(self, key):
+ """Get an item from the cache. Moves the item up so that it has the
+ highest priority then.
+
+ Raise an `KeyError` if it does not exist.
+ """
+ rv = self._mapping[key]
+ if self._queue[-1] != key:
+ try:
+ self._remove(key)
+ except ValueError:
+ # if something removed the key from the container
+ # when we read, ignore the ValueError that we would
+ # get otherwise.
+ pass
+ self._append(key)
+ return rv
+
+ def __setitem__(self, key, value):
+ """Sets the value for an item. Moves the item up so that it
+ has the highest priority then.
+ """
+ self._wlock.acquire()
+ try:
+ if key in self._mapping:
+ try:
+ self._remove(key)
+ except ValueError:
+ # __getitem__ is not locked, it might happen
+ pass
+ elif len(self._mapping) == self.capacity:
+ del self._mapping[self._popleft()]
+ self._append(key)
+ self._mapping[key] = value
+ finally:
+ self._wlock.release()
+
+ def __delitem__(self, key):
+ """Remove an item from the cache dict.
+ Raise an `KeyError` if it does not exist.
+ """
+ self._wlock.acquire()
+ try:
+ del self._mapping[key]
+ try:
+ self._remove(key)
+ except ValueError:
+ # __getitem__ is not locked, it might happen
+ pass
+ finally:
+ self._wlock.release()
+
+ def items(self):
+ """Return a list of items."""
+ result = [(key, self._mapping[key]) for key in list(self._queue)]
+ result.reverse()
+ return result
+
+ def iteritems(self):
+ """Iterate over all items."""
+ return iter(self.items())
+
+ def values(self):
+ """Return a list of all values."""
+ return [x[1] for x in self.items()]
+
+ def itervalue(self):
+ """Iterate over all values."""
+ return iter(self.values())
+
+ def keys(self):
+ """Return a list of all keys ordered by most recent usage."""
+ return list(self)
+
+ def iterkeys(self):
+ """Iterate over all keys in the cache dict, ordered by
+ the most recent usage.
+ """
+ return reversed(tuple(self._queue))
+
+ __iter__ = iterkeys
+
+ def __reversed__(self):
+ """Iterate over the values in the cache dict, oldest items
+ coming first.
+ """
+ return iter(tuple(self._queue))
+
+ __copy__ = copy
+
+
+# register the LRU cache as mutable mapping if possible
+try:
+ from collections import MutableMapping
+ MutableMapping.register(LRUCache)
+except ImportError:
+ pass
+
+
+class Cycler(object):
+ """A cycle helper for templates."""
+
+ def __init__(self, *items):
+ if not items:
+ raise RuntimeError('at least one item has to be provided')
+ self.items = items
+ self.reset()
+
+ def reset(self):
+ """Resets the cycle."""
+ self.pos = 0
+
+ @property
+ def current(self):
+ """Returns the current item."""
+ return self.items[self.pos]
+
+ def next(self):
+ """Goes one item ahead and returns it."""
+ rv = self.current
+ self.pos = (self.pos + 1) % len(self.items)
+ return rv
+
+
+class Joiner(object):
+ """A joining helper for templates."""
+
+ def __init__(self, sep=u', '):
+ self.sep = sep
+ self.used = False
+
+ def __call__(self):
+ if not self.used:
+ self.used = True
+ return u''
+ return self.sep
+
+
+# try markupsafe first, if that fails go with Jinja2's bundled version
+# of markupsafe. Markupsafe was previously Jinja2's implementation of
+# the Markup object but was moved into a separate package in a patchleve
+# release
+try:
+ from markupsafe import Markup, escape, soft_unicode
+except ImportError:
+ from jinja2._markupsafe import Markup, escape, soft_unicode
+
+
+# partials
+try:
+ from functools import partial
+except ImportError:
+ class partial(object):
+ def __init__(self, _func, *args, **kwargs):
+ self._func = _func
+ self._args = args
+ self._kwargs = kwargs
+ def __call__(self, *args, **kwargs):
+ kwargs.update(self._kwargs)
+ return self._func(*(self._args + args), **kwargs)
diff --git a/slider-agent/src/main/python/jinja2/jinja2/visitor.py b/slider-agent/src/main/python/jinja2/jinja2/visitor.py
new file mode 100644
index 0000000..413e7c3
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/jinja2/visitor.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2.visitor
+ ~~~~~~~~~~~~~~
+
+ This module implements a visitor for the nodes.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD.
+"""
+from jinja2.nodes import Node
+
+
+class NodeVisitor(object):
+ """Walks the abstract syntax tree and call visitor functions for every
+ node found. The visitor functions may return values which will be
+ forwarded by the `visit` method.
+
+ Per default the visitor functions for the nodes are ``'visit_'`` +
+ class name of the node. So a `TryFinally` node visit function would
+ be `visit_TryFinally`. This behavior can be changed by overriding
+ the `get_visitor` function. If no visitor function exists for a node
+ (return value `None`) the `generic_visit` visitor is used instead.
+ """
+
+ def get_visitor(self, node):
+ """Return the visitor function for this node or `None` if no visitor
+ exists for this node. In that case the generic visit function is
+ used instead.
+ """
+ method = 'visit_' + node.__class__.__name__
+ return getattr(self, method, None)
+
+ def visit(self, node, *args, **kwargs):
+ """Visit a node."""
+ f = self.get_visitor(node)
+ if f is not None:
+ return f(node, *args, **kwargs)
+ return self.generic_visit(node, *args, **kwargs)
+
+ def generic_visit(self, node, *args, **kwargs):
+ """Called if no explicit visitor function exists for a node."""
+ for node in node.iter_child_nodes():
+ self.visit(node, *args, **kwargs)
+
+
+class NodeTransformer(NodeVisitor):
+ """Walks the abstract syntax tree and allows modifications of nodes.
+
+ The `NodeTransformer` will walk the AST and use the return value of the
+ visitor functions to replace or remove the old node. If the return
+ value of the visitor function is `None` the node will be removed
+ from the previous location otherwise it's replaced with the return
+ value. The return value may be the original node in which case no
+ replacement takes place.
+ """
+
+ def generic_visit(self, node, *args, **kwargs):
+ for field, old_value in node.iter_fields():
+ if isinstance(old_value, list):
+ new_values = []
+ for value in old_value:
+ if isinstance(value, Node):
+ value = self.visit(value, *args, **kwargs)
+ if value is None:
+ continue
+ elif not isinstance(value, Node):
+ new_values.extend(value)
+ continue
+ new_values.append(value)
+ old_value[:] = new_values
+ elif isinstance(old_value, Node):
+ new_node = self.visit(old_value, *args, **kwargs)
+ if new_node is None:
+ delattr(node, field)
+ else:
+ setattr(node, field, new_node)
+ return node
+
+ def visit_list(self, node, *args, **kwargs):
+ """As transformers may return lists in some places this method
+ can be used to enforce a list as return value.
+ """
+ rv = self.visit(node, *args, **kwargs)
+ if not isinstance(rv, list):
+ rv = [rv]
+ return rv
diff --git a/slider-agent/src/main/python/jinja2/scripts/pylintrc b/slider-agent/src/main/python/jinja2/scripts/pylintrc
new file mode 100644
index 0000000..4f85b49
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/scripts/pylintrc
@@ -0,0 +1,301 @@
+# lint Python modules using external checkers.
+#
+# This is the main checker controling the other ones and the reports
+# generation. It is itself both a raw checker and an astng checker in order
+# to:
+# * handle message activation / deactivation at the module level
+# * handle some basic but necessary stats'data (number of classes, methods...)
+#
+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Profiled execution.
+profile=no
+
+# Add <file or directory> to the black list. It should be a base name, not a
+# path. You may set this option multiple times.
+ignore=.svn
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Set the cache size for astng objects.
+cache-size=500
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+
+[MESSAGES CONTROL]
+
+# Enable only checker(s) with the given id(s). This option conflict with the
+# disable-checker option
+#enable-checker=
+
+# Enable all checker(s) except those with the given id(s). This option conflict
+# with the disable-checker option
+#disable-checker=
+
+# Enable all messages in the listed categories.
+#enable-msg-cat=
+
+# Disable all messages in the listed categories.
+#disable-msg-cat=
+
+# Enable the message(s) with the given id(s).
+#enable-msg=
+
+# Disable the message(s) with the given id(s).
+disable-msg=C0323,W0142,C0301,C0103,C0111,E0213,C0302,C0203,W0703,R0201
+
+
+[REPORTS]
+
+# set the output format. Available formats are text, parseable, colorized and
+# html
+output-format=colorized
+
+# Include message's id in output
+include-ids=yes
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]".
+files-output=no
+
+# Tells wether to display a full report or only the messages
+reports=yes
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note).You have access to the variables errors warning, statement which
+# respectivly contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (R0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Add a comment according to your evaluation note. This is used by the global
+# evaluation report (R0004).
+comment=no
+
+# Enable the report(s) with the given id(s).
+#enable-report=
+
+# Disable the report(s) with the given id(s).
+#disable-report=
+
+
+# checks for
+# * unused variables / imports
+# * undefined variables
+# * redefinition of variable from builtins or from an outer scope
+# * use of variable before assigment
+#
+[VARIABLES]
+
+# Tells wether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching names used for dummy variables (i.e. not used).
+dummy-variables-rgx=_|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+
+# try to find bugs in the code using type inference
+#
+[TYPECHECK]
+
+# Tells wether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# When zope mode is activated, consider the acquired-members option to ignore
+# access to some undefined attributes.
+zope=no
+
+# List of members which are usually get through zope's acquisition mecanism and
+# so shouldn't trigger E0201 when accessed (need zope=yes to be considered).
+acquired-members=REQUEST,acl_users,aq_parent
+
+
+# checks for :
+# * doc strings
+# * modules / classes / functions / methods / arguments / variables name
+# * number of arguments, local variables, branchs, returns and statements in
+# functions, methods
+# * required module attributes
+# * dangerous default values as arguments
+# * redefinition of function / method / class
+# * uses of the global statement
+#
+[BASIC]
+
+# Required attributes for module, separated by a comma
+required-attributes=
+
+# Regular expression which should only match functions or classes name which do
+# not require a docstring
+no-docstring-rgx=__.*__
+
+# Regular expression which should only match correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression which should only match correct module level names
+const-rgx=(([A-Z_][A-Z1-9_]*)|(__.*__))$
+
+# Regular expression which should only match correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression which should only match correct function names
+function-rgx=[a-z_][a-z0-9_]*$
+
+# Regular expression which should only match correct method names
+method-rgx=[a-z_][a-z0-9_]*$
+
+# Regular expression which should only match correct instance attribute names
+attr-rgx=[a-z_][a-z0-9_]*$
+
+# Regular expression which should only match correct argument names
+argument-rgx=[a-z_][a-z0-9_]*$
+
+# Regular expression which should only match correct variable names
+variable-rgx=[a-z_][a-z0-9_]*$
+
+# Regular expression which should only match correct list comprehension /
+# generator expression variable names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=apply,input
+
+
+# checks for sign of poor/misdesign:
+# * number of methods, attributes, local variables...
+# * size, complexity of functions, methods
+#
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=12
+
+# Maximum number of locals for function / method body
+max-locals=30
+
+# Maximum number of return / yield for function / method body
+max-returns=12
+
+# Maximum number of branch for function / method body
+max-branchs=30
+
+# Maximum number of statements in function / method body
+max-statements=60
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=20
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=0
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+
+# checks for
+# * external modules dependencies
+# * relative / wildcard imports
+# * cyclic imports
+# * uses of deprecated modules
+#
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report R0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report R0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report R0402 must
+# not be disabled)
+int-import-graph=
+
+
+# checks for :
+# * methods without self as first argument
+# * overridden methods signature
+# * access only to existant members via self
+# * attributes not defined in the __init__ method
+# * supported interfaces implementation
+# * unreachable code
+#
+[CLASSES]
+
+# List of interface methods to ignore, separated by a comma. This is used for
+# instance to not check methods defines in Zope's Interface base class.
+ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+
+# checks for similarities and duplicated code. This computation may be
+# memory / CPU intensive, so you should disable it if you experiments some
+# problems.
+#
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=10
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+
+# checks for:
+# * warning notes in the code like FIXME, XXX
+# * PEP 263: source code with non ascii character but no encoding declaration
+#
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+# checks for :
+# * unauthorized constructions
+# * strict indentation
+# * line length
+# * use of <> instead of !=
+#
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=90
+
+# Maximum number of lines in a module
+max-module-lines=1000
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string=' '
diff --git a/slider-agent/src/main/python/jinja2/setup.cfg b/slider-agent/src/main/python/jinja2/setup.cfg
new file mode 100644
index 0000000..2d74c58
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/setup.cfg
@@ -0,0 +1,6 @@
+[egg_info]
+tag_build = dev
+tag_date = true
+
+[aliases]
+release = egg_info -RDb ''
diff --git a/slider-agent/src/main/python/jinja2/setup.py b/slider-agent/src/main/python/jinja2/setup.py
new file mode 100644
index 0000000..7956d19
--- /dev/null
+++ b/slider-agent/src/main/python/jinja2/setup.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+"""
+Jinja2
+~~~~~~
+
+Jinja2 is a template engine written in pure Python. It provides a
+`Django`_ inspired non-XML syntax but supports inline expressions and
+an optional `sandboxed`_ environment.
+
+Nutshell
+--------
+
+Here a small example of a Jinja template::
+
+ {% extends 'base.html' %}
+ {% block title %}Memberlist{% endblock %}
+ {% block content %}
+ <ul>
+ {% for user in users %}
+ <li><a href="{{ user.url }}">{{ user.username }}</a></li>
+ {% endfor %}
+ </ul>
+ {% endblock %}
+
+Philosophy
+----------
+
+Application logic is for the controller but don't try to make the life
+for the template designer too hard by giving him too few functionality.
+
+For more informations visit the new `Jinja2 webpage`_ and `documentation`_.
+
+.. _sandboxed: http://en.wikipedia.org/wiki/Sandbox_(computer_security)
+.. _Django: http://www.djangoproject.com/
+.. _Jinja2 webpage: http://jinja.pocoo.org/
+.. _documentation: http://jinja.pocoo.org/2/documentation/
+"""
+import sys
+
+from setuptools import setup, Extension, Feature
+
+debugsupport = Feature(
+ 'optional C debug support',
+ standard=False,
+ ext_modules = [
+ Extension('jinja2._debugsupport', ['jinja2/_debugsupport.c']),
+ ],
+)
+
+
+# tell distribute to use 2to3 with our own fixers.
+extra = {}
+if sys.version_info >= (3, 0):
+ extra.update(
+ use_2to3=True,
+ use_2to3_fixers=['custom_fixers']
+ )
+
+# ignore the old '--with-speedups' flag
+try:
+ speedups_pos = sys.argv.index('--with-speedups')
+except ValueError:
+ pass
+else:
+ sys.argv[speedups_pos] = '--with-debugsupport'
+ sys.stderr.write('*' * 74 + '\n')
+ sys.stderr.write('WARNING:\n')
+ sys.stderr.write(' the --with-speedups flag is deprecated, assuming '
+ '--with-debugsupport\n')
+ sys.stderr.write(' For the actual speedups install the MarkupSafe '
+ 'package.\n')
+ sys.stderr.write('*' * 74 + '\n')
+
+
+setup(
+ name='Jinja2',
+ version='2.5.5',
+ url='http://jinja.pocoo.org/',
+ license='BSD',
+ author='Armin Ronacher',
+ author_email='armin.ronacher@active-4.com',
+ description='A small but fast and easy to use stand-alone template '
+ 'engine written in pure python.',
+ long_description=__doc__,
+ # jinja is egg safe. But we hate eggs
+ zip_safe=False,
+ classifiers=[
+ 'Development Status :: 5 - Production/Stable',
+ 'Environment :: Web Environment',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: BSD License',
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 3',
+ 'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ 'Topic :: Text Processing :: Markup :: HTML'
+ ],
+ packages=['jinja2', 'jinja2.testsuite', 'jinja2.testsuite.res',
+ 'jinja2._markupsafe'],
+ extras_require={'i18n': ['Babel>=0.8']},
+ test_suite='jinja2.testsuite.suite',
+ include_package_data=True,
+ entry_points="""
+ [babel.extractors]
+ jinja2 = jinja2.ext:babel_extract[i18n]
+ """,
+ features={'debugsupport': debugsupport},
+ **extra
+)
diff --git a/slider-agent/src/main/python/resource_management/__init__.py b/slider-agent/src/main/python/resource_management/__init__.py
new file mode 100644
index 0000000..19902a4
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/__init__.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.libraries import *
+from resource_management.core import *
+
+
diff --git a/slider-agent/src/main/python/resource_management/core/__init__.py b/slider-agent/src/main/python/resource_management/core/__init__.py
new file mode 100644
index 0000000..6c3e1ed
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/__init__.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.core.base import *
+from resource_management.core.environment import *
+from resource_management.core.exceptions import *
+from resource_management.core.providers import *
+from resource_management.core.resources import *
+from resource_management.core.source import *
+from resource_management.core.system import *
+from resource_management.core.shell import *
+from resource_management.core.logger import *
+
+__version__ = "0.4.1"
diff --git a/slider-agent/src/main/python/resource_management/core/base.py b/slider-agent/src/main/python/resource_management/core/base.py
new file mode 100644
index 0000000..4171e95
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/base.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["Resource", "ResourceArgument", "ForcedListArgument",
+ "BooleanArgument"]
+
+from resource_management.core.exceptions import Fail, InvalidArgument
+from resource_management.core.environment import Environment
+from resource_management.core.logger import Logger
+
+class ResourceArgument(object):
+ def __init__(self, default=None, required=False):
+ self.required = False # Prevents the initial validate from failing
+ if hasattr(default, '__call__'):
+ self.default = default
+ else:
+ self.default = self.validate(default)
+ self.required = required
+
+ def validate(self, value):
+ if self.required and value is None:
+ raise InvalidArgument("Required argument %s missing" % self.name)
+ return value
+
+
+class ForcedListArgument(ResourceArgument):
+ def validate(self, value):
+ value = super(ForcedListArgument, self).validate(value)
+ if not isinstance(value, (tuple, list)):
+ value = [value]
+ return value
+
+
+class BooleanArgument(ResourceArgument):
+ def validate(self, value):
+ value = super(BooleanArgument, self).validate(value)
+ if not value in (True, False):
+ raise InvalidArgument(
+ "Expected a boolean for %s received %r" % (self.name, value))
+ return value
+
+
+class Accessor(object):
+ def __init__(self, name):
+ self.name = name
+
+ def __get__(self, obj, cls):
+ try:
+ return obj.arguments[self.name]
+ except KeyError:
+ val = obj._arguments[self.name].default
+ if hasattr(val, '__call__'):
+ val = val(obj)
+ return val
+
+ def __set__(self, obj, value):
+ obj.arguments[self.name] = obj._arguments[self.name].validate(value)
+
+
+class ResourceMetaclass(type):
+ # def __new__(cls, name, bases, attrs):
+ # super_new = super(ResourceMetaclass, cls).__new__
+ # return super_new(cls, name, bases, attrs)
+
+ def __init__(mcs, _name, bases, attrs):
+ mcs._arguments = getattr(bases[0], '_arguments', {}).copy()
+ for key, value in list(attrs.items()):
+ if isinstance(value, ResourceArgument):
+ value.name = key
+ mcs._arguments[key] = value
+ setattr(mcs, key, Accessor(key))
+
+
+class Resource(object):
+ __metaclass__ = ResourceMetaclass
+
+ action = ForcedListArgument(default="nothing")
+ ignore_failures = BooleanArgument(default=False)
+ not_if = ResourceArgument() # pass command e.g. not_if = ('ls','/root/jdk')
+ only_if = ResourceArgument() # pass command
+ initial_wait = ResourceArgument() # in seconds
+
+ actions = ["nothing"]
+
+ def __new__(cls, name, env=None, provider=None, **kwargs):
+ if isinstance(name, list):
+ while len(name) != 1:
+ cls(name.pop(0), env, provider, **kwargs)
+
+ name = name[0]
+
+ env = env or Environment.get_instance()
+ provider = provider or getattr(cls, 'provider', None)
+
+ r_type = cls.__name__
+ if r_type not in env.resources:
+ env.resources[r_type] = {}
+
+ obj = super(Resource, cls).__new__(cls)
+ env.resources[r_type][name] = obj
+ env.resource_list.append(obj)
+ return obj
+
+ def __init__(self, name, env=None, provider=None, **kwargs):
+ if isinstance(name, list):
+ name = name.pop(0)
+
+ if hasattr(self, 'name'):
+ return
+
+ self.env = env or Environment.get_instance()
+ self.name = name
+
+ self.provider = provider or getattr(self, 'provider', None)
+
+ self.arguments = {}
+ for key, value in kwargs.items():
+ try:
+ arg = self._arguments[key]
+ except KeyError:
+ raise Fail("%s received unsupported argument %s" % (self, key))
+ else:
+ try:
+ self.arguments[key] = arg.validate(value)
+ except InvalidArgument, exc:
+ raise InvalidArgument("%s %s" % (self, exc))
+
+ if not self.env.test_mode:
+ self.env.run()
+
+ def validate(self):
+ pass
+
+ def __repr__(self):
+ return "%s['%s']" % (self.__class__.__name__, self.name)
+
+ def __unicode__(self):
+ return u"%s['%s']" % (self.__class__.__name__, self.name)
+
+ def __getstate__(self):
+ return dict(
+ name=self.name,
+ provider=self.provider,
+ arguments=self.arguments,
+ env=self.env,
+ )
+
+ def __setstate__(self, state):
+ self.name = state['name']
+ self.provider = state['provider']
+ self.arguments = state['arguments']
+ self.env = state['env']
+
+ self.validate()
diff --git a/slider-agent/src/main/python/resource_management/core/environment.py b/slider-agent/src/main/python/resource_management/core/environment.py
new file mode 100644
index 0000000..04d4b78
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/environment.py
@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["Environment"]
+
+import os
+import shutil
+import time
+from datetime import datetime
+
+from resource_management.core import shell
+from resource_management.core.exceptions import Fail
+from resource_management.core.providers import find_provider
+from resource_management.core.utils import AttributeDictionary
+from resource_management.core.system import System
+from resource_management.core.logger import Logger
+
+
+class Environment(object):
+ _instances = []
+
+ def __init__(self, basedir=None, test_mode=False):
+ """
+ @param basedir: basedir/files, basedir/templates are the places where templates / static files
+ are looked up
+ @param test_mode: if this is enabled, resources won't be executed until manualy running env.run().
+ """
+ self.reset(basedir, test_mode)
+
+ def reset(self, basedir, test_mode):
+ self.system = System.get_instance()
+ self.config = AttributeDictionary()
+ self.resources = {}
+ self.resource_list = []
+ self.delayed_actions = set()
+ self.test_mode = test_mode
+ self.update_config({
+ # current time
+ 'date': datetime.now(),
+ # backups here files which were rewritten while executing File resource
+ 'backup.path': '/tmp/resource_management/backup',
+ # prefix for this files
+ 'backup.prefix': datetime.now().strftime("%Y%m%d%H%M%S"),
+ # dir where templates,failes dirs are
+ 'basedir': basedir,
+ # variables, which can be used in templates
+ 'params': {},
+ })
+
+ def backup_file(self, path):
+ if self.config.backup:
+ if not os.path.exists(self.config.backup.path):
+ os.makedirs(self.config.backup.path, 0700)
+ new_name = self.config.backup.prefix + path.replace('/', '-')
+ backup_path = os.path.join(self.config.backup.path, new_name)
+ Logger.info("backing up %s to %s" % (path, backup_path))
+ shutil.copy(path, backup_path)
+
+ def update_config(self, attributes, overwrite=True):
+ for key, value in attributes.items():
+ attr = self.config
+ path = key.split('.')
+ for pth in path[:-1]:
+ if pth not in attr:
+ attr[pth] = AttributeDictionary()
+ attr = attr[pth]
+ if overwrite or path[-1] not in attr:
+ attr[path[-1]] = value
+
+ def set_params(self, arg):
+ """
+ @param arg: is a dictionary of configurations, or a module with the configurations
+ """
+ if isinstance(arg, dict):
+ variables = arg
+ else:
+ variables = dict((var, getattr(arg, var)) for var in dir(arg))
+
+ for variable, value in variables.iteritems():
+ # don't include system variables, methods, classes, modules
+ if not variable.startswith("__") and \
+ not hasattr(value, '__call__')and \
+ not hasattr(value, '__file__'):
+ self.config.params[variable] = value
+
+ def run_action(self, resource, action):
+ Logger.debug("Performing action %s on %s" % (action, resource))
+
+ provider_class = find_provider(self, resource.__class__.__name__,
+ resource.provider)
+ provider = provider_class(resource)
+ try:
+ provider_action = getattr(provider, 'action_%s' % action)
+ except AttributeError:
+ raise Fail("%r does not implement action %s" % (provider, action))
+ provider_action()
+
+ def _check_condition(self, cond):
+ if hasattr(cond, '__call__'):
+ return cond()
+
+ if isinstance(cond, basestring):
+ ret, out = shell.call(cond)
+ return ret == 0
+
+ raise Exception("Unknown condition type %r" % cond)
+
+ def run(self):
+ with self:
+ # Run resource actions
+ while self.resource_list:
+ resource = self.resource_list.pop(0)
+ Logger.info_resource(resource)
+
+ if resource.initial_wait:
+ time.sleep(resource.initial_wait)
+
+ if resource.not_if is not None and self._check_condition(
+ resource.not_if):
+ Logger.info("Skipping %s due to not_if" % resource)
+ continue
+
+ if resource.only_if is not None and not self._check_condition(
+ resource.only_if):
+ Logger.info("Skipping %s due to only_if" % resource)
+ continue
+
+ for action in resource.action:
+ if not resource.ignore_failures:
+ self.run_action(resource, action)
+ else:
+ try:
+ self.run_action(resource, action)
+ except Exception as ex:
+ Logger.info("Skipping failure of %s due to ignore_failures. Failure reason: %s" % (resource, str(ex)))
+ pass
+
+ # Run delayed actions
+ while self.delayed_actions:
+ action, resource = self.delayed_actions.pop()
+ self.run_action(resource, action)
+
+ @classmethod
+ def get_instance(cls):
+ return cls._instances[-1]
+
+ @classmethod
+ def get_instance_copy(cls):
+ """
+ Copy only configurations, but not resources execution state
+ """
+ old_instance = cls.get_instance()
+ new_instance = Environment()
+ new_instance.config = old_instance.config.copy()
+
+ return new_instance
+
+ def __enter__(self):
+ self.__class__._instances.append(self)
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.__class__._instances.pop()
+ return False
+
+ def __getstate__(self):
+ return dict(
+ config=self.config,
+ resources=self.resources,
+ resource_list=self.resource_list,
+ delayed_actions=self.delayed_actions,
+ )
+
+ def __setstate__(self, state):
+ self.__init__()
+ self.config = state['config']
+ self.resources = state['resources']
+ self.resource_list = state['resource_list']
+ self.delayed_actions = state['delayed_actions']
diff --git a/slider-agent/src/main/python/resource_management/core/exceptions.py b/slider-agent/src/main/python/resource_management/core/exceptions.py
new file mode 100644
index 0000000..697bd74
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/exceptions.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+class Fail(Exception):
+ pass
+
+
+class ExecuteTimeoutException(Exception):
+ pass
+
+class InvalidArgument(Fail):
+ pass
+
+class ClientComponentHasNoStatus(Fail):
+ """
+ Thrown when status() method is called for a CLIENT component.
+ The only valid status for CLIENT component is installed,
+ that's why exception is thrown and later silently processed at script.py
+ """
+ pass
+
+class ComponentIsNotRunning(Fail):
+ """
+ Thrown when status() method is called for a component (only
+ in situations when component process is not running).
+ Later exception is silently processed at script.py
+ """
+ pass
diff --git a/slider-agent/src/main/python/resource_management/core/logger.py b/slider-agent/src/main/python/resource_management/core/logger.py
new file mode 100644
index 0000000..7124ef0
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/logger.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["Logger"]
+import logging
+from resource_management.libraries.script.config_dictionary import UnknownConfiguration
+
+class Logger:
+ logger = logging.getLogger("resource_management")
+
+ # unprotected_strings : protected_strings map
+ sensitive_strings = {}
+
+ @staticmethod
+ def info(text):
+ Logger.logger.info(Logger.get_protected_text(text))
+
+ @staticmethod
+ def debug(text):
+ Logger.logger.debug(Logger.get_protected_text(text))
+
+ @staticmethod
+ def info_resource(resource):
+ Logger.info(Logger.get_protected_text(Logger._get_resource_repr(resource)))
+
+ @staticmethod
+ def debug_resource(resource):
+ Logger.debug(Logger.get_protected_text(Logger._get_resource_repr(resource)))
+
+ @staticmethod
+ def get_protected_text(text):
+ """
+ Replace passwords with [PROTECTED]
+ """
+ for unprotected_string, protected_string in Logger.sensitive_strings.iteritems():
+ text = text.replace(unprotected_string, protected_string)
+
+ return text
+
+ @staticmethod
+ def _get_resource_repr(resource):
+ MESSAGE_MAX_LEN = 256
+ arguments_str = ""
+ for x,y in resource.arguments.iteritems():
+
+ # strip unicode 'u' sign
+ if isinstance(y, unicode):
+ # don't show long messages
+ if len(y) > MESSAGE_MAX_LEN:
+ y = '...'
+ val = repr(y).lstrip('u')
+ # don't show dicts of configurations
+ # usually too long
+ elif isinstance(y, dict):
+ val = "..."
+ # for configs which didn't come
+ elif isinstance(y, UnknownConfiguration):
+ val = "[EMPTY]"
+ # correctly output 'mode' (as they are octal values like 0755)
+ elif y and x == 'mode':
+ val = oct(y)
+ else:
+ val = repr(y)
+
+
+ arguments_str += "'{0}': {1}, ".format(x, val)
+
+ if arguments_str:
+ arguments_str = arguments_str[:-2]
+
+ return "{0} {{{1}}}".format(resource, arguments_str)
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/core/providers/__init__.py b/slider-agent/src/main/python/resource_management/core/providers/__init__.py
new file mode 100644
index 0000000..0c170e7
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/__init__.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["Provider", "find_provider"]
+
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.providers import PROVIDERS as LIBRARY_PROVIDERS
+
+
+class Provider(object):
+ def __init__(self, resource):
+ self.resource = resource
+
+ def action_nothing(self):
+ pass
+
+ def __repr__(self):
+ return self.__unicode__()
+
+ def __unicode__(self):
+ return u"%s[%s]" % (self.__class__.__name__, self.resource)
+
+
+PROVIDERS = dict(
+ redhat=dict(
+ Package="resource_management.core.providers.package.yumrpm.YumProvider",
+ ),
+ suse=dict(
+ Package="resource_management.core.providers.package.zypper.ZypperProvider",
+ ),
+ debian=dict(
+ Package="resource_management.core.providers.package.apt.AptProvider",
+ ),
+ default=dict(
+ File="resource_management.core.providers.system.FileProvider",
+ Directory="resource_management.core.providers.system.DirectoryProvider",
+ Link="resource_management.core.providers.system.LinkProvider",
+ Execute="resource_management.core.providers.system.ExecuteProvider",
+ ExecuteScript="resource_management.core.providers.system.ExecuteScriptProvider",
+ Mount="resource_management.core.providers.mount.MountProvider",
+ User="resource_management.core.providers.accounts.UserProvider",
+ Group="resource_management.core.providers.accounts.GroupProvider",
+ Service="resource_management.core.providers.service.ServiceProvider",
+ Tarball="resource_management.core.providers.tarball.TarballProvider",
+ ),
+)
+
+
+def find_provider(env, resource, class_path=None):
+ if not class_path:
+ providers = [PROVIDERS, LIBRARY_PROVIDERS]
+ for provider in providers:
+ if resource in provider[env.system.os_family]:
+ class_path = provider[env.system.os_family][resource]
+ break
+ if resource in provider["default"]:
+ class_path = provider["default"][resource]
+ break
+
+ try:
+ mod_path, class_name = class_path.rsplit('.', 1)
+ except ValueError:
+ raise Fail("Unable to find provider for %s as %s" % (resource, class_path))
+ mod = __import__(mod_path, {}, {}, [class_name])
+ return getattr(mod, class_name)
diff --git a/slider-agent/src/main/python/resource_management/core/providers/accounts.py b/slider-agent/src/main/python/resource_management/core/providers/accounts.py
new file mode 100644
index 0000000..747f120
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/accounts.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from __future__ import with_statement
+
+import grp
+import pwd
+from resource_management.core import shell
+from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
+
+
+class UserProvider(Provider):
+ def action_create(self):
+ if not self.user:
+ command = ['useradd', "-m"]
+ Logger.info("Adding user %s" % self.resource)
+ else:
+ command = ['usermod']
+ Logger.info("Modifying user %s" % (self.resource.username))
+
+ options = dict(
+ comment="-c",
+ gid="-g",
+ uid="-u",
+ shell="-s",
+ password="-p",
+ home="-d",
+ )
+
+ if self.resource.system and not self.user:
+ command.append("--system")
+
+ if self.resource.groups:
+ command += ["-G", ",".join(self.resource.groups)]
+
+ for option_name, option_flag in options.items():
+ option_value = getattr(self.resource, option_name)
+ if option_flag and option_value:
+ command += [option_flag, str(option_value)]
+
+ command.append(self.resource.username)
+
+ shell.checked_call(command)
+
+ def action_remove(self):
+ if self.user:
+ command = ['userdel', self.resource.username]
+ shell.checked_call(command)
+ Logger.info("Removed user %s" % self.resource)
+
+ @property
+ def user(self):
+ try:
+ return pwd.getpwnam(self.resource.username)
+ except KeyError:
+ return None
+
+
+class GroupProvider(Provider):
+ def action_create(self):
+ group = self.group
+ if not group:
+ command = ['groupadd']
+ Logger.info("Adding group %s" % self.resource)
+ else:
+ command = ['groupmod']
+ Logger.info("Modifying group %s" % (self.resource.group_name))
+
+ options = dict(
+ gid="-g",
+ password="-p",
+ )
+
+ for option_name, option_flag in options.items():
+ option_value = getattr(self.resource, option_name)
+ if option_flag and option_value:
+ command += [option_flag, str(option_value)]
+
+ command.append(self.resource.group_name)
+
+ shell.checked_call(command)
+
+ group = self.group
+
+ def action_remove(self):
+ if self.group:
+ command = ['groupdel', self.resource.group_name]
+ shell.checked_call(command)
+ Logger.info("Removed group %s" % self.resource)
+
+ @property
+ def group(self):
+ try:
+ return grp.getgrnam(self.resource.group_name)
+ except KeyError:
+ return None
diff --git a/slider-agent/src/main/python/resource_management/core/providers/mount.py b/slider-agent/src/main/python/resource_management/core/providers/mount.py
new file mode 100644
index 0000000..3fbb1e9
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/mount.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from __future__ import with_statement
+
+import os
+import re
+from resource_management.core.base import Fail
+from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
+
+
+class MountProvider(Provider):
+ def action_mount(self):
+ if not os.path.exists(self.resource.mount_point):
+ os.makedirs(self.resource.mount_point)
+
+ if self.is_mounted():
+ Logger.debug("%s already mounted" % self)
+ else:
+ args = ["mount"]
+ if self.resource.fstype:
+ args += ["-t", self.resource.fstype]
+ if self.resource.options:
+ args += ["-o", ",".join(self.resource.options)]
+ if self.resource.device:
+ args.append(self.resource.device)
+ args.append(self.resource.mount_point)
+
+ check_call(args)
+
+ Logger.info("%s mounted" % self)
+
+ def action_umount(self):
+ if self.is_mounted():
+ check_call(["umount", self.resource.mount_point])
+
+ Logger.info("%s unmounted" % self)
+ else:
+ Logger.debug("%s is not mounted" % self)
+
+ def action_enable(self):
+ if self.is_enabled():
+ Logger.debug("%s already enabled" % self)
+ else:
+ if not self.resource.device:
+ raise Fail("[%s] device not set but required for enable action" % self)
+ if not self.resource.fstype:
+ raise Fail("[%s] fstype not set but required for enable action" % self)
+
+ with open("/etc/fstab", "a") as fp:
+ fp.write("%s %s %s %s %d %d\n" % (
+ self.resource.device,
+ self.resource.mount_point,
+ self.resource.fstype,
+ ",".join(self.resource.options or ["defaults"]),
+ self.resource.dump,
+ self.resource.passno,
+ ))
+
+ Logger.info("%s enabled" % self)
+
+ def action_disable(self):
+ pass # TODO
+
+ def is_mounted(self):
+ if not os.path.exists(self.resource.mount_point):
+ return False
+
+ if self.resource.device and not os.path.exists(self.resource.device):
+ raise Fail("%s Device %s does not exist" % (self, self.resource.device))
+
+ mounts = self.get_mounted()
+ for m in mounts:
+ if m['mount_point'] == self.resource.mount_point:
+ return True
+
+ return False
+
+ def is_enabled(self):
+ mounts = self.get_fstab()
+ for m in mounts:
+ if m['mount_point'] == self.resource.mount_point:
+ return True
+
+ return False
+
+ def get_mounted(self):
+ p = Popen("mount", stdout=PIPE, stderr=STDOUT, shell=True)
+ out = p.communicate()[0]
+ if p.wait() != 0:
+ raise Fail("[%s] Getting list of mounts (calling mount) failed" % self)
+
+ mounts = [x.split(' ') for x in out.strip().split('\n')]
+
+ return [dict(
+ device=m[0],
+ mount_point=m[2],
+ fstype=m[4],
+ options=m[5][1:-1].split(','),
+ ) for m in mounts if m[1] == "on" and m[3] == "type"]
+
+ def get_fstab(self):
+ mounts = []
+ with open("/etc/fstab", "r") as fp:
+ for line in fp:
+ line = line.split('#', 1)[0].strip()
+ mount = re.split('\s+', line)
+ if len(mount) == 6:
+ mounts.append(dict(
+ device=mount[0],
+ mount_point=mount[1],
+ fstype=mount[2],
+ options=mount[3].split(","),
+ dump=int(mount[4]),
+ passno=int(mount[5]),
+ ))
+ return mounts
diff --git a/slider-agent/src/main/python/resource_management/core/providers/package/__init__.py b/slider-agent/src/main/python/resource_management/core/providers/package/__init__.py
new file mode 100644
index 0000000..f45ab41
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/package/__init__.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.core.base import Fail
+from resource_management.core.providers import Provider
+
+
+class PackageProvider(Provider):
+ def __init__(self, *args, **kwargs):
+ super(PackageProvider, self).__init__(*args, **kwargs)
+
+ def install_package(self, name, version):
+ raise NotImplementedError()
+ def remove_package(self, name):
+ raise NotImplementedError()
+ def upgrade_package(self, name, version):
+ raise NotImplementedError()
+
+ def action_install(self):
+ package_name = self.get_package_name_with_version()
+ self.install_package(package_name)
+
+ def action_upgrade(self):
+ package_name = self.get_package_name_with_version()
+ self.upgrade_package(package_name)
+
+ def action_remove(self):
+ package_name = self.get_package_name_with_version()
+ self.remove_package(package_name)
+
+ def get_package_name_with_version(self):
+ if self.resource.version:
+ return self.resource.package_name + '-' + self.resource.version
+ else:
+ return self.resource.package_name
+
diff --git a/slider-agent/src/main/python/resource_management/core/providers/package/apt.py b/slider-agent/src/main/python/resource_management/core/providers/package/apt.py
new file mode 100644
index 0000000..8d53f96
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/package/apt.py
@@ -0,0 +1,52 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management.core.providers.package import PackageProvider
+from resource_management.core import shell
+from resource_management.core.logger import Logger
+
+INSTALL_CMD = "/usr/bin/apt-get --assume-yes install %s"
+REMOVE_CMD = "/usr/bin/apt-get -y -q remove %s"
+CHECK_CMD = "dpkg --get-selections %s | grep -v deinstall"
+
+class AptProvider(PackageProvider):
+ def install_package(self, name):
+ if not self._check_existence(name):
+ cmd = INSTALL_CMD % (name)
+ Logger.info("Installing package %s ('%s')" % (name, cmd))
+ shell.checked_call(cmd)
+ else:
+ Logger.info("Skipping installing existent package %s" % (name))
+
+ def upgrade_package(self, name):
+ return self.install_package(name)
+
+ def remove_package(self, name):
+ if self._check_existence(name):
+ cmd = REMOVE_CMD % (name)
+ Logger.info("Removing package %s ('%s')" % (name, cmd))
+ shell.checked_call(cmd)
+ else:
+ Logger.info("Skipping removing non-existent package %s" % (name))
+
+ def _check_existence(self, name):
+ code, out = shell.call(CHECK_CMD % name)
+ return not bool(code)
diff --git a/slider-agent/src/main/python/resource_management/core/providers/package/yumrpm.py b/slider-agent/src/main/python/resource_management/core/providers/package/yumrpm.py
new file mode 100644
index 0000000..37b3d2f
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/package/yumrpm.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.core.providers.package import PackageProvider
+from resource_management.core import shell
+from resource_management.core.logger import Logger
+
+INSTALL_CMD = "/usr/bin/yum -d 0 -e 0 -y install %s"
+REMOVE_CMD = "/usr/bin/yum -d 0 -e 0 -y erase %s"
+CHECK_CMD = "rpm -q --quiet %s"
+
+class YumProvider(PackageProvider):
+ def install_package(self, name):
+ if not self._check_existence(name):
+ cmd = INSTALL_CMD % (name)
+ Logger.info("Installing package %s ('%s')" % (name, cmd))
+ shell.checked_call(cmd)
+ else:
+ Logger.info("Skipping installing existent package %s" % (name))
+
+ def upgrade_package(self, name):
+ return self.install_package(name)
+
+ def remove_package(self, name):
+ if self._check_existence(name):
+ cmd = REMOVE_CMD % (name)
+ Logger.info("Removing package %s ('%s')" % (name, cmd))
+ shell.checked_call(cmd)
+ else:
+ Logger.info("Skipping removing non-existent package %s" % (name))
+
+ def _check_existence(self, name):
+ code, out = shell.call(CHECK_CMD % name)
+ return not bool(code)
diff --git a/slider-agent/src/main/python/resource_management/core/providers/package/zypper.py b/slider-agent/src/main/python/resource_management/core/providers/package/zypper.py
new file mode 100644
index 0000000..bb32454
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/package/zypper.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.core.providers.package import PackageProvider
+from resource_management.core import shell
+from resource_management.core.logger import Logger
+
+INSTALL_CMD = "/usr/bin/zypper --quiet install --auto-agree-with-licenses --no-confirm %s"
+REMOVE_CMD = "/usr/bin/zypper --quiet remove --no-confirm %s"
+CHECK_CMD = "rpm -q --quiet %s"
+
+class ZypperProvider(PackageProvider):
+ def install_package(self, name):
+ if not self._check_existence(name):
+ cmd = INSTALL_CMD % (name)
+ Logger.info("Installing package %s ('%s')" % (name, cmd))
+ shell.checked_call(cmd)
+ else:
+ Logger.info("Skipping installing existent package %s" % (name))
+
+ def upgrade_package(self, name):
+ return self.install_package(name)
+
+ def remove_package(self, name):
+ if self._check_existence(name):
+ cmd = REMOVE_CMD % (name)
+ Logger.info("Removing package %s ('%s')" % (name, cmd))
+ shell.checked_call(cmd)
+ else:
+ Logger.info("Skipping removing non-existent package %s" % (name))
+
+ def _check_existence(self, name):
+ code, out = shell.call(CHECK_CMD % name)
+ return not bool(code)
diff --git a/slider-agent/src/main/python/resource_management/core/providers/service.py b/slider-agent/src/main/python/resource_management/core/providers/service.py
new file mode 100644
index 0000000..57f65ec
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/service.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+import os
+
+from resource_management.core import shell
+from resource_management.core.base import Fail
+from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
+
+
+class ServiceProvider(Provider):
+ def action_start(self):
+ if not self.status():
+ self._exec_cmd("start", 0)
+
+ def action_stop(self):
+ if self.status():
+ self._exec_cmd("stop", 0)
+
+ def action_restart(self):
+ if not self.status():
+ self._exec_cmd("start", 0)
+ else:
+ self._exec_cmd("restart", 0)
+
+ def action_reload(self):
+ if not self.status():
+ self._exec_cmd("start", 0)
+ else:
+ self._exec_cmd("reload", 0)
+
+ def status(self):
+ return self._exec_cmd("status") == 0
+
+ def _exec_cmd(self, command, expect=None):
+ if command != "status":
+ Logger.info("%s command '%s'" % (self.resource, command))
+
+ custom_cmd = getattr(self.resource, "%s_command" % command, None)
+ if custom_cmd:
+ Logger.debug("%s executing '%s'" % (self.resource, custom_cmd))
+ if hasattr(custom_cmd, "__call__"):
+ if custom_cmd():
+ ret = 0
+ else:
+ ret = 1
+ else:
+ ret,out = shell.call(custom_cmd)
+ else:
+ ret = self._init_cmd(command)
+
+ if expect is not None and expect != ret:
+ raise Fail("%r command %s for service %s failed with return code: %d. %s" % (
+ self, command, self.resource.service_name, ret, out))
+ return ret
+
+ def _init_cmd(self, command):
+ if self._upstart:
+ if command == "status":
+ ret,out = shell.call(["/sbin/" + command, self.resource.service_name])
+ _proc, state = out.strip().split(' ', 1)
+ ret = 0 if state != "stop/waiting" else 1
+ else:
+ ret,out = shell.call(["/sbin/" + command, self.resource.service_name])
+ else:
+ ret,out = shell.call(["/etc/init.d/%s" % self.resource.service_name, command])
+ return ret
+
+ @property
+ def _upstart(self):
+ try:
+ return self.__upstart
+ except AttributeError:
+ self.__upstart = os.path.exists("/sbin/start") \
+ and os.path.exists("/etc/init/%s.conf" % self.resource.service_name)
+ return self.__upstart
diff --git a/slider-agent/src/main/python/resource_management/core/providers/system.py b/slider-agent/src/main/python/resource_management/core/providers/system.py
new file mode 100644
index 0000000..3475d6a
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/system.py
@@ -0,0 +1,265 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from __future__ import with_statement
+
+import grp
+import os
+import pwd
+import time
+import shutil
+from resource_management.core import shell
+from resource_management.core.base import Fail
+from resource_management.core import ExecuteTimeoutException
+from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
+
+
+def _coerce_uid(user):
+ try:
+ uid = int(user)
+ except ValueError:
+ try:
+ uid = pwd.getpwnam(user).pw_uid
+ except KeyError:
+ raise Fail("User %s doesn't exist." % user)
+ return uid
+
+
+def _coerce_gid(group):
+ try:
+ gid = int(group)
+ except ValueError:
+ try:
+ gid = grp.getgrnam(group).gr_gid
+ except KeyError:
+ raise Fail("Group %s doesn't exist." % group)
+ return gid
+
+
+def _ensure_metadata(path, user, group, mode=None):
+ stat = os.stat(path)
+
+ if mode:
+ existing_mode = stat.st_mode & 07777
+ if existing_mode != mode:
+ Logger.info("Changing permission for %s from %o to %o" % (
+ path, existing_mode, mode))
+ os.chmod(path, mode)
+
+ if user:
+ uid = _coerce_uid(user)
+ if stat.st_uid != uid:
+ Logger.info(
+ "Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
+ os.chown(path, uid, -1)
+
+ if group:
+ gid = _coerce_gid(group)
+ if stat.st_gid != gid:
+ Logger.info(
+ "Changing group for %s from %d to %s" % (path, stat.st_gid, group))
+ os.chown(path, -1, gid)
+
+
+class FileProvider(Provider):
+ def action_create(self):
+ path = self.resource.path
+
+ if os.path.isdir(path):
+ raise Fail("Applying %s failed, directory with name %s exists" % (self.resource, path))
+
+ dirname = os.path.dirname(path)
+ if not os.path.isdir(dirname):
+ raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
+
+ write = False
+ content = self._get_content()
+ if not os.path.exists(path):
+ write = True
+ reason = "it doesn't exist"
+ elif self.resource.replace:
+ if content is not None:
+ with open(path, "rb") as fp:
+ old_content = fp.read()
+ if content != old_content:
+ write = True
+ reason = "contents don't match"
+ if self.resource.backup:
+ self.resource.env.backup_file(path)
+
+ if write:
+ Logger.info("Writing %s because %s" % (self.resource, reason))
+ with open(path, "wb") as fp:
+ if content:
+ fp.write(content)
+
+ _ensure_metadata(self.resource.path, self.resource.owner,
+ self.resource.group, mode=self.resource.mode)
+
+ def action_delete(self):
+ path = self.resource.path
+
+ if os.path.isdir(path):
+ raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
+
+ if os.path.exists(path):
+ Logger.info("Deleting %s" % self.resource)
+ os.unlink(path)
+
+ def _get_content(self):
+ content = self.resource.content
+ if content is None:
+ return None
+ elif isinstance(content, basestring):
+ return content
+ elif hasattr(content, "__call__"):
+ return content()
+ raise Fail("Unknown source type for %s: %r" % (self, content))
+
+
+class DirectoryProvider(Provider):
+ def action_create(self):
+ path = self.resource.path
+ if not os.path.exists(path):
+ Logger.info("Creating directory %s" % self.resource)
+ if self.resource.recursive:
+ os.makedirs(path, self.resource.mode or 0755)
+ else:
+ dirname = os.path.dirname(path)
+ if not os.path.isdir(dirname):
+ raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
+
+ os.mkdir(path, self.resource.mode or 0755)
+
+ if not os.path.isdir(path):
+ raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
+
+ _ensure_metadata(path, self.resource.owner, self.resource.group,
+ mode=self.resource.mode)
+
+ def action_delete(self):
+ path = self.resource.path
+ if os.path.exists(path):
+ if not os.path.isdir(path):
+ raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path))
+
+ Logger.info("Removing directory %s and all its content" % self.resource)
+ shutil.rmtree(path)
+
+
+class LinkProvider(Provider):
+ def action_create(self):
+ path = self.resource.path
+
+ if os.path.lexists(path):
+ oldpath = os.path.realpath(path)
+ if oldpath == self.resource.to:
+ return
+ if not os.path.islink(path):
+ raise Fail(
+ "%s trying to create a symlink with the same name as an existing file or directory" % self)
+ Logger.info("%s replacing old symlink to %s" % (self.resource, oldpath))
+ os.unlink(path)
+
+ if self.resource.hard:
+ if not os.path.exists(self.resource.to):
+ raise Fail("Failed to apply %s, linking to nonexistent location %s" % (self.resource, self.resource.to))
+ if os.path.isdir(self.resource.to):
+ raise Fail("Failed to apply %s, cannot create hard link to a directory (%s)" % (self.resource, self.resource.to))
+
+ Logger.info("Creating hard %s" % self.resource)
+ os.link(self.resource.to, path)
+ else:
+ if not os.path.exists(self.resource.to):
+ Logger.info("Warning: linking to nonexistent location %s" % self.resource.to)
+
+ Logger.info("Creating symbolic %s" % self.resource)
+ os.symlink(self.resource.to, path)
+
+ def action_delete(self):
+ path = self.resource.path
+ if os.path.exists(path):
+ Logger.info("Deleting %s" % self.resource)
+ os.unlink(path)
+
+
+def _preexec_fn(resource):
+ def preexec():
+ if resource.group:
+ gid = _coerce_gid(resource.group)
+ os.setgid(gid)
+ os.setegid(gid)
+
+ return preexec
+
+
+class ExecuteProvider(Provider):
+ def action_run(self):
+ if self.resource.creates:
+ if os.path.exists(self.resource.creates):
+ return
+
+ Logger.debug("Executing %s" % self.resource)
+
+ if self.resource.path != []:
+ if not self.resource.environment:
+ self.resource.environment = {}
+
+ self.resource.environment['PATH'] = os.pathsep.join(self.resource.path)
+
+ for i in range (0, self.resource.tries):
+ try:
+ shell.checked_call(self.resource.command, logoutput=self.resource.logoutput,
+ cwd=self.resource.cwd, env=self.resource.environment,
+ preexec_fn=_preexec_fn(self.resource), user=self.resource.user,
+ wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout)
+ break
+ except Fail as ex:
+ if i == self.resource.tries-1: # last try
+ raise ex
+ else:
+ Logger.info("Retrying after %d seconds. Reason: %s" % (self.resource.try_sleep, str(ex)))
+ time.sleep(self.resource.try_sleep)
+ except ExecuteTimeoutException:
+ err_msg = ("Execution of '%s' was killed due timeout after %d seconds") % (self.resource.command, self.resource.timeout)
+
+ if self.resource.on_timeout:
+ Logger.info("Executing '%s'. Reason: %s" % (self.resource.on_timeout, err_msg))
+ shell.checked_call(self.resource.on_timeout)
+ else:
+ raise Fail(err_msg)
+
+
+class ExecuteScriptProvider(Provider):
+ def action_run(self):
+ from tempfile import NamedTemporaryFile
+
+ Logger.info("Running script %s" % self.resource)
+ with NamedTemporaryFile(prefix="resource_management-script", bufsize=0) as tf:
+ tf.write(self.resource.code)
+ tf.flush()
+
+ _ensure_metadata(tf.name, self.resource.user, self.resource.group)
+ shell.call([self.resource.interpreter, tf.name],
+ cwd=self.resource.cwd, env=self.resource.environment,
+ preexec_fn=_preexec_fn(self.resource))
diff --git a/slider-agent/src/main/python/resource_management/core/providers/tarball.py b/slider-agent/src/main/python/resource_management/core/providers/tarball.py
new file mode 100644
index 0000000..dc66bd9
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/providers/tarball.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from __future__ import with_statement
+
+from resource_management.core import shell
+from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
+
+TAR_CMD = "tar -xvf %s -C %s"
+ZIP_CMD = "unzip %s -d %s"
+
+class TarballProvider(Provider):
+ def action_install(self):
+ package_name = self.resource.package_name
+ location = self.resource.location
+ if not self._check_existence(package_name, location):
+ cmd = TAR_CMD % (package_name, location)
+ if package_name.lower().endswith("zip"):
+ cmd = ZIP_CMD % (package_name, location)
+ Logger.info("Installing tarball %s at %s (%s)" % (package_name, location, cmd))
+ shell.checked_call(cmd)
+
+ def _check_existence(self, name, location):
+ return False
+
diff --git a/slider-agent/src/main/python/resource_management/core/resources/__init__.py b/slider-agent/src/main/python/resource_management/core/resources/__init__.py
new file mode 100644
index 0000000..51172ee
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/resources/__init__.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.core.resources.accounts import *
+from resource_management.core.resources.packaging import *
+from resource_management.core.resources.service import *
+from resource_management.core.resources.system import *
+from resource_management.core.resources.tarball import *
diff --git a/slider-agent/src/main/python/resource_management/core/resources/accounts.py b/slider-agent/src/main/python/resource_management/core/resources/accounts.py
new file mode 100644
index 0000000..c4bd76f
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/resources/accounts.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+__all__ = ["Group", "User"]
+
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+
+class Group(Resource):
+ action = ForcedListArgument(default="create")
+ group_name = ResourceArgument(default=lambda obj: obj.name)
+ gid = ResourceArgument()
+ password = ResourceArgument()
+
+ actions = Resource.actions + ["create", "remove"]
+
+
+class User(Resource):
+ action = ForcedListArgument(default="create")
+ username = ResourceArgument(default=lambda obj: obj.name)
+ comment = ResourceArgument()
+ uid = ResourceArgument()
+ gid = ResourceArgument()
+ groups = ForcedListArgument(default=[]) # supplementary groups
+ home = ResourceArgument()
+ shell = ResourceArgument(default="/bin/bash")
+ password = ResourceArgument()
+ system = BooleanArgument(default=False)
+
+ actions = Resource.actions + ["create", "remove"]
diff --git a/slider-agent/src/main/python/resource_management/core/resources/packaging.py b/slider-agent/src/main/python/resource_management/core/resources/packaging.py
new file mode 100644
index 0000000..bfd76ca
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/resources/packaging.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["Package"]
+
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument
+
+
+class Package(Resource):
+ action = ForcedListArgument(default="install")
+ package_name = ResourceArgument(default=lambda obj: obj.name)
+ location = ResourceArgument(default=lambda obj: obj.package_name)
+ version = ResourceArgument()
+ actions = ["install", "upgrade", "remove"]
+ build_vars = ForcedListArgument(default=[])
diff --git a/slider-agent/src/main/python/resource_management/core/resources/service.py b/slider-agent/src/main/python/resource_management/core/resources/service.py
new file mode 100644
index 0000000..1da8e97
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/resources/service.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["Service"]
+
+from resource_management.core.base import Resource, ResourceArgument, ForcedListArgument
+
+
+class Service(Resource):
+ action = ForcedListArgument(default="start")
+ service_name = ResourceArgument(default=lambda obj: obj.name)
+ #enabled = ResourceArgument() # Maybe add support to put in/out autostart.
+ start_command = ResourceArgument()
+ stop_command = ResourceArgument()
+ restart_command = ResourceArgument()
+ reload_command = ResourceArgument() # reload the config file without interrupting pending operations
+ status_command = ResourceArgument()
+
+ actions = ["nothing", "start", "stop", "restart", "reload"]
diff --git a/slider-agent/src/main/python/resource_management/core/resources/system.py b/slider-agent/src/main/python/resource_management/core/resources/system.py
new file mode 100644
index 0000000..2c832a4
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/resources/system.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["File", "Directory", "Link", "Execute", "ExecuteScript", "Mount"]
+
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+
+class File(Resource):
+ action = ForcedListArgument(default="create")
+ path = ResourceArgument(default=lambda obj: obj.name)
+ backup = ResourceArgument()
+ mode = ResourceArgument()
+ owner = ResourceArgument()
+ group = ResourceArgument()
+ content = ResourceArgument()
+ # whether to replace files with different content
+ replace = ResourceArgument(default=True)
+
+ actions = Resource.actions + ["create", "delete"]
+
+
+class Directory(Resource):
+ action = ForcedListArgument(default="create")
+ path = ResourceArgument(default=lambda obj: obj.name)
+ mode = ResourceArgument()
+ owner = ResourceArgument()
+ group = ResourceArgument()
+ recursive = BooleanArgument(default=False) # this work for 'create', 'delete' is anyway recursive
+
+ actions = Resource.actions + ["create", "delete"]
+
+
+class Link(Resource):
+ action = ForcedListArgument(default="create")
+ path = ResourceArgument(default=lambda obj: obj.name)
+ to = ResourceArgument(required=True)
+ hard = BooleanArgument(default=False)
+
+ actions = Resource.actions + ["create", "delete"]
+
+
+class Execute(Resource):
+ action = ForcedListArgument(default="run")
+
+ """
+ Recommended:
+ command = ('rm','-f','myfile')
+ Not recommended:
+ command = 'rm -f myfile'
+
+ The first one helps to stop escaping issues
+ """
+ command = ResourceArgument(default=lambda obj: obj.name)
+
+ creates = ResourceArgument()
+ cwd = ResourceArgument()
+ # this runs command with a specific env variables, env={'JAVA_HOME': '/usr/jdk'}
+ environment = ResourceArgument()
+ user = ResourceArgument()
+ group = ResourceArgument()
+ returns = ForcedListArgument(default=0)
+ tries = ResourceArgument(default=1)
+ try_sleep = ResourceArgument(default=0) # seconds
+ path = ForcedListArgument(default=[])
+ actions = Resource.actions + ["run"]
+ logoutput = BooleanArgument(default=False)
+ """
+ if on_timeout is not set leads to failing after x seconds,
+ otherwise calls on_timeout
+ """
+ timeout = ResourceArgument() # seconds
+ on_timeout = ResourceArgument()
+ """
+ Wait for command to finish or not.
+
+ NOTE:
+ In case of False, since any command results are skipped, it disables some functionality:
+ - non-zero return code failure
+ - logoutput
+ - tries
+ - try_sleep
+ """
+ wait_for_finish = BooleanArgument(default=True)
+
+
+class ExecuteScript(Resource):
+ action = ForcedListArgument(default="run")
+ code = ResourceArgument(required=True)
+ cwd = ResourceArgument()
+ environment = ResourceArgument()
+ interpreter = ResourceArgument(default="/bin/bash")
+ user = ResourceArgument()
+ group = ResourceArgument()
+
+ actions = Resource.actions + ["run"]
+
+
+class Mount(Resource):
+ action = ForcedListArgument(default="mount")
+ mount_point = ResourceArgument(default=lambda obj: obj.name)
+ device = ResourceArgument()
+ fstype = ResourceArgument()
+ options = ResourceArgument(default=["defaults"])
+ dump = ResourceArgument(default=0)
+ passno = ResourceArgument(default=2)
+
+ actions = Resource.actions + ["mount", "umount", "remount", "enable",
+ "disable"]
diff --git a/slider-agent/src/main/python/resource_management/core/resources/tarball.py b/slider-agent/src/main/python/resource_management/core/resources/tarball.py
new file mode 100644
index 0000000..7b09842
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/resources/tarball.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["Tarball"]
+
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument
+
+
+class Tarball(Resource):
+ action = ForcedListArgument(default="install")
+ package_name = ResourceArgument(default=lambda obj: obj.name)
+ location = ResourceArgument()
+ actions = ["install"]
diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py
new file mode 100644
index 0000000..32e917c
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/shell.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["checked_call", "call"]
+
+import pipes
+import subprocess
+import threading
+from multiprocessing import Queue
+from exceptions import Fail
+from exceptions import ExecuteTimeoutException
+from resource_management.core.logger import Logger
+
+def checked_call(command, logoutput=False,
+ cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None):
+ return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout)
+
+def call(command, logoutput=False,
+ cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None):
+ return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout)
+
+def _call(command, logoutput=False, throw_on_failure=True,
+ cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None):
+ """
+ Execute shell command
+
+ @param command: list/tuple of arguments (recommended as more safe - don't need to escape)
+ or string of the command to execute
+ @param logoutput: boolean, whether command output should be logged of not
+ @param throw_on_failure: if true, when return code is not zero exception is thrown
+
+ @return: retrun_code, stdout
+ """
+ # convert to string and escape
+ if isinstance(command, (list, tuple)):
+ command = ' '.join(pipes.quote(x) for x in command)
+
+ """
+ Do not su to the supplied user (need to differentiate between when to call su and when not to)
+ if user:
+ command = ["su", "-", user, "-c", command]
+ else:
+ command = ["/bin/bash","--login","-c", command]
+ """
+ command = ["/bin/bash","--login","-c", command]
+ proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ cwd=cwd, env=env, shell=False,
+ preexec_fn=preexec_fn)
+
+ if not wait_for_finish:
+ return None, None
+
+ if timeout:
+ q = Queue()
+ t = threading.Timer( timeout, on_timeout, [proc, q] )
+ t.start()
+
+ out = proc.communicate()[0].strip('\n')
+
+ if timeout:
+ if q.empty():
+ t.cancel()
+ # timeout occurred
+ else:
+ raise ExecuteTimeoutException()
+
+ code = proc.returncode
+
+ if logoutput and out:
+ Logger.info(out)
+
+ if throw_on_failure and code:
+ err_msg = ("Execution of '%s' returned %d. %s") % (command[-1], code, out)
+ raise Fail(err_msg)
+
+ return code, out
+
+def on_timeout(proc, q):
+ q.put(True)
+ if proc.poll() == None:
+ try:
+ proc.terminate()
+ except:
+ pass
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/core/source.py b/slider-agent/src/main/python/resource_management/core/source.py
new file mode 100644
index 0000000..874c750
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/source.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from __future__ import with_statement
+from resource_management.core.environment import Environment
+from resource_management.core.utils import checked_unite
+
+__all__ = ["Source", "Template", "InlineTemplate", "StaticFile", "DownloadSource"]
+
+import hashlib
+import os
+import urllib2
+import urlparse
+
+
+class Source(object):
+ def __init__(self, name):
+ self.env = Environment.get_instance()
+ self.name = name
+
+ def get_content(self):
+ raise NotImplementedError()
+
+ def get_checksum(self):
+ return None
+
+ def __call__(self):
+ return self.get_content()
+
+ def __repr__(self):
+ return self.__class__.__name__+"('"+self.name+"')"
+
+ def __eq__(self, other):
+ return (isinstance(other, self.__class__)
+ and self.get_content() == other.get_content())
+
+
+class StaticFile(Source):
+ def __init__(self, name):
+ super(StaticFile, self).__init__(name)
+
+ def get_content(self):
+ # absolute path
+ if self.name.startswith(os.path.sep):
+ path = self.name
+ # relative path
+ else:
+ basedir = self.env.config.basedir
+ path = os.path.join(basedir, "files", self.name)
+
+ with open(path, "rb") as fp:
+ return fp.read()
+
+
+try:
+ from jinja2 import Environment as JinjaEnvironment, BaseLoader, TemplateNotFound, FunctionLoader, StrictUndefined
+except ImportError:
+ class Template(Source):
+ def __init__(self, name, variables=None, env=None):
+ raise Exception("Jinja2 required for Template/InlineTemplate")
+
+ class InlineTemplate(Source):
+ def __init__(self, name, variables=None, env=None):
+ raise Exception("Jinja2 required for Template/InlineTemplate")
+else:
+ class TemplateLoader(BaseLoader):
+ def __init__(self, env=None):
+ self.env = env or Environment.get_instance()
+
+ def get_source(self, environment, template_name):
+ # absolute path
+ if template_name.startswith(os.path.sep):
+ path = template_name
+ # relative path
+ else:
+ basedir = self.env.config.basedir
+ path = os.path.join(basedir, "templates", template_name)
+
+ if not os.path.exists(path):
+ raise TemplateNotFound("%s at %s" % (template_name, path))
+ mtime = os.path.getmtime(path)
+ with open(path, "rb") as fp:
+ source = fp.read().decode('utf-8')
+ return source, path, lambda: mtime == os.path.getmtime(path)
+
+ class Template(Source):
+ def __init__(self, name, extra_imports=[], **kwargs):
+ """
+ @param kwargs: Additional variables passed to template
+ """
+ super(Template, self).__init__(name)
+ params = self.env.config.params
+ variables = checked_unite(params, kwargs)
+ self.imports_dict = dict((module.__name__, module) for module in extra_imports)
+ self.context = variables.copy() if variables else {}
+ if not hasattr(self, 'template_env'):
+ self.template_env = JinjaEnvironment(loader=TemplateLoader(self.env),
+ autoescape=False, undefined=StrictUndefined, trim_blocks=True)
+
+ self.template = self.template_env.get_template(self.name)
+
+ def get_content(self):
+ default_variables = { 'env':self.env, 'repr':repr, 'str':str, 'bool':bool }
+ variables = checked_unite(default_variables, self.imports_dict)
+ self.context.update(variables)
+
+ rendered = self.template.render(self.context)
+ return rendered + "\n" if not rendered.endswith('\n') else rendered
+
+ class InlineTemplate(Template):
+ def __init__(self, name, extra_imports=[], **kwargs):
+ self.template_env = JinjaEnvironment(loader=FunctionLoader(lambda text: text))
+ super(InlineTemplate, self).__init__(name, extra_imports, **kwargs)
+
+ def __repr__(self):
+ return "InlineTemplate(...)"
+
+
+class DownloadSource(Source):
+ def __init__(self, name, cache=True, md5sum=None):
+ super(DownloadSource, self).__init__(name)
+ self.url = self.name
+ self.md5sum = md5sum
+ self.cache = cache
+ if not 'download_path' in self.env.config:
+ self.env.config.download_path = '/var/tmp/downloads'
+ if not os.path.exists(self.env.config.download_path):
+ os.makedirs(self.env.config.download_path)
+
+ def get_content(self):
+ filepath = os.path.basename(urlparse.urlparse(self.url).path)
+ content = None
+ if not self.cache or not os.path.exists(
+ os.path.join(self.env.config.download_path, filepath)):
+ web_file = urllib2.urlopen(self.url)
+ content = web_file.read()
+ else:
+ update = False
+ with open(os.path.join(self.env.config.download_path, filepath)) as fp:
+ content = fp.read()
+ if self.md5sum:
+ m = hashlib.md5(content)
+ md5 = m.hexdigest()
+ if md5 != self.md5sum:
+ web_file = urllib2.urlopen(self.url)
+ content = web_file.read()
+ update = True
+ if self.cache and update:
+ with open(os.path.join(self.env.config.download_path, filepath),
+ 'w') as fp:
+ fp.write(content)
+ return content
diff --git a/slider-agent/src/main/python/resource_management/core/system.py b/slider-agent/src/main/python/resource_management/core/system.py
new file mode 100644
index 0000000..c05cf3c
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/system.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["System"]
+
+import os
+import sys
+import platform
+from resource_management.core import shell
+from resource_management.core.exceptions import Fail
+from functools import wraps
+from resource_management.libraries.functions.os_check import OSCheck
+
+def lazy_property(undecorated):
+ name = '_' + undecorated.__name__
+
+ @property
+ @wraps(undecorated)
+ def decorated(self):
+ try:
+ return getattr(self, name)
+ except AttributeError:
+ v = undecorated(self)
+ setattr(self, name, v)
+ return v
+
+ return decorated
+
+class System(object):
+ @lazy_property
+ def os(self):
+ """
+ Return values:
+ linux, unknown
+
+ In case cannot detect raises 'unknown'
+ """
+ platform = sys.platform
+ if platform.startswith('linux'):
+ return "linux"
+ else:
+ return "unknown"
+
+ @lazy_property
+ def os_version(self):
+ """
+ Example return value:
+ "6.3" for "Centos 6.3"
+
+ In case cannot detect --> Fail
+ """
+ return OSCheck().get_os_version()
+
+ @lazy_property
+ def os_release_name(self):
+ """
+ For Ubuntu 12.04:
+ precise
+ """
+ return OSCheck().get_os_release_name()
+
+ @lazy_property
+ def os_type(self):
+ """
+ Return values:
+ redhat, fedora, centos, oraclelinux, ascendos,
+ amazon, xenserver, oel, ovs, cloudlinux, slc, scientific, psbm,
+ ubuntu, debian, sles, sled, opensuse, suse ... and others
+
+ In case cannot detect raises exception.
+ """
+ return OSCheck().get_os_type()
+
+ @lazy_property
+ def os_family(self):
+ """
+ Return values:
+ redhat, debian, suse
+
+ In case cannot detect raises exception
+ """
+ return OSCheck().get_os_family()
+
+ @lazy_property
+ def ec2(self):
+ if not os.path.exists("/proc/xen"):
+ return False
+ if os.path.exists("/etc/ec2_version"):
+ return True
+ return False
+
+ @lazy_property
+ def vm(self):
+ if os.path.exists("/usr/bin/VBoxControl"):
+ return "vbox"
+ elif os.path.exists("/usr/bin/vmware-toolbox-cmd") or os.path.exists(
+ "/usr/sbin/vmware-toolbox-cmd"):
+ return "vmware"
+ elif os.path.exists("/proc/xen"):
+ return "xen"
+ return None
+
+ @lazy_property
+ def arch(self):
+ machine = self.machine
+ if machine in ("i386", "i486", "i686"):
+ return "x86_32"
+ return machine
+
+ @lazy_property
+ def machine(self):
+ code, out = shell.call(["/bin/uname", "-m"])
+ return out.strip()
+
+ @lazy_property
+ def locales(self):
+ code, out = shell.call("locale -a")
+ return out.strip().split("\n")
+
+ @classmethod
+ def get_instance(cls):
+ try:
+ return cls._instance
+ except AttributeError:
+ cls._instance = cls()
+ return cls._instance
+
+ def unquote(self, val):
+ if val[0] == '"':
+ val = val[1:-1]
+ return val
diff --git a/slider-agent/src/main/python/resource_management/core/utils.py b/slider-agent/src/main/python/resource_management/core/utils.py
new file mode 100644
index 0000000..0bfc459
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/core/utils.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.core.exceptions import Fail
+
+class AttributeDictionary(object):
+ def __init__(self, *args, **kwargs):
+ d = kwargs
+ if args:
+ d = args[0]
+ super(AttributeDictionary, self).__setattr__("_dict", d)
+
+ def __setattr__(self, name, value):
+ self[name] = value
+
+ def __getattr__(self, name):
+ if name in self.__dict__:
+ return self.__dict__[name]
+ try:
+ return self[name]
+ except KeyError:
+ raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name))
+
+ def __setitem__(self, name, value):
+ self._dict[name] = self._convert_value(value)
+
+ def __getitem__(self, name):
+ return self._convert_value(self._dict[name])
+
+ def _convert_value(self, value):
+ if isinstance(value, dict) and not isinstance(value, AttributeDictionary):
+ return AttributeDictionary(value)
+ return value
+
+ def copy(self):
+ return self.__class__(self._dict.copy())
+
+ def update(self, *args, **kwargs):
+ self._dict.update(*args, **kwargs)
+
+ def items(self):
+ return self._dict.items()
+
+ def iteritems(self):
+ return self._dict.iteritems()
+
+ def values(self):
+ return self._dict.values()
+
+ def keys(self):
+ return self._dict.keys()
+
+ def pop(self, *args, **kwargs):
+ return self._dict.pop(*args, **kwargs)
+
+ def get(self, *args, **kwargs):
+ return self._dict.get(*args, **kwargs)
+
+ def __repr__(self):
+ return self._dict.__repr__()
+
+ def __unicode__(self):
+ if isinstance(self._dict, str):
+ return self._dict.__unicode__()
+ else:
+ return str(self._dict)
+
+ def __str__(self):
+ return self._dict.__str__()
+
+ def __iter__(self):
+ return self._dict.__iter__()
+
+ def __getstate__(self):
+ return self._dict
+
+ def __setstate__(self, state):
+ super(AttributeDictionary, self).__setattr__("_dict", state)
+
+def checked_unite(dict1, dict2):
+ for key in dict1:
+ if key in dict2:
+ if not dict2[key] is dict1[key]: # it's not a big deal if this is the same variable
+ raise Fail("Variable '%s' already exists more than once as a variable/configuration/kwarg parameter. Cannot evaluate it." % key)
+
+ result = dict1.copy()
+ result.update(dict2)
+
+ return result
diff --git a/slider-agent/src/main/python/resource_management/libraries/__init__.py b/slider-agent/src/main/python/resource_management/libraries/__init__.py
new file mode 100644
index 0000000..629995e
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/__init__.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.libraries.functions import *
+from resource_management.libraries.resources import *
+from resource_management.libraries.providers import *
+from resource_management.libraries.script import *
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py b/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py
new file mode 100644
index 0000000..41cf666
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.libraries.functions.default import *
+from resource_management.libraries.functions.format import *
+from resource_management.libraries.functions.get_kinit_path import *
+from resource_management.libraries.functions.get_unique_id_and_date import *
+from resource_management.libraries.functions.check_process_status import *
+from resource_management.libraries.functions.is_empty import *
+from resource_management.libraries.functions.substitute_vars import *
+from resource_management.libraries.functions.os_check import *
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
new file mode 100644
index 0000000..e8ed9f0
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+__all__ = ["check_process_status"]
+
+import os
+
+def check_process_status(pid_file):
+ """
+ Function checks whether process is running.
+ Process is considered running, if pid file exists, and process with
+ a pid, mentioned in pid file is running
+ If process is not running, will throw ComponentIsNotRunning exception
+
+ @param pid_file: path to service pid file
+ """
+ if not pid_file or not os.path.isfile(pid_file):
+ raise ComponentIsNotRunning()
+ with open(pid_file, "r") as f:
+ try:
+ pid = int(f.read())
+ except:
+ Logger.debug("Pid file {0} does not exist".format(pid_file))
+ raise ComponentIsNotRunning()
+ try:
+ # Kill will not actually kill the process
+ # From the doc:
+ # If sig is 0, then no signal is sent, but error checking is still
+ # performed; this can be used to check for the existence of a
+ # process ID or process group ID.
+ os.kill(pid, 0)
+ except OSError:
+ Logger.debug("Process with pid {0} is not running. Stale pid file"
+ " at {1}".format(pid, pid_file))
+ raise ComponentIsNotRunning()
+ pass
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/default.py b/slider-agent/src/main/python/resource_management/libraries/functions/default.py
new file mode 100644
index 0000000..9796b93
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/default.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["default"]
+from resource_management.libraries.script import Script
+from resource_management.libraries.script.config_dictionary import UnknownConfiguration
+from resource_management.core.logger import Logger
+
+default_subdict='/configurations/global'
+
+def default(name, default_value):
+ subdicts = filter(None, name.split('/'))
+
+ if not name.startswith('/'):
+ subdicts = filter(None, default_subdict.split('/')) + subdicts
+
+ curr_dict = Script.get_config()
+ for x in subdicts:
+ if x in curr_dict:
+ curr_dict = curr_dict[x]
+ else:
+ if not isinstance(default_value, UnknownConfiguration):
+ Logger.debug("Cannot find configuration: '%s'. Using '%s' value as default" % (name, default_value))
+ return default_value
+
+ return curr_dict
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/format.py b/slider-agent/src/main/python/resource_management/libraries/functions/format.py
new file mode 100644
index 0000000..9594791
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/format.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["format"]
+import sys
+import pipes
+from string import Formatter
+from resource_management.core.exceptions import Fail
+from resource_management.core.utils import checked_unite
+from resource_management.core.environment import Environment
+from resource_management.core.logger import Logger
+
+
+class ConfigurationFormatter(Formatter):
+ """
+ Flags:
+ !e - escape bash properties flag
+ !h - hide sensitive information from the logs
+ !p - password flag, !p=!s+!e. Has both !e, !h effect
+ """
+ def format(self, format_string, *args, **kwargs):
+ env = Environment.get_instance()
+ variables = kwargs
+ params = env.config.params
+ all_params = checked_unite(variables, params)
+
+ self.convert_field = self.convert_field_protected
+ result_protected = self.vformat(format_string, args, all_params)
+
+ self.convert_field = self.convert_field_unprotected
+ result_unprotected = self.vformat(format_string, args, all_params)
+
+ if result_protected != result_unprotected:
+ Logger.sensitive_strings[result_unprotected] = result_protected
+
+ return result_unprotected
+
+ def convert_field_unprotected(self, value, conversion):
+ return self._convert_field(value, conversion, False)
+
+ def convert_field_protected(self, value, conversion):
+ """
+ Enable masking sensitive information like
+ passwords from logs via !p (password) format flag.
+ """
+ return self._convert_field(value, conversion, True)
+
+ def _convert_field(self, value, conversion, is_protected):
+ if conversion == 'e':
+ return pipes.quote(str(value))
+ elif conversion == 'h':
+ return "[PROTECTED]" if is_protected else value
+ elif conversion == 'p':
+ return "[PROTECTED]" if is_protected else self._convert_field(value, 'e', is_protected)
+
+ return super(ConfigurationFormatter, self).convert_field(value, conversion)
+
+
+def format(format_string, *args, **kwargs):
+ variables = sys._getframe(1).f_locals
+
+ result = checked_unite(kwargs, variables)
+ result.pop("self", None) # self kwarg would result in an error
+ return ConfigurationFormatter().format(format_string, args, **result)
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/get_kinit_path.py b/slider-agent/src/main/python/resource_management/libraries/functions/get_kinit_path.py
new file mode 100644
index 0000000..4285694
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/get_kinit_path.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["get_kinit_path"]
+import os
+
+def get_kinit_path(pathes_list):
+ """
+ @param pathes: comma separated list
+ """
+ kinit_path = ""
+
+ for x in pathes_list:
+ if not x:
+ continue
+
+ path = os.path.join(x,"kinit")
+
+ if os.path.isfile(path):
+ kinit_path = path
+ break
+
+ return kinit_path
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py b/slider-agent/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py
new file mode 100644
index 0000000..d23d097
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+__all__ = ["get_unique_id_and_date"]
+import datetime
+from resource_management.core import shell
+
+def get_unique_id_and_date():
+ out = shell.checked_call("hostid")[1]
+ id = out.strip()
+
+ now = datetime.datetime.now()
+ date = now.strftime("%M%d%y")
+
+ return "id{id}_date{date}".format(id=id, date=date)
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/is_empty.py b/slider-agent/src/main/python/resource_management/libraries/functions/is_empty.py
new file mode 100644
index 0000000..88af145
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/is_empty.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+from resource_management.libraries.script.config_dictionary import UnknownConfiguration
+
+def is_empty(var):
+ """
+ Check if certain configuration sent from the server has been received.
+ """
+ return isinstance(var, UnknownConfiguration)
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py b/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
new file mode 100644
index 0000000..7a72bc8
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import sys
+import platform
+
+__all__ = [
+ 'OSCheck',
+ ]
+
+class OSCheck(object):
+ def __init__(self):
+ pass
+
+ def get_os_type(self):
+ """
+ Return values:
+ redhat, fedora, centos, oraclelinux, ascendos,
+ amazon, xenserver, oel, ovs, cloudlinux, slc, scientific, psbm,
+ ubuntu, debian, sles, sled, opensuse, suse ... and others
+
+ In case cannot detect - exit.
+ """
+ # Read content from /etc/*-release file
+ # Full release name
+ dist = platform.linux_distribution()
+ operatingSystem = dist[0].lower()
+
+ # special cases
+ if os.path.exists('/etc/oracle-release'):
+ return 'oraclelinux'
+ elif operatingSystem.startswith('suse linux enterprise server'):
+ return 'sles'
+ elif operatingSystem.startswith('red hat enterprise linux server'):
+ return 'redhat'
+
+ if operatingSystem != '':
+ return operatingSystem
+ else:
+ print "Cannot detect os type. Exiting..."
+ sys.exit(1)
+
+
+ def get_os_family(self):
+ """
+ Return values:
+ redhat, debian, suse ... and others
+
+ In case cannot detect raises exception( from self.get_operating_system_type() ).
+ """
+ os_family = self.get_os_type()
+ if os_family in ['redhat', 'fedora', 'centos', 'oraclelinux', 'ascendos',
+ 'amazon', 'xenserver', 'oel', 'ovs', 'cloudlinux',
+ 'slc', 'scientific', 'psbm', 'centos linux']:
+ os_family = 'RedHat'
+ elif os_family in ['ubuntu', 'debian']:
+ os_family = 'Debian'
+ elif os_family in ['sles', 'sled', 'opensuse', 'suse']:
+ os_family = 'Suse'
+ #else: os_family = self.get_os_type()
+ return os_family.lower()
+
+
+ def get_os_version(self):
+ """
+ Returns the OS version
+
+ In case cannot detect raises exception.
+ """
+ # Read content from /etc/*-release file
+ # Full release name
+ dist = platform.linux_distribution()
+ dist = dist[1]
+
+ if dist:
+ return dist
+ else:
+ print "Cannot detect os version. Exiting..."
+ sys.exit(1)
+
+ def get_os_major_version(self):
+ """
+ Returns the main OS version like
+ Centos 6.5 --> 6
+ RedHat 1.2.3 --> 1
+ """
+ return self.get_os_version().split('.')[0]
+
+ def get_os_release_name(self):
+ """
+ Returns the OS release name
+
+ In case cannot detect raises exception.
+ """
+ dist = platform.linux_distribution()
+ dist = dist[2].lower()
+
+ if dist:
+ return dist
+ else:
+ print "Cannot detect os release name. Exiting..."
+ sys.exit(1)
+
+
+def main(argv=None):
+ # Same logic that was in "os_type_check.sh"
+ if len(sys.argv) != 2:
+ print "Usage: <cluster_os>"
+ sys.exit(2)
+ pass
+
+ cluster_os = sys.argv[1]
+ current_os = OSCheck().get_os_family() + OSCheck().get_os_major_version()
+
+ # If agent/server have the same {"family","main_version"} - then ok.
+ print "Cluster primary/cluster OS type is %s and local/current OS type is %s" % (
+ cluster_os, current_os)
+ if current_os == cluster_os:
+ sys.exit(0)
+ else:
+ print "Local OS is not compatible with cluster primary OS. Please perform manual bootstrap on this host."
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py b/slider-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py
new file mode 100644
index 0000000..3d558ac
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+import re
+
+_MAX_SUBST = 20
+
+def substitute_vars(raw, config):
+ """
+ @param raw: str (e.g '${hbase.tmp.dir}/local')
+ @param config: dict (e.g {'hbase.tmp.dir': '/hadoop/hbase'})
+ """
+ result = raw
+
+ pattern = re.compile("\$\{[^\}\$\x0020]+\}")
+
+ for depth in range(0, _MAX_SUBST - 1):
+ match = pattern.search(result)
+
+ if match:
+ start = match.start()
+ end = match.end()
+
+ name = result[start + 2 : end - 1]
+
+ try:
+ value = config[name]
+ except KeyError:
+ return result
+
+ result = result[:start] + value + result[end:]
+ else:
+ break
+
+ return result
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/__init__.py b/slider-agent/src/main/python/resource_management/libraries/providers/__init__.py
new file mode 100644
index 0000000..973958b
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/__init__.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+PROVIDERS = dict(
+ redhat=dict(
+ Repository="resource_management.libraries.providers.repository.RhelSuseRepositoryProvider",
+ ),
+ suse=dict(
+ Repository="resource_management.libraries.providers.repository.RhelSuseRepositoryProvider",
+ ),
+ debian=dict(
+ Repository="resource_management.libraries.providers.repository.DebianRepositoryProvider",
+ ),
+ default=dict(
+ ExecuteHadoop="resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider",
+ TemplateConfig="resource_management.libraries.providers.template_config.TemplateConfigProvider",
+ XmlConfig="resource_management.libraries.providers.xml_config.XmlConfigProvider",
+ PropertiesFile="resource_management.libraries.providers.properties_file.PropertiesFileProvider",
+ MonitorWebserver="resource_management.libraries.providers.monitor_webserver.MonitorWebserverProvider",
+ HdfsDirectory="resource_management.libraries.providers.hdfs_directory.HdfsDirectoryProvider",
+ CopyFromLocal="resource_management.libraries.providers.copy_from_local.CopyFromLocalProvider"
+ ),
+)
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py b/slider-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
new file mode 100644
index 0000000..0dfecd7
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+import os
+from resource_management import *
+
+class CopyFromLocalProvider(Provider):
+ def action_run(self):
+ path = self.resource.path
+ dest_dir = self.resource.dest_dir
+ kinnit_if_needed = self.resource.kinnit_if_needed
+ owner = self.resource.owner
+ group = self.resource.group
+ mode = self.resource.mode
+ hdfs_usr=self.resource.hdfs_user
+ hadoop_conf_path = self.resource.hadoop_conf_dir
+
+ copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
+ dest_file_name = os.path.split(path)[1]
+ dest_path = dest_dir + dest_file_name if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file_name
+
+ unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_path} >/dev/null 2>&1")
+
+ ExecuteHadoop(copy_cmd,
+ not_if=unless_cmd,
+ user=owner,
+ conf_dir=hadoop_conf_path
+ )
+
+ if not owner:
+ chown = None
+ else:
+ if not group:
+ chown = owner
+ else:
+ chown = format('{owner}:{group}')
+
+ if chown:
+ chown_cmd = format("fs -chown {chown} {dest_path}")
+
+ ExecuteHadoop(chown_cmd,
+ user=hdfs_usr,
+ conf_dir=hadoop_conf_path)
+ pass
+
+ if mode:
+ dir_mode = oct(mode)
+ chmod_cmd = format('fs -chmod {dir_mode} {dest_path}')
+
+ ExecuteHadoop(chmod_cmd,
+ user=hdfs_usr,
+ conf_dir=hadoop_conf_path)
+ pass
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py b/slider-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
new file mode 100644
index 0000000..61b2ddb
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+import pipes
+from resource_management import *
+
+class ExecuteHadoopProvider(Provider):
+ def action_run(self):
+ kinit__path_local = self.resource.kinit_path_local
+ keytab = self.resource.keytab
+ conf_dir = self.resource.conf_dir
+ command = self.resource.command
+ principal = self.resource.principal
+
+ if isinstance(command, (list, tuple)):
+ command = ' '.join(pipes.quote(x) for x in command)
+
+ with Environment.get_instance_copy() as env:
+ if self.resource.security_enabled and not self.resource.kinit_override:
+ Execute (format("{kinit__path_local} -kt {keytab} {principal}"),
+ path = ['/bin'],
+ user = self.resource.user
+ )
+
+ Execute (format("hadoop --config {conf_dir} {command}"),
+ user = self.resource.user,
+ tries = self.resource.tries,
+ try_sleep = self.resource.try_sleep,
+ logoutput = self.resource.logoutput,
+ )
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/hdfs_directory.py b/slider-agent/src/main/python/resource_management/libraries/providers/hdfs_directory.py
new file mode 100644
index 0000000..c4450df
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/hdfs_directory.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management import *
+directories_list = [] #direcotries list for mkdir
+chmod_map = {} #(mode,recursive):dir_list map
+chown_map = {} #(owner,group,recursive):dir_list map
+class HdfsDirectoryProvider(Provider):
+ def action_create_delayed(self):
+ global delayed_directories
+ global chmod_map
+ global chown_map
+
+ if not self.resource.dir_name:
+ return
+
+ dir_name = self.resource.dir_name
+ dir_owner = self.resource.owner
+ dir_group = self.resource.group
+ dir_mode = oct(self.resource.mode)[1:] if self.resource.mode else None
+ directories_list.append(self.resource.dir_name)
+
+ recursive_chown_str = "-R" if self.resource.recursive_chown else ""
+ recursive_chmod_str = "-R" if self.resource.recursive_chmod else ""
+ # grouping directories by mode/owner/group to modify them in one 'chXXX' call
+ if dir_mode:
+ chmod_key = (dir_mode,recursive_chmod_str)
+ if chmod_map.has_key(chmod_key):
+ chmod_map[chmod_key].append(dir_name)
+ else:
+ chmod_map[chmod_key] = [dir_name]
+
+ if dir_owner:
+ owner_key = (dir_owner,dir_group,recursive_chown_str)
+ if chown_map.has_key(owner_key):
+ chown_map[owner_key].append(dir_name)
+ else:
+ chown_map[owner_key] = [dir_name]
+
+ def action_create(self):
+ global delayed_directories
+ global chmod_map
+ global chown_map
+
+ self.action_create_delayed()
+
+ hdp_conf_dir = self.resource.conf_dir
+ hdp_hdfs_user = self.resource.hdfs_user
+ secured = self.resource.security_enabled
+ keytab_file = self.resource.keytab
+ kinit_path = self.resource.kinit_path_local
+
+ chmod_commands = []
+ chown_commands = []
+
+ for chmod_key, chmod_dirs in chmod_map.items():
+ mode = chmod_key[0]
+ recursive = chmod_key[1]
+ chmod_dirs_str = ' '.join(chmod_dirs)
+ chmod_commands.append(format("hadoop fs -chmod {recursive} {mode} {chmod_dirs_str}"))
+
+ for chown_key, chown_dirs in chown_map.items():
+ owner = chown_key[0]
+ group = chown_key[1]
+ recursive = chown_key[2]
+ chown_dirs_str = ' '.join(chown_dirs)
+ if owner:
+ chown = owner
+ if group:
+ chown = format("{owner}:{group}")
+ chown_commands.append(format("hadoop fs -chown {recursive} {chown} {chown_dirs_str}"))
+
+ if secured:
+ Execute(format("{kinit_path} -kt {keytab_file} {hdp_hdfs_user}"),
+ user=hdp_hdfs_user)
+ #create all directories in one 'mkdir' call
+ dir_list_str = ' '.join(directories_list)
+ #for hadoop 2 we need to specify -p to create directories recursively
+ parent_flag = '`rpm -q hadoop | grep -q "hadoop-1" || echo "-p"`'
+
+ exec_command = format('hadoop fs -mkdir {parent_flag} {dir_list_str}')
+ if len(chmod_commands) != 0:
+ exec_command = exec_command + ' && ' + ' && '.join(chmod_commands)
+ if len(chown_commands) != 0:
+ exec_command = exec_command + ' && ' + ' && '.join(chown_commands)
+
+ Execute(exec_command,
+ user=hdp_hdfs_user,
+ not_if=format("hadoop fs -ls {dir_list_str}")
+ )
+
+ directories_list[:] = []
+ chmod_map.clear()
+ chown_map.clear()
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/monitor_webserver.py b/slider-agent/src/main/python/resource_management/libraries/providers/monitor_webserver.py
new file mode 100644
index 0000000..5817879
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/monitor_webserver.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management import *
+from resource_management.core.system import System
+
+
+class MonitorWebserverProvider(Provider):
+ def action_start(self):
+ self.get_serivice_params()
+ self.enable_keep_alive()
+ service_name = self.service_name
+ Execute(format("/etc/init.d/{service_name} start"))
+
+ def action_stop(self):
+ self.get_serivice_params()
+ service_name = self.service_name
+ Execute(format("/etc/init.d/{service_name} stop"))
+
+ def action_restart(self):
+ self.action_stop()
+ self.action_start()
+
+ def get_serivice_params(self):
+ self.system = System.get_instance()
+ if self.system.os_family == "suse":
+ self.service_name = "apache2"
+ self.httpd_conf_dir = '/etc/apache2'
+ else:
+ self.service_name = "httpd"
+ self.httpd_conf_dir = '/etc/httpd/conf'
+
+ def enable_keep_alive(self):
+ httpd_conf_dir = self.httpd_conf_dir
+ Execute(format(
+ "grep -E 'KeepAlive (On|Off)' {httpd_conf_dir}/httpd.conf && sed -i 's/KeepAlive Off/KeepAlive On/' {httpd_conf_dir}/httpd.conf || echo 'KeepAlive On' >> {httpd_conf_dir}/httpd.conf"))
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/properties_file.py b/slider-agent/src/main/python/resource_management/libraries/providers/properties_file.py
new file mode 100644
index 0000000..47cb7fd
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/properties_file.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+import time
+import os
+from resource_management import *
+
+class PropertiesFileProvider(Provider):
+ def action_create(self):
+ filename = self.resource.filename
+ dir = self.resource.dir
+ if dir == None:
+ filepath = filename
+ else:
+ filepath = os.path.join(dir, filename)
+
+ config_content = InlineTemplate('''# Generated by Apache Slider. {{time.asctime(time.localtime())}}
+ {% for key, value in properties_dict|dictsort %}
+{{key}}={{value}}{% endfor %}
+ ''', extra_imports=[time], properties_dict=self.resource.properties)
+
+ Logger.info(format("Generating properties file: {filepath}"))
+
+ with Environment.get_instance_copy() as env:
+ File (format("{filepath}"),
+ content = config_content,
+ owner = self.resource.owner,
+ group = self.resource.group,
+ mode = self.resource.mode
+ )
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/repository.py b/slider-agent/src/main/python/resource_management/libraries/providers/repository.py
new file mode 100644
index 0000000..c5ba083
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/repository.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+import os
+import filecmp
+import tempfile
+from resource_management import *
+
+class RhelSuseRepositoryProvider(Provider):
+ def action_create(self):
+ with Environment.get_instance_copy() as env:
+ repo_file_name = self.resource.repo_file_name
+ repo_dir = repos_dirs[env.system.os_family]
+
+ File(format("{repo_dir}/{repo_file_name}.repo"),
+ content = InlineTemplate("""[{{repo_id}}]
+name={{repo_file_name}}
+{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}
+path=/
+enabled=1
+gpgcheck=0""", repo_id=self.resource.repo_id, repo_file_name=self.resource.repo_file_name, base_url=self.resource.base_url, mirror_list=self.resource.mirror_list)
+ )
+
+ def action_remove(self):
+ with Environment.get_instance_copy() as env:
+ repo_file_name = self.resource.repo_file_name
+ repo_dir = repos_dirs[env.system.os_family]
+
+ File(format("{repo_dir}/{repo_file_name}.repo"),
+ action = "delete")
+
+
+repos_dirs = {
+ 'redhat': '/etc/yum.repos.d',
+ 'suse': '/etc/zypp/repos.d'
+}
+
+
+class DebianRepositoryProvider(Provider):
+ package_type = "deb"
+ repo_dir = "/etc/apt/sources.list.d"
+ update_cmd = 'apt-get update -o Dir::Etc::sourcelist="sources.list.d/{repo_file_name}" -o APT::Get::List-Cleanup="0"'
+
+ def action_create(self):
+ with Environment.get_instance_copy() as env:
+ with tempfile.NamedTemporaryFile() as tmpf:
+ File(tmpf.name,
+ content = InlineTemplate("{{package_type}} {{base_url}} {{relase_name}} {{components}}",
+ package_type=self.package_type, base_url=self.resource.base_url, relase_name=env.system.os_release_name, components=' '.join(self.resource.components))
+ )
+
+ repo_file_name = format("{repo_file_name}.list",repo_file_name = self.resource.repo_file_name)
+ repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir = self.repo_dir)
+
+ if not os.path.isfile(repo_file_path) or not filecmp.cmp(tmpf.name, repo_file_path):
+ File(repo_file_path,
+ content = StaticFile(tmpf.name)
+ )
+
+ # this is time expensive
+ Execute(format(self.update_cmd))
+
+ def action_remove(self):
+ with Environment.get_instance_copy() as env:
+ repo_file_name = format("{repo_file_name}.list",repo_file_name = self.resource.repo_file_name)
+ repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir = self.repo_dir)
+
+ if os.path.isfile(repo_file_path):
+ File(repo_file_path,
+ action = "delete")
+
+ # this is time expensive
+ Execute(format(self.update_cmd))
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/template_config.py b/slider-agent/src/main/python/resource_management/libraries/providers/template_config.py
new file mode 100644
index 0000000..bc867e4
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/template_config.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+import os
+from resource_management import *
+
+class TemplateConfigProvider(Provider):
+ def action_create(self):
+ template_tag = self.resource.template_tag
+ qualified_file_name = self.resource.name
+ file_name = os.path.basename(qualified_file_name)
+
+ if not template_tag:
+ template_name = format("{file_name}.j2")
+ else:
+ template_name = format("{file_name}-{template_tag}.j2")
+
+ with Environment.get_instance_copy() as env:
+ File( qualified_file_name,
+ owner = self.resource.owner,
+ group = self.resource.group,
+ mode = self.resource.mode,
+ content = Template(template_name, extra_imports=self.resource.extra_imports)
+ )
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/xml_config.py b/slider-agent/src/main/python/resource_management/libraries/providers/xml_config.py
new file mode 100644
index 0000000..4983602
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/xml_config.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+import time
+from resource_management import *
+
+class XmlConfigProvider(Provider):
+ def action_create(self):
+ filename = self.resource.filename
+ conf_dir = self.resource.conf_dir
+
+ # |e - for html-like escaping of <,>,',"
+ config_content = InlineTemplate('''<!--{{time.asctime(time.localtime())}}-->
+ <configuration>
+ {% for key, value in configurations_dict.items() %}
+ <property>
+ <name>{{ key|e }}</name>
+ <value>{{ value|e }}</value>
+ </property>
+ {% endfor %}
+ </configuration>''', extra_imports=[time], configurations_dict=self.resource.configurations)
+
+
+ Logger.info(format("Generating config: {conf_dir}/{filename}"))
+
+ with Environment.get_instance_copy() as env:
+ File (format("{conf_dir}/{filename}"),
+ content = config_content,
+ owner = self.resource.owner,
+ group = self.resource.group,
+ mode = self.resource.mode
+ )
diff --git a/slider-agent/src/main/python/resource_management/libraries/resources/__init__.py b/slider-agent/src/main/python/resource_management/libraries/resources/__init__.py
new file mode 100644
index 0000000..628454b
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/resources/__init__.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.libraries.resources.execute_hadoop import *
+from resource_management.libraries.resources.template_config import *
+from resource_management.libraries.resources.xml_config import *
+from resource_management.libraries.resources.properties_file import *
+from resource_management.libraries.resources.repository import *
+from resource_management.libraries.resources.monitor_webserver import *
+from resource_management.libraries.resources.hdfs_directory import *
+from resource_management.libraries.resources.copy_from_local import *
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/libraries/resources/copy_from_local.py b/slider-agent/src/main/python/resource_management/libraries/resources/copy_from_local.py
new file mode 100644
index 0000000..decf207
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/resources/copy_from_local.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+_all__ = ["CopyFromLocal"]
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+class CopyFromLocal(Resource):
+ action = ForcedListArgument(default="run")
+
+ path = ResourceArgument(default=lambda obj: obj.name)
+ dest_dir = ResourceArgument(required=True)
+ owner = ResourceArgument(required=True)
+ group = ResourceArgument()
+ mode = ResourceArgument()
+ kinnit_if_needed = ResourceArgument(default='')
+ hadoop_conf_dir = ResourceArgument(default='/etc/hadoop/conf')
+ hdfs_user = ResourceArgument(default='hdfs')
+
+ actions = Resource.actions + ["run"]
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/libraries/resources/execute_hadoop.py b/slider-agent/src/main/python/resource_management/libraries/resources/execute_hadoop.py
new file mode 100644
index 0000000..9283cf8
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/resources/execute_hadoop.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+_all__ = ["ExecuteHadoop"]
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+class ExecuteHadoop(Resource):
+ action = ForcedListArgument(default="run")
+ command = ResourceArgument(default=lambda obj: obj.name)
+ kinit_override = BooleanArgument(default=False)
+ tries = ResourceArgument(default=1)
+ try_sleep = ResourceArgument(default=0) # seconds
+ user = ResourceArgument()
+ logoutput = BooleanArgument(default=False)
+ principal = ResourceArgument(default=lambda obj: obj.user)
+
+ conf_dir = ResourceArgument()
+
+ security_enabled = BooleanArgument(default=False)
+ keytab = ResourceArgument()
+ kinit_path_local = ResourceArgument()
+
+ actions = Resource.actions + ["run"]
+
diff --git a/slider-agent/src/main/python/resource_management/libraries/resources/hdfs_directory.py b/slider-agent/src/main/python/resource_management/libraries/resources/hdfs_directory.py
new file mode 100644
index 0000000..7bc1725
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/resources/hdfs_directory.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+_all__ = ["HdfsDirectory"]
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+class HdfsDirectory(Resource):
+ action = ForcedListArgument()
+
+ dir_name = ResourceArgument(default=lambda obj: obj.name)
+ owner = ResourceArgument()
+ group = ResourceArgument()
+ mode = ResourceArgument()
+ recursive_chown = BooleanArgument(default=False)
+ recursive_chmod = BooleanArgument(default=False)
+
+ conf_dir = ResourceArgument()
+ security_enabled = BooleanArgument(default=False)
+ keytab = ResourceArgument()
+ kinit_path_local = ResourceArgument()
+ hdfs_user = ResourceArgument()
+
+ #action 'create' immediately creates all pending directory in efficient manner
+ #action 'create_delayed' add directory to list of pending directories
+ actions = Resource.actions + ["create","create_delayed"]
diff --git a/slider-agent/src/main/python/resource_management/libraries/resources/monitor_webserver.py b/slider-agent/src/main/python/resource_management/libraries/resources/monitor_webserver.py
new file mode 100644
index 0000000..f5a139a
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/resources/monitor_webserver.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+_all__ = ["MonitorWebserver"]
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+
+class MonitorWebserver(Resource):
+ action = ForcedListArgument(default=lambda obj: [obj.name])
+ actions = Resource.actions + ["start", "stop", "restart"]
diff --git a/slider-agent/src/main/python/resource_management/libraries/resources/properties_file.py b/slider-agent/src/main/python/resource_management/libraries/resources/properties_file.py
new file mode 100644
index 0000000..ce606e3
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/resources/properties_file.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+_all__ = ["PropertiesFile"]
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+class PropertiesFile(Resource):
+ action = ForcedListArgument(default="create")
+ filename = ResourceArgument(default=lambda obj: obj.name)
+
+ properties = ResourceArgument()
+ dir = ResourceArgument()
+
+ mode = ResourceArgument()
+ owner = ResourceArgument()
+ group = ResourceArgument()
+
+ actions = Resource.actions + ["create"]
diff --git a/slider-agent/src/main/python/resource_management/libraries/resources/repository.py b/slider-agent/src/main/python/resource_management/libraries/resources/repository.py
new file mode 100644
index 0000000..023073c
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/resources/repository.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+_all__ = ["Repository"]
+
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+class Repository(Resource):
+ action = ForcedListArgument(default="create")
+ repo_id = ResourceArgument(default=lambda obj: obj.name)
+ base_url = ResourceArgument()
+ mirror_list = ResourceArgument()
+ repo_file_name = ResourceArgument()
+ components = ForcedListArgument(default=[]) # ubuntu specific
+
+ actions = Resource.actions + ["create","remove"]
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/libraries/resources/template_config.py b/slider-agent/src/main/python/resource_management/libraries/resources/template_config.py
new file mode 100644
index 0000000..e13eef7
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/resources/template_config.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+_all__ = ["TemplateConfig"]
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+class TemplateConfig(Resource):
+ action = ForcedListArgument(default="create")
+ path = ResourceArgument(default=lambda obj: obj.name)
+ mode = ResourceArgument()
+ owner = ResourceArgument()
+ group = ResourceArgument()
+ template_tag = ResourceArgument()
+ extra_imports = ResourceArgument(default=[])
+
+ actions = Resource.actions + ["create"]
diff --git a/slider-agent/src/main/python/resource_management/libraries/resources/xml_config.py b/slider-agent/src/main/python/resource_management/libraries/resources/xml_config.py
new file mode 100644
index 0000000..0a76a32
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/resources/xml_config.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+_all__ = ["XmlConfig"]
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+class XmlConfig(Resource):
+ action = ForcedListArgument(default="create")
+ filename = ResourceArgument(default=lambda obj: obj.name)
+
+ configurations = ResourceArgument()
+ conf_dir = ResourceArgument()
+
+ mode = ResourceArgument()
+ owner = ResourceArgument()
+ group = ResourceArgument()
+
+ actions = Resource.actions + ["create"]
diff --git a/slider-agent/src/main/python/resource_management/libraries/script/__init__.py b/slider-agent/src/main/python/resource_management/libraries/script/__init__.py
new file mode 100644
index 0000000..f391681
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/script/__init__.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management.libraries.script.script import *
+from resource_management.libraries.script.hook import *
+from resource_management.libraries.script.config_dictionary import *
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/libraries/script/config_dictionary.py b/slider-agent/src/main/python/resource_management/libraries/script/config_dictionary.py
new file mode 100644
index 0000000..453c546
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/script/config_dictionary.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from resource_management.core.exceptions import Fail
+
+class ConfigDictionary(dict):
+ """
+ Immutable config dictionary
+ """
+
+ def __init__(self, dictionary):
+ """
+ Recursively turn dict to ConfigDictionary
+ """
+ for k, v in dictionary.iteritems():
+ if isinstance(v, dict):
+ dictionary[k] = ConfigDictionary(v)
+
+ super(ConfigDictionary, self).__init__(dictionary)
+
+ def __setitem__(self, name, value):
+ raise Fail("Configuration dictionary is immutable!")
+
+ def __getitem__(self, name):
+ """
+ - use Python types
+ - enable lazy failure for unknown configs.
+ """
+ try:
+ value = super(ConfigDictionary, self).__getitem__(name)
+ except KeyError:
+ return UnknownConfiguration(name)
+
+
+ if value == "true":
+ value = True
+ elif value == "false":
+ value = False
+ else:
+ try:
+ value = int(value)
+ except (ValueError, TypeError):
+ try:
+ value = float(value)
+ except (ValueError, TypeError):
+ pass
+
+ return value
+
+
+class UnknownConfiguration():
+ """
+ Lazy failing for unknown configs.
+ """
+ def __init__(self, name):
+ self.name = name
+
+ def __getattr__(self, name):
+ raise Fail("Configuration parameter '"+self.name+"' was not found in configurations dictionary!")
+
+ def __getitem__(self, name):
+ """
+ Allow []
+ """
+ return self
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/libraries/script/hook.py b/slider-agent/src/main/python/resource_management/libraries/script/hook.py
new file mode 100644
index 0000000..19b5204
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/script/hook.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+__all__ = ["Hook"]
+
+from resource_management.libraries.script import Script
+import subprocess
+import sys
+
+class Hook(Script):
+ """
+ Executes a hook for acommand for custom service. stdout and stderr are written to
+ tmpoutfile and to tmperrfile respectively.
+ """
+
+ HOOK_METHOD_NAME = "hook" # This method is always executed at hooks
+
+
+ def choose_method_to_execute(self, command_name):
+ """
+ Changes logics of resolving method name
+ """
+ return super(Hook, self).choose_method_to_execute(self.HOOK_METHOD_NAME)
+
+
+ def run_custom_hook(self, command):
+ """
+ Runs custom hook
+ """
+ args = sys.argv
+ #Hook script to run
+ args[0] = args[0].replace(args[1], command)
+ #Hook script base directory
+ args[3] = args[3].replace(args[1], command)
+
+
+ cmd = [sys.executable]
+ cmd.extend(args)
+
+ if subprocess.call(cmd) != 0:
+ self.fail_with_error("Error: Unable to run the custom hook script " +
+ cmd.__str__())
+
diff --git a/slider-agent/src/main/python/resource_management/libraries/script/repo_installer.py b/slider-agent/src/main/python/resource_management/libraries/script/repo_installer.py
new file mode 100644
index 0000000..e3a5aea
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/script/repo_installer.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import json
+from resource_management.libraries.resources.repository import Repository
+
+class RepoInstaller():
+ @classmethod
+ def install_repos(cls, config):
+ cls._alter_repo("create", config['hostLevelParams']['repo_info'])
+
+ if 'service_repo_info' in config['hostLevelParams']:
+ cls._alter_repo("create", config['hostLevelParams']['service_repo_info'])
+
+ @classmethod
+ def remove_repos(cls, config):
+ cls._alter_repo("remove", config['hostLevelParams']['repo_info'])
+
+ if 'service_repo_info' in config['hostLevelParams']:
+ cls._alter_repo("remove", config['hostLevelParams']['service_repo_info'])
+
+ @staticmethod
+ def _alter_repo(action, repo_string):
+ """
+ @param action: "delete" or "create"
+ @param repo_string: e.g. "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"
+ """
+ repo_dicts = json.loads(repo_string)
+
+ if not isinstance(repo_dicts, list):
+ repo_dicts = [repo_dicts]
+
+ for repo in repo_dicts:
+ if not 'baseUrl' in repo:
+ repo['baseUrl'] = None
+ if not 'mirrorsList' in repo:
+ repo['mirrorsList'] = None
+
+ Repository(repo['repoId'],
+ action = action,
+ base_url = repo['baseUrl'],
+ mirror_list = repo['mirrorsList'],
+ repo_file_name = repo['repoName'])
\ No newline at end of file
diff --git a/slider-agent/src/main/python/resource_management/libraries/script/script.py b/slider-agent/src/main/python/resource_management/libraries/script/script.py
new file mode 100644
index 0000000..12fda80
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/script/script.py
@@ -0,0 +1,220 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+__all__ = ["Script"]
+
+import os
+import sys
+import json
+import logging
+
+from resource_management.core.environment import Environment
+from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus, ComponentIsNotRunning
+from resource_management.core.resources.packaging import Package
+from resource_management.core.resources import Tarball
+from resource_management.core.resources import Directory
+from resource_management.libraries.script.config_dictionary import ConfigDictionary
+from resource_management.libraries.script.repo_installer import RepoInstaller
+
+class Script(object):
+ """
+ Executes a command for custom service. stdout and stderr are written to
+ tmpoutfile and to tmperrfile respectively.
+ Script instances share configuration as a class parameter and therefore
+ different Script instances can not be used from different threads at
+ the same time within a single python process
+
+ Accepted command line arguments mapping:
+ 1 command type (START/STOP/...)
+ 2 path to command json file
+ 3 path to service metadata dir (Directory "package" inside service directory)
+ 4 path to file with structured command output (file will be created)
+ """
+ structuredOut = {}
+
+ def put_structured_out(self, sout):
+ Script.structuredOut.update(sout)
+ try:
+ structuredOut = json.dumps(Script.structuredOut)
+ with open(self.stroutfile, 'w') as fp:
+ json.dump(structuredOut, fp)
+ except IOError:
+ Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
+
+ def execute(self):
+ """
+ Sets up logging;
+ Parses command parameters and executes method relevant to command type
+ """
+ # set up logging (two separate loggers for stderr and stdout with different loglevels)
+ logger = logging.getLogger('resource_management')
+ logger.setLevel(logging.DEBUG)
+ formatter = logging.Formatter('%(asctime)s - %(message)s')
+ chout = logging.StreamHandler(sys.stdout)
+ chout.setLevel(logging.INFO)
+ chout.setFormatter(formatter)
+ cherr = logging.StreamHandler(sys.stderr)
+ cherr.setLevel(logging.ERROR)
+ cherr.setFormatter(formatter)
+ logger.addHandler(cherr)
+ logger.addHandler(chout)
+ # parse arguments
+ if len(sys.argv) < 5:
+ logger.error("Script expects at least 4 arguments")
+ sys.exit(1)
+ command_name = str.lower(sys.argv[1])
+ # parse command parameters
+ command_data_file = sys.argv[2]
+ basedir = sys.argv[3]
+ self.stroutfile = sys.argv[4]
+ try:
+ with open(command_data_file, "r") as f:
+ pass
+ Script.config = ConfigDictionary(json.load(f))
+ except IOError:
+ logger.exception("Can not read json file with command parameters: ")
+ sys.exit(1)
+ # Run class method depending on a command type
+ try:
+ method = self.choose_method_to_execute(command_name)
+ with Environment(basedir) as env:
+ method(env)
+ except ClientComponentHasNoStatus or ComponentIsNotRunning:
+ # Support of component status checks.
+ # Non-zero exit code is interpreted as an INSTALLED status of a component
+ sys.exit(1)
+ except Fail:
+ logger.exception("Error while executing command '{0}':".format(command_name))
+ sys.exit(1)
+
+
+ def choose_method_to_execute(self, command_name):
+ """
+ Returns a callable object that should be executed for a given command.
+ """
+ self_methods = dir(self)
+ if not command_name in self_methods:
+ raise Fail("Script '{0}' has no method '{1}'".format(sys.argv[0], command_name))
+ method = getattr(self, command_name)
+ return method
+
+
+ @staticmethod
+ def get_config():
+ """
+ HACK. Uses static field to store configuration. This is a workaround for
+ "circular dependency" issue when importing params.py file and passing to
+ it a configuration instance.
+ """
+ return Script.config
+
+
+ def install(self, env):
+ """
+ Default implementation of install command is to install all packages
+ from a list, received from the server.
+ Feel free to override install() method with your implementation. It
+ usually makes sense to call install_packages() manually in this case
+ """
+ self.install_packages(env)
+
+
+ def install_packages(self, env):
+ """
+ List of packages that are required by service is received from the server
+ as a command parameter. The method installs all packages
+ from this list
+ """
+ config = self.get_config()
+ repo_installed = False
+
+ try:
+ package_list_str = config['hostLevelParams']['package_list']
+ if isinstance(package_list_str,basestring) and len(package_list_str) > 0:
+ package_list = json.loads(package_list_str)
+ for package in package_list:
+ name = package['name']
+ type = package['type']
+ if type.lower() == "tarball":
+ if name.startswith(os.path.sep):
+ tarball = name
+ else:
+ basedir = env.config.basedir
+ tarball = os.path.join(basedir, name)
+ install_location = config['configurations']['global']['app_install_dir']
+ Directory(install_location, action = "delete")
+ Directory(install_location)
+ Tarball(tarball, location=install_location)
+ else:
+ if not repo_installed:
+ RepoInstaller.install_repos(config)
+ repo_installed = True
+ Package(name)
+ except KeyError:
+ pass # No reason to worry
+
+ #RepoInstaller.remove_repos(config)
+
+
+
+ def fail_with_error(self, message):
+ """
+ Prints error message and exits with non-zero exit code
+ """
+ print("Error: " + message)
+ sys.stderr.write("Error: " + message)
+ sys.exit(1)
+
+ def start(self, env):
+ """
+ To be overridden by subclasses
+ """
+ self.fail_with_error('start method isn\'t implemented')
+
+ def stop(self, env):
+ """
+ To be overridden by subclasses
+ """
+ self.fail_with_error('stop method isn\'t implemented')
+
+ def restart(self, env):
+ """
+ Default implementation of restart command is to call stop and start methods
+ Feel free to override restart() method with your implementation.
+ For client components we call install
+ """
+ config = self.get_config()
+ componentCategory = None
+ try :
+ componentCategory = config['roleParams']['component_category']
+ except KeyError:
+ pass
+
+ if componentCategory and componentCategory.strip().lower() == 'CLIENT'.lower():
+ self.install(env)
+ else:
+ self.stop(env)
+ self.start(env)
+
+ def configure(self, env):
+ """
+ To be overridden by subclasses
+ """
+ self.fail_with_error('configure method isn\'t implemented')
diff --git a/slider-agent/src/main/python/setup.cfg b/slider-agent/src/main/python/setup.cfg
new file mode 100644
index 0000000..a73754e
--- /dev/null
+++ b/slider-agent/src/main/python/setup.cfg
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[egg_info]
+tag_build =
+tag_date = 0
+tag_svn_revision = 0
diff --git a/slider-agent/src/main/python/setup.py b/slider-agent/src/main/python/setup.py
new file mode 100644
index 0000000..d024dbe
--- /dev/null
+++ b/slider-agent/src/main/python/setup.py
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from setuptools import setup
+
+setup(
+ name = "slider-agent",
+ version = "0.13.0-SNAPSHOT",
+ packages = ['agent'],
+ # metadata for upload to PyPI
+ author = "Apache Software Foundation",
+ author_email = "slider-dev@incubator.apache.org",
+ description = "Slider agent",
+ license = "Apache License v2.0",
+ keywords = "hadoop, slider",
+ url = "http://incubator.apache.org/slider",
+ long_description = "This package implements Slider for deploying and managing Apps on Yarn.",
+ platforms=["any"],
+ entry_points = {
+ "console_scripts": [
+ "slider-agent = agent.main:main",
+ ],
+ }
+)
diff --git a/slider-agent/src/packages/tarball/all.xml b/slider-agent/src/packages/tarball/all.xml
new file mode 100644
index 0000000..a5b6ba0
--- /dev/null
+++ b/slider-agent/src/packages/tarball/all.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd">
+ <!--This 'all' id is not appended to the produced bundle because we do this:
+ http://maven.apache.org/plugins/maven-assembly-plugin/faq.html#required-classifiers
+ -->
+ <formats>
+ <format>tar</format>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>src/main/python</directory>
+ <outputDirectory>slider-agent</outputDirectory>
+ <excludes>
+ <exclude>setup.cfg</exclude>
+ <exclude>setup.py</exclude>
+ </excludes>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
new file mode 100644
index 0000000..d52a6db
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -0,0 +1,413 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from Queue import Queue
+from unittest import TestCase
+import unittest
+from agent.ActionQueue import ActionQueue
+from agent.AgentConfig import AgentConfig
+import os
+import errno
+import time
+import pprint
+import tempfile
+import threading
+import StringIO
+import sys
+import logging
+from threading import Thread
+from mock.mock import patch, MagicMock, call
+from agent.CustomServiceOrchestrator import CustomServiceOrchestrator
+from agent.PythonExecutor import PythonExecutor
+from agent.CommandStatusDict import CommandStatusDict
+
+
+class TestActionQueue(TestCase):
+ def setUp(self):
+ out = StringIO.StringIO()
+ sys.stdout = out
+ # save original open() method for later use
+ self.original_open = open
+
+
+ def tearDown(self):
+ sys.stdout = sys.__stdout__
+
+ datanode_install_command = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'HBASE_MASTER',
+ 'roleCommand': u'INSTALL',
+ 'commandId': '1-1',
+ 'taskId': 3,
+ 'clusterName': u'cc',
+ 'serviceName': u'HBASE',
+ 'configurations': {'global': {}},
+ 'configurationTags': {'global': {'tag': 'v1'}},
+ 'commandParams': {'script_type': 'PYTHON',
+ 'script': 'scripts/abc.py',
+ 'command_timeout': '600'}
+ }
+
+ hbase_install_command = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'HBASE',
+ 'roleCommand': u'INSTALL',
+ 'commandId': '1-1',
+ 'taskId': 7,
+ 'clusterName': u'cc',
+ 'serviceName': u'HDFS',
+ }
+
+ status_command = {
+ "serviceName": 'ACCUMULO',
+ "commandType": "STATUS_COMMAND",
+ "clusterName": "c1",
+ "componentName": "ACCUMULO_MASTER",
+ 'configurations': {},
+ 'roleCommand': "STATUS"
+ }
+
+ status_command_with_config = {
+ "serviceName": 'ACCUMULO',
+ "commandType": "STATUS_COMMAND",
+ "clusterName": "c1",
+ "componentName": "ACCUMULO_MASTER",
+ 'configurations': {},
+ "commandParams": {"retrieve_config": "false"},
+ 'roleCommand': "GET_CONFIG"
+ }
+
+ @patch.object(ActionQueue, "process_command")
+ @patch.object(Queue, "get")
+ @patch.object(CustomServiceOrchestrator, "__init__")
+ def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock,
+ get_mock, process_command_mock):
+ CustomServiceOrchestrator_mock.return_value = None
+ dummy_controller = MagicMock()
+ config = MagicMock()
+ actionQueue = ActionQueue(config, dummy_controller)
+ actionQueue.start()
+ time.sleep(3)
+ actionQueue.stop()
+ actionQueue.join()
+ self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
+ self.assertTrue(process_command_mock.call_count > 1)
+
+
+ @patch("traceback.print_exc")
+ @patch.object(ActionQueue, "execute_command")
+ def test_process_command(self,
+ execute_command_mock, print_exc_mock):
+ dummy_controller = MagicMock()
+ actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
+ execution_command = {
+ 'commandType': ActionQueue.EXECUTION_COMMAND,
+ }
+ wrong_command = {
+ 'commandType': "SOME_WRONG_COMMAND",
+ }
+ # Try wrong command
+ actionQueue.process_command(wrong_command)
+ self.assertFalse(execute_command_mock.called)
+ self.assertFalse(print_exc_mock.called)
+
+ execute_command_mock.reset_mock()
+ print_exc_mock.reset_mock()
+ # Try normal execution
+ actionQueue.process_command(execution_command)
+ self.assertTrue(execute_command_mock.called)
+ self.assertFalse(print_exc_mock.called)
+
+ execute_command_mock.reset_mock()
+ print_exc_mock.reset_mock()
+
+ execute_command_mock.reset_mock()
+ print_exc_mock.reset_mock()
+
+ # Try exception to check proper logging
+ def side_effect(self):
+ raise Exception("TerribleException")
+
+ execute_command_mock.side_effect = side_effect
+ actionQueue.process_command(execution_command)
+ self.assertTrue(print_exc_mock.called)
+
+ print_exc_mock.reset_mock()
+
+ actionQueue.process_command(execution_command)
+ self.assertTrue(print_exc_mock.called)
+
+
+ @patch.object(ActionQueue, "status_update_callback")
+ @patch.object(CustomServiceOrchestrator, "requestComponentStatus")
+ @patch.object(ActionQueue, "execute_command")
+ @patch.object(CustomServiceOrchestrator, "__init__")
+ def test_execute_status_command(self, CustomServiceOrchestrator_mock,
+ execute_command_mock,
+ requestComponentStatus_mock,
+ status_update_callback):
+ CustomServiceOrchestrator_mock.return_value = None
+ dummy_controller = MagicMock()
+ actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
+
+ requestComponentStatus_mock.return_value = {'exitcode': 'dummy report'}
+ actionQueue.execute_status_command(self.status_command)
+ report = actionQueue.result()
+ expected = 'dummy report'
+ self.assertEqual(len(report['componentStatus']), 1)
+ self.assertEqual(report['componentStatus'][0]["status"], expected)
+ self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER")
+ self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO")
+ self.assertEqual(report['componentStatus'][0]["clusterName"], "c1")
+ self.assertTrue(requestComponentStatus_mock.called)
+
+ @patch.object(ActionQueue, "status_update_callback")
+ @patch.object(CustomServiceOrchestrator, "runCommand")
+ @patch.object(ActionQueue, "execute_command")
+ @patch('CustomServiceOrchestrator.CustomServiceOrchestrator', autospec=True)
+ def test_execute_status_command_expect_config(self, CustomServiceOrchestrator_mock,
+ execute_command_mock,
+ runCommand_mock,
+ status_update_callback):
+ csoMocks = [MagicMock()]
+ CustomServiceOrchestrator_mock.side_effect = csoMocks
+ csoMocks[0].status_commands_stdout = None
+ csoMocks[0].status_commands_stderr = None
+ dummy_controller = MagicMock()
+ actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
+
+ runCommand_mock.return_value = {'configurations': {}}
+ actionQueue.execute_status_command(self.status_command_with_config)
+ report = actionQueue.result()
+ self.assertEqual(len(report['componentStatus']), 1)
+ self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER")
+ self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO")
+ self.assertEqual(report['componentStatus'][0]["clusterName"], "c1")
+ self.assertEqual(report['componentStatus'][0]["configurations"], {})
+ self.assertFalse(runCommand_mock.called)
+
+
+ @patch("traceback.print_exc")
+ @patch.object(ActionQueue, "execute_command")
+ @patch.object(ActionQueue, "execute_status_command")
+ def test_process_command(self, execute_status_command_mock,
+ execute_command_mock, print_exc_mock):
+ dummy_controller = MagicMock()
+ actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
+ execution_command = {
+ 'commandType': ActionQueue.EXECUTION_COMMAND,
+ }
+ status_command = {
+ 'commandType': ActionQueue.STATUS_COMMAND,
+ }
+ wrong_command = {
+ 'commandType': "SOME_WRONG_COMMAND",
+ }
+ # Try wrong command
+ actionQueue.process_command(wrong_command)
+ self.assertFalse(execute_command_mock.called)
+ self.assertFalse(execute_status_command_mock.called)
+ self.assertFalse(print_exc_mock.called)
+
+ execute_command_mock.reset_mock()
+ execute_status_command_mock.reset_mock()
+ print_exc_mock.reset_mock()
+ # Try normal execution
+ actionQueue.process_command(execution_command)
+ self.assertTrue(execute_command_mock.called)
+ self.assertFalse(execute_status_command_mock.called)
+ self.assertFalse(print_exc_mock.called)
+
+ execute_command_mock.reset_mock()
+ execute_status_command_mock.reset_mock()
+ print_exc_mock.reset_mock()
+
+ actionQueue.process_command(status_command)
+ self.assertFalse(execute_command_mock.called)
+ self.assertTrue(execute_status_command_mock.called)
+ self.assertFalse(print_exc_mock.called)
+
+ execute_command_mock.reset_mock()
+ execute_status_command_mock.reset_mock()
+ print_exc_mock.reset_mock()
+
+ # Try exception to check proper logging
+ def side_effect(self):
+ raise Exception("TerribleException")
+
+ execute_command_mock.side_effect = side_effect
+ actionQueue.process_command(execution_command)
+ self.assertTrue(print_exc_mock.called)
+
+ print_exc_mock.reset_mock()
+
+ execute_status_command_mock.side_effect = side_effect
+ actionQueue.process_command(execution_command)
+ self.assertTrue(print_exc_mock.called)
+
+
+ @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+ @patch("json.load")
+ @patch("__builtin__.open")
+ @patch.object(ActionQueue, "status_update_callback")
+ def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock,
+ resolve_script_path_mock):
+
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ # Make file read calls visible
+ def open_side_effect(file, mode):
+ if mode == 'r':
+ file_mock = MagicMock()
+ file_mock.read.return_value = "Read from " + str(file)
+ return file_mock
+ else:
+ return self.original_open(file, mode)
+
+ open_mock.side_effect = open_side_effect
+ json_load_mock.return_value = ''
+ resolve_script_path_mock.return_value = "abc.py"
+
+ dummy_controller = MagicMock()
+ actionQueue = ActionQueue(config, dummy_controller)
+ unfreeze_flag = threading.Event()
+ python_execution_result_dict = {
+ 'stdout': 'out',
+ 'stderr': 'stderr',
+ 'structuredOut': ''
+ }
+
+ def side_effect(py_file, script_params,
+ tmpoutfile, tmperrfile, timeout,
+ tmpstrucoutfile,
+ override_output_files,
+ environment_vars):
+ unfreeze_flag.wait()
+ return python_execution_result_dict
+
+ def patched_aq_execute_command(command):
+ # We have to perform patching for separate thread in the same thread
+ with patch.object(PythonExecutor, "run_file") as runCommand_mock:
+ runCommand_mock.side_effect = side_effect
+ actionQueue.execute_command(command)
+ ### Test install/start/stop command ###
+
+ ## Test successful execution with configuration tags
+
+ python_execution_result_dict['status'] = 'COMPLETE'
+ python_execution_result_dict['exitcode'] = 0
+ # We call method in a separate thread
+ execution_thread = Thread(target=patched_aq_execute_command,
+ args=(self.datanode_install_command, ))
+ execution_thread.start()
+ # check in progress report
+ # wait until ready
+ while True:
+ time.sleep(0.1)
+ report = actionQueue.result()
+ if len(report['reports']) != 0:
+ break
+ expected = {'status': 'IN_PROGRESS',
+ 'stderr': 'Read from {0}/errors-3.txt'.format(tempdir),
+ 'stdout': 'Read from {0}/output-3.txt'.format(tempdir),
+ 'structuredOut': '',
+ 'clusterName': u'cc',
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HBASE',
+ 'role': u'HBASE_MASTER',
+ 'actionId': '1-1',
+ 'taskId': 3,
+ 'exitCode': 777}
+ self.assertEqual(report['reports'][0], expected)
+ # Continue command execution
+ unfreeze_flag.set()
+ # wait until ready
+ while report['reports'][0]['status'] == 'IN_PROGRESS':
+ time.sleep(0.1)
+ report = actionQueue.result()
+ # check report
+ configname = os.path.join(tempdir, 'command-3.json')
+ expected = {'status': 'COMPLETED',
+ 'stderr': 'stderr',
+ 'stdout': 'out',
+ 'clusterName': u'cc',
+ 'configurationTags': {'global': {'tag': 'v1'}},
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HBASE',
+ 'role': u'HBASE_MASTER',
+ 'actionId': '1-1',
+ 'taskId': 3,
+ 'structuredOut': '',
+ 'exitCode': 0}
+ self.assertEqual(len(report['reports']), 1)
+ self.assertEqual(report['reports'][0], expected)
+ self.assertTrue(os.path.isfile(configname))
+ # Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE)
+ self.assertEqual(status_update_callback_mock.call_count, 2)
+ os.remove(configname)
+
+ # now should not have reports (read complete/failed reports are deleted)
+ report = actionQueue.result()
+ self.assertEqual(len(report['reports']), 0)
+
+ ## Test failed execution
+ python_execution_result_dict['status'] = 'FAILED'
+ python_execution_result_dict['exitcode'] = 13
+ # We call method in a separate thread
+ execution_thread = Thread(target=patched_aq_execute_command,
+ args=(self.datanode_install_command, ))
+ execution_thread.start()
+ unfreeze_flag.set()
+ # check in progress report
+ # wait until ready
+ report = actionQueue.result()
+ while len(report['reports']) == 0 or \
+ report['reports'][0]['status'] == 'IN_PROGRESS':
+ time.sleep(0.1)
+ report = actionQueue.result()
+ # check report
+ expected = {'status': 'FAILED',
+ 'stderr': 'stderr',
+ 'stdout': 'out',
+ 'clusterName': u'cc',
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HBASE',
+ 'role': u'HBASE_MASTER',
+ 'actionId': '1-1',
+ 'taskId': 3,
+ 'structuredOut': '',
+ 'exitCode': 13}
+ self.assertEqual(len(report['reports']), 1)
+ self.assertEqual(report['reports'][0], expected)
+
+ # now should not have reports (read complete/failed reports are deleted)
+ report = actionQueue.result()
+ self.assertEqual(len(report['reports']), 0)
+
+
+if __name__ == "__main__":
+ logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
+ unittest.main()
+
diff --git a/slider-agent/src/test/python/agent/TestController.py b/slider-agent/src/test/python/agent/TestController.py
new file mode 100644
index 0000000..c4e18e9
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestController.py
@@ -0,0 +1,566 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import StringIO
+import ssl
+import unittest, threading
+from agent import Controller, ActionQueue
+from agent import hostname
+import sys
+from agent.Controller import AGENT_AUTO_RESTART_EXIT_CODE
+from agent.Controller import State
+from agent.AgentConfig import AgentConfig
+from mock.mock import patch, MagicMock, call, Mock
+import logging
+from threading import Event
+
+class TestController(unittest.TestCase):
+
+ logger = logging.getLogger()
+
+ @patch("threading.Thread")
+ @patch("threading.Lock")
+ @patch.object(Controller, "NetUtil")
+ @patch.object(hostname, "hostname")
+ def setUp(self, hostname_method, NetUtil_mock, lockMock, threadMock):
+
+ Controller.logger = MagicMock()
+ lockMock.return_value = MagicMock()
+ NetUtil_mock.return_value = MagicMock()
+ hostname_method.return_value = "test_hostname"
+
+
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = "something"
+
+ self.controller = Controller.Controller(config)
+ self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS = 0.1
+ self.controller.netutil.HEARTBEAT_NOT_IDDLE_INTERVAL_SEC = 0.1
+
+
+ @patch("json.dumps")
+ @patch("time.sleep")
+ @patch("pprint.pformat")
+ @patch.object(Controller, "randint")
+ def test_registerWithServer(self, randintMock, pformatMock, sleepMock,
+ dumpsMock):
+
+ out = StringIO.StringIO()
+ sys.stdout = out
+
+ register = MagicMock()
+ self.controller.register = register
+
+ self.controller.sendRequest = MagicMock()
+
+ dumpsMock.return_value = "request"
+ self.controller.sendRequest.return_value = '{"log":"Error text", "exitstatus":"1"}'
+
+ self.assertEqual({u'exitstatus': u'1', u'log': u'Error text'}, self.controller.registerWithServer())
+
+ self.controller.sendRequest.return_value = '{"responseId":1}'
+ self.assertEqual({"responseId":1}, self.controller.registerWithServer())
+
+ self.controller.sendRequest.return_value = '{"responseId":1, "statusCommands": "commands", "log":"", "exitstatus":"0"}'
+ self.controller.isRegistered = False
+ self.assertEqual({'exitstatus': '0', 'responseId': 1, 'log': '', 'statusCommands': 'commands'}, self.controller.registerWithServer())
+
+ calls = []
+
+ def side_effect(*args):
+ if len(calls) == 0:
+ calls.append(1)
+ raise Exception("test")
+ return "request"
+
+ self.controller.sendRequest.return_value = '{"responseId":1}'
+
+ dumpsMock.side_effect = side_effect
+ self.controller.isRegistered = False
+ self.assertEqual({"responseId":1}, self.controller.registerWithServer())
+ self.assertTrue(randintMock.called)
+ self.assertTrue(sleepMock.called)
+
+ sys.stdout = sys.__stdout__
+
+ self.controller.sendRequest = Controller.Controller.sendRequest
+
+
+ @patch("pprint.pformat")
+ def test_addToQueue(self, pformatMock):
+
+ actionQueue = MagicMock()
+ self.controller.actionQueue = actionQueue
+ self.controller.addToQueue(None)
+ self.assertFalse(actionQueue.put.called)
+ self.controller.addToQueue("cmd")
+ self.assertTrue(actionQueue.put.called)
+
+
+ @patch("urllib2.build_opener")
+ @patch("urllib2.install_opener")
+ @patch.object(Controller, "ActionQueue")
+ def test_run(self, ActionQueue_mock, installMock, buildMock):
+ aq = MagicMock()
+ ActionQueue_mock.return_value = aq
+
+ buildMock.return_value = "opener"
+ registerAndHeartbeat = MagicMock("registerAndHeartbeat")
+ calls = []
+ def side_effect():
+ if len(calls) == 0:
+ self.controller.repeatRegistration = True
+ calls.append(1)
+ registerAndHeartbeat.side_effect = side_effect
+ self.controller.registerAndHeartbeat = registerAndHeartbeat
+
+ # repeat registration
+ self.controller.run()
+
+ self.assertTrue(buildMock.called)
+ installMock.called_once_with("opener")
+ self.assertEqual(2, registerAndHeartbeat.call_count)
+
+ # one call, +1
+ registerAndHeartbeat.side_effect = None
+ self.controller.run()
+ self.assertEqual(3, registerAndHeartbeat.call_count)
+
+ # Action queue should be started during calls
+ self.assertTrue(ActionQueue_mock.called)
+ self.assertTrue(aq.start.called)
+
+
+ @patch("urllib2.build_opener")
+ @patch("urllib2.install_opener")
+ @patch.object(ActionQueue.ActionQueue, "run")
+ def test_repeatRegistration(self,
+ run_mock, installMock, buildMock):
+
+ registerAndHeartbeat = MagicMock(name="registerAndHeartbeat")
+
+ self.controller.registerAndHeartbeat = registerAndHeartbeat
+ self.controller.run()
+ self.assertTrue(installMock.called)
+ self.assertTrue(buildMock.called)
+ self.controller.registerAndHeartbeat.assert_called_once_with()
+
+ calls = []
+ def switchBool():
+ if len(calls) == 0:
+ self.controller.repeatRegistration = True
+ calls.append(1)
+ self.controller.repeatRegistration = False
+
+ registerAndHeartbeat.side_effect = switchBool
+ self.controller.run()
+ self.assertEqual(2, registerAndHeartbeat.call_count)
+
+ self.controller.registerAndHeartbeat = \
+ Controller.Controller.registerAndHeartbeat
+
+
+ @patch("time.sleep")
+ def test_registerAndHeartbeatWithException(self, sleepMock):
+
+ registerWithServer = MagicMock(name="registerWithServer")
+ registerWithServer.return_value = {"response":"resp"}
+ self.controller.registerWithServer = registerWithServer
+ heartbeatWithServer = MagicMock(name="heartbeatWithServer")
+ self.controller.heartbeatWithServer = heartbeatWithServer
+
+ Controller.Controller.__sendRequest__ = MagicMock(side_effect=Exception())
+
+ self.controller.isRegistered = True
+ self.controller.registerAndHeartbeat()
+ registerWithServer.assert_called_once_with()
+ heartbeatWithServer.assert_called_once_with()
+
+ self.controller.registerWithServer =\
+ Controller.Controller.registerWithServer
+ self.controller.heartbeatWithServer =\
+ Controller.Controller.registerWithServer
+
+ @patch("time.sleep")
+ def test_registerAndHeartbeat(self, sleepMock):
+
+ registerWithServer = MagicMock(name="registerWithServer")
+ registerWithServer.return_value = {"response":"resp"}
+ self.controller.registerWithServer = registerWithServer
+ heartbeatWithServer = MagicMock(name="heartbeatWithServer")
+ self.controller.heartbeatWithServer = heartbeatWithServer
+
+ listener1 = MagicMock()
+ listener2 = MagicMock()
+ self.controller.registration_listeners.append(listener1)
+ self.controller.registration_listeners.append(listener2)
+ self.controller.isRegistered = True
+ self.controller.registerAndHeartbeat()
+ registerWithServer.assert_called_once_with()
+ heartbeatWithServer.assert_called_once_with()
+ self.assertTrue(listener1.called)
+ self.assertTrue(listener2.called)
+
+ self.controller.registerWithServer = \
+ Controller.Controller.registerWithServer
+ self.controller.heartbeatWithServer = \
+ Controller.Controller.registerWithServer
+
+
+ @patch("time.sleep")
+ def test_registerAndHeartbeat_check_registration_listener(self, sleepMock):
+ registerWithServer = MagicMock(name="registerWithServer")
+ registerWithServer.return_value = {"response":"resp"}
+ self.controller.registerWithServer = registerWithServer
+ heartbeatWithServer = MagicMock(name="heartbeatWithServer")
+ self.controller.heartbeatWithServer = heartbeatWithServer
+
+ self.controller.isRegistered = True
+ self.controller.registerAndHeartbeat()
+ registerWithServer.assert_called_once_with()
+ heartbeatWithServer.assert_called_once_with()
+
+ self.controller.registerWithServer = \
+ Controller.Controller.registerWithServer
+ self.controller.heartbeatWithServer = \
+ Controller.Controller.registerWithServer
+
+
+ @patch("os._exit")
+ def test_restartAgent(self, os_exit_mock):
+
+ self.controller.restartAgent()
+ self.assertTrue(os_exit_mock.called)
+ self.assertTrue(os_exit_mock.call_args[0][0] == AGENT_AUTO_RESTART_EXIT_CODE)
+
+
+ @patch("urllib2.urlopen")
+ def test_sendRequest(self, requestMock):
+
+ conMock = MagicMock()
+ conMock.read.return_value = "response"
+ url = "url"
+ data = "data"
+ requestMock.return_value = conMock
+
+ self.assertEqual("response", self.controller.sendRequest(url, data))
+ requestMock.called_once_with(url, data,
+ {'Content-Type': 'application/json'})
+
+
+ @patch.object(threading._Event, "wait")
+ @patch("time.sleep")
+ @patch("json.loads")
+ @patch("json.dumps")
+ def test_heartbeatWithServer(self, dumpsMock, loadsMock, sleepMock, event_mock):
+ original_value = self.controller.config
+ self.controller.config = AgentConfig("", "")
+ out = StringIO.StringIO()
+ sys.stdout = out
+
+ hearbeat = MagicMock()
+ self.controller.heartbeat = hearbeat
+
+ dumpsMock.return_value = "data"
+
+ sendRequest = MagicMock(name="sendRequest")
+ self.controller.sendRequest = sendRequest
+
+ self.controller.responseId = 1
+ response = {"responseId":"2", "restartAgent":"false"}
+ loadsMock.return_value = response
+
+ def one_heartbeat(*args, **kwargs):
+ self.controller.DEBUG_STOP_HEARTBEATING = True
+ return "data"
+
+ sendRequest.side_effect = one_heartbeat
+
+ actionQueue = MagicMock()
+ actionQueue.isIdle.return_value = True
+
+ # one successful request, after stop
+ self.controller.actionQueue = actionQueue
+ self.controller.heartbeatWithServer()
+ self.assertTrue(sendRequest.called)
+
+ calls = []
+ def retry(*args, **kwargs):
+ if len(calls) == 0:
+ calls.append(1)
+ response["responseId"] = "3"
+ raise Exception()
+ if len(calls) > 0:
+ self.controller.DEBUG_STOP_HEARTBEATING = True
+ return "data"
+
+ # exception, retry, successful and stop
+ sendRequest.side_effect = retry
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ self.assertEqual(1, self.controller.DEBUG_SUCCESSFULL_HEARTBEATS)
+
+ # retry registration
+ response["registrationCommand"] = "true"
+ sendRequest.side_effect = one_heartbeat
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ self.assertTrue(self.controller.repeatRegistration)
+
+ # components are not mapped
+ response["registrationCommand"] = "false"
+ response["hasMappedComponents"] = False
+ sendRequest.side_effect = one_heartbeat
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ self.assertFalse(self.controller.hasMappedComponents)
+
+ # components are mapped
+ response["hasMappedComponents"] = True
+ sendRequest.side_effect = one_heartbeat
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ self.assertTrue(self.controller.hasMappedComponents)
+
+ # components are mapped
+ del response["hasMappedComponents"]
+ sendRequest.side_effect = one_heartbeat
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ self.assertTrue(self.controller.hasMappedComponents)
+
+ # wrong responseId => restart
+ response = {"responseId":"2", "restartAgent":"false"}
+ loadsMock.return_value = response
+
+ restartAgent = MagicMock(name="restartAgent")
+ self.controller.restartAgent = restartAgent
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ restartAgent.assert_called_once_with()
+
+ # executionCommands
+ self.controller.responseId = 1
+ addToQueue = MagicMock(name="addToQueue")
+ self.controller.addToQueue = addToQueue
+ response["executionCommands"] = "executionCommands"
+ self.controller.statusCommand = ["statusCommand"]
+ updateStateBasedOnCommand = MagicMock(name="updateStateBasedOnCommand")
+ self.controller.updateStateBasedOnCommand = updateStateBasedOnCommand
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ addToQueue.assert_has_calls([call("executionCommands")])
+ updateStateBasedOnCommand.assert_has_calls([call("executionCommands")])
+
+ # just status command when state = STARTED
+ self.controller.responseId = 1
+ response = {"responseId":"2", "restartAgent":"false"}
+ loadsMock.return_value = response
+ addToQueue = MagicMock(name="addToQueue")
+ self.controller.addToQueue = addToQueue
+ self.controller.statusCommand = "statusCommand"
+ self.controller.componentActualState = State.STARTED
+ self.controller.componentExpectedState = State.STARTED
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ addToQueue.assert_has_calls([call(["statusCommand"])])
+
+ # just status command when state = FAILED
+ self.controller.responseId = 1
+ response = {"responseId":"2", "restartAgent":"false"}
+ loadsMock.return_value = response
+ addToQueue = MagicMock(name="addToQueue")
+ self.controller.addToQueue = addToQueue
+ self.controller.statusCommand = "statusCommand"
+ self.controller.componentActualState = State.FAILED
+ self.controller.componentExpectedState = State.STARTED
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ addToQueue.assert_has_calls([call(["statusCommand"])])
+
+ # no status command when state = STARTING
+ self.controller.responseId = 1
+ response = {"responseId":"2", "restartAgent":"false"}
+ loadsMock.return_value = response
+ addToQueue = MagicMock(name="addToQueue")
+ self.controller.addToQueue = addToQueue
+ self.controller.statusCommand = "statusCommand"
+ self.controller.componentActualState = State.STARTING
+ self.controller.componentExpectedState = State.STARTED
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ addToQueue.assert_has_calls([])
+
+ # statusCommands
+ response["statusCommands"] = "statusCommands"
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ self.controller.heartbeatWithServer()
+
+ addToQueue.assert_has_calls([call("statusCommands")])
+
+ # restartAgent command
+ self.controller.responseId = 1
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ response["restartAgent"] = "true"
+ restartAgent = MagicMock(name="restartAgent")
+ self.controller.restartAgent = restartAgent
+ self.controller.heartbeatWithServer()
+
+ restartAgent.assert_called_once_with()
+
+ # actionQueue not idle
+ self.controller.responseId = 1
+ self.controller.DEBUG_STOP_HEARTBEATING = False
+ actionQueue.isIdle.return_value = False
+ response["restartAgent"] = "false"
+ self.controller.heartbeatWithServer()
+
+ sleepMock.assert_called_with(
+ self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS)
+
+ sys.stdout = sys.__stdout__
+ self.controller.sendRequest = Controller.Controller.sendRequest
+ self.controller.sendRequest = Controller.Controller.addToQueue
+
+ self.controller.config = original_value
+ pass
+
+
+ @patch.object(Controller.Controller, "createStatusCommand")
+ def test_updateStateBasedOnResult(self, mock_createStatusCommand):
+ commands = []
+ commands.append({u'roleCommand': u'INSTALL'})
+ self.controller.updateStateBasedOnCommand(commands)
+
+ commandResult = {"commandStatus": "COMPLETED"}
+ self.controller.updateStateBasedOnResult(commandResult)
+ self.assertEqual(State.INSTALLED, self.controller.componentActualState)
+ self.assertEqual(State.INSTALLED, self.controller.componentExpectedState)
+
+ commands = []
+ commands.append({u'roleCommand': u'START'})
+ self.controller.updateStateBasedOnCommand(commands)
+
+ commandResult = {"commandStatus": "COMPLETED"}
+ self.controller.updateStateBasedOnResult(commandResult)
+ self.assertEqual(State.STARTED, self.controller.componentActualState)
+ self.assertEqual(State.STARTED, self.controller.componentExpectedState)
+
+ commandResult = {"healthStatus": "STARTED"}
+ self.controller.updateStateBasedOnResult(commandResult)
+ self.assertEqual(State.STARTED, self.controller.componentActualState)
+ self.assertEqual(State.STARTED, self.controller.componentExpectedState)
+
+ commandResult = {"healthStatus": "INSTALLED"}
+ self.controller.updateStateBasedOnResult(commandResult)
+ self.assertEqual(State.FAILED, self.controller.componentActualState)
+ self.assertEqual(State.STARTED, self.controller.componentExpectedState)
+ self.assertEqual(1, self.controller.failureCount)
+
+ commandResult = {"healthStatus": "INSTALLED"}
+ self.controller.updateStateBasedOnResult(commandResult)
+ self.assertEqual(State.FAILED, self.controller.componentActualState)
+ self.assertEqual(State.STARTED, self.controller.componentExpectedState)
+ self.assertEqual(2, self.controller.failureCount)
+
+ self.assertTrue(mock_createStatusCommand.called)
+
+
+ def test_updateStateBasedOnCommand(self):
+ commands = []
+ self.controller.updateStateBasedOnCommand(commands)
+ self.assertEqual(State.INIT, self.controller.componentActualState)
+ self.assertEqual(State.INIT, self.controller.componentExpectedState)
+
+ commands.append({u'roleCommand': u'INSTALL'})
+ self.controller.updateStateBasedOnCommand(commands)
+ self.assertEqual(State.INSTALLING, self.controller.componentActualState)
+ self.assertEqual(State.INSTALLED, self.controller.componentExpectedState)
+
+ commands = []
+ commands.append({
+ u'roleCommand': u'START',
+ "clusterName": "c1",
+ "commandParams": ["cp"],
+ "role": "HBASE_MASTER",
+ "configurations": {"global": {"a": "b"}, "abc-site": {"c": "d"}},
+ "hostLevelParams": [],
+ "serviceName": "HBASE"
+ })
+ self.controller.updateStateBasedOnCommand(commands)
+ self.assertEqual(State.STARTING, self.controller.componentActualState)
+ self.assertEqual(State.STARTED, self.controller.componentExpectedState)
+
+ self.assertEqual(self.controller.statusCommand["clusterName"], "c1")
+ self.assertEqual(self.controller.statusCommand["commandParams"], ["cp"])
+ self.assertEqual(self.controller.statusCommand["commandType"], "STATUS_COMMAND")
+ self.assertEqual(self.controller.statusCommand["roleCommand"], "STATUS")
+ self.assertEqual(self.controller.statusCommand["componentName"], "HBASE_MASTER")
+ self.assertEqual(self.controller.statusCommand["configurations"], {"global": {"a": "b"}})
+ self.assertEqual(self.controller.statusCommand["hostLevelParams"], [])
+ self.assertEqual(self.controller.statusCommand["serviceName"], "HBASE")
+ self.assertEqual(self.controller.statusCommand["taskId"], "status")
+ self.assertEqual(self.controller.statusCommand["auto_generated"], True)
+ self.assertEqual(len(self.controller.statusCommand), 10)
+
+ commands = []
+ self.controller.updateStateBasedOnCommand(commands)
+ self.assertEqual(State.STARTING, self.controller.componentActualState)
+ self.assertEqual(State.STARTED, self.controller.componentExpectedState)
+
+ @patch("pprint.pformat")
+ @patch("time.sleep")
+ @patch("json.loads")
+ @patch("json.dumps")
+ def test_certSigningFailed(self, dumpsMock, loadsMock, sleepMock, pformatMock):
+ register = MagicMock()
+ self.controller.register = register
+
+ dumpsMock.return_value = "request"
+ response = {"responseId":1,}
+ loadsMock.return_value = response
+
+ self.controller.sendRequest = Mock(side_effect=ssl.SSLError())
+
+ self.controller.repeatRegistration=True
+ self.controller.registerWithServer()
+
+ #Conroller thread and the agent stop if the repeatRegistration flag is False
+ self.assertFalse(self.controller.repeatRegistration)
+
+if __name__ == "__main__":
+ logging.basicConfig(format='%(asctime)s %(message)s',level=logging.DEBUG)
+ unittest.main()
+
+
+
+
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
new file mode 100644
index 0000000..9f04298
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -0,0 +1,321 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import ConfigParser
+import os
+import pprint
+
+from unittest import TestCase
+import unittest
+import threading
+import tempfile
+import time
+import logging
+from threading import Thread
+
+from PythonExecutor import PythonExecutor
+from CustomServiceOrchestrator import CustomServiceOrchestrator
+from mock.mock import MagicMock, patch
+import StringIO
+import sys
+from AgentException import AgentException
+
+
+class TestCustomServiceOrchestrator(TestCase):
+ def setUp(self):
+ # disable stdout
+ out = StringIO.StringIO()
+ sys.stdout = out
+ # generate sample config
+ tmpdir = tempfile.gettempdir()
+
+
+ @patch("hostname.public_hostname")
+ @patch("os.path.isfile")
+ @patch("os.unlink")
+ def test_dump_command_to_json(self, unlink_mock,
+ isfile_mock, hostname_mock):
+ hostname_mock.return_value = "test.hst"
+ command = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'DATANODE',
+ 'roleCommand': u'INSTALL',
+ 'commandId': '1-1',
+ 'taskId': 3,
+ 'clusterName': u'cc',
+ 'serviceName': u'HDFS',
+ 'configurations': {'global': {}},
+ 'configurationTags': {'global': {'tag': 'v1'}},
+ 'clusterHostInfo': {'namenode_host': ['1'],
+ 'slave_hosts': ['0', '1'],
+ 'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
+ 'all_ping_ports': ['8670:0,1']}
+ }
+
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ isfile_mock.return_value = True
+ # Test dumping EXECUTION_COMMAND
+ json_file = orchestrator.dump_command_to_json(command)
+ self.assertTrue(os.path.exists(json_file))
+ self.assertTrue(os.path.getsize(json_file) > 0)
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ self.assertTrue(json_file.endswith("command-3.json"))
+ os.unlink(json_file)
+ # Test dumping STATUS_COMMAND
+ command['commandType'] = 'STATUS_COMMAND'
+ json_file = orchestrator.dump_command_to_json(command)
+ self.assertTrue(os.path.exists(json_file))
+ self.assertTrue(os.path.getsize(json_file) > 0)
+ self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+ self.assertTrue(json_file.endswith("status_command.json"))
+ os.unlink(json_file)
+ # Testing side effect of dump_command_to_json
+ self.assertEquals(command['public_hostname'], "test.hst")
+ self.assertTrue(unlink_mock.called)
+
+
+ @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+ @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+ @patch.object(PythonExecutor, "run_file")
+ def test_runCommand(self,
+ run_file_mock, dump_command_to_json_mock,
+ resolve_script_path_mock):
+ command = {
+ 'role': 'REGION_SERVER',
+ 'hostLevelParams': {
+ 'stack_name': 'HDP',
+ 'stack_version': '2.0.7',
+ 'jdk_location': 'some_location'
+ },
+ 'commandParams': {
+ 'script_type': 'PYTHON',
+ 'script': 'scripts/hbase_regionserver.py',
+ 'command_timeout': '600',
+ 'service_package_folder': 'HBASE'
+ },
+ 'taskId': '3',
+ 'roleCommand': 'INSTALL'
+ }
+
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ resolve_script_path_mock.return_value = "/basedir/scriptpath"
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ # normal run case
+ run_file_mock.return_value = {
+ 'stdout': 'sss',
+ 'stderr': 'eee',
+ 'exitcode': 0,
+ }
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ self.assertEqual(ret['exitcode'], 0)
+ self.assertTrue(run_file_mock.called)
+ self.assertEqual(run_file_mock.call_count, 1)
+
+ run_file_mock.reset_mock()
+
+ # Case when we force another command
+ run_file_mock.return_value = {
+ 'stdout': 'sss',
+ 'stderr': 'eee',
+ 'exitcode': 0,
+ }
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ ## Check that override_output_files was true only during first call
+ self.assertEquals(run_file_mock.call_args_list[0][0][6], True)
+
+ run_file_mock.reset_mock()
+
+ # unknown script type case
+ command['commandParams']['script_type'] = "PUPPET"
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+ self.assertEqual(ret['exitcode'], 1)
+ self.assertFalse(run_file_mock.called)
+ self.assertTrue("Unknown script type" in ret['stdout'])
+
+ #By default returns empty dictionary
+ self.assertEqual(ret['structuredOut'], '{}')
+ pass
+
+
+ @patch("hostname.public_hostname")
+ @patch("os.path.isfile")
+ @patch("os.unlink")
+ @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+ @patch.object(PythonExecutor, "run_file")
+ def test_runCommand_with_config(self,
+ run_file_mock,
+ resolve_script_path_mock, unlink_mock,
+ isfile_mock, hostname_mock):
+ hostname_mock.return_value = "test.hst"
+ isfile_mock.return_value = True
+ command = {
+ 'role': 'REGION_SERVER',
+ 'hostLevelParams': {
+ 'stack_name': 'HDP',
+ 'stack_version': '2.0.7',
+ 'jdk_location': 'some_location'
+ },
+ 'commandParams': {
+ 'script_type': 'PYTHON',
+ 'script': 'scripts/hbase_regionserver.py',
+ 'command_timeout': '600',
+ 'service_package_folder': 'HBASE'
+ },
+ 'configurations': {
+ "hbase-site": {
+ "hbase.log": "${AGENT_LOG_ROOT}",
+ "hbase.number": "10485760"},
+ "hbase-log4j": {"a": "b"}
+ },
+ 'taskId': '3',
+ 'roleCommand': 'INSTALL',
+ 'commandType': 'EXECUTION_COMMAND',
+ 'commandId': '1-1'
+ }
+
+ command_get = {
+ 'roleCommand': 'GET_CONFIG',
+ 'commandType': 'STATUS_COMMAND'
+ }
+
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ resolve_script_path_mock.return_value = "/basedir/scriptpath"
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ # normal run case
+ run_file_mock.return_value = {
+ 'stdout': 'sss',
+ 'stderr': 'eee',
+ 'exitcode': 0,
+ }
+
+ expected = {
+ 'hbase-site': {
+ 'hbase.log': tempdir, 'hbase.number': '10485760'},
+ 'hbase-log4j': {'a': 'b'}}
+
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt", True, True)
+ self.assertEqual(ret['exitcode'], 0)
+ self.assertTrue(run_file_mock.called)
+ self.assertEqual(orchestrator.applied_configs, expected)
+
+ ret = orchestrator.requestComponentStatus(command_get)
+ self.assertEqual(ret['configurations'], expected)
+ pass
+
+ @patch.object(CustomServiceOrchestrator, "runCommand")
+ def test_requestComponentStatus(self, runCommand_mock):
+ status_command = {
+ "serviceName": 'HDFS',
+ "commandType": "STATUS_COMMAND",
+ "clusterName": "",
+ "componentName": "DATANODE",
+ 'configurations': {},
+ 'roleCommand' : "STATUS"
+ }
+ dummy_controller = MagicMock()
+
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ # Test alive case
+ runCommand_mock.return_value = {
+ "exitcode": 0
+ }
+ status = orchestrator.requestComponentStatus(status_command)
+ self.assertEqual(CustomServiceOrchestrator.LIVE_STATUS, status['exitcode'])
+
+ # Test dead case
+ runCommand_mock.return_value = {
+ "exitcode": 1
+ }
+ status = orchestrator.requestComponentStatus(status_command)
+ self.assertEqual(CustomServiceOrchestrator.DEAD_STATUS, status['exitcode'])
+
+ def test_finalize_command(self):
+ dummy_controller = MagicMock()
+ tempdir = tempfile.gettempdir()
+ tempWorkDir = tempdir + "W"
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempWorkDir
+ config.getLogPath.return_value = tempdir
+
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ command = {}
+ command['configurations'] = {}
+ command['configurations']['hbase-site'] = {}
+ command['configurations']['hbase-site']['a'] = 'b'
+ command['configurations']['hbase-site']['work_root'] = "${AGENT_WORK_ROOT}"
+ command['configurations']['hbase-site']['log_root'] = "${AGENT_LOG_ROOT}/log"
+ command['configurations']['hbase-site']['blog_root'] = "/b/${AGENT_LOG_ROOT}/log"
+ command['configurations']['oozie-site'] = {}
+ command['configurations']['oozie-site']['log_root'] = "${AGENT_LOG_ROOT}"
+
+ orchestrator.finalize_command(command, False)
+ self.assertEqual(command['configurations']['hbase-site']['work_root'], tempWorkDir)
+ self.assertEqual(command['configurations']['oozie-site']['log_root'], tempdir)
+ self.assertEqual(orchestrator.applied_configs, {})
+
+ command['configurations']['hbase-site']['work_root'] = "${AGENT_WORK_ROOT}"
+ command['configurations']['hbase-site']['log_root'] = "${AGENT_LOG_ROOT}/log"
+ command['configurations']['hbase-site']['blog_root'] = "/b/${AGENT_LOG_ROOT}/log"
+ command['configurations']['oozie-site']['log_root'] = "${AGENT_LOG_ROOT}"
+
+ orchestrator.finalize_command(command, True)
+ self.assertEqual(command['configurations']['hbase-site']['log_root'], tempdir + "/log")
+ self.assertEqual(command['configurations']['hbase-site']['blog_root'], "/b/" + tempdir + "/log")
+ self.assertEqual(orchestrator.applied_configs, command['configurations'])
+
+ def tearDown(self):
+ # enable stdout
+ sys.stdout = sys.__stdout__
+
+
+if __name__ == "__main__":
+ logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
+ unittest.main()
+
diff --git a/slider-agent/src/test/python/agent/TestGrep.py b/slider-agent/src/test/python/agent/TestGrep.py
new file mode 100644
index 0000000..75f0093
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestGrep.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from agent.Grep import Grep
+import socket
+import os, sys
+import logging
+
+class TestGrep(TestCase):
+
+ logger = logging.getLogger()
+ string_good = None
+ string_bad = None
+ grep = Grep()
+
+ def setUp(self):
+ self.string_good = open('agent' + os.sep + 'dummy_output_good.txt', 'r').read().replace("\n", os.linesep)
+ self.string_bad = open('agent' + os.sep + 'dummy_output_error.txt', 'r').read().replace("\n", os.linesep)
+ pass
+
+ def test_grep_many_lines(self):
+ fragment = self.grep.grep(self.string_bad, "err", 1000, 1000)
+ desired = self.string_bad.strip()
+ self.assertEquals(fragment, desired, "Grep grep function should return all lines if there are less lines than n")
+
+
+ def test_grep_few_lines(self):
+ fragment = self.grep.grep(self.string_bad, "Err", 3, 3)
+ desired = """
+debug: /Schedule[never]: Skipping device resources because running on a host
+debug: Exec[command_good](provider=posix): Executing 'wget e432423423xample.com/badurl444111'
+debug: Executing 'wget e432423423xample.com/badurl444111'
+err: /Stage[main]//Exec[command_good]/returns: change from notrun to 0 failed: wget e432423423xample.com/badurl444111 returned 4 instead of one of [0] at /root/puppet-learn/2-bad.pp:5
+debug: /Schedule[weekly]: Skipping device resources because running on a host
+debug: /Schedule[puppet]: Skipping device resources because running on a host
+debug: Finishing transaction 70171639726240
+""".strip()
+ self.assertEquals(fragment, desired, "Grep grep function should return only last 3 lines of file")
+
+ def test_grep_no_result(self):
+ fragment = self.grep.grep(self.string_good, "Err", 3, 3)
+ desired = None
+ self.assertEquals(fragment, desired, 'Grep grep function should return None if result is not found')
+
+ def test_grep_empty_string(self):
+ fragment = self.grep.grep("", "Err", 1000, 1000)
+ desired = None
+ self.assertEquals(fragment, desired, 'Grep grep function should return None for empty string')
+
+ def test_grep_all(self):
+ fragment = self.grep.grep(self.string_bad, "Err", 35, 9)
+ desired = self.string_bad.strip()
+ self.assertEquals(fragment, desired, 'Grep grep function contains bug in index arithmetics')
+
+
+ def test_tail_many_lines(self):
+ fragment = self.grep.tail(self.string_good, 1000)
+ desired = self.string_good.strip()
+ self.assertEquals(fragment, desired, "Grep tail function should return all lines if there are less lines than n")
+
+ def test_tail_few_lines(self):
+ fragment = self.grep.tail(self.string_good, 3)
+ desired = """
+debug: Finishing transaction 70060456663980
+debug: Received report to process from ambari-dmi
+debug: Processing report from ambari-dmi with processor Puppet::Reports::Store
+""".strip()
+ self.assertEquals(fragment, desired, "Grep tail function should return only last 3 lines of file")
+
+ def test_tail_no_lines(self):
+ fragment = self.grep.tail("", 3)
+ desired = ''
+ self.assertEquals(fragment, desired, 'Grep tail function should return "" for empty string')
+
+ def test_tail_all(self):
+ fragment = self.grep.tail("", 47)
+ desired = ''
+ self.assertEquals(fragment, desired, 'Grep tail function contains bug in index arithmetics')
+
+ def test_filterMarkup(self):
+ string = """[0;36mnotice: /Stage[main]/Hdp-hadoop/Hdp-hadoop::Package[hadoop]/Hdp::Package[hadoop 64]/Hdp::Package::Process_pkg[hadoop 64]/Package[hadoop-libhdfs]/ensure: created[0m"""
+ desired="""notice: /Stage[main]/Hdp-hadoop/Hdp-hadoop::Package[hadoop]/Hdp::Package[hadoop 64]/Hdp::Package::Process_pkg[hadoop 64]/Package[hadoop-libhdfs]/ensure: created"""
+ filtered = self.grep.filterMarkup(string)
+ #sys.stderr.write(filtered)
+ self.assertEquals(filtered, desired)
+
+ def tearDown(self):
+ pass
+
+ def test_cleanByTemplate(self):
+ fragment = self.grep.cleanByTemplate(self.string_bad, "debug")
+ desired = """
+info: Applying configuration version '1352127563'
+err: /Stage[main]//Exec[command_good]/returns: change from notrun to 0 failed: wget e432423423xample.com/badurl444111 returned 4 instead of one of [0] at /root/puppet-learn/2-bad.pp:5
+notice: Finished catalog run in 0.23 seconds
+""".strip()
+ self.assertEquals(fragment, desired, 'Grep cleanByTemplate function should return string without debug lines.')
+
+
diff --git a/slider-agent/src/test/python/agent/TestHeartbeat.py b/slider-agent/src/test/python/agent/TestHeartbeat.py
new file mode 100644
index 0000000..68bbc91
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestHeartbeat.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+import unittest
+from agent.Heartbeat import Heartbeat
+from agent.ActionQueue import ActionQueue
+from agent.AgentConfig import AgentConfig
+import socket
+import os
+import time
+from mock.mock import patch, MagicMock, call
+import StringIO
+import sys
+import logging
+
+
+class TestHeartbeat(TestCase):
+ def setUp(self):
+ # disable stdout
+ out = StringIO.StringIO()
+ sys.stdout = out
+
+
+ def tearDown(self):
+ # enable stdout
+ sys.stdout = sys.__stdout__
+
+
+ def test_build(self):
+ config = AgentConfig("", "")
+ config.set('agent', 'prefix', 'tmp')
+ dummy_controller = MagicMock()
+ actionQueue = ActionQueue(config, dummy_controller)
+ heartbeat = Heartbeat(actionQueue, config)
+ result = heartbeat.build({}, 100)
+ print "Heartbeat: " + str(result)
+ self.assertEquals(result['hostname'] != '', True,
+ "hostname should not be empty")
+ self.assertEquals(result['responseId'], 100)
+ self.assertEquals('componentStatus' not in result, True,
+ "Heartbeat should contain componentStatus")
+ self.assertEquals(result['reports'] is not None, True,
+ "Heartbeat should contain reports")
+ self.assertEquals(result['timestamp'] >= 1353679373880L, True)
+ self.assertEquals(len(result['nodeStatus']), 2)
+ self.assertEquals(result['nodeStatus']['cause'], "NONE")
+ self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
+ # result may or may NOT have an agentEnv structure in it
+ self.assertEquals((len(result) is 4) or (len(result) is 5), True)
+ self.assertEquals(not heartbeat.reports, True,
+ "Heartbeat should not contain task in progress")
+
+
+ @patch.object(ActionQueue, "result")
+ def test_build_long_result(self, result_mock):
+ config = AgentConfig("", "")
+ config.set('agent', 'prefix', 'tmp')
+ dummy_controller = MagicMock()
+ actionQueue = ActionQueue(config, dummy_controller)
+ result_mock.return_value = {
+ 'reports': [{'status': 'IN_PROGRESS',
+ 'stderr': 'Read from /tmp/errors-3.txt',
+ 'stdout': 'Read from /tmp/output-3.txt',
+ 'clusterName': u'cc',
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HDFS',
+ 'role': u'DATANODE',
+ 'actionId': '1-1',
+ 'taskId': 3,
+ 'exitCode': 777},
+
+ {'status': 'COMPLETED',
+ 'stderr': 'stderr',
+ 'stdout': 'out',
+ 'clusterName': 'clusterName',
+ 'roleCommand': 'UPGRADE',
+ 'serviceName': 'serviceName',
+ 'role': 'role',
+ 'actionId': 17,
+ 'taskId': 'taskId',
+ 'exitCode': 0},
+
+ {'status': 'FAILED',
+ 'stderr': 'stderr',
+ 'stdout': 'out',
+ 'clusterName': u'cc',
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HDFS',
+ 'role': u'DATANODE',
+ 'actionId': '1-1',
+ 'taskId': 3,
+ 'exitCode': 13},
+
+ {'status': 'COMPLETED',
+ 'stderr': 'stderr',
+ 'stdout': 'out',
+ 'clusterName': u'cc',
+ 'configurationTags': {'global': {'tag': 'v1'}},
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HDFS',
+ 'role': u'DATANODE',
+ 'actionId': '1-1',
+ 'taskId': 3,
+ 'exitCode': 0}
+
+ ],
+ 'componentStatus': [
+ {'status': 'HEALTHY', 'componentName': 'DATANODE', 'reportResult' : True},
+ {'status': 'UNHEALTHY', 'componentName': 'NAMENODE', 'reportResult' : True},
+ {'status': 'UNHEALTHY', 'componentName': 'HBASE_MASTER', 'reportResult' : False},
+ ],
+ }
+ heartbeat = Heartbeat(actionQueue, config)
+ hb = heartbeat.build({}, 10)
+ hb['hostname'] = 'hostname'
+ hb['timestamp'] = 'timestamp'
+ expected = {'nodeStatus':
+ {'status': 'HEALTHY',
+ 'cause': 'NONE'},
+ 'timestamp': 'timestamp', 'hostname': 'hostname',
+ 'responseId': 10, 'reports': [
+ {'status': 'IN_PROGRESS', 'roleCommand': u'INSTALL',
+ 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
+ 'stderr': 'Read from /tmp/errors-3.txt',
+ 'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc',
+ 'taskId': 3, 'exitCode': 777},
+ {'status': 'COMPLETED', 'roleCommand': 'UPGRADE',
+ 'serviceName': 'serviceName', 'role': 'role', 'actionId': 17,
+ 'stderr': 'stderr', 'stdout': 'out', 'clusterName': 'clusterName',
+ 'taskId': 'taskId', 'exitCode': 0},
+ {'status': 'FAILED', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS',
+ 'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'stderr',
+ 'stdout': 'out', 'clusterName': u'cc', 'taskId': 3, 'exitCode': 13},
+ {'status': 'COMPLETED', 'stdout': 'out',
+ 'configurationTags': {'global': {'tag': 'v1'}}, 'taskId': 3,
+ 'exitCode': 0, 'roleCommand': u'INSTALL', 'clusterName': u'cc',
+ 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
+ 'stderr': 'stderr'}], 'componentStatus': [
+ {'status': 'HEALTHY', 'componentName': 'DATANODE'},
+ {'status': 'UNHEALTHY', 'componentName': 'NAMENODE'}]}
+ self.assertEquals(hb, expected)
+
+
+if __name__ == "__main__":
+ logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
+ unittest.main()
diff --git a/slider-agent/src/test/python/agent/TestMain.py b/slider-agent/src/test/python/agent/TestMain.py
new file mode 100644
index 0000000..5273623
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestMain.py
@@ -0,0 +1,305 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import StringIO
+import sys
+
+from agent import NetUtil, security
+from mock.mock import MagicMock, patch, ANY
+import unittest
+from agent import ProcessHelper, main
+import logging
+import signal
+from agent.AgentConfig import AgentConfig
+import ConfigParser
+import os
+import tempfile
+from agent.Controller import Controller
+from optparse import OptionParser
+
+
+class TestMain(unittest.TestCase):
+ def setUp(self):
+ # disable stdout
+ out = StringIO.StringIO()
+ sys.stdout = out
+
+
+ def tearDown(self):
+ # enable stdout
+ sys.stdout = sys.__stdout__
+
+
+ @patch("os._exit")
+ @patch("os.getpid")
+ @patch.object(ProcessHelper, "stopAgent")
+ def test_signal_handler(self, stopAgent_mock, os_getpid_mock, os_exit_mock):
+ # testing exit of children
+ main.agentPid = 4444
+ os_getpid_mock.return_value = 5555
+ main.signal_handler("signum", "frame")
+ self.assertTrue(os_exit_mock.called)
+
+ os_exit_mock.reset_mock()
+
+ # testing exit of main process
+ os_getpid_mock.return_value = main.agentPid
+ main.signal_handler("signum", "frame")
+ self.assertFalse(os_exit_mock.called)
+ self.assertTrue(stopAgent_mock.called)
+
+
+ @patch.object(main.logger, "addHandler")
+ @patch.object(main.logger, "setLevel")
+ @patch("logging.handlers.RotatingFileHandler")
+ def test_setup_logging(self, rfh_mock, setLevel_mock, addHandler_mock):
+ # Testing silent mode
+ _, tmpoutfile = tempfile.mkstemp()
+ main.setup_logging(False, tmpoutfile)
+ self.assertTrue(addHandler_mock.called)
+ setLevel_mock.assert_called_with(logging.INFO)
+
+ addHandler_mock.reset_mock()
+ setLevel_mock.reset_mock()
+
+ # Testing verbose mode
+ main.setup_logging(True, tmpoutfile)
+ self.assertTrue(addHandler_mock.called)
+ setLevel_mock.assert_called_with(logging.DEBUG)
+
+
+ @patch.object(main.logger, "setLevel")
+ @patch("logging.basicConfig")
+ def test_update_log_level(self, basicConfig_mock, setLevel_mock):
+ config = AgentConfig("", "")
+ _, tmpoutfile = tempfile.mkstemp()
+
+ # Testing with default setup (config file does not contain loglevel entry)
+ # Log level should not be changed
+ config.set('agent', 'log_level', None)
+ main.update_log_level(config, tmpoutfile)
+ self.assertFalse(setLevel_mock.called)
+
+ setLevel_mock.reset_mock()
+
+ # Testing debug mode
+ config.set('agent', 'log_level', 'DEBUG')
+ main.update_log_level(config, tmpoutfile)
+ setLevel_mock.assert_called_with(logging.DEBUG)
+ setLevel_mock.reset_mock()
+
+ # Testing any other mode
+ config.set('agent', 'log_level', 'INFO')
+ main.update_log_level(config, tmpoutfile)
+ setLevel_mock.assert_called_with(logging.INFO)
+
+ setLevel_mock.reset_mock()
+
+ config.set('agent', 'log_level', 'WRONG')
+ main.update_log_level(config, tmpoutfile)
+ setLevel_mock.assert_called_with(logging.INFO)
+
+
+ @patch("signal.signal")
+ def test_bind_signal_handlers(self, signal_mock):
+ main.bind_signal_handlers()
+ # Check if on SIGINT/SIGTERM agent is configured to terminate
+ signal_mock.assert_any_call(signal.SIGINT, main.signal_handler)
+ signal_mock.assert_any_call(signal.SIGTERM, main.signal_handler)
+ # Check if on SIGUSR1 agent is configured to fall into debug
+ signal_mock.assert_any_call(signal.SIGUSR1, main.debug)
+
+
+ @patch("os.path.exists")
+ @patch("ConfigParser.RawConfigParser.read")
+ def test_resolve_config(self, read_mock, exists_mock):
+ config = AgentConfig("", "")
+ # Trying case if conf file exists
+ exists_mock.return_value = True
+ main.update_config_from_file(config)
+ self.assertTrue(read_mock.called)
+
+ exists_mock.reset_mock()
+ read_mock.reset_mock()
+
+ # Trying case if conf file does not exist
+ exists_mock.return_value = False
+ main.update_config_from_file(config)
+ self.assertFalse(read_mock.called)
+
+
+ @patch("os.remove")
+ @patch("sys.exit")
+ @patch("os.path.isfile")
+ @patch("os.path.isdir")
+ @patch("hostname.hostname")
+ def test_perform_prestart_checks(self, hostname_mock, isdir_mock, isfile_mock,
+ exit_mock, remove_mock):
+ main.config = AgentConfig("", "")
+
+ # Trying case if there is another instance running
+ isfile_mock.return_value = True
+ isdir_mock.return_value = True
+ main.perform_prestart_checks(main.config)
+ self.assertFalse(exit_mock.called)
+ self.assertTrue(remove_mock.called)
+
+ isfile_mock.reset_mock()
+ isdir_mock.reset_mock()
+ exit_mock.reset_mock()
+
+ # Trying case if agent prefix dir does not exist
+ isfile_mock.return_value = False
+ isdir_mock.return_value = False
+ main.perform_prestart_checks(main.config)
+ self.assertTrue(exit_mock.called)
+
+ isfile_mock.reset_mock()
+ isdir_mock.reset_mock()
+ exit_mock.reset_mock()
+
+ # Trying normal case
+ isfile_mock.return_value = False
+ isdir_mock.return_value = True
+ main.perform_prestart_checks(main.config)
+ self.assertFalse(exit_mock.called)
+
+
+ @patch("time.sleep")
+ @patch("os.kill")
+ @patch("os._exit")
+ @patch("os.path.exists")
+ def test_daemonize_and_stop(self, exists_mock, _exit_mock, kill_mock,
+ sleep_mock):
+ oldpid = ProcessHelper.pidfile
+ pid = str(os.getpid())
+ _, tmpoutfile = tempfile.mkstemp()
+ ProcessHelper.pidfile = tmpoutfile
+
+ # Test daemonization
+ main.write_pid()
+ saved = open(ProcessHelper.pidfile, 'r').read()
+ self.assertEqual(pid, saved)
+
+ # Reuse pid file when testing agent stop
+ # Testing normal exit
+ exists_mock.return_value = False
+ main.stop_agent()
+ kill_mock.assert_called_with(int(pid), signal.SIGTERM)
+
+ # Restore
+ kill_mock.reset_mock()
+ _exit_mock.reset_mock()
+
+ # Testing exit when failed to remove pid file
+ exists_mock.return_value = True
+ main.stop_agent()
+ kill_mock.assert_any_call(int(pid), signal.SIGTERM)
+ kill_mock.assert_any_call(int(pid), signal.SIGKILL)
+ _exit_mock.assert_called_with(1)
+
+ # Restore
+ ProcessHelper.pidfile = oldpid
+ os.remove(tmpoutfile)
+
+
+ @patch.object(main, "setup_logging")
+ @patch.object(main, "bind_signal_handlers")
+ @patch.object(main, "stop_agent")
+ @patch.object(main, "update_config_from_file")
+ @patch.object(main, "perform_prestart_checks")
+ @patch.object(main, "write_pid")
+ @patch.object(main, "update_log_level")
+ @patch.object(NetUtil.NetUtil, "try_to_connect")
+ @patch.object(Controller, "__init__")
+ @patch.object(Controller, "start")
+ @patch.object(Controller, "join")
+ @patch("optparse.OptionParser.parse_args")
+ def test_main(self, parse_args_mock, join_mock, start_mock,
+ Controller_init_mock, try_to_connect_mock,
+ update_log_level_mock, write_pid_mock,
+ perform_prestart_checks_mock,
+ update_config_from_file_mock, stop_mock,
+ bind_signal_handlers_mock, setup_logging_mock):
+ Controller_init_mock.return_value = None
+ options = MagicMock()
+ parse_args_mock.return_value = (options, MagicMock)
+
+ tmpdir = tempfile.gettempdir()
+
+ #testing call without command-line arguments
+ os.environ["AGENT_WORK_ROOT"] = os.path.join(tmpdir, "work")
+ os.environ["AGENT_LOG_ROOT"] = os.path.join(tmpdir, "log")
+ main.main()
+
+ self.assertTrue(setup_logging_mock.called)
+ self.assertTrue(bind_signal_handlers_mock.called)
+ self.assertTrue(stop_mock.called)
+ self.assertTrue(update_config_from_file_mock.called)
+ self.assertTrue(perform_prestart_checks_mock.called)
+ self.assertTrue(write_pid_mock.called)
+ self.assertTrue(update_log_level_mock.called)
+ try_to_connect_mock.assert_called_once_with(ANY, -1, ANY)
+ self.assertTrue(start_mock.called)
+
+ class AgentOptions:
+ def __init__(self, label, host, port, verbose):
+ self.label = label
+ self.host = host
+ self.port = port
+ self.verbose = verbose
+
+ @patch.object(main, "setup_logging")
+ @patch.object(main, "bind_signal_handlers")
+ @patch.object(main, "stop_agent")
+ @patch.object(main, "update_config_from_file")
+ @patch.object(main, "perform_prestart_checks")
+ @patch.object(main, "write_pid")
+ @patch.object(main, "update_log_level")
+ @patch.object(NetUtil.NetUtil, "try_to_connect")
+ @patch.object(AgentConfig, 'set')
+ @patch.object(Controller, "__init__")
+ @patch.object(Controller, "start")
+ @patch.object(Controller, "join")
+ @patch.object(Controller, "is_alive")
+ @patch("optparse.OptionParser.parse_args")
+ def test_main(self, parse_args_mock, isAlive_mock, join_mock, start_mock,
+ Controller_init_mock, AgentConfig_set_mock,
+ try_to_connect_mock,
+ update_log_level_mock, write_pid_mock,
+ perform_prestart_checks_mock,
+ update_config_from_file_mock, stop_mock,
+ bind_signal_handlers_mock, setup_logging_mock):
+ Controller_init_mock.return_value = None
+ isAlive_mock.return_value = False
+ parse_args_mock.return_value = (
+ TestMain.AgentOptions("agent", "host1", "8080", True), [])
+ tmpdir = tempfile.gettempdir()
+
+ #testing call without command-line arguments
+ os.environ["AGENT_WORK_ROOT"] = os.path.join(tmpdir, "work")
+ os.environ["AGENT_LOG_ROOT"] = os.path.join(tmpdir, "log")
+ main.main()
+ self.assertTrue(AgentConfig_set_mock.call_count == 2)
+ AgentConfig_set_mock.assert_any_call("server", "hostname", "host1")
+ AgentConfig_set_mock.assert_any_call("server", "port", "8080")
+
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file
diff --git a/slider-agent/src/test/python/agent/TestNetUtil.py b/slider-agent/src/test/python/agent/TestNetUtil.py
new file mode 100644
index 0000000..c19ec19
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestNetUtil.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from agent import NetUtil
+from mock.mock import MagicMock, patch
+import unittest
+
+class TestNetUtil(unittest.TestCase):
+
+ @patch("urlparse.urlparse")
+ @patch("httplib.HTTPConnection")
+ def test_checkURL(self, httpsConMock, parseMock):
+
+ NetUtil.logger = MagicMock()
+ parseMock.return_value = [1, 2]
+ ca_connection = MagicMock()
+ response = MagicMock()
+ response.status = 200
+ ca_connection.getresponse.return_value = response
+ httpsConMock.return_value = ca_connection
+
+ # test 200
+ netutil = NetUtil.NetUtil()
+ self.assertTrue(netutil.checkURL("url"))
+
+ # test fail
+ response.status = 404
+ self.assertFalse(netutil.checkURL("url"))
+
+ # test Exception
+ response.status = 200
+ httpsConMock.side_effect = Exception("test")
+ self.assertFalse(netutil.checkURL("url"))
+
+
+ @patch("time.sleep")
+ def test_try_to_connect(self, sleepMock):
+
+ netutil = NetUtil.NetUtil()
+ checkURL = MagicMock(name="checkURL")
+ checkURL.return_value = True
+ netutil.checkURL = checkURL
+ l = MagicMock()
+
+ # one successful get
+ self.assertEqual(0, netutil.try_to_connect("url", 10))
+
+ # got successful after N retries
+ gets = [True, False, False]
+ def side_effect(*args):
+ return gets.pop()
+ checkURL.side_effect = side_effect
+ self.assertEqual(2, netutil.try_to_connect("url", 10))
+
+ # max retries
+ checkURL.side_effect = None
+ checkURL.return_value = False
+ self.assertEqual(5, netutil.try_to_connect("url", 5))
+
+
diff --git a/slider-agent/src/test/python/agent/TestPythonExecutor.py b/slider-agent/src/test/python/agent/TestPythonExecutor.py
new file mode 100644
index 0000000..7cec409
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestPythonExecutor.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import pprint
+
+from unittest import TestCase
+import threading
+import tempfile
+import time
+from threading import Thread
+import unittest
+
+from PythonExecutor import PythonExecutor
+from AgentConfig import AgentConfig
+from mock.mock import MagicMock, patch
+
+
+class TestPythonExecutor(TestCase):
+
+ @patch("shell.kill_process_with_children")
+ def test_watchdog_1(self, kill_process_with_children_mock):
+ """
+ Tests whether watchdog works
+ """
+ subproc_mock = self.Subprocess_mockup()
+ executor = PythonExecutor("/tmp", AgentConfig("", ""))
+ _, tmpoutfile = tempfile.mkstemp()
+ _, tmperrfile = tempfile.mkstemp()
+ _, tmpstrucout = tempfile.mkstemp()
+ PYTHON_TIMEOUT_SECONDS = 0.1
+ kill_process_with_children_mock.side_effect = lambda pid : subproc_mock.terminate()
+
+ def launch_python_subprocess_method(command, tmpout, tmperr, environment_vars):
+ subproc_mock.tmpout = tmpout
+ subproc_mock.tmperr = tmperr
+ return subproc_mock
+ executor.launch_python_subprocess = launch_python_subprocess_method
+ runShellKillPgrp_method = MagicMock()
+ runShellKillPgrp_method.side_effect = lambda python : python.terminate()
+ executor.runShellKillPgrp = runShellKillPgrp_method
+ subproc_mock.returncode = None
+ thread = Thread(target = executor.run_file, args = ("fake_puppetFile",
+ ["arg1", "arg2"], tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstrucout))
+ thread.start()
+ time.sleep(0.1)
+ subproc_mock.finished_event.wait()
+ self.assertEquals(subproc_mock.was_terminated, True, "Subprocess should be terminated due to timeout")
+
+
+ def test_watchdog_2(self):
+ """
+ Tries to catch false positive watchdog invocations
+ """
+ subproc_mock = self.Subprocess_mockup()
+ executor = PythonExecutor("/tmp", AgentConfig("", ""))
+ _, tmpoutfile = tempfile.mkstemp()
+ _, tmperrfile = tempfile.mkstemp()
+ _, tmpstrucout = tempfile.mkstemp()
+ PYTHON_TIMEOUT_SECONDS = 5
+
+ environment_vars = [("PYTHONPATH", "a:b")]
+ def launch_python_subprocess_method(command, tmpout, tmperr, environment_vars):
+ subproc_mock.tmpout = tmpout
+ subproc_mock.tmperr = tmperr
+ return subproc_mock
+ executor.launch_python_subprocess = launch_python_subprocess_method
+ runShellKillPgrp_method = MagicMock()
+ runShellKillPgrp_method.side_effect = lambda python : python.terminate()
+ executor.runShellKillPgrp = runShellKillPgrp_method
+ subproc_mock.returncode = 0
+ thread = Thread(target = executor.run_file, args = ("fake_puppetFile", ["arg1", "arg2"],
+ tmpoutfile, tmperrfile,
+ PYTHON_TIMEOUT_SECONDS, tmpstrucout))
+ thread.start()
+ time.sleep(0.1)
+ subproc_mock.should_finish_event.set()
+ subproc_mock.finished_event.wait()
+ self.assertEquals(subproc_mock.was_terminated, False, "Subprocess should not be terminated before timeout")
+ self.assertEquals(subproc_mock.returncode, 0, "Subprocess should not be terminated before timeout")
+
+ @patch("__builtin__.open")
+ @patch("subprocess.Popen")
+ @patch("os.environ.copy")
+ def test_set_env_values(self, os_env_copy_mock, subprocess_mock, open_mock):
+ actual_vars = {"someOther" : "value1"}
+ executor = PythonExecutor("/tmp", AgentConfig("", ""))
+ environment_vars = [("PYTHONPATH", "a:b")]
+ os_env_copy_mock.return_value = actual_vars
+ executor.run_file("script.pynot", ["a","b"], "", "", 10, "", True, environment_vars)
+ self.assertEquals(2, len(os_env_copy_mock.return_value))
+
+ def test_execution_results(self):
+ subproc_mock = self.Subprocess_mockup()
+ executor = PythonExecutor("/tmp", AgentConfig("", ""))
+ _, tmpoutfile = tempfile.mkstemp()
+ _, tmperrfile = tempfile.mkstemp()
+ _, tmpstroutfile = tempfile.mkstemp()
+ PYTHON_TIMEOUT_SECONDS = 5
+
+ def launch_python_subprocess_method(command, tmpout, tmperr, environment_vars):
+ subproc_mock.tmpout = tmpout
+ subproc_mock.tmperr = tmperr
+ return subproc_mock
+ executor.launch_python_subprocess = launch_python_subprocess_method
+ runShellKillPgrp_method = MagicMock()
+ runShellKillPgrp_method.side_effect = lambda python : python.terminate()
+ executor.runShellKillPgrp = runShellKillPgrp_method
+ subproc_mock.returncode = 0
+ subproc_mock.should_finish_event.set()
+ result = executor.run_file("file", ["arg1", "arg2"], tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstroutfile)
+ self.assertEquals(result, {'exitcode': 0, 'stderr': 'Dummy err', 'stdout': 'Dummy output',
+ 'structuredOut': {'msg': 'Unable to read structured output from ' + tmpstroutfile}})
+
+
+ def test_is_successfull(self):
+ executor = PythonExecutor("/tmp", AgentConfig("", ""))
+
+ executor.python_process_has_been_killed = False
+ self.assertTrue(executor.isSuccessfull(0))
+ self.assertFalse(executor.isSuccessfull(1))
+
+ executor.python_process_has_been_killed = True
+ self.assertFalse(executor.isSuccessfull(0))
+ self.assertFalse(executor.isSuccessfull(1))
+
+
+ def test_python_command(self):
+ executor = PythonExecutor("/tmp", AgentConfig("", ""))
+ command = executor.python_command("script", ["script_param1"])
+ self.assertEqual(4, len(command))
+ self.assertTrue("python" in command[0])
+ self.assertEquals("-S", command[1])
+ self.assertEquals("script", command[2])
+ self.assertEquals("script_param1", command[3])
+
+
+ class Subprocess_mockup():
+ """
+ It's not trivial to use PyMock instead of class here because we need state
+ and complex logics
+ """
+
+ returncode = 0
+
+ started_event = threading.Event()
+ should_finish_event = threading.Event()
+ finished_event = threading.Event()
+ was_terminated = False
+ tmpout = None
+ tmperr = None
+ pid=-1
+
+ def communicate(self):
+ self.started_event.set()
+ self.tmpout.write("Dummy output")
+ self.tmpout.flush()
+
+ self.tmperr.write("Dummy err")
+ self.tmperr.flush()
+ self.should_finish_event.wait()
+ self.finished_event.set()
+ pass
+
+ def terminate(self):
+ self.was_terminated = True
+ self.returncode = 17
+ self.should_finish_event.set()
+
+if __name__ == "__main__":
+ unittest.main()
+
diff --git a/slider-agent/src/test/python/agent/TestRegistration.py b/slider-agent/src/test/python/agent/TestRegistration.py
new file mode 100644
index 0000000..356e480
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestRegistration.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+import unittest
+import os
+import errno
+import tempfile
+from mock.mock import patch
+from mock.mock import MagicMock
+from agent.Register import Register
+from agent.AgentConfig import AgentConfig
+
+class TestRegistration(TestCase):
+
+ def test_registration_build(self):
+ tmpdir = tempfile.gettempdir()
+ ver_dir = os.path.join(tmpdir, "infra")
+ config = AgentConfig(tmpdir, ver_dir)
+ config.set('agent', 'prefix', tmpdir)
+ config.set('agent', 'current_ping_port', '33777')
+ try:
+ os.mkdir(ver_dir)
+ except OSError as exception:
+ if exception.errno != errno.EEXIST:
+ raise
+ pass
+ ver_file = os.path.join(ver_dir, "version")
+ with open(ver_file, "w") as text_file:
+ text_file.write("1.3.0")
+
+ register = Register(config)
+ data = register.build(1)
+ #print ("Register: " + pprint.pformat(data))
+ self.assertEquals(data['hostname'] != "", True, "hostname should not be empty")
+ self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty")
+ self.assertEquals(data['responseId'], 1)
+ self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp should not be empty")
+ self.assertEquals(data['agentVersion'], '1.3.0', "agentVersion should not be empty")
+ self.assertEquals(len(data), 5)
+
+ self.assertEquals(os.path.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir"))
+ self.assertEquals(os.path.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir"))
+ self.assertEquals(os.path.join(ver_dir, "app/log"), config.getResolvedPath("app_log_dir"))
+ self.assertEquals(os.path.join(ver_dir, "infra/log"), config.getResolvedPath("log_dir"))
+ self.assertEquals(os.path.join(ver_dir, "app/command-log"), config.getResolvedPath("app_task_dir"))
+
+ os.remove(ver_file)
+ os.removedirs(ver_dir)
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file
diff --git a/slider-agent/src/test/python/agent/TestShell.py b/slider-agent/src/test/python/agent/TestShell.py
new file mode 100644
index 0000000..7377457
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestShell.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import unittest
+import tempfile
+from mock.mock import patch, MagicMock, call
+from agent import shell
+from shell import shellRunner
+from sys import platform as _platform
+import subprocess, time
+
+class TestShell(unittest.TestCase):
+
+
+ @patch("os.setuid")
+ def test_changeUid(self, os_setUIDMock):
+ shell.threadLocal.uid = 9999
+ shell.changeUid()
+ self.assertTrue(os_setUIDMock.called)
+
+
+ @patch("pwd.getpwnam")
+ def test_shellRunner_run(self, getpwnamMock):
+ sh = shellRunner()
+ result = sh.run(['echo'])
+ self.assertEquals(result['exitCode'], 0)
+ self.assertEquals(result['error'], '')
+
+ getpwnamMock.return_value = [os.getuid(), os.getuid(), os.getuid()]
+ result = sh.run(['echo'], 'non_exist_user_name')
+ self.assertEquals(result['exitCode'], 0)
+ self.assertEquals(result['error'], '')
+
+ def test_kill_process_with_children(self):
+ if _platform == "linux" or _platform == "linux2": # Test is Linux-specific
+ gracefull_kill_delay_old = shell.gracefull_kill_delay
+ shell.gracefull_kill_delay = 0.1
+ sleep_cmd = "sleep 314159265"
+ test_cmd = """ (({0}) | ({0} | {0})) """.format(sleep_cmd)
+ # Starting process tree (multiple process groups)
+ test_process = subprocess.Popen(test_cmd, shell=True)
+ time.sleep(0.3) # Delay to allow subprocess to start
+ # Check if processes are running
+ ps_cmd = """ps aux | grep "{0}" | grep -v grep """.format(sleep_cmd)
+ ps_process = subprocess.Popen(ps_cmd, stdout=subprocess.PIPE, shell=True)
+ (out, err) = ps_process.communicate()
+ self.assertTrue(sleep_cmd in out)
+ # Kill test process
+ shell.kill_process_with_children(test_process.pid)
+ test_process.communicate()
+ # Now test process should not be running
+ ps_process = subprocess.Popen(ps_cmd, stdout=subprocess.PIPE, shell=True)
+ (out, err) = ps_process.communicate()
+ self.assertFalse(sleep_cmd in out)
+ shell.gracefull_kill_delay = gracefull_kill_delay_old
+ else:
+ # Do not run under other systems
+ pass
diff --git a/slider-agent/src/test/python/agent/dummy_output_error.txt b/slider-agent/src/test/python/agent/dummy_output_error.txt
new file mode 100644
index 0000000..5efa5af
--- /dev/null
+++ b/slider-agent/src/test/python/agent/dummy_output_error.txt
@@ -0,0 +1,45 @@
+debug: Creating default schedules
+debug: Puppet::Type::User::ProviderDirectoryservice: file /usr/bin/dscl does not exist
+debug: Puppet::Type::User::ProviderUser_role_add: file roledel does not exist
+debug: Puppet::Type::User::ProviderPw: file pw does not exist
+debug: Failed to load library 'ldap' for feature 'ldap'
+debug: Puppet::Type::User::ProviderLdap: feature ldap is missing
+debug: /File[/var/lib/puppet/state/graphs]: Autorequiring File[/var/lib/puppet/state]
+debug: /File[/var/lib/puppet/ssl/crl.pem]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/ssl/private_keys]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/state/resources.txt]: Autorequiring File[/var/lib/puppet/state]
+debug: /File[/var/lib/puppet/ssl/certificate_requests]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/ssl/certs]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/state/state.yaml]: Autorequiring File[/var/lib/puppet/state]
+debug: /File[/var/lib/puppet/client_data]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/facts]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/ssl]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/state]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/ssl/private]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/state/last_run_report.yaml]: Autorequiring File[/var/lib/puppet/state]
+debug: /File[/var/lib/puppet/lib]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/ssl/public_keys]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/client_yaml]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/ssl/certs/ca.pem]: Autorequiring File[/var/lib/puppet/ssl/certs]
+debug: /File[/var/lib/puppet/clientbucket]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/state/last_run_summary.yaml]: Autorequiring File[/var/lib/puppet/state]
+debug: Finishing transaction 70171638648540
+debug: Loaded state in 0.00 seconds
+debug: Loaded state in 0.00 seconds
+info: Applying configuration version '1352127563'
+debug: /Schedule[daily]: Skipping device resources because running on a host
+debug: /Schedule[monthly]: Skipping device resources because running on a host
+debug: /Schedule[hourly]: Skipping device resources because running on a host
+debug: /Schedule[never]: Skipping device resources because running on a host
+debug: Exec[command_good](provider=posix): Executing 'wget e432423423xample.com/badurl444111'
+debug: Executing 'wget e432423423xample.com/badurl444111'
+err: /Stage[main]//Exec[command_good]/returns: change from notrun to 0 failed: wget e432423423xample.com/badurl444111 returned 4 instead of one of [0] at /root/puppet-learn/2-bad.pp:5
+debug: /Schedule[weekly]: Skipping device resources because running on a host
+debug: /Schedule[puppet]: Skipping device resources because running on a host
+debug: Finishing transaction 70171639726240
+debug: Storing state
+debug: Stored state in 0.01 seconds
+notice: Finished catalog run in 0.23 seconds
+debug: Finishing transaction 70171638871060
+debug: Received report to process from ambari-dmi
+debug: Processing report from ambari-dmi with processor Puppet::Reports::Store
diff --git a/slider-agent/src/test/python/agent/dummy_output_good.txt b/slider-agent/src/test/python/agent/dummy_output_good.txt
new file mode 100644
index 0000000..320f316
--- /dev/null
+++ b/slider-agent/src/test/python/agent/dummy_output_good.txt
@@ -0,0 +1,47 @@
+debug: Creating default schedules
+debug: Puppet::Type::User::ProviderDirectoryservice: file /usr/bin/dscl does not exist
+debug: Puppet::Type::User::ProviderUser_role_add: file roledel does not exist
+debug: Puppet::Type::User::ProviderPw: file pw does not exist
+debug: Failed to load library 'ldap' for feature 'ldap'
+debug: Puppet::Type::User::ProviderLdap: feature ldap is missing
+debug: /File[/var/lib/puppet/ssl/certs/ca.pem]: Autorequiring File[/var/lib/puppet/ssl/certs]
+debug: /File[/var/lib/puppet/ssl/public_keys]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/ssl/crl.pem]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/state/last_run_report.yaml]: Autorequiring File[/var/lib/puppet/state]
+debug: /File[/var/lib/puppet/ssl/certificate_requests]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/state/last_run_summary.yaml]: Autorequiring File[/var/lib/puppet/state]
+debug: /File[/var/lib/puppet/client_data]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/state]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/ssl/private]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/state/graphs]: Autorequiring File[/var/lib/puppet/state]
+debug: /File[/var/lib/puppet/ssl]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/state/state.yaml]: Autorequiring File[/var/lib/puppet/state]
+debug: /File[/var/lib/puppet/client_yaml]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/facts]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/ssl/private_keys]: Autorequiring File[/var/lib/puppet/ssl]
+debug: /File[/var/lib/puppet/state/resources.txt]: Autorequiring File[/var/lib/puppet/state]
+debug: /File[/var/lib/puppet/clientbucket]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/lib]: Autorequiring File[/var/lib/puppet]
+debug: /File[/var/lib/puppet/ssl/certs]: Autorequiring File[/var/lib/puppet/ssl]
+debug: Finishing transaction 70060456464420
+debug: Loaded state in 0.00 seconds
+debug: Loaded state in 0.00 seconds
+info: Applying configuration version '1352127399'
+debug: /Schedule[daily]: Skipping device resources because running on a host
+debug: /Schedule[monthly]: Skipping device resources because running on a host
+debug: /Schedule[hourly]: Skipping device resources because running on a host
+debug: /Schedule[never]: Skipping device resources because running on a host
+debug: Exec[command_good](provider=posix): Executing 'wget example.com'
+debug: Executing 'wget example.com'
+notice: /Stage[main]//Exec[command_good]/returns: executed successfully
+debug: /Stage[main]//Exec[command_good]: The container Class[Main] will propagate my refresh event
+debug: /Schedule[weekly]: Skipping device resources because running on a host
+debug: /Schedule[puppet]: Skipping device resources because running on a host
+debug: Class[Main]: The container Stage[main] will propagate my refresh event
+debug: Finishing transaction 70060457541680
+debug: Storing state
+debug: Stored state in 0.01 seconds
+notice: Finished catalog run in 0.59 seconds
+debug: Finishing transaction 70060456663980
+debug: Received report to process from ambari-dmi
+debug: Processing report from ambari-dmi with processor Puppet::Reports::Store
diff --git a/slider-agent/src/test/python/mock/LICENSE.txt b/slider-agent/src/test/python/mock/LICENSE.txt
new file mode 100644
index 0000000..7891703
--- /dev/null
+++ b/slider-agent/src/test/python/mock/LICENSE.txt
@@ -0,0 +1,26 @@
+Copyright (c) 2003-2012, Michael Foord
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/slider-agent/src/test/python/mock/MANIFEST.in b/slider-agent/src/test/python/mock/MANIFEST.in
new file mode 100644
index 0000000..d52b301
--- /dev/null
+++ b/slider-agent/src/test/python/mock/MANIFEST.in
@@ -0,0 +1,2 @@
+include LICENSE.txt tox.ini tests/*.py
+recursive-include docs *.txt *.py *.png *.css *.html *.js
diff --git a/slider-agent/src/test/python/mock/README.txt b/slider-agent/src/test/python/mock/README.txt
new file mode 100644
index 0000000..677faf9
--- /dev/null
+++ b/slider-agent/src/test/python/mock/README.txt
@@ -0,0 +1,179 @@
+mock is a library for testing in Python. It allows you to replace parts of
+your system under test with mock objects and make assertions about how they
+have been used.
+
+mock is now part of the Python standard library, available as `unittest.mock
+<http://docs.python.org/py3k/library/unittest.mock.html#module-unittest.mock>`_
+in Python 3.3 onwards.
+
+mock provides a core `MagicMock` class removing the need to create a host of
+stubs throughout your test suite. After performing an action, you can make
+assertions about which methods / attributes were used and arguments they were
+called with. You can also specify return values and set needed attributes in
+the normal way.
+
+mock is tested on Python versions 2.5-2.7 and Python 3. mock is also tested
+with the latest versions of Jython and pypy.
+
+The mock module also provides utility functions / objects to assist with
+testing, particularly monkey patching.
+
+* `PDF documentation for 1.0.1
+ <http://www.voidspace.org.uk/downloads/mock-1.0.1.pdf>`_
+* `mock on google code (repository and issue tracker)
+ <http://code.google.com/p/mock/>`_
+* `mock documentation
+ <http://www.voidspace.org.uk/python/mock/>`_
+* `mock on PyPI <http://pypi.python.org/pypi/mock/>`_
+* `Mailing list (testing-in-python@lists.idyll.org)
+ <http://lists.idyll.org/listinfo/testing-in-python>`_
+
+Mock is very easy to use and is designed for use with
+`unittest <http://pypi.python.org/pypi/unittest2>`_. Mock is based on
+the 'action -> assertion' pattern instead of 'record -> replay' used by many
+mocking frameworks. See the `mock documentation`_ for full details.
+
+Mock objects create all attributes and methods as you access them and store
+details of how they have been used. You can configure them, to specify return
+values or limit what attributes are available, and then make assertions about
+how they have been used::
+
+ >>> from mock import Mock
+ >>> real = ProductionClass()
+ >>> real.method = Mock(return_value=3)
+ >>> real.method(3, 4, 5, key='value')
+ 3
+ >>> real.method.assert_called_with(3, 4, 5, key='value')
+
+`side_effect` allows you to perform side effects, return different values or
+raise an exception when a mock is called::
+
+ >>> mock = Mock(side_effect=KeyError('foo'))
+ >>> mock()
+ Traceback (most recent call last):
+ ...
+ KeyError: 'foo'
+ >>> values = {'a': 1, 'b': 2, 'c': 3}
+ >>> def side_effect(arg):
+ ... return values[arg]
+ ...
+ >>> mock.side_effect = side_effect
+ >>> mock('a'), mock('b'), mock('c')
+ (3, 2, 1)
+ >>> mock.side_effect = [5, 4, 3, 2, 1]
+ >>> mock(), mock(), mock()
+ (5, 4, 3)
+
+Mock has many other ways you can configure it and control its behaviour. For
+example the `spec` argument configures the mock to take its specification from
+another object. Attempting to access attributes or methods on the mock that
+don't exist on the spec will fail with an `AttributeError`.
+
+The `patch` decorator / context manager makes it easy to mock classes or
+objects in a module under test. The object you specify will be replaced with a
+mock (or other object) during the test and restored when the test ends::
+
+ >>> from mock import patch
+ >>> @patch('test_module.ClassName1')
+ ... @patch('test_module.ClassName2')
+ ... def test(MockClass2, MockClass1):
+ ... test_module.ClassName1()
+ ... test_module.ClassName2()
+
+ ... assert MockClass1.called
+ ... assert MockClass2.called
+ ...
+ >>> test()
+
+.. note::
+
+ When you nest patch decorators the mocks are passed in to the decorated
+ function in the same order they applied (the normal *python* order that
+ decorators are applied). This means from the bottom up, so in the example
+ above the mock for `test_module.ClassName2` is passed in first.
+
+ With `patch` it matters that you patch objects in the namespace where they
+ are looked up. This is normally straightforward, but for a quick guide
+ read `where to patch
+ <http://www.voidspace.org.uk/python/mock/patch.html#where-to-patch>`_.
+
+As well as a decorator `patch` can be used as a context manager in a with
+statement::
+
+ >>> with patch.object(ProductionClass, 'method') as mock_method:
+ ... mock_method.return_value = None
+ ... real = ProductionClass()
+ ... real.method(1, 2, 3)
+ ...
+ >>> mock_method.assert_called_once_with(1, 2, 3)
+
+There is also `patch.dict` for setting values in a dictionary just during the
+scope of a test and restoring the dictionary to its original state when the
+test ends::
+
+ >>> foo = {'key': 'value'}
+ >>> original = foo.copy()
+ >>> with patch.dict(foo, {'newkey': 'newvalue'}, clear=True):
+ ... assert foo == {'newkey': 'newvalue'}
+ ...
+ >>> assert foo == original
+
+Mock supports the mocking of Python magic methods. The easiest way of
+using magic methods is with the `MagicMock` class. It allows you to do
+things like::
+
+ >>> from mock import MagicMock
+ >>> mock = MagicMock()
+ >>> mock.__str__.return_value = 'foobarbaz'
+ >>> str(mock)
+ 'foobarbaz'
+ >>> mock.__str__.assert_called_once_with()
+
+Mock allows you to assign functions (or other Mock instances) to magic methods
+and they will be called appropriately. The MagicMock class is just a Mock
+variant that has all of the magic methods pre-created for you (well - all the
+useful ones anyway).
+
+The following is an example of using magic methods with the ordinary Mock
+class::
+
+ >>> from mock import Mock
+ >>> mock = Mock()
+ >>> mock.__str__ = Mock(return_value = 'wheeeeee')
+ >>> str(mock)
+ 'wheeeeee'
+
+For ensuring that the mock objects your tests use have the same api as the
+objects they are replacing, you can use "auto-speccing". Auto-speccing can
+be done through the `autospec` argument to patch, or the `create_autospec`
+function. Auto-speccing creates mock objects that have the same attributes
+and methods as the objects they are replacing, and any functions and methods
+(including constructors) have the same call signature as the real object.
+
+This ensures that your mocks will fail in the same way as your production
+code if they are used incorrectly::
+
+ >>> from mock import create_autospec
+ >>> def function(a, b, c):
+ ... pass
+ ...
+ >>> mock_function = create_autospec(function, return_value='fishy')
+ >>> mock_function(1, 2, 3)
+ 'fishy'
+ >>> mock_function.assert_called_once_with(1, 2, 3)
+ >>> mock_function('wrong arguments')
+ Traceback (most recent call last):
+ ...
+ TypeError: <lambda>() takes exactly 3 arguments (1 given)
+
+`create_autospec` can also be used on classes, where it copies the signature of
+the `__init__` method, and on callable objects where it copies the signature of
+the `__call__` method.
+
+The distribution contains tests and documentation. The tests require
+`unittest2 <http://pypi.python.org/pypi/unittest2>`_ to run on Python 2.5, 2.6
+or 3.1. For Python 2.7 and 3.2 they can be run with
+`python -m unittest discover`.
+
+Docs from the in-development version of `mock` can be found at
+`mock.readthedocs.org <http://mock.readthedocs.org>`_.
diff --git a/slider-agent/src/test/python/mock/__init__.py b/slider-agent/src/test/python/mock/__init__.py
new file mode 100644
index 0000000..3c61c75
--- /dev/null
+++ b/slider-agent/src/test/python/mock/__init__.py
@@ -0,0 +1 @@
+__author__ = 'Michael Foord'
diff --git a/slider-agent/src/test/python/mock/docs/changelog.txt b/slider-agent/src/test/python/mock/docs/changelog.txt
new file mode 100644
index 0000000..823341a
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/changelog.txt
@@ -0,0 +1,737 @@
+.. currentmodule:: mock
+
+
+CHANGELOG
+=========
+
+2012/11/05 Version 1.0.1
+------------------------
+
+* Functions decorated with `patch` variants have a `__wrapped__` attribute
+ pointing to the original function. This brings compatibility with the
+ default behaviour in Python 3.3 (due to a new feature in `functools.wraps`).
+
+Note that due to changes in `tox`, `mock` is no longer tested with Python 2.4.
+The compatibility code has not been removed so it probably still works, but
+tests are no longer run.
+
+
+2012/10/07 Version 1.0.0
+------------------------
+
+No changes since 1.0.0 beta 1. This version has feature parity with
+`unittest.mock
+<http://docs.python.org/py3k/library/unittest.mock.html#module-unittest.mock>`_
+in Python 3.3.
+
+Full list of changes since 0.8:
+
+* `mocksignature`, along with the `mocksignature` argument to `patch`, removed
+* Support for deleting attributes (accessing deleted attributes will raise an
+ `AttributeError`)
+* Added the `mock_open` helper function for mocking the builtin `open`
+* `__class__` is assignable, so a mock can pass an `isinstance` check without
+ requiring a spec
+* Addition of `PropertyMock`, for mocking properties
+* `MagicMocks` made unorderable by default (in Python 3). The comparison
+ methods (other than equality and inequality) now return `NotImplemented`
+* Propagate traceback info to support subclassing of `_patch` by other
+ libraries
+* `create_autospec` works with attributes present in results of `dir` that
+ can't be fetched from the object's class. Contributed by Konstantine Rybnikov
+* Any exceptions in an iterable `side_effect` will be raised instead of
+ returned
+* In Python 3, `create_autospec` now supports keyword only arguments
+* Added `patch.stopall` method to stop all active patches created by `start`
+* BUGFIX: calling `MagicMock.reset_mock` wouldn't reset magic method mocks
+* BUGFIX: calling `reset_mock` on a `MagicMock` created with autospec could
+ raise an exception
+* BUGFIX: passing multiple spec arguments to patchers (`spec` , `spec_set` and
+ `autospec`) had unpredictable results, now it is an error
+* BUGFIX: using `spec=True` *and* `create=True` as arguments to patchers could
+ result in using `DEFAULT` as the spec. Now it is an error instead
+* BUGFIX: using `spec` or `autospec` arguments to patchers, along with
+ `spec_set=True` did not work correctly
+* BUGFIX: using an object that evaluates to False as a spec could be ignored
+* BUGFIX: a list as the `spec` argument to a patcher would always result in a
+ non-callable mock. Now if `__call__` is in the spec the mock is callable
+
+
+2012/07/13 Version 1.0.0 beta 1
+--------------------------------
+
+* Added `patch.stopall` method to stop all active patches created by `start`
+* BUGFIX: calling `MagicMock.reset_mock` wouldn't reset magic method mocks
+* BUGFIX: calling `reset_mock` on a `MagicMock` created with autospec could
+ raise an exception
+
+
+2012/05/04 Version 1.0.0 alpha 2
+--------------------------------
+
+* `PropertyMock` attributes are now standard `MagicMocks`
+* `create_autospec` works with attributes present in results of `dir` that
+ can't be fetched from the object's class. Contributed by Konstantine Rybnikov
+* Any exceptions in an iterable `side_effect` will be raised instead of
+ returned
+* In Python 3, `create_autospec` now supports keyword only arguments
+
+
+2012/03/25 Version 1.0.0 alpha 1
+--------------------------------
+
+The standard library version!
+
+* `mocksignature`, along with the `mocksignature` argument to `patch`, removed
+* Support for deleting attributes (accessing deleted attributes will raise an
+ `AttributeError`)
+* Added the `mock_open` helper function for mocking the builtin `open`
+* `__class__` is assignable, so a mock can pass an `isinstance` check without
+ requiring a spec
+* Addition of `PropertyMock`, for mocking properties
+* `MagicMocks` made unorderable by default (in Python 3). The comparison
+ methods (other than equality and inequality) now return `NotImplemented`
+* Propagate traceback info to support subclassing of `_patch` by other
+ libraries
+* BUGFIX: passing multiple spec arguments to patchers (`spec` , `spec_set` and
+ `autospec`) had unpredictable results, now it is an error
+* BUGFIX: using `spec=True` *and* `create=True` as arguments to patchers could
+ result in using `DEFAULT` as the spec. Now it is an error instead
+* BUGFIX: using `spec` or `autospec` arguments to patchers, along with
+ `spec_set=True` did not work correctly
+* BUGFIX: using an object that evaluates to False as a spec could be ignored
+* BUGFIX: a list as the `spec` argument to a patcher would always result in a
+ non-callable mock. Now if `__call__` is in the spec the mock is callable
+
+
+2012/02/13 Version 0.8.0
+------------------------
+
+The only changes since 0.8rc2 are:
+
+* Improved repr of :data:`sentinel` objects
+* :data:`ANY` can be used for comparisons against :data:`call` objects
+* The return value of `MagicMock.__iter__` method can be set to
+ any iterable and isn't required to be an iterator
+
+Full List of changes since 0.7:
+
+mock 0.8.0 is the last version that will support Python 2.4.
+
+* Addition of :attr:`~Mock.mock_calls` list for *all* calls (including magic
+ methods and chained calls)
+* :func:`patch` and :func:`patch.object` now create a :class:`MagicMock`
+ instead of a :class:`Mock` by default
+* The patchers (`patch`, `patch.object` and `patch.dict`), plus `Mock` and
+ `MagicMock`, take arbitrary keyword arguments for configuration
+* New mock method :meth:`~Mock.configure_mock` for setting attributes and
+ return values / side effects on the mock and its attributes
+* New mock assert methods :meth:`~Mock.assert_any_call` and
+ :meth:`~Mock.assert_has_calls`
+* Implemented :ref:`auto-speccing` (recursive, lazy speccing of mocks with
+ mocked signatures for functions/methods), as the `autospec` argument to
+ `patch`
+* Added the :func:`create_autospec` function for manually creating
+ 'auto-specced' mocks
+* :func:`patch.multiple` for doing multiple patches in a single call, using
+ keyword arguments
+* Setting :attr:`~Mock.side_effect` to an iterable will cause calls to the mock
+ to return the next value from the iterable
+* New `new_callable` argument to `patch` and `patch.object` allowing you to
+ pass in a class or callable object (instead of `MagicMock`) that will be
+ called to replace the object being patched
+* Addition of :class:`NonCallableMock` and :class:`NonCallableMagicMock`, mocks
+ without a `__call__` method
+* Addition of :meth:`~Mock.mock_add_spec` method for adding (or changing) a
+ spec on an existing mock
+* Protocol methods on :class:`MagicMock` are magic mocks, and are created
+ lazily on first lookup. This means the result of calling a protocol method is
+ a `MagicMock` instead of a `Mock` as it was previously
+* Addition of :meth:`~Mock.attach_mock` method
+* Added :data:`ANY` for ignoring arguments in :meth:`~Mock.assert_called_with`
+ calls
+* Addition of :data:`call` helper object
+* Improved repr for mocks
+* Improved repr for :attr:`Mock.call_args` and entries in
+ :attr:`Mock.call_args_list`, :attr:`Mock.method_calls` and
+ :attr:`Mock.mock_calls`
+* Improved repr for :data:`sentinel` objects
+* `patch` lookup is done at use time not at decoration time
+* In Python 2.6 or more recent, `dir` on a mock will report all the dynamically
+ created attributes (or the full list of attributes if there is a spec) as
+ well as all the mock methods and attributes.
+* Module level :data:`FILTER_DIR` added to control whether `dir(mock)` filters
+ private attributes. `True` by default.
+* `patch.TEST_PREFIX` for controlling how patchers recognise test methods when
+ used to decorate a class
+* Support for using Java exceptions as a :attr:`~Mock.side_effect` on Jython
+* `Mock` call lists (`call_args_list`, `method_calls` & `mock_calls`) are now
+ custom list objects that allow membership tests for "sub lists" and have
+ a nicer representation if you `str` or `print` them
+* Mocks attached as attributes or return values to other mocks have calls
+ recorded in `method_calls` and `mock_calls` of the parent (unless a name is
+ already set on the child)
+* Improved failure messages for `assert_called_with` and
+ `assert_called_once_with`
+* The return value of the :class:`MagicMock` `__iter__` method can be set to
+ any iterable and isn't required to be an iterator
+* Added the Mock API (`assert_called_with` etc) to functions created by
+ :func:`mocksignature`
+* Tuples as well as lists can be used to specify allowed methods for `spec` &
+ `spec_set` arguments
+* Calling `stop` on an unstarted patcher fails with a more meaningful error
+ message
+* Renamed the internal classes `Sentinel` and `SentinelObject` to prevent abuse
+* BUGFIX: an error creating a patch, with nested patch decorators, won't leave
+ patches in place
+* BUGFIX: `__truediv__` and `__rtruediv__` not available as magic methods on
+ mocks in Python 3
+* BUGFIX: `assert_called_with` / `assert_called_once_with` can be used with
+ `self` as a keyword argument
+* BUGFIX: when patching a class with an explicit spec / spec_set (not a
+ boolean) it applies "spec inheritance" to the return value of the created
+ mock (the "instance")
+* BUGFIX: remove the `__unittest` marker causing traceback truncation
+* Removal of deprecated `patch_object`
+* Private attributes `_name`, `_methods`, '_children', `_wraps` and `_parent`
+ (etc) renamed to reduce likelihood of clash with user attributes.
+* Added license file to the distribution
+
+
+2012/01/10 Version 0.8.0 release candidate 2
+--------------------------------------------
+
+* Removed the `configure` keyword argument to `create_autospec` and allow
+ arbitrary keyword arguments (for the `Mock` constructor) instead
+* Fixed `ANY` equality with some types in `assert_called_with` calls
+* Switched to a standard Sphinx theme (compatible with
+ `readthedocs.org <http://mock.readthedocs.org>`_)
+
+
+2011/12/29 Version 0.8.0 release candidate 1
+--------------------------------------------
+
+* `create_autospec` on the return value of a mocked class will use `__call__`
+ for the signature rather than `__init__`
+* Performance improvement instantiating `Mock` and `MagicMock`
+* Mocks used as magic methods have the same type as their parent instead of
+ being hardcoded to `MagicMock`
+
+Special thanks to Julian Berman for his help with diagnosing and improving
+performance in this release.
+
+
+2011/10/09 Version 0.8.0 beta 4
+-------------------------------
+
+* `patch` lookup is done at use time not at decoration time
+* When attaching a Mock to another Mock as a magic method, calls are recorded
+ in mock_calls
+* Addition of `attach_mock` method
+* Renamed the internal classes `Sentinel` and `SentinelObject` to prevent abuse
+* BUGFIX: various issues around circular references with mocks (setting a mock
+ return value to be itself etc)
+
+
+2011/08/15 Version 0.8.0 beta 3
+-------------------------------
+
+* Mocks attached as attributes or return values to other mocks have calls
+ recorded in `method_calls` and `mock_calls` of the parent (unless a name is
+ already set on the child)
+* Addition of `mock_add_spec` method for adding (or changing) a spec on an
+ existing mock
+* Improved repr for `Mock.call_args` and entries in `Mock.call_args_list`,
+ `Mock.method_calls` and `Mock.mock_calls`
+* Improved repr for mocks
+* BUGFIX: minor fixes in the way `mock_calls` is worked out,
+ especially for "intermediate" mocks in a call chain
+
+
+2011/08/05 Version 0.8.0 beta 2
+-------------------------------
+
+* Setting `side_effect` to an iterable will cause calls to the mock to return
+ the next value from the iterable
+* Added `assert_any_call` method
+* Moved `assert_has_calls` from call lists onto mocks
+* BUGFIX: `call_args` and all members of `call_args_list` are two tuples of
+ `(args, kwargs)` again instead of three tuples of `(name, args, kwargs)`
+
+
+2011/07/25 Version 0.8.0 beta 1
+-------------------------------
+
+* `patch.TEST_PREFIX` for controlling how patchers recognise test methods when
+ used to decorate a class
+* `Mock` call lists (`call_args_list`, `method_calls` & `mock_calls`) are now
+ custom list objects that allow membership tests for "sub lists" and have
+ an `assert_has_calls` method for unordered call checks
+* `callargs` changed to *always* be a three-tuple of `(name, args, kwargs)`
+* Addition of `mock_calls` list for *all* calls (including magic methods and
+ chained calls)
+* Extension of `call` object to support chained calls and `callargs` for better
+ comparisons with or without names. `call` object has a `call_list` method for
+ chained calls
+* Added the public `instance` argument to `create_autospec`
+* Support for using Java exceptions as a `side_effect` on Jython
+* Improved failure messages for `assert_called_with` and
+ `assert_called_once_with`
+* Tuples as well as lists can be used to specify allowed methods for `spec` &
+ `spec_set` arguments
+* BUGFIX: Fixed bug in `patch.multiple` for argument passing when creating
+ mocks
+* Added license file to the distribution
+
+
+2011/07/16 Version 0.8.0 alpha 2
+--------------------------------
+
+* `patch.multiple` for doing multiple patches in a single call, using keyword
+ arguments
+* New `new_callable` argument to `patch` and `patch.object` allowing you to
+ pass in a class or callable object (instead of `MagicMock`) that will be
+ called to replace the object being patched
+* Addition of `NonCallableMock` and `NonCallableMagicMock`, mocks without a
+ `__call__` method
+* Mocks created by `patch` have a `MagicMock` as the `return_value` where a
+ class is being patched
+* `create_autospec` can create non-callable mocks for non-callable objects.
+ `return_value` mocks of classes will be non-callable unless the class has
+ a `__call__` method
+* `autospec` creates a `MagicMock` without a spec for properties and slot
+ descriptors, because we don't know the type of object they return
+* Removed the "inherit" argument from `create_autospec`
+* Calling `stop` on an unstarted patcher fails with a more meaningful error
+ message
+* BUGFIX: an error creating a patch, with nested patch decorators, won't leave
+ patches in place
+* BUGFIX: `__truediv__` and `__rtruediv__` not available as magic methods on
+ mocks in Python 3
+* BUGFIX: `assert_called_with` / `assert_called_once_with` can be used with
+ `self` as a keyword argument
+* BUGFIX: autospec for functions / methods with an argument named self that
+ isn't the first argument no longer broken
+* BUGFIX: when patching a class with an explicit spec / spec_set (not a
+ boolean) it applies "spec inheritance" to the return value of the created
+ mock (the "instance")
+* BUGFIX: remove the `__unittest` marker causing traceback truncation
+
+
+2011/06/14 Version 0.8.0 alpha 1
+--------------------------------
+
+mock 0.8.0 is the last version that will support Python 2.4.
+
+* The patchers (`patch`, `patch.object` and `patch.dict`), plus `Mock` and
+ `MagicMock`, take arbitrary keyword arguments for configuration
+* New mock method `configure_mock` for setting attributes and return values /
+ side effects on the mock and its attributes
+* In Python 2.6 or more recent, `dir` on a mock will report all the dynamically
+ created attributes (or the full list of attributes if there is a spec) as
+ well as all the mock methods and attributes.
+* Module level `FILTER_DIR` added to control whether `dir(mock)` filters
+ private attributes. `True` by default. Note that `vars(Mock())` can still be
+ used to get all instance attributes and `dir(type(Mock())` will still return
+ all the other attributes (irrespective of `FILTER_DIR`)
+* `patch` and `patch.object` now create a `MagicMock` instead of a `Mock` by
+ default
+* Added `ANY` for ignoring arguments in `assert_called_with` calls
+* Addition of `call` helper object
+* Protocol methods on `MagicMock` are magic mocks, and are created lazily on
+ first lookup. This means the result of calling a protocol method is a
+ MagicMock instead of a Mock as it was previously
+* Added the Mock API (`assert_called_with` etc) to functions created by
+ `mocksignature`
+* Private attributes `_name`, `_methods`, '_children', `_wraps` and `_parent`
+ (etc) renamed to reduce likelihood of clash with user attributes.
+* Implemented auto-speccing (recursive, lazy speccing of mocks with mocked
+ signatures for functions/methods)
+
+ Limitations:
+
+ - Doesn't mock magic methods or attributes (it creates MagicMocks, so the
+ magic methods are *there*, they just don't have the signature mocked nor
+ are attributes followed)
+ - Doesn't mock function / method attributes
+ - Uses object traversal on the objects being mocked to determine types - so
+ properties etc may be triggered
+ - The return value of mocked classes (the 'instance') has the same call
+ signature as the class __init__ (as they share the same spec)
+
+ You create auto-specced mocks by passing `autospec=True` to `patch`.
+
+ Note that attributes that are None are special cased and mocked without a
+ spec (so any attribute / method can be used). This is because None is
+ typically used as a default value for attributes that may be of some other
+ type, and as we don't know what type that may be we allow all access.
+
+ Note that the `autospec` option to `patch` obsoletes the `mocksignature`
+ option.
+
+* Added the `create_autospec` function for manually creating 'auto-specced'
+ mocks
+* Removal of deprecated `patch_object`
+
+
+2011/05/30 Version 0.7.2
+------------------------
+
+* BUGFIX: instances of list subclasses can now be used as mock specs
+* BUGFIX: MagicMock equality / inequality protocol methods changed to use the
+ default equality / inequality. This is done through a `side_effect` on
+ the mocks used for `__eq__` / `__ne__`
+
+
+2011/05/06 Version 0.7.1
+------------------------
+
+Package fixes contributed by Michael Fladischer. No code changes.
+
+* Include template in package
+* Use isolated binaries for the tox tests
+* Unset executable bit on docs
+* Fix DOS line endings in getting-started.txt
+
+
+2011/03/05 Version 0.7.0
+------------------------
+
+No API changes since 0.7.0 rc1. Many documentation changes including a stylish
+new `Sphinx theme <https://github.com/coordt/ADCtheme/>`_.
+
+The full set of changes since 0.6.0 are:
+
+* Python 3 compatibility
+* Ability to mock magic methods with `Mock` and addition of `MagicMock`
+ with pre-created magic methods
+* Addition of `mocksignature` and `mocksignature` argument to `patch` and
+ `patch.object`
+* Addition of `patch.dict` for changing dictionaries during a test
+* Ability to use `patch`, `patch.object` and `patch.dict` as class decorators
+* Renamed ``patch_object`` to `patch.object` (``patch_object`` is
+ deprecated)
+* Addition of soft comparisons: `call_args`, `call_args_list` and `method_calls`
+ now return tuple-like objects which compare equal even when empty args
+ or kwargs are skipped
+* patchers (`patch`, `patch.object` and `patch.dict`) have start and stop
+ methods
+* Addition of `assert_called_once_with` method
+* Mocks can now be named (`name` argument to constructor) and the name is used
+ in the repr
+* repr of a mock with a spec includes the class name of the spec
+* `assert_called_with` works with `python -OO`
+* New `spec_set` keyword argument to `Mock` and `patch`. If used,
+ attempting to *set* an attribute on a mock not on the spec will raise an
+ `AttributeError`
+* Mocks created with a spec can now pass `isinstance` tests (`__class__`
+ returns the type of the spec)
+* Added docstrings to all objects
+* Improved failure message for `Mock.assert_called_with` when the mock
+ has not been called at all
+* Decorated functions / methods have their docstring and `__module__`
+ preserved on Python 2.4.
+* BUGFIX: `mock.patch` now works correctly with certain types of objects that
+ proxy attribute access, like the django settings object
+* BUGFIX: mocks are now copyable (thanks to Ned Batchelder for reporting and
+ diagnosing this)
+* BUGFIX: `spec=True` works with old style classes
+* BUGFIX: ``help(mock)`` works now (on the module). Can no longer use ``__bases__``
+ as a valid sentinel name (thanks to Stephen Emslie for reporting and
+ diagnosing this)
+* BUGFIX: ``side_effect`` now works with ``BaseException`` exceptions like
+ ``KeyboardInterrupt``
+* BUGFIX: `reset_mock` caused infinite recursion when a mock is set as its own
+ return value
+* BUGFIX: patching the same object twice now restores the patches correctly
+* with statement tests now skipped on Python 2.4
+* Tests require unittest2 (or unittest2-py3k) to run
+* Tested with `tox <http://pypi.python.org/pypi/tox>`_ on Python 2.4 - 3.2,
+ jython and pypy (excluding 3.0)
+* Added 'build_sphinx' command to setup.py (requires setuptools or distribute)
+ Thanks to Florian Bauer
+* Switched from subversion to mercurial for source code control
+* `Konrad Delong <http://konryd.blogspot.com/>`_ added as co-maintainer
+
+
+2011/02/16 Version 0.7.0 RC 1
+-----------------------------
+
+Changes since beta 4:
+
+* Tested with jython, pypy and Python 3.2 and 3.1
+* Decorated functions / methods have their docstring and `__module__`
+ preserved on Python 2.4
+* BUGFIX: `mock.patch` now works correctly with certain types of objects that
+ proxy attribute access, like the django settings object
+* BUGFIX: `reset_mock` caused infinite recursion when a mock is set as its own
+ return value
+
+
+2010/11/12 Version 0.7.0 beta 4
+-------------------------------
+
+* patchers (`patch`, `patch.object` and `patch.dict`) have start and stop
+ methods
+* Addition of `assert_called_once_with` method
+* repr of a mock with a spec includes the class name of the spec
+* `assert_called_with` works with `python -OO`
+* New `spec_set` keyword argument to `Mock` and `patch`. If used,
+ attempting to *set* an attribute on a mock not on the spec will raise an
+ `AttributeError`
+* Attributes and return value of a `MagicMock` are `MagicMock` objects
+* Attempting to set an unsupported magic method now raises an `AttributeError`
+* `patch.dict` works as a class decorator
+* Switched from subversion to mercurial for source code control
+* BUGFIX: mocks are now copyable (thanks to Ned Batchelder for reporting and
+ diagnosing this)
+* BUGFIX: `spec=True` works with old style classes
+* BUGFIX: `mocksignature=True` can now patch instance methods via
+ `patch.object`
+
+
+2010/09/18 Version 0.7.0 beta 3
+-------------------------------
+
+* Using spec with :class:`MagicMock` only pre-creates magic methods in the spec
+* Setting a magic method on a mock with a ``spec`` can only be done if the
+ spec has that method
+* Mocks can now be named (`name` argument to constructor) and the name is used
+ in the repr
+* `mocksignature` can now be used with classes (signature based on `__init__`)
+ and callable objects (signature based on `__call__`)
+* Mocks created with a spec can now pass `isinstance` tests (`__class__`
+ returns the type of the spec)
+* Default numeric value for MagicMock is 1 rather than zero (because the
+ MagicMock bool defaults to True and 0 is False)
+* Improved failure message for :meth:`~Mock.assert_called_with` when the mock
+ has not been called at all
+* Adding the following to the set of supported magic methods:
+
+ - ``__getformat__`` and ``__setformat__``
+ - pickle methods
+ - ``__trunc__``, ``__ceil__`` and ``__floor__``
+ - ``__sizeof__``
+
+* Added 'build_sphinx' command to setup.py (requires setuptools or distribute)
+ Thanks to Florian Bauer
+* with statement tests now skipped on Python 2.4
+* Tests require unittest2 to run on Python 2.7
+* Improved several docstrings and documentation
+
+
+2010/06/23 Version 0.7.0 beta 2
+-------------------------------
+
+* :func:`patch.dict` works as a context manager as well as a decorator
+* ``patch.dict`` takes a string to specify dictionary as well as a dictionary
+ object. If a string is supplied the name specified is imported
+* BUGFIX: ``patch.dict`` restores dictionary even when an exception is raised
+
+
+2010/06/22 Version 0.7.0 beta 1
+-------------------------------
+
+* Addition of :func:`mocksignature`
+* Ability to mock magic methods
+* Ability to use ``patch`` and ``patch.object`` as class decorators
+* Renamed ``patch_object`` to :func:`patch.object` (``patch_object`` is
+ deprecated)
+* Addition of :class:`MagicMock` class with all magic methods pre-created for you
+* Python 3 compatibility (tested with 3.2 but should work with 3.0 & 3.1 as
+ well)
+* Addition of :func:`patch.dict` for changing dictionaries during a test
+* Addition of ``mocksignature`` argument to ``patch`` and ``patch.object``
+* ``help(mock)`` works now (on the module). Can no longer use ``__bases__``
+ as a valid sentinel name (thanks to Stephen Emslie for reporting and
+ diagnosing this)
+* Addition of soft comparisons: `call_args`, `call_args_list` and `method_calls`
+ now return tuple-like objects which compare equal even when empty args
+ or kwargs are skipped
+* Added docstrings.
+* BUGFIX: ``side_effect`` now works with ``BaseException`` exceptions like
+ ``KeyboardInterrupt``
+* BUGFIX: patching the same object twice now restores the patches correctly
+* The tests now require `unittest2 <http://pypi.python.org/pypi/unittest2>`_
+ to run
+* `Konrad Delong <http://konryd.blogspot.com/>`_ added as co-maintainer
+
+
+2009/08/22 Version 0.6.0
+------------------------
+
+* New test layout compatible with test discovery
+* Descriptors (static methods / class methods etc) can now be patched and
+ restored correctly
+* Mocks can raise exceptions when called by setting ``side_effect`` to an
+ exception class or instance
+* Mocks that wrap objects will not pass on calls to the underlying object if
+ an explicit return_value is set
+
+
+2009/04/17 Version 0.5.0
+------------------------
+
+* Made DEFAULT part of the public api.
+* Documentation built with Sphinx.
+* ``side_effect`` is now called with the same arguments as the mock is called with and
+ if returns a non-DEFAULT value that is automatically set as the ``mock.return_value``.
+* ``wraps`` keyword argument used for wrapping objects (and passing calls through to the wrapped object).
+* ``Mock.reset`` renamed to ``Mock.reset_mock``, as reset is a common API name.
+* ``patch`` / ``patch_object`` are now context managers and can be used with ``with``.
+* A new 'create' keyword argument to patch and patch_object that allows them to patch
+ (and unpatch) attributes that don't exist. (Potentially unsafe to use - it can allow
+ you to have tests that pass when they are testing an API that doesn't exist - use at
+ your own risk!)
+* The methods keyword argument to Mock has been removed and merged with spec. The spec
+ argument can now be a list of methods or an object to take the spec from.
+* Nested patches may now be applied in a different order (created mocks passed
+ in the opposite order). This is actually a bugfix.
+* patch and patch_object now take a spec keyword argument. If spec is
+ passed in as 'True' then the Mock created will take the object it is replacing
+ as its spec object. If the object being replaced is a class, then the return
+ value for the mock will also use the class as a spec.
+* A Mock created without a spec will not attempt to mock any magic methods / attributes
+ (they will raise an ``AttributeError`` instead).
+
+
+2008/10/12 Version 0.4.0
+------------------------
+
+* Default return value is now a new mock rather than None
+* return_value added as a keyword argument to the constructor
+* New method 'assert_called_with'
+* Added 'side_effect' attribute / keyword argument called when mock is called
+* patch decorator split into two decorators:
+
+ - ``patch_object`` which takes an object and an attribute name to patch
+ (plus optionally a value to patch with which defaults to a mock object)
+ - ``patch`` which takes a string specifying a target to patch; in the form
+ 'package.module.Class.attribute'. (plus optionally a value to
+ patch with which defaults to a mock object)
+
+* Can now patch objects with ``None``
+* Change to patch for nose compatibility with error reporting in wrapped functions
+* Reset no longer clears children / return value etc - it just resets
+ call count and call args. It also calls reset on all children (and
+ the return value if it is a mock).
+
+Thanks to Konrad Delong, Kevin Dangoor and others for patches and suggestions.
+
+
+2007/12/03 Version 0.3.1
+-------------------------
+
+``patch`` maintains the name of decorated functions for compatibility with nose
+test autodiscovery.
+
+Tests decorated with ``patch`` that use the two argument form (implicit mock
+creation) will receive the mock(s) passed in as extra arguments.
+
+Thanks to Kevin Dangoor for these changes.
+
+
+2007/11/30 Version 0.3.0
+-------------------------
+
+Removed ``patch_module``. ``patch`` can now take a string as the first
+argument for patching modules.
+
+The third argument to ``patch`` is optional - a mock will be created by
+default if it is not passed in.
+
+
+2007/11/21 Version 0.2.1
+-------------------------
+
+Bug fix, allows reuse of functions decorated with ``patch`` and ``patch_module``.
+
+
+2007/11/20 Version 0.2.0
+-------------------------
+
+Added ``spec`` keyword argument for creating ``Mock`` objects from a
+specification object.
+
+Added ``patch`` and ``patch_module`` monkey patching decorators.
+
+Added ``sentinel`` for convenient access to unique objects.
+
+Distribution includes unit tests.
+
+
+2007/11/19 Version 0.1.0
+-------------------------
+
+Initial release.
+
+
+TODO and Limitations
+====================
+
+Contributions, bug reports and comments welcomed!
+
+Feature requests and bug reports are handled on the issue tracker:
+
+ * `mock issue tracker <http://code.google.com/p/mock/issues/list>`_
+
+`wraps` is not integrated with magic methods.
+
+`patch` could auto-do the patching in the constructor and unpatch in the
+destructor. This would be useful in itself, but violates TOOWTDI and would be
+unsafe for IronPython & PyPy (non-deterministic calling of destructors).
+Destructors aren't called in CPython where there are cycles, but a weak
+reference with a callback can be used to get round this.
+
+`Mock` has several attributes. This makes it unsuitable for mocking objects
+that use these attribute names. A way round this would be to provide methods
+that *hide* these attributes when needed. In 0.8 many, but not all, of these
+attributes are renamed to gain a `_mock` prefix, making it less likely that
+they will clash. Any outstanding attributes that haven't been modified with
+the prefix should be changed.
+
+If a patch is started using `patch.start` and then not stopped correctly then
+the unpatching is not done. Using weak references it would be possible to
+detect and fix this when the patch object itself is garbage collected. This
+would be tricky to get right though.
+
+When a `Mock` is created by `patch`, arbitrary keywords can be used to set
+attributes. If `patch` is created with a `spec`, and is replacing a class, then
+a `return_value` mock is created. The keyword arguments are not applied to the
+child mock, but could be.
+
+When mocking a class with `patch`, passing in `spec=True` or `autospec=True`,
+the mock class has an instance created from the same spec. Should this be the
+default behaviour for mocks anyway (mock return values inheriting the spec
+from their parent), or should it be controlled by an additional keyword
+argument (`inherit`) to the Mock constructor? `create_autospec` does this, so
+an additional keyword argument to Mock is probably unnecessary.
+
+The `mocksignature` argument to `patch` with a non `Mock` passed into
+`new_callable` will *probably* cause an error. Should it just be invalid?
+
+Note that `NonCallableMock` and `NonCallableMagicMock` still have the unused
+(and unusable) attributes: `return_value`, `side_effect`, `call_count`,
+`call_args` and `call_args_list`. These could be removed or raise errors on
+getting / setting. They also have the `assert_called_with` and
+`assert_called_once_with` methods. Removing these would be pointless as
+fetching them would create a mock (attribute) that could be called without
+error.
+
+Some outstanding technical debt. The way autospeccing mocks function
+signatures was copied and modified from `mocksignature`. This could all be
+refactored into one set of functions instead of two. The way we tell if
+patchers are started and if a patcher is being used for a `patch.multiple`
+call are both horrible. There are now a host of helper functions that should
+be rationalised. (Probably time to split mock into a package instead of a
+module.)
+
+Passing arbitrary keyword arguments to `create_autospec`, or `patch` with
+`autospec`, when mocking a *function* works fine. However, the arbitrary
+attributes are set on the created mock - but `create_autospec` returns a
+real function (which doesn't have those attributes). However, what is the use
+case for using autospec to create functions with attributes that don't exist
+on the original?
+
+`mocksignature`, plus the `call_args_list` and `method_calls` attributes of
+`Mock` could all be deprecated.
diff --git a/slider-agent/src/test/python/mock/docs/compare.txt b/slider-agent/src/test/python/mock/docs/compare.txt
new file mode 100644
index 0000000..4155530
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/compare.txt
@@ -0,0 +1,628 @@
+=========================
+ Mock Library Comparison
+=========================
+
+
+.. testsetup::
+
+ def assertEqual(a, b):
+ assert a == b, ("%r != %r" % (a, b))
+
+ def assertRaises(Exc, func):
+ try:
+ func()
+ except Exc:
+ return
+ assert False, ("%s not raised" % Exc)
+
+ sys.modules['somemodule'] = somemodule = mock.Mock(name='somemodule')
+ class SomeException(Exception):
+ some_method = method1 = method2 = None
+ some_other_object = SomeObject = SomeException
+
+
+A side-by-side comparison of how to accomplish some basic tasks with mock and
+some other popular Python mocking libraries and frameworks.
+
+These are:
+
+* `flexmock <http://pypi.python.org/pypi/flexmock>`_
+* `mox <http://pypi.python.org/pypi/mox>`_
+* `Mocker <http://niemeyer.net/mocker>`_
+* `dingus <http://pypi.python.org/pypi/dingus>`_
+* `fudge <http://pypi.python.org/pypi/fudge>`_
+
+Popular python mocking frameworks not yet represented here include
+`MiniMock <http://pypi.python.org/pypi/MiniMock>`_.
+
+`pMock <http://pmock.sourceforge.net/>`_ (last release 2004 and doesn't import
+in recent versions of Python) and
+`python-mock <http://python-mock.sourceforge.net/>`_ (last release 2005) are
+intentionally omitted.
+
+.. note::
+
+ A more up to date, and tested for all mock libraries (only the mock
+ examples on this page can be executed as doctests) version of this
+ comparison is maintained by Gary Bernhardt:
+
+ * `Python Mock Library Comparison
+ <http://garybernhardt.github.com/python-mock-comparison/>`_
+
+This comparison is by no means complete, and also may not be fully idiomatic
+for all the libraries represented. *Please* contribute corrections, missing
+comparisons, or comparisons for additional libraries to the `mock issue
+tracker <https://code.google.com/p/mock/issues/list>`_.
+
+This comparison page was originally created by the `Mox project
+<https://code.google.com/p/pymox/wiki/MoxComparison>`_ and then extended for
+`flexmock and mock <http://has207.github.com/flexmock/compare.html>`_ by
+Herman Sheremetyev. Dingus examples written by `Gary Bernhadt
+<http://garybernhardt.github.com/python-mock-comparison/>`_. fudge examples
+provided by `Kumar McMillan <http://farmdev.com/>`_.
+
+.. note::
+
+ The examples tasks here were originally created by Mox which is a mocking
+ *framework* rather than a library like mock. The tasks shown naturally
+ exemplify tasks that frameworks are good at and not the ones they make
+ harder. In particular you can take a `Mock` or `MagicMock` object and use
+ it in any way you want with no up-front configuration. The same is also
+ true for Dingus.
+
+ The examples for mock here assume version 0.7.0.
+
+
+Simple fake object
+~~~~~~~~~~~~~~~~~~
+
+.. doctest::
+
+ >>> # mock
+ >>> my_mock = mock.Mock()
+ >>> my_mock.some_method.return_value = "calculated value"
+ >>> my_mock.some_attribute = "value"
+ >>> assertEqual("calculated value", my_mock.some_method())
+ >>> assertEqual("value", my_mock.some_attribute)
+
+::
+
+ # Flexmock
+ mock = flexmock(some_method=lambda: "calculated value", some_attribute="value")
+ assertEqual("calculated value", mock.some_method())
+ assertEqual("value", mock.some_attribute)
+
+ # Mox
+ mock = mox.MockAnything()
+ mock.some_method().AndReturn("calculated value")
+ mock.some_attribute = "value"
+ mox.Replay(mock)
+ assertEqual("calculated value", mock.some_method())
+ assertEqual("value", mock.some_attribute)
+
+ # Mocker
+ mock = mocker.mock()
+ mock.some_method()
+ mocker.result("calculated value")
+ mocker.replay()
+ mock.some_attribute = "value"
+ assertEqual("calculated value", mock.some_method())
+ assertEqual("value", mock.some_attribute)
+
+::
+
+ >>> # Dingus
+ >>> my_dingus = dingus.Dingus(some_attribute="value",
+ ... some_method__returns="calculated value")
+ >>> assertEqual("calculated value", my_dingus.some_method())
+ >>> assertEqual("value", my_dingus.some_attribute)
+
+::
+
+ >>> # fudge
+ >>> my_fake = (fudge.Fake()
+ ... .provides('some_method')
+ ... .returns("calculated value")
+ ... .has_attr(some_attribute="value"))
+ ...
+ >>> assertEqual("calculated value", my_fake.some_method())
+ >>> assertEqual("value", my_fake.some_attribute)
+
+
+Simple mock
+~~~~~~~~~~~
+
+.. doctest::
+
+ >>> # mock
+ >>> my_mock = mock.Mock()
+ >>> my_mock.some_method.return_value = "value"
+ >>> assertEqual("value", my_mock.some_method())
+ >>> my_mock.some_method.assert_called_once_with()
+
+::
+
+ # Flexmock
+ mock = flexmock()
+ mock.should_receive("some_method").and_return("value").once
+ assertEqual("value", mock.some_method())
+
+ # Mox
+ mock = mox.MockAnything()
+ mock.some_method().AndReturn("value")
+ mox.Replay(mock)
+ assertEqual("value", mock.some_method())
+ mox.Verify(mock)
+
+ # Mocker
+ mock = mocker.mock()
+ mock.some_method()
+ mocker.result("value")
+ mocker.replay()
+ assertEqual("value", mock.some_method())
+ mocker.verify()
+
+::
+
+ >>> # Dingus
+ >>> my_dingus = dingus.Dingus(some_method__returns="value")
+ >>> assertEqual("value", my_dingus.some_method())
+ >>> assert my_dingus.some_method.calls().once()
+
+::
+
+ >>> # fudge
+ >>> @fudge.test
+ ... def test():
+ ... my_fake = (fudge.Fake()
+ ... .expects('some_method')
+ ... .returns("value")
+ ... .times_called(1))
+ ...
+ >>> test()
+ Traceback (most recent call last):
+ ...
+ AssertionError: fake:my_fake.some_method() was not called
+
+
+Creating partial mocks
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. doctest::
+
+ >>> # mock
+ >>> SomeObject.some_method = mock.Mock(return_value='value')
+ >>> assertEqual("value", SomeObject.some_method())
+
+::
+
+ # Flexmock
+ flexmock(SomeObject).should_receive("some_method").and_return('value')
+ assertEqual("value", mock.some_method())
+
+ # Mox
+ mock = mox.MockObject(SomeObject)
+ mock.some_method().AndReturn("value")
+ mox.Replay(mock)
+ assertEqual("value", mock.some_method())
+ mox.Verify(mock)
+
+ # Mocker
+ mock = mocker.mock(SomeObject)
+ mock.Get()
+ mocker.result("value")
+ mocker.replay()
+ assertEqual("value", mock.some_method())
+ mocker.verify()
+
+::
+
+ >>> # Dingus
+ >>> object = SomeObject
+ >>> object.some_method = dingus.Dingus(return_value="value")
+ >>> assertEqual("value", object.some_method())
+
+::
+
+ >>> # fudge
+ >>> fake = fudge.Fake().is_callable().returns("<fudge-value>")
+ >>> with fudge.patched_context(SomeObject, 'some_method', fake):
+ ... s = SomeObject()
+ ... assertEqual("<fudge-value>", s.some_method())
+ ...
+
+
+Ensure calls are made in specific order
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. doctest::
+
+ >>> # mock
+ >>> my_mock = mock.Mock(spec=SomeObject)
+ >>> my_mock.method1()
+ <Mock name='mock.method1()' id='...'>
+ >>> my_mock.method2()
+ <Mock name='mock.method2()' id='...'>
+ >>> assertEqual(my_mock.mock_calls, [call.method1(), call.method2()])
+
+::
+
+ # Flexmock
+ mock = flexmock(SomeObject)
+ mock.should_receive('method1').once.ordered.and_return('first thing')
+ mock.should_receive('method2').once.ordered.and_return('second thing')
+
+ # Mox
+ mock = mox.MockObject(SomeObject)
+ mock.method1().AndReturn('first thing')
+ mock.method2().AndReturn('second thing')
+ mox.Replay(mock)
+ mox.Verify(mock)
+
+ # Mocker
+ mock = mocker.mock()
+ with mocker.order():
+ mock.method1()
+ mocker.result('first thing')
+ mock.method2()
+ mocker.result('second thing')
+ mocker.replay()
+ mocker.verify()
+
+::
+
+ >>> # Dingus
+ >>> my_dingus = dingus.Dingus()
+ >>> my_dingus.method1()
+ <Dingus ...>
+ >>> my_dingus.method2()
+ <Dingus ...>
+ >>> assertEqual(['method1', 'method2'], [call.name for call in my_dingus.calls])
+
+::
+
+ >>> # fudge
+ >>> @fudge.test
+ ... def test():
+ ... my_fake = (fudge.Fake()
+ ... .remember_order()
+ ... .expects('method1')
+ ... .expects('method2'))
+ ... my_fake.method2()
+ ... my_fake.method1()
+ ...
+ >>> test()
+ Traceback (most recent call last):
+ ...
+ AssertionError: Call #1 was fake:my_fake.method2(); Expected: #1 fake:my_fake.method1(), #2 fake:my_fake.method2(), end
+
+
+Raising exceptions
+~~~~~~~~~~~~~~~~~~
+
+.. doctest::
+
+ >>> # mock
+ >>> my_mock = mock.Mock()
+ >>> my_mock.some_method.side_effect = SomeException("message")
+ >>> assertRaises(SomeException, my_mock.some_method)
+
+::
+
+ # Flexmock
+ mock = flexmock()
+ mock.should_receive("some_method").and_raise(SomeException("message"))
+ assertRaises(SomeException, mock.some_method)
+
+ # Mox
+ mock = mox.MockAnything()
+ mock.some_method().AndRaise(SomeException("message"))
+ mox.Replay(mock)
+ assertRaises(SomeException, mock.some_method)
+ mox.Verify(mock)
+
+ # Mocker
+ mock = mocker.mock()
+ mock.some_method()
+ mocker.throw(SomeException("message"))
+ mocker.replay()
+ assertRaises(SomeException, mock.some_method)
+ mocker.verify()
+
+::
+
+ >>> # Dingus
+ >>> my_dingus = dingus.Dingus()
+ >>> my_dingus.some_method = dingus.exception_raiser(SomeException)
+ >>> assertRaises(SomeException, my_dingus.some_method)
+
+::
+
+ >>> # fudge
+ >>> my_fake = (fudge.Fake()
+ ... .is_callable()
+ ... .raises(SomeException("message")))
+ ...
+ >>> my_fake()
+ Traceback (most recent call last):
+ ...
+ SomeException: message
+
+
+Override new instances of a class
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. doctest::
+
+ >>> # mock
+ >>> with mock.patch('somemodule.Someclass') as MockClass:
+ ... MockClass.return_value = some_other_object
+ ... assertEqual(some_other_object, somemodule.Someclass())
+ ...
+
+
+::
+
+ # Flexmock
+ flexmock(some_module.SomeClass, new_instances=some_other_object)
+ assertEqual(some_other_object, some_module.SomeClass())
+
+ # Mox
+ # (you will probably have mox.Mox() available as self.mox in a real test)
+ mox.Mox().StubOutWithMock(some_module, 'SomeClass', use_mock_anything=True)
+ some_module.SomeClass().AndReturn(some_other_object)
+ mox.ReplayAll()
+ assertEqual(some_other_object, some_module.SomeClass())
+
+ # Mocker
+ instance = mocker.mock()
+ klass = mocker.replace(SomeClass, spec=None)
+ klass('expected', 'args')
+ mocker.result(instance)
+
+::
+
+ >>> # Dingus
+ >>> MockClass = dingus.Dingus(return_value=some_other_object)
+ >>> with dingus.patch('somemodule.SomeClass', MockClass):
+ ... assertEqual(some_other_object, somemodule.SomeClass())
+ ...
+
+::
+
+ >>> # fudge
+ >>> @fudge.patch('somemodule.SomeClass')
+ ... def test(FakeClass):
+ ... FakeClass.is_callable().returns(some_other_object)
+ ... assertEqual(some_other_object, somemodule.SomeClass())
+ ...
+ >>> test()
+
+
+Call the same method multiple times
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. note::
+
+ You don't need to do *any* configuration to call `mock.Mock()` methods
+ multiple times. Attributes like `call_count`, `call_args_list` and
+ `method_calls` provide various different ways of making assertions about
+ how the mock was used.
+
+.. doctest::
+
+ >>> # mock
+ >>> my_mock = mock.Mock()
+ >>> my_mock.some_method()
+ <Mock name='mock.some_method()' id='...'>
+ >>> my_mock.some_method()
+ <Mock name='mock.some_method()' id='...'>
+ >>> assert my_mock.some_method.call_count >= 2
+
+::
+
+ # Flexmock # (verifies that the method gets called at least twice)
+ flexmock(some_object).should_receive('some_method').at_least.twice
+
+ # Mox
+ # (does not support variable number of calls, so you need to create a new entry for each explicit call)
+ mock = mox.MockObject(some_object)
+ mock.some_method(mox.IgnoreArg(), mox.IgnoreArg())
+ mock.some_method(mox.IgnoreArg(), mox.IgnoreArg())
+ mox.Replay(mock)
+ mox.Verify(mock)
+
+ # Mocker
+ # (TODO)
+
+::
+
+ >>> # Dingus
+ >>> my_dingus = dingus.Dingus()
+ >>> my_dingus.some_method()
+ <Dingus ...>
+ >>> my_dingus.some_method()
+ <Dingus ...>
+ >>> assert len(my_dingus.calls('some_method')) == 2
+
+::
+
+ >>> # fudge
+ >>> @fudge.test
+ ... def test():
+ ... my_fake = fudge.Fake().expects('some_method').times_called(2)
+ ... my_fake.some_method()
+ ...
+ >>> test()
+ Traceback (most recent call last):
+ ...
+ AssertionError: fake:my_fake.some_method() was called 1 time(s). Expected 2.
+
+
+Mock chained methods
+~~~~~~~~~~~~~~~~~~~~
+
+.. doctest::
+
+ >>> # mock
+ >>> my_mock = mock.Mock()
+ >>> method3 = my_mock.method1.return_value.method2.return_value.method3
+ >>> method3.return_value = 'some value'
+ >>> assertEqual('some value', my_mock.method1().method2().method3(1, 2))
+ >>> method3.assert_called_once_with(1, 2)
+
+::
+
+ # Flexmock
+ # (intermediate method calls are automatically assigned to temporary fake objects
+ # and can be called with any arguments)
+ flexmock(some_object).should_receive(
+ 'method1.method2.method3'
+ ).with_args(arg1, arg2).and_return('some value')
+ assertEqual('some_value', some_object.method1().method2().method3(arg1, arg2))
+
+::
+
+ # Mox
+ mock = mox.MockObject(some_object)
+ mock2 = mox.MockAnything()
+ mock3 = mox.MockAnything()
+ mock.method1().AndReturn(mock1)
+ mock2.method2().AndReturn(mock2)
+ mock3.method3(arg1, arg2).AndReturn('some_value')
+ self.mox.ReplayAll()
+ assertEqual("some_value", some_object.method1().method2().method3(arg1, arg2))
+ self.mox.VerifyAll()
+
+ # Mocker
+ # (TODO)
+
+::
+
+ >>> # Dingus
+ >>> my_dingus = dingus.Dingus()
+ >>> method3 = my_dingus.method1.return_value.method2.return_value.method3
+ >>> method3.return_value = 'some value'
+ >>> assertEqual('some value', my_dingus.method1().method2().method3(1, 2))
+ >>> assert method3.calls('()', 1, 2).once()
+
+::
+
+ >>> # fudge
+ >>> @fudge.test
+ ... def test():
+ ... my_fake = fudge.Fake()
+ ... (my_fake
+ ... .expects('method1')
+ ... .returns_fake()
+ ... .expects('method2')
+ ... .returns_fake()
+ ... .expects('method3')
+ ... .with_args(1, 2)
+ ... .returns('some value'))
+ ... assertEqual('some value', my_fake.method1().method2().method3(1, 2))
+ ...
+ >>> test()
+
+
+Mocking a context manager
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Examples for mock, Dingus and fudge only (so far):
+
+.. doctest::
+
+ >>> # mock
+ >>> my_mock = mock.MagicMock()
+ >>> with my_mock:
+ ... pass
+ ...
+ >>> my_mock.__enter__.assert_called_with()
+ >>> my_mock.__exit__.assert_called_with(None, None, None)
+
+::
+
+
+ >>> # Dingus (nothing special here; all dinguses are "magic mocks")
+ >>> my_dingus = dingus.Dingus()
+ >>> with my_dingus:
+ ... pass
+ ...
+ >>> assert my_dingus.__enter__.calls()
+ >>> assert my_dingus.__exit__.calls('()', None, None, None)
+
+::
+
+ >>> # fudge
+ >>> my_fake = fudge.Fake().provides('__enter__').provides('__exit__')
+ >>> with my_fake:
+ ... pass
+ ...
+
+
+Mocking the builtin open used as a context manager
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Example for mock only (so far):
+
+.. doctest::
+
+ >>> # mock
+ >>> my_mock = mock.MagicMock()
+ >>> with mock.patch('__builtin__.open', my_mock):
+ ... manager = my_mock.return_value.__enter__.return_value
+ ... manager.read.return_value = 'some data'
+ ... with open('foo') as h:
+ ... data = h.read()
+ ...
+ >>> data
+ 'some data'
+ >>> my_mock.assert_called_once_with('foo')
+
+*or*:
+
+.. doctest::
+
+ >>> # mock
+ >>> with mock.patch('__builtin__.open') as my_mock:
+ ... my_mock.return_value.__enter__ = lambda s: s
+ ... my_mock.return_value.__exit__ = mock.Mock()
+ ... my_mock.return_value.read.return_value = 'some data'
+ ... with open('foo') as h:
+ ... data = h.read()
+ ...
+ >>> data
+ 'some data'
+ >>> my_mock.assert_called_once_with('foo')
+
+::
+
+ >>> # Dingus
+ >>> my_dingus = dingus.Dingus()
+ >>> with dingus.patch('__builtin__.open', my_dingus):
+ ... file_ = open.return_value.__enter__.return_value
+ ... file_.read.return_value = 'some data'
+ ... with open('foo') as h:
+ ... data = f.read()
+ ...
+ >>> data
+ 'some data'
+ >>> assert my_dingus.calls('()', 'foo').once()
+
+::
+
+ >>> # fudge
+ >>> from contextlib import contextmanager
+ >>> from StringIO import StringIO
+ >>> @contextmanager
+ ... def fake_file(filename):
+ ... yield StringIO('sekrets')
+ ...
+ >>> with fudge.patch('__builtin__.open') as fake_open:
+ ... fake_open.is_callable().calls(fake_file)
+ ... with open('/etc/password') as f:
+ ... data = f.read()
+ ...
+ fake:__builtin__.open
+ >>> data
+ 'sekrets'
\ No newline at end of file
diff --git a/slider-agent/src/test/python/mock/docs/conf.py b/slider-agent/src/test/python/mock/docs/conf.py
new file mode 100644
index 0000000..62f0491
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/conf.py
@@ -0,0 +1,209 @@
+# -*- coding: utf-8 -*-
+#
+# Mock documentation build configuration file, created by
+# sphinx-quickstart on Mon Nov 17 18:12:00 2008.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# The contents of this file are pickled, so don't put values in the namespace
+# that aren't pickleable (module imports are okay, they're removed automatically).
+#
+# All configuration values have a default value; values that are commented out
+# serve to show the default value.
+
+import sys, os
+sys.path.insert(0, os.path.abspath('..'))
+from mock import __version__
+
+# If your extensions are in another directory, add it here. If the directory
+# is relative to the documentation root, use os.path.abspath to make it
+# absolute, like shown here.
+#sys.path.append(os.path.abspath('some/directory'))
+
+# General configuration
+# ---------------------
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.doctest']
+
+doctest_global_setup = """
+import os
+import sys
+import mock
+from mock import * # yeah, I know :-/
+import unittest2
+import __main__
+
+if os.getcwd() not in sys.path:
+ sys.path.append(os.getcwd())
+
+# keep a reference to __main__
+sys.modules['__main'] = __main__
+
+class ProxyModule(object):
+ def __init__(self):
+ self.__dict__ = globals()
+
+sys.modules['__main__'] = ProxyModule()
+"""
+
+doctest_global_cleanup = """
+sys.modules['__main__'] = sys.modules['__main']
+"""
+
+html_theme = 'nature'
+html_theme_options = {}
+
+# Add any paths that contain templates here, relative to this directory.
+#templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.txt'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General substitutions.
+project = u'Mock'
+copyright = u'2007-2012, Michael Foord & the mock team'
+
+# The default replacements for |version| and |release|, also used in various
+# other places throughout the built documents.
+#
+# The short X.Y version.
+version = __version__[:3]
+# The full version, including alpha/beta/rc tags.
+release = __version__
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+today_fmt = '%B %d, %Y'
+
+# List of documents that shouldn't be included in the build.
+#unused_docs = []
+
+# List of directories, relative to source directories, that shouldn't be searched
+# for source files.
+exclude_trees = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+add_module_names = False
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'friendly'
+
+
+# Options for HTML output
+# -----------------------
+
+# The style sheet to use for HTML and HTML Help pages. A file of that name
+# must exist either in Sphinx' static/ path, or in one of the custom paths
+# given in html_static_path.
+#html_style = 'adctheme.css'
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+html_use_modindex = False
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, the reST sources are included in the HTML build as _sources/<name>.
+#html_copy_source = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = ''
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Mockdoc'
+
+
+# Options for LaTeX output
+# ------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+latex_font_size = '12pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, document class [howto/manual]).
+latex_documents = [
+ ('index', 'Mock.tex', u'Mock Documentation',
+ u'Michael Foord', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+latex_use_modindex = False
\ No newline at end of file
diff --git a/slider-agent/src/test/python/mock/docs/examples.txt b/slider-agent/src/test/python/mock/docs/examples.txt
new file mode 100644
index 0000000..ecb994b
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/examples.txt
@@ -0,0 +1,1063 @@
+.. _further-examples:
+
+==================
+ Further Examples
+==================
+
+.. currentmodule:: mock
+
+.. testsetup::
+
+ from datetime import date
+
+ BackendProvider = Mock()
+ sys.modules['mymodule'] = mymodule = Mock(name='mymodule')
+
+ def grob(val):
+ "First frob and then clear val"
+ mymodule.frob(val)
+ val.clear()
+
+ mymodule.frob = lambda val: val
+ mymodule.grob = grob
+ mymodule.date = date
+
+ class TestCase(unittest2.TestCase):
+ def run(self):
+ result = unittest2.TestResult()
+ out = unittest2.TestCase.run(self, result)
+ assert result.wasSuccessful()
+
+ from mock import inPy3k
+
+
+
+For comprehensive examples, see the unit tests included in the full source
+distribution.
+
+Here are some more examples for some slightly more advanced scenarios than in
+the :ref:`getting started <getting-started>` guide.
+
+
+Mocking chained calls
+=====================
+
+Mocking chained calls is actually straightforward with mock once you
+understand the :attr:`~Mock.return_value` attribute. When a mock is called for
+the first time, or you fetch its `return_value` before it has been called, a
+new `Mock` is created.
+
+This means that you can see how the object returned from a call to a mocked
+object has been used by interrogating the `return_value` mock:
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> mock().foo(a=2, b=3)
+ <Mock name='mock().foo()' id='...'>
+ >>> mock.return_value.foo.assert_called_with(a=2, b=3)
+
+From here it is a simple step to configure and then make assertions about
+chained calls. Of course another alternative is writing your code in a more
+testable way in the first place...
+
+So, suppose we have some code that looks a little bit like this:
+
+.. doctest::
+
+ >>> class Something(object):
+ ... def __init__(self):
+ ... self.backend = BackendProvider()
+ ... def method(self):
+ ... response = self.backend.get_endpoint('foobar').create_call('spam', 'eggs').start_call()
+ ... # more code
+
+Assuming that `BackendProvider` is already well tested, how do we test
+`method()`? Specifically, we want to test that the code section `# more
+code` uses the response object in the correct way.
+
+As this chain of calls is made from an instance attribute we can monkey patch
+the `backend` attribute on a `Something` instance. In this particular case
+we are only interested in the return value from the final call to
+`start_call` so we don't have much configuration to do. Let's assume the
+object it returns is 'file-like', so we'll ensure that our response object
+uses the builtin `file` as its `spec`.
+
+To do this we create a mock instance as our mock backend and create a mock
+response object for it. To set the response as the return value for that final
+`start_call` we could do this:
+
+ `mock_backend.get_endpoint.return_value.create_call.return_value.start_call.return_value = mock_response`.
+
+We can do that in a slightly nicer way using the :meth:`~Mock.configure_mock`
+method to directly set the return value for us:
+
+.. doctest::
+
+ >>> something = Something()
+ >>> mock_response = Mock(spec=file)
+ >>> mock_backend = Mock()
+ >>> config = {'get_endpoint.return_value.create_call.return_value.start_call.return_value': mock_response}
+ >>> mock_backend.configure_mock(**config)
+
+With these we monkey patch the "mock backend" in place and can make the real
+call:
+
+.. doctest::
+
+ >>> something.backend = mock_backend
+ >>> something.method()
+
+Using :attr:`~Mock.mock_calls` we can check the chained call with a single
+assert. A chained call is several calls in one line of code, so there will be
+several entries in `mock_calls`. We can use :meth:`call.call_list` to create
+this list of calls for us:
+
+.. doctest::
+
+ >>> chained = call.get_endpoint('foobar').create_call('spam', 'eggs').start_call()
+ >>> call_list = chained.call_list()
+ >>> assert mock_backend.mock_calls == call_list
+
+
+Partial mocking
+===============
+
+In some tests I wanted to mock out a call to `datetime.date.today()
+<http://docs.python.org/library/datetime.html#datetime.date.today>`_ to return
+a known date, but I didn't want to prevent the code under test from
+creating new date objects. Unfortunately `datetime.date` is written in C, and
+so I couldn't just monkey-patch out the static `date.today` method.
+
+I found a simple way of doing this that involved effectively wrapping the date
+class with a mock, but passing through calls to the constructor to the real
+class (and returning real instances).
+
+The :func:`patch decorator <patch>` is used here to
+mock out the `date` class in the module under test. The :attr:`side_effect`
+attribute on the mock date class is then set to a lambda function that returns
+a real date. When the mock date class is called a real date will be
+constructed and returned by `side_effect`.
+
+.. doctest::
+
+ >>> from datetime import date
+ >>> with patch('mymodule.date') as mock_date:
+ ... mock_date.today.return_value = date(2010, 10, 8)
+ ... mock_date.side_effect = lambda *args, **kw: date(*args, **kw)
+ ...
+ ... assert mymodule.date.today() == date(2010, 10, 8)
+ ... assert mymodule.date(2009, 6, 8) == date(2009, 6, 8)
+ ...
+
+Note that we don't patch `datetime.date` globally, we patch `date` in the
+module that *uses* it. See :ref:`where to patch <where-to-patch>`.
+
+When `date.today()` is called a known date is returned, but calls to the
+`date(...)` constructor still return normal dates. Without this you can find
+yourself having to calculate an expected result using exactly the same
+algorithm as the code under test, which is a classic testing anti-pattern.
+
+Calls to the date constructor are recorded in the `mock_date` attributes
+(`call_count` and friends) which may also be useful for your tests.
+
+An alternative way of dealing with mocking dates, or other builtin classes,
+is discussed in `this blog entry
+<http://williamjohnbert.com/2011/07/how-to-unit-testing-in-django-with-mocking-and-patching/>`_.
+
+
+Mocking a Generator Method
+==========================
+
+A Python generator is a function or method that uses the `yield statement
+<http://docs.python.org/reference/simple_stmts.html#the-yield-statement>`_ to
+return a series of values when iterated over [#]_.
+
+A generator method / function is called to return the generator object. It is
+the generator object that is then iterated over. The protocol method for
+iteration is `__iter__
+<http://docs.python.org/library/stdtypes.html#container.__iter__>`_, so we can
+mock this using a `MagicMock`.
+
+Here's an example class with an "iter" method implemented as a generator:
+
+.. doctest::
+
+ >>> class Foo(object):
+ ... def iter(self):
+ ... for i in [1, 2, 3]:
+ ... yield i
+ ...
+ >>> foo = Foo()
+ >>> list(foo.iter())
+ [1, 2, 3]
+
+
+How would we mock this class, and in particular its "iter" method?
+
+To configure the values returned from the iteration (implicit in the call to
+`list`), we need to configure the object returned by the call to `foo.iter()`.
+
+.. doctest::
+
+ >>> mock_foo = MagicMock()
+ >>> mock_foo.iter.return_value = iter([1, 2, 3])
+ >>> list(mock_foo.iter())
+ [1, 2, 3]
+
+.. [#] There are also generator expressions and more `advanced uses
+ <http://www.dabeaz.com/coroutines/index.html>`_ of generators, but we aren't
+ concerned about them here. A very good introduction to generators and how
+ powerful they are is: `Generator Tricks for Systems Programmers
+ <http://www.dabeaz.com/generators/>`_.
+
+
+Applying the same patch to every test method
+============================================
+
+If you want several patches in place for multiple test methods the obvious way
+is to apply the patch decorators to every method. This can feel like unnecessary
+repetition. For Python 2.6 or more recent you can use `patch` (in all its
+various forms) as a class decorator. This applies the patches to all test
+methods on the class. A test method is identified by methods whose names start
+with `test`:
+
+.. doctest::
+
+ >>> @patch('mymodule.SomeClass')
+ ... class MyTest(TestCase):
+ ...
+ ... def test_one(self, MockSomeClass):
+ ... self.assertTrue(mymodule.SomeClass is MockSomeClass)
+ ...
+ ... def test_two(self, MockSomeClass):
+ ... self.assertTrue(mymodule.SomeClass is MockSomeClass)
+ ...
+ ... def not_a_test(self):
+ ... return 'something'
+ ...
+ >>> MyTest('test_one').test_one()
+ >>> MyTest('test_two').test_two()
+ >>> MyTest('test_two').not_a_test()
+ 'something'
+
+An alternative way of managing patches is to use the :ref:`start-and-stop`.
+These allow you to move the patching into your `setUp` and `tearDown` methods.
+
+.. doctest::
+
+ >>> class MyTest(TestCase):
+ ... def setUp(self):
+ ... self.patcher = patch('mymodule.foo')
+ ... self.mock_foo = self.patcher.start()
+ ...
+ ... def test_foo(self):
+ ... self.assertTrue(mymodule.foo is self.mock_foo)
+ ...
+ ... def tearDown(self):
+ ... self.patcher.stop()
+ ...
+ >>> MyTest('test_foo').run()
+
+If you use this technique you must ensure that the patching is "undone" by
+calling `stop`. This can be fiddlier than you might think, because if an
+exception is raised in the setUp then tearDown is not called. `unittest2
+<http://pypi.python.org/pypi/unittest2>`_ cleanup functions make this simpler:
+
+
+.. doctest::
+
+ >>> class MyTest(TestCase):
+ ... def setUp(self):
+ ... patcher = patch('mymodule.foo')
+ ... self.addCleanup(patcher.stop)
+ ... self.mock_foo = patcher.start()
+ ...
+ ... def test_foo(self):
+ ... self.assertTrue(mymodule.foo is self.mock_foo)
+ ...
+ >>> MyTest('test_foo').run()
+
+
+Mocking Unbound Methods
+=======================
+
+Whilst writing tests today I needed to patch an *unbound method* (patching the
+method on the class rather than on the instance). I needed self to be passed
+in as the first argument because I want to make asserts about which objects
+were calling this particular method. The issue is that you can't patch with a
+mock for this, because if you replace an unbound method with a mock it doesn't
+become a bound method when fetched from the instance, and so it doesn't get
+self passed in. The workaround is to patch the unbound method with a real
+function instead. The :func:`patch` decorator makes it so simple to
+patch out methods with a mock that having to create a real function becomes a
+nuisance.
+
+If you pass `autospec=True` to patch then it does the patching with a
+*real* function object. This function object has the same signature as the one
+it is replacing, but delegates to a mock under the hood. You still get your
+mock auto-created in exactly the same way as before. What it means though, is
+that if you use it to patch out an unbound method on a class the mocked
+function will be turned into a bound method if it is fetched from an instance.
+It will have `self` passed in as the first argument, which is exactly what I
+wanted:
+
+.. doctest::
+
+ >>> class Foo(object):
+ ... def foo(self):
+ ... pass
+ ...
+ >>> with patch.object(Foo, 'foo', autospec=True) as mock_foo:
+ ... mock_foo.return_value = 'foo'
+ ... foo = Foo()
+ ... foo.foo()
+ ...
+ 'foo'
+ >>> mock_foo.assert_called_once_with(foo)
+
+If we don't use `autospec=True` then the unbound method is patched out
+with a Mock instance instead, and isn't called with `self`.
+
+
+Checking multiple calls with mock
+=================================
+
+mock has a nice API for making assertions about how your mock objects are used.
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> mock.foo_bar.return_value = None
+ >>> mock.foo_bar('baz', spam='eggs')
+ >>> mock.foo_bar.assert_called_with('baz', spam='eggs')
+
+If your mock is only being called once you can use the
+:meth:`assert_called_once_with` method that also asserts that the
+:attr:`call_count` is one.
+
+.. doctest::
+
+ >>> mock.foo_bar.assert_called_once_with('baz', spam='eggs')
+ >>> mock.foo_bar()
+ >>> mock.foo_bar.assert_called_once_with('baz', spam='eggs')
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected to be called once. Called 2 times.
+
+Both `assert_called_with` and `assert_called_once_with` make assertions about
+the *most recent* call. If your mock is going to be called several times, and
+you want to make assertions about *all* those calls you can use
+:attr:`~Mock.call_args_list`:
+
+.. doctest::
+
+ >>> mock = Mock(return_value=None)
+ >>> mock(1, 2, 3)
+ >>> mock(4, 5, 6)
+ >>> mock()
+ >>> mock.call_args_list
+ [call(1, 2, 3), call(4, 5, 6), call()]
+
+The :data:`call` helper makes it easy to make assertions about these calls. You
+can build up a list of expected calls and compare it to `call_args_list`. This
+looks remarkably similar to the repr of the `call_args_list`:
+
+.. doctest::
+
+ >>> expected = [call(1, 2, 3), call(4, 5, 6), call()]
+ >>> mock.call_args_list == expected
+ True
+
+
+Coping with mutable arguments
+=============================
+
+Another situation is rare, but can bite you, is when your mock is called with
+mutable arguments. `call_args` and `call_args_list` store *references* to the
+arguments. If the arguments are mutated by the code under test then you can no
+longer make assertions about what the values were when the mock was called.
+
+Here's some example code that shows the problem. Imagine the following functions
+defined in 'mymodule'::
+
+ def frob(val):
+ pass
+
+ def grob(val):
+ "First frob and then clear val"
+ frob(val)
+ val.clear()
+
+When we try to test that `grob` calls `frob` with the correct argument look
+what happens:
+
+.. doctest::
+
+ >>> with patch('mymodule.frob') as mock_frob:
+ ... val = set([6])
+ ... mymodule.grob(val)
+ ...
+ >>> val
+ set([])
+ >>> mock_frob.assert_called_with(set([6]))
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected: ((set([6]),), {})
+ Called with: ((set([]),), {})
+
+One possibility would be for mock to copy the arguments you pass in. This
+could then cause problems if you do assertions that rely on object identity
+for equality.
+
+Here's one solution that uses the :attr:`side_effect`
+functionality. If you provide a `side_effect` function for a mock then
+`side_effect` will be called with the same args as the mock. This gives us an
+opportunity to copy the arguments and store them for later assertions. In this
+example I'm using *another* mock to store the arguments so that I can use the
+mock methods for doing the assertion. Again a helper function sets this up for
+me.
+
+.. doctest::
+
+ >>> from copy import deepcopy
+ >>> from mock import Mock, patch, DEFAULT
+ >>> def copy_call_args(mock):
+ ... new_mock = Mock()
+ ... def side_effect(*args, **kwargs):
+ ... args = deepcopy(args)
+ ... kwargs = deepcopy(kwargs)
+ ... new_mock(*args, **kwargs)
+ ... return DEFAULT
+ ... mock.side_effect = side_effect
+ ... return new_mock
+ ...
+ >>> with patch('mymodule.frob') as mock_frob:
+ ... new_mock = copy_call_args(mock_frob)
+ ... val = set([6])
+ ... mymodule.grob(val)
+ ...
+ >>> new_mock.assert_called_with(set([6]))
+ >>> new_mock.call_args
+ call(set([6]))
+
+`copy_call_args` is called with the mock that will be called. It returns a new
+mock that we do the assertion on. The `side_effect` function makes a copy of
+the args and calls our `new_mock` with the copy.
+
+.. note::
+
+ If your mock is only going to be used once there is an easier way of
+ checking arguments at the point they are called. You can simply do the
+ checking inside a `side_effect` function.
+
+ .. doctest::
+
+ >>> def side_effect(arg):
+ ... assert arg == set([6])
+ ...
+ >>> mock = Mock(side_effect=side_effect)
+ >>> mock(set([6]))
+ >>> mock(set())
+ Traceback (most recent call last):
+ ...
+ AssertionError
+
+An alternative approach is to create a subclass of `Mock` or `MagicMock` that
+copies (using `copy.deepcopy
+<http://docs.python.org/library/copy.html#copy.deepcopy>`_) the arguments.
+Here's an example implementation:
+
+.. doctest::
+
+ >>> from copy import deepcopy
+ >>> class CopyingMock(MagicMock):
+ ... def __call__(self, *args, **kwargs):
+ ... args = deepcopy(args)
+ ... kwargs = deepcopy(kwargs)
+ ... return super(CopyingMock, self).__call__(*args, **kwargs)
+ ...
+ >>> c = CopyingMock(return_value=None)
+ >>> arg = set()
+ >>> c(arg)
+ >>> arg.add(1)
+ >>> c.assert_called_with(set())
+ >>> c.assert_called_with(arg)
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected call: mock(set([1]))
+ Actual call: mock(set([]))
+ >>> c.foo
+ <CopyingMock name='mock.foo' id='...'>
+
+When you subclass `Mock` or `MagicMock` all dynamically created attributes,
+and the `return_value` will use your subclass automatically. That means all
+children of a `CopyingMock` will also have the type `CopyingMock`.
+
+
+Raising exceptions on attribute access
+======================================
+
+You can use :class:`PropertyMock` to mimic the behaviour of properties. This
+includes raising exceptions when an attribute is accessed.
+
+Here's an example raising a `ValueError` when the 'foo' attribute is accessed:
+
+.. doctest::
+
+ >>> m = MagicMock()
+ >>> p = PropertyMock(side_effect=ValueError)
+ >>> type(m).foo = p
+ >>> m.foo
+ Traceback (most recent call last):
+ ....
+ ValueError
+
+Because every mock object has its own type, a new subclass of whichever mock
+class you're using, all mock objects are isolated from each other. You can
+safely attach properties (or other descriptors or whatever you want in fact)
+to `type(mock)` without affecting other mock objects.
+
+
+Multiple calls with different effects
+=====================================
+
+.. note::
+
+ In mock 1.0 the handling of iterable `side_effect` was changed. Any
+ exceptions in the iterable will be raised instead of returned.
+
+Handling code that needs to behave differently on subsequent calls during the
+test can be tricky. For example you may have a function that needs to raise
+an exception the first time it is called but returns a response on the second
+call (testing retry behaviour).
+
+One approach is to use a :attr:`side_effect` function that replaces itself. The
+first time it is called the `side_effect` sets a new `side_effect` that will
+be used for the second call. It then raises an exception:
+
+.. doctest::
+
+ >>> def side_effect(*args):
+ ... def second_call(*args):
+ ... return 'response'
+ ... mock.side_effect = second_call
+ ... raise Exception('boom')
+ ...
+ >>> mock = Mock(side_effect=side_effect)
+ >>> mock('first')
+ Traceback (most recent call last):
+ ...
+ Exception: boom
+ >>> mock('second')
+ 'response'
+ >>> mock.assert_called_with('second')
+
+Another perfectly valid way would be to pop return values from a list. If the
+return value is an exception, raise it instead of returning it:
+
+.. doctest::
+
+ >>> returns = [Exception('boom'), 'response']
+ >>> def side_effect(*args):
+ ... result = returns.pop(0)
+ ... if isinstance(result, Exception):
+ ... raise result
+ ... return result
+ ...
+ >>> mock = Mock(side_effect=side_effect)
+ >>> mock('first')
+ Traceback (most recent call last):
+ ...
+ Exception: boom
+ >>> mock('second')
+ 'response'
+ >>> mock.assert_called_with('second')
+
+Which approach you prefer is a matter of taste. The first approach is actually
+a line shorter but maybe the second approach is more readable.
+
+
+Nesting Patches
+===============
+
+Using patch as a context manager is nice, but if you do multiple patches you
+can end up with nested with statements indenting further and further to the
+right:
+
+.. doctest::
+
+ >>> class MyTest(TestCase):
+ ...
+ ... def test_foo(self):
+ ... with patch('mymodule.Foo') as mock_foo:
+ ... with patch('mymodule.Bar') as mock_bar:
+ ... with patch('mymodule.Spam') as mock_spam:
+ ... assert mymodule.Foo is mock_foo
+ ... assert mymodule.Bar is mock_bar
+ ... assert mymodule.Spam is mock_spam
+ ...
+ >>> original = mymodule.Foo
+ >>> MyTest('test_foo').test_foo()
+ >>> assert mymodule.Foo is original
+
+With unittest2_ `cleanup` functions and the :ref:`start-and-stop` we can
+achieve the same effect without the nested indentation. A simple helper
+method, `create_patch`, puts the patch in place and returns the created mock
+for us:
+
+.. doctest::
+
+ >>> class MyTest(TestCase):
+ ...
+ ... def create_patch(self, name):
+ ... patcher = patch(name)
+ ... thing = patcher.start()
+ ... self.addCleanup(patcher.stop)
+ ... return thing
+ ...
+ ... def test_foo(self):
+ ... mock_foo = self.create_patch('mymodule.Foo')
+ ... mock_bar = self.create_patch('mymodule.Bar')
+ ... mock_spam = self.create_patch('mymodule.Spam')
+ ...
+ ... assert mymodule.Foo is mock_foo
+ ... assert mymodule.Bar is mock_bar
+ ... assert mymodule.Spam is mock_spam
+ ...
+ >>> original = mymodule.Foo
+ >>> MyTest('test_foo').run()
+ >>> assert mymodule.Foo is original
+
+
+Mocking a dictionary with MagicMock
+===================================
+
+You may want to mock a dictionary, or other container object, recording all
+access to it whilst having it still behave like a dictionary.
+
+We can do this with :class:`MagicMock`, which will behave like a dictionary,
+and using :data:`~Mock.side_effect` to delegate dictionary access to a real
+underlying dictionary that is under our control.
+
+When the `__getitem__` and `__setitem__` methods of our `MagicMock` are called
+(normal dictionary access) then `side_effect` is called with the key (and in
+the case of `__setitem__` the value too). We can also control what is returned.
+
+After the `MagicMock` has been used we can use attributes like
+:data:`~Mock.call_args_list` to assert about how the dictionary was used:
+
+.. doctest::
+
+ >>> my_dict = {'a': 1, 'b': 2, 'c': 3}
+ >>> def getitem(name):
+ ... return my_dict[name]
+ ...
+ >>> def setitem(name, val):
+ ... my_dict[name] = val
+ ...
+ >>> mock = MagicMock()
+ >>> mock.__getitem__.side_effect = getitem
+ >>> mock.__setitem__.side_effect = setitem
+
+.. note::
+
+ An alternative to using `MagicMock` is to use `Mock` and *only* provide
+ the magic methods you specifically want:
+
+ .. doctest::
+
+ >>> mock = Mock()
+ >>> mock.__setitem__ = Mock(side_effect=getitem)
+ >>> mock.__getitem__ = Mock(side_effect=setitem)
+
+ A *third* option is to use `MagicMock` but passing in `dict` as the `spec`
+ (or `spec_set`) argument so that the `MagicMock` created only has
+ dictionary magic methods available:
+
+ .. doctest::
+
+ >>> mock = MagicMock(spec_set=dict)
+ >>> mock.__getitem__.side_effect = getitem
+ >>> mock.__setitem__.side_effect = setitem
+
+With these side effect functions in place, the `mock` will behave like a normal
+dictionary but recording the access. It even raises a `KeyError` if you try
+to access a key that doesn't exist.
+
+.. doctest::
+
+ >>> mock['a']
+ 1
+ >>> mock['c']
+ 3
+ >>> mock['d']
+ Traceback (most recent call last):
+ ...
+ KeyError: 'd'
+ >>> mock['b'] = 'fish'
+ >>> mock['d'] = 'eggs'
+ >>> mock['b']
+ 'fish'
+ >>> mock['d']
+ 'eggs'
+
+After it has been used you can make assertions about the access using the normal
+mock methods and attributes:
+
+.. doctest::
+
+ >>> mock.__getitem__.call_args_list
+ [call('a'), call('c'), call('d'), call('b'), call('d')]
+ >>> mock.__setitem__.call_args_list
+ [call('b', 'fish'), call('d', 'eggs')]
+ >>> my_dict
+ {'a': 1, 'c': 3, 'b': 'fish', 'd': 'eggs'}
+
+
+Mock subclasses and their attributes
+====================================
+
+There are various reasons why you might want to subclass `Mock`. One reason
+might be to add helper methods. Here's a silly example:
+
+.. doctest::
+
+ >>> class MyMock(MagicMock):
+ ... def has_been_called(self):
+ ... return self.called
+ ...
+ >>> mymock = MyMock(return_value=None)
+ >>> mymock
+ <MyMock id='...'>
+ >>> mymock.has_been_called()
+ False
+ >>> mymock()
+ >>> mymock.has_been_called()
+ True
+
+The standard behaviour for `Mock` instances is that attributes and the return
+value mocks are of the same type as the mock they are accessed on. This ensures
+that `Mock` attributes are `Mocks` and `MagicMock` attributes are `MagicMocks`
+[#]_. So if you're subclassing to add helper methods then they'll also be
+available on the attributes and return value mock of instances of your
+subclass.
+
+.. doctest::
+
+ >>> mymock.foo
+ <MyMock name='mock.foo' id='...'>
+ >>> mymock.foo.has_been_called()
+ False
+ >>> mymock.foo()
+ <MyMock name='mock.foo()' id='...'>
+ >>> mymock.foo.has_been_called()
+ True
+
+Sometimes this is inconvenient. For example, `one user
+<https://code.google.com/p/mock/issues/detail?id=105>`_ is subclassing mock to
+created a `Twisted adaptor
+<http://twistedmatrix.com/documents/11.0.0/api/twisted.python.components.html>`_.
+Having this applied to attributes too actually causes errors.
+
+`Mock` (in all its flavours) uses a method called `_get_child_mock` to create
+these "sub-mocks" for attributes and return values. You can prevent your
+subclass being used for attributes by overriding this method. The signature is
+that it takes arbitrary keyword arguments (`**kwargs`) which are then passed
+onto the mock constructor:
+
+.. doctest::
+
+ >>> class Subclass(MagicMock):
+ ... def _get_child_mock(self, **kwargs):
+ ... return MagicMock(**kwargs)
+ ...
+ >>> mymock = Subclass()
+ >>> mymock.foo
+ <MagicMock name='mock.foo' id='...'>
+ >>> assert isinstance(mymock, Subclass)
+ >>> assert not isinstance(mymock.foo, Subclass)
+ >>> assert not isinstance(mymock(), Subclass)
+
+.. [#] An exception to this rule are the non-callable mocks. Attributes use the
+ callable variant because otherwise non-callable mocks couldn't have callable
+ methods.
+
+
+Mocking imports with patch.dict
+===============================
+
+One situation where mocking can be hard is where you have a local import inside
+a function. These are harder to mock because they aren't using an object from
+the module namespace that we can patch out.
+
+Generally local imports are to be avoided. They are sometimes done to prevent
+circular dependencies, for which there is *usually* a much better way to solve
+the problem (refactor the code) or to prevent "up front costs" by delaying the
+import. This can also be solved in better ways than an unconditional local
+import (store the module as a class or module attribute and only do the import
+on first use).
+
+That aside there is a way to use `mock` to affect the results of an import.
+Importing fetches an *object* from the `sys.modules` dictionary. Note that it
+fetches an *object*, which need not be a module. Importing a module for the
+first time results in a module object being put in `sys.modules`, so usually
+when you import something you get a module back. This need not be the case
+however.
+
+This means you can use :func:`patch.dict` to *temporarily* put a mock in place
+in `sys.modules`. Any imports whilst this patch is active will fetch the mock.
+When the patch is complete (the decorated function exits, the with statement
+body is complete or `patcher.stop()` is called) then whatever was there
+previously will be restored safely.
+
+Here's an example that mocks out the 'fooble' module.
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> with patch.dict('sys.modules', {'fooble': mock}):
+ ... import fooble
+ ... fooble.blob()
+ ...
+ <Mock name='mock.blob()' id='...'>
+ >>> assert 'fooble' not in sys.modules
+ >>> mock.blob.assert_called_once_with()
+
+As you can see the `import fooble` succeeds, but on exit there is no 'fooble'
+left in `sys.modules`.
+
+This also works for the `from module import name` form:
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> with patch.dict('sys.modules', {'fooble': mock}):
+ ... from fooble import blob
+ ... blob.blip()
+ ...
+ <Mock name='mock.blob.blip()' id='...'>
+ >>> mock.blob.blip.assert_called_once_with()
+
+With slightly more work you can also mock package imports:
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> modules = {'package': mock, 'package.module': mock.module}
+ >>> with patch.dict('sys.modules', modules):
+ ... from package.module import fooble
+ ... fooble()
+ ...
+ <Mock name='mock.module.fooble()' id='...'>
+ >>> mock.module.fooble.assert_called_once_with()
+
+
+Tracking order of calls and less verbose call assertions
+========================================================
+
+The :class:`Mock` class allows you to track the *order* of method calls on
+your mock objects through the :attr:`~Mock.method_calls` attribute. This
+doesn't allow you to track the order of calls between separate mock objects,
+however we can use :attr:`~Mock.mock_calls` to achieve the same effect.
+
+Because mocks track calls to child mocks in `mock_calls`, and accessing an
+arbitrary attribute of a mock creates a child mock, we can create our separate
+mocks from a parent one. Calls to those child mock will then all be recorded,
+in order, in the `mock_calls` of the parent:
+
+.. doctest::
+
+ >>> manager = Mock()
+ >>> mock_foo = manager.foo
+ >>> mock_bar = manager.bar
+
+ >>> mock_foo.something()
+ <Mock name='mock.foo.something()' id='...'>
+ >>> mock_bar.other.thing()
+ <Mock name='mock.bar.other.thing()' id='...'>
+
+ >>> manager.mock_calls
+ [call.foo.something(), call.bar.other.thing()]
+
+We can then assert about the calls, including the order, by comparing with
+the `mock_calls` attribute on the manager mock:
+
+.. doctest::
+
+ >>> expected_calls = [call.foo.something(), call.bar.other.thing()]
+ >>> manager.mock_calls == expected_calls
+ True
+
+If `patch` is creating, and putting in place, your mocks then you can attach
+them to a manager mock using the :meth:`~Mock.attach_mock` method. After
+attaching calls will be recorded in `mock_calls` of the manager.
+
+.. doctest::
+
+ >>> manager = MagicMock()
+ >>> with patch('mymodule.Class1') as MockClass1:
+ ... with patch('mymodule.Class2') as MockClass2:
+ ... manager.attach_mock(MockClass1, 'MockClass1')
+ ... manager.attach_mock(MockClass2, 'MockClass2')
+ ... MockClass1().foo()
+ ... MockClass2().bar()
+ ...
+ <MagicMock name='mock.MockClass1().foo()' id='...'>
+ <MagicMock name='mock.MockClass2().bar()' id='...'>
+ >>> manager.mock_calls
+ [call.MockClass1(),
+ call.MockClass1().foo(),
+ call.MockClass2(),
+ call.MockClass2().bar()]
+
+If many calls have been made, but you're only interested in a particular
+sequence of them then an alternative is to use the
+:meth:`~Mock.assert_has_calls` method. This takes a list of calls (constructed
+with the :data:`call` object). If that sequence of calls are in
+:attr:`~Mock.mock_calls` then the assert succeeds.
+
+.. doctest::
+
+ >>> m = MagicMock()
+ >>> m().foo().bar().baz()
+ <MagicMock name='mock().foo().bar().baz()' id='...'>
+ >>> m.one().two().three()
+ <MagicMock name='mock.one().two().three()' id='...'>
+ >>> calls = call.one().two().three().call_list()
+ >>> m.assert_has_calls(calls)
+
+Even though the chained call `m.one().two().three()` aren't the only calls that
+have been made to the mock, the assert still succeeds.
+
+Sometimes a mock may have several calls made to it, and you are only interested
+in asserting about *some* of those calls. You may not even care about the
+order. In this case you can pass `any_order=True` to `assert_has_calls`:
+
+.. doctest::
+
+ >>> m = MagicMock()
+ >>> m(1), m.two(2, 3), m.seven(7), m.fifty('50')
+ (...)
+ >>> calls = [call.fifty('50'), call(1), call.seven(7)]
+ >>> m.assert_has_calls(calls, any_order=True)
+
+
+More complex argument matching
+==============================
+
+Using the same basic concept as `ANY` we can implement matchers to do more
+complex assertions on objects used as arguments to mocks.
+
+Suppose we expect some object to be passed to a mock that by default
+compares equal based on object identity (which is the Python default for user
+defined classes). To use :meth:`~Mock.assert_called_with` we would need to pass
+in the exact same object. If we are only interested in some of the attributes
+of this object then we can create a matcher that will check these attributes
+for us.
+
+You can see in this example how a 'standard' call to `assert_called_with` isn't
+sufficient:
+
+.. doctest::
+
+ >>> class Foo(object):
+ ... def __init__(self, a, b):
+ ... self.a, self.b = a, b
+ ...
+ >>> mock = Mock(return_value=None)
+ >>> mock(Foo(1, 2))
+ >>> mock.assert_called_with(Foo(1, 2))
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected: call(<__main__.Foo object at 0x...>)
+ Actual call: call(<__main__.Foo object at 0x...>)
+
+A comparison function for our `Foo` class might look something like this:
+
+.. doctest::
+
+ >>> def compare(self, other):
+ ... if not type(self) == type(other):
+ ... return False
+ ... if self.a != other.a:
+ ... return False
+ ... if self.b != other.b:
+ ... return False
+ ... return True
+ ...
+
+And a matcher object that can use comparison functions like this for its
+equality operation would look something like this:
+
+.. doctest::
+
+ >>> class Matcher(object):
+ ... def __init__(self, compare, some_obj):
+ ... self.compare = compare
+ ... self.some_obj = some_obj
+ ... def __eq__(self, other):
+ ... return self.compare(self.some_obj, other)
+ ...
+
+Putting all this together:
+
+.. doctest::
+
+ >>> match_foo = Matcher(compare, Foo(1, 2))
+ >>> mock.assert_called_with(match_foo)
+
+The `Matcher` is instantiated with our compare function and the `Foo` object
+we want to compare against. In `assert_called_with` the `Matcher` equality
+method will be called, which compares the object the mock was called with
+against the one we created our matcher with. If they match then
+`assert_called_with` passes, and if they don't an `AssertionError` is raised:
+
+.. doctest::
+
+ >>> match_wrong = Matcher(compare, Foo(3, 4))
+ >>> mock.assert_called_with(match_wrong)
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected: ((<Matcher object at 0x...>,), {})
+ Called with: ((<Foo object at 0x...>,), {})
+
+With a bit of tweaking you could have the comparison function raise the
+`AssertionError` directly and provide a more useful failure message.
+
+As of version 1.5, the Python testing library `PyHamcrest
+<http://pypi.python.org/pypi/PyHamcrest>`_ provides similar functionality,
+that may be useful here, in the form of its equality matcher
+(`hamcrest.library.integration.match_equality
+<http://packages.python.org/PyHamcrest/integration.html#hamcrest.library.integration.match_equality>`_).
+
+
+Less verbose configuration of mock objects
+==========================================
+
+This recipe, for easier configuration of mock objects, is now part of `Mock`.
+See the :meth:`~Mock.configure_mock` method.
+
+
+Matching any argument in assertions
+===================================
+
+This example is now built in to mock. See :data:`ANY`.
+
+
+Mocking Properties
+==================
+
+This example is now built in to mock. See :class:`PropertyMock`.
+
+
+Mocking open
+============
+
+This example is now built in to mock. See :func:`mock_open`.
+
+
+Mocks without some attributes
+=============================
+
+This example is now built in to mock. See :ref:`deleting-attributes`.
diff --git a/slider-agent/src/test/python/mock/docs/getting-started.txt b/slider-agent/src/test/python/mock/docs/getting-started.txt
new file mode 100644
index 0000000..1b5d289
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/getting-started.txt
@@ -0,0 +1,479 @@
+===========================
+ Getting Started with Mock
+===========================
+
+.. _getting-started:
+
+.. index:: Getting Started
+
+.. testsetup::
+
+ class SomeClass(object):
+ static_method = None
+ class_method = None
+ attribute = None
+
+ sys.modules['package'] = package = Mock(name='package')
+ sys.modules['package.module'] = module = package.module
+ sys.modules['module'] = package.module
+
+
+Using Mock
+==========
+
+Mock Patching Methods
+---------------------
+
+Common uses for :class:`Mock` objects include:
+
+* Patching methods
+* Recording method calls on objects
+
+You might want to replace a method on an object to check that
+it is called with the correct arguments by another part of the system:
+
+.. doctest::
+
+ >>> real = SomeClass()
+ >>> real.method = MagicMock(name='method')
+ >>> real.method(3, 4, 5, key='value')
+ <MagicMock name='method()' id='...'>
+
+Once our mock has been used (`real.method` in this example) it has methods
+and attributes that allow you to make assertions about how it has been used.
+
+.. note::
+
+ In most of these examples the :class:`Mock` and :class:`MagicMock` classes
+ are interchangeable. As the `MagicMock` is the more capable class it makes
+ a sensible one to use by default.
+
+Once the mock has been called its :attr:`~Mock.called` attribute is set to
+`True`. More importantly we can use the :meth:`~Mock.assert_called_with` or
+:meth:`~Mock.assert_called_once_with` method to check that it was called with
+the correct arguments.
+
+This example tests that calling `ProductionClass().method` results in a call to
+the `something` method:
+
+.. doctest::
+
+ >>> from mock import MagicMock
+ >>> class ProductionClass(object):
+ ... def method(self):
+ ... self.something(1, 2, 3)
+ ... def something(self, a, b, c):
+ ... pass
+ ...
+ >>> real = ProductionClass()
+ >>> real.something = MagicMock()
+ >>> real.method()
+ >>> real.something.assert_called_once_with(1, 2, 3)
+
+
+
+Mock for Method Calls on an Object
+----------------------------------
+
+In the last example we patched a method directly on an object to check that it
+was called correctly. Another common use case is to pass an object into a
+method (or some part of the system under test) and then check that it is used
+in the correct way.
+
+The simple `ProductionClass` below has a `closer` method. If it is called with
+an object then it calls `close` on it.
+
+.. doctest::
+
+ >>> class ProductionClass(object):
+ ... def closer(self, something):
+ ... something.close()
+ ...
+
+So to test it we need to pass in an object with a `close` method and check
+that it was called correctly.
+
+.. doctest::
+
+ >>> real = ProductionClass()
+ >>> mock = Mock()
+ >>> real.closer(mock)
+ >>> mock.close.assert_called_with()
+
+We don't have to do any work to provide the 'close' method on our mock.
+Accessing close creates it. So, if 'close' hasn't already been called then
+accessing it in the test will create it, but :meth:`~Mock.assert_called_with`
+will raise a failure exception.
+
+
+Mocking Classes
+---------------
+
+A common use case is to mock out classes instantiated by your code under test.
+When you patch a class, then that class is replaced with a mock. Instances
+are created by *calling the class*. This means you access the "mock instance"
+by looking at the return value of the mocked class.
+
+In the example below we have a function `some_function` that instantiates `Foo`
+and calls a method on it. The call to `patch` replaces the class `Foo` with a
+mock. The `Foo` instance is the result of calling the mock, so it is configured
+by modifying the mock :attr:`~Mock.return_value`.
+
+.. doctest::
+
+ >>> def some_function():
+ ... instance = module.Foo()
+ ... return instance.method()
+ ...
+ >>> with patch('module.Foo') as mock:
+ ... instance = mock.return_value
+ ... instance.method.return_value = 'the result'
+ ... result = some_function()
+ ... assert result == 'the result'
+
+
+Naming your mocks
+-----------------
+
+It can be useful to give your mocks a name. The name is shown in the repr of
+the mock and can be helpful when the mock appears in test failure messages. The
+name is also propagated to attributes or methods of the mock:
+
+.. doctest::
+
+ >>> mock = MagicMock(name='foo')
+ >>> mock
+ <MagicMock name='foo' id='...'>
+ >>> mock.method
+ <MagicMock name='foo.method' id='...'>
+
+
+Tracking all Calls
+------------------
+
+Often you want to track more than a single call to a method. The
+:attr:`~Mock.mock_calls` attribute records all calls
+to child attributes of the mock - and also to their children.
+
+.. doctest::
+
+ >>> mock = MagicMock()
+ >>> mock.method()
+ <MagicMock name='mock.method()' id='...'>
+ >>> mock.attribute.method(10, x=53)
+ <MagicMock name='mock.attribute.method()' id='...'>
+ >>> mock.mock_calls
+ [call.method(), call.attribute.method(10, x=53)]
+
+If you make an assertion about `mock_calls` and any unexpected methods
+have been called, then the assertion will fail. This is useful because as well
+as asserting that the calls you expected have been made, you are also checking
+that they were made in the right order and with no additional calls:
+
+You use the :data:`call` object to construct lists for comparing with
+`mock_calls`:
+
+.. doctest::
+
+ >>> expected = [call.method(), call.attribute.method(10, x=53)]
+ >>> mock.mock_calls == expected
+ True
+
+
+Setting Return Values and Attributes
+------------------------------------
+
+Setting the return values on a mock object is trivially easy:
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> mock.return_value = 3
+ >>> mock()
+ 3
+
+Of course you can do the same for methods on the mock:
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> mock.method.return_value = 3
+ >>> mock.method()
+ 3
+
+The return value can also be set in the constructor:
+
+.. doctest::
+
+ >>> mock = Mock(return_value=3)
+ >>> mock()
+ 3
+
+If you need an attribute setting on your mock, just do it:
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> mock.x = 3
+ >>> mock.x
+ 3
+
+Sometimes you want to mock up a more complex situation, like for example
+`mock.connection.cursor().execute("SELECT 1")`. If we wanted this call to
+return a list, then we have to configure the result of the nested call.
+
+We can use :data:`call` to construct the set of calls in a "chained call" like
+this for easy assertion afterwards:
+
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> cursor = mock.connection.cursor.return_value
+ >>> cursor.execute.return_value = ['foo']
+ >>> mock.connection.cursor().execute("SELECT 1")
+ ['foo']
+ >>> expected = call.connection.cursor().execute("SELECT 1").call_list()
+ >>> mock.mock_calls
+ [call.connection.cursor(), call.connection.cursor().execute('SELECT 1')]
+ >>> mock.mock_calls == expected
+ True
+
+It is the call to `.call_list()` that turns our call object into a list of
+calls representing the chained calls.
+
+
+
+Raising exceptions with mocks
+-----------------------------
+
+A useful attribute is :attr:`~Mock.side_effect`. If you set this to an
+exception class or instance then the exception will be raised when the mock
+is called.
+
+.. doctest::
+
+ >>> mock = Mock(side_effect=Exception('Boom!'))
+ >>> mock()
+ Traceback (most recent call last):
+ ...
+ Exception: Boom!
+
+
+Side effect functions and iterables
+-----------------------------------
+
+`side_effect` can also be set to a function or an iterable. The use case for
+`side_effect` as an iterable is where your mock is going to be called several
+times, and you want each call to return a different value. When you set
+`side_effect` to an iterable every call to the mock returns the next value
+from the iterable:
+
+.. doctest::
+
+ >>> mock = MagicMock(side_effect=[4, 5, 6])
+ >>> mock()
+ 4
+ >>> mock()
+ 5
+ >>> mock()
+ 6
+
+
+For more advanced use cases, like dynamically varying the return values
+depending on what the mock is called with, `side_effect` can be a function.
+The function will be called with the same arguments as the mock. Whatever the
+function returns is what the call returns:
+
+.. doctest::
+
+ >>> vals = {(1, 2): 1, (2, 3): 2}
+ >>> def side_effect(*args):
+ ... return vals[args]
+ ...
+ >>> mock = MagicMock(side_effect=side_effect)
+ >>> mock(1, 2)
+ 1
+ >>> mock(2, 3)
+ 2
+
+
+Creating a Mock from an Existing Object
+---------------------------------------
+
+One problem with over use of mocking is that it couples your tests to the
+implementation of your mocks rather than your real code. Suppose you have a
+class that implements `some_method`. In a test for another class, you
+provide a mock of this object that *also* provides `some_method`. If later
+you refactor the first class, so that it no longer has `some_method` - then
+your tests will continue to pass even though your code is now broken!
+
+`Mock` allows you to provide an object as a specification for the mock,
+using the `spec` keyword argument. Accessing methods / attributes on the
+mock that don't exist on your specification object will immediately raise an
+attribute error. If you change the implementation of your specification, then
+tests that use that class will start failing immediately without you having to
+instantiate the class in those tests.
+
+.. doctest::
+
+ >>> mock = Mock(spec=SomeClass)
+ >>> mock.old_method()
+ Traceback (most recent call last):
+ ...
+ AttributeError: object has no attribute 'old_method'
+
+If you want a stronger form of specification that prevents the setting
+of arbitrary attributes as well as the getting of them then you can use
+`spec_set` instead of `spec`.
+
+
+
+Patch Decorators
+================
+
+.. note::
+
+ With `patch` it matters that you patch objects in the namespace where they
+ are looked up. This is normally straightforward, but for a quick guide
+ read :ref:`where to patch <where-to-patch>`.
+
+
+A common need in tests is to patch a class attribute or a module attribute,
+for example patching a builtin or patching a class in a module to test that it
+is instantiated. Modules and classes are effectively global, so patching on
+them has to be undone after the test or the patch will persist into other
+tests and cause hard to diagnose problems.
+
+mock provides three convenient decorators for this: `patch`, `patch.object` and
+`patch.dict`. `patch` takes a single string, of the form
+`package.module.Class.attribute` to specify the attribute you are patching. It
+also optionally takes a value that you want the attribute (or class or
+whatever) to be replaced with. 'patch.object' takes an object and the name of
+the attribute you would like patched, plus optionally the value to patch it
+with.
+
+`patch.object`:
+
+.. doctest::
+
+ >>> original = SomeClass.attribute
+ >>> @patch.object(SomeClass, 'attribute', sentinel.attribute)
+ ... def test():
+ ... assert SomeClass.attribute == sentinel.attribute
+ ...
+ >>> test()
+ >>> assert SomeClass.attribute == original
+
+ >>> @patch('package.module.attribute', sentinel.attribute)
+ ... def test():
+ ... from package.module import attribute
+ ... assert attribute is sentinel.attribute
+ ...
+ >>> test()
+
+If you are patching a module (including `__builtin__`) then use `patch`
+instead of `patch.object`:
+
+.. doctest::
+
+ >>> mock = MagicMock(return_value = sentinel.file_handle)
+ >>> with patch('__builtin__.open', mock):
+ ... handle = open('filename', 'r')
+ ...
+ >>> mock.assert_called_with('filename', 'r')
+ >>> assert handle == sentinel.file_handle, "incorrect file handle returned"
+
+The module name can be 'dotted', in the form `package.module` if needed:
+
+.. doctest::
+
+ >>> @patch('package.module.ClassName.attribute', sentinel.attribute)
+ ... def test():
+ ... from package.module import ClassName
+ ... assert ClassName.attribute == sentinel.attribute
+ ...
+ >>> test()
+
+A nice pattern is to actually decorate test methods themselves:
+
+.. doctest::
+
+ >>> class MyTest(unittest2.TestCase):
+ ... @patch.object(SomeClass, 'attribute', sentinel.attribute)
+ ... def test_something(self):
+ ... self.assertEqual(SomeClass.attribute, sentinel.attribute)
+ ...
+ >>> original = SomeClass.attribute
+ >>> MyTest('test_something').test_something()
+ >>> assert SomeClass.attribute == original
+
+If you want to patch with a Mock, you can use `patch` with only one argument
+(or `patch.object` with two arguments). The mock will be created for you and
+passed into the test function / method:
+
+.. doctest::
+
+ >>> class MyTest(unittest2.TestCase):
+ ... @patch.object(SomeClass, 'static_method')
+ ... def test_something(self, mock_method):
+ ... SomeClass.static_method()
+ ... mock_method.assert_called_with()
+ ...
+ >>> MyTest('test_something').test_something()
+
+You can stack up multiple patch decorators using this pattern:
+
+.. doctest::
+
+ >>> class MyTest(unittest2.TestCase):
+ ... @patch('package.module.ClassName1')
+ ... @patch('package.module.ClassName2')
+ ... def test_something(self, MockClass2, MockClass1):
+ ... self.assertTrue(package.module.ClassName1 is MockClass1)
+ ... self.assertTrue(package.module.ClassName2 is MockClass2)
+ ...
+ >>> MyTest('test_something').test_something()
+
+When you nest patch decorators the mocks are passed in to the decorated
+function in the same order they applied (the normal *python* order that
+decorators are applied). This means from the bottom up, so in the example
+above the mock for `test_module.ClassName2` is passed in first.
+
+There is also :func:`patch.dict` for setting values in a dictionary just
+during a scope and restoring the dictionary to its original state when the test
+ends:
+
+.. doctest::
+
+ >>> foo = {'key': 'value'}
+ >>> original = foo.copy()
+ >>> with patch.dict(foo, {'newkey': 'newvalue'}, clear=True):
+ ... assert foo == {'newkey': 'newvalue'}
+ ...
+ >>> assert foo == original
+
+`patch`, `patch.object` and `patch.dict` can all be used as context managers.
+
+Where you use `patch` to create a mock for you, you can get a reference to the
+mock using the "as" form of the with statement:
+
+.. doctest::
+
+ >>> class ProductionClass(object):
+ ... def method(self):
+ ... pass
+ ...
+ >>> with patch.object(ProductionClass, 'method') as mock_method:
+ ... mock_method.return_value = None
+ ... real = ProductionClass()
+ ... real.method(1, 2, 3)
+ ...
+ >>> mock_method.assert_called_with(1, 2, 3)
+
+
+As an alternative `patch`, `patch.object` and `patch.dict` can be used as
+class decorators. When used in this way it is the same as applying the
+decorator indvidually to every method whose name starts with "test".
+
+For some more advanced examples, see the :ref:`further-examples` page.
diff --git a/slider-agent/src/test/python/mock/docs/helpers.txt b/slider-agent/src/test/python/mock/docs/helpers.txt
new file mode 100644
index 0000000..571b71d
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/helpers.txt
@@ -0,0 +1,583 @@
+=========
+ Helpers
+=========
+
+.. currentmodule:: mock
+
+.. testsetup::
+
+ mock.FILTER_DIR = True
+ from pprint import pprint as pp
+ original_dir = dir
+ def dir(obj):
+ print pp(original_dir(obj))
+
+ import urllib2
+ __main__.urllib2 = urllib2
+
+.. testcleanup::
+
+ dir = original_dir
+ mock.FILTER_DIR = True
+
+
+
+call
+====
+
+.. function:: call(*args, **kwargs)
+
+ `call` is a helper object for making simpler assertions, for comparing
+ with :attr:`~Mock.call_args`, :attr:`~Mock.call_args_list`,
+ :attr:`~Mock.mock_calls` and :attr: `~Mock.method_calls`. `call` can also be
+ used with :meth:`~Mock.assert_has_calls`.
+
+ .. doctest::
+
+ >>> m = MagicMock(return_value=None)
+ >>> m(1, 2, a='foo', b='bar')
+ >>> m()
+ >>> m.call_args_list == [call(1, 2, a='foo', b='bar'), call()]
+ True
+
+.. method:: call.call_list()
+
+ For a call object that represents multiple calls, `call_list`
+ returns a list of all the intermediate calls as well as the
+ final call.
+
+`call_list` is particularly useful for making assertions on "chained calls". A
+chained call is multiple calls on a single line of code. This results in
+multiple entries in :attr:`~Mock.mock_calls` on a mock. Manually constructing
+the sequence of calls can be tedious.
+
+:meth:`~call.call_list` can construct the sequence of calls from the same
+chained call:
+
+.. doctest::
+
+ >>> m = MagicMock()
+ >>> m(1).method(arg='foo').other('bar')(2.0)
+ <MagicMock name='mock().method().other()()' id='...'>
+ >>> kall = call(1).method(arg='foo').other('bar')(2.0)
+ >>> kall.call_list()
+ [call(1),
+ call().method(arg='foo'),
+ call().method().other('bar'),
+ call().method().other()(2.0)]
+ >>> m.mock_calls == kall.call_list()
+ True
+
+.. _calls-as-tuples:
+
+A `call` object is either a tuple of (positional args, keyword args) or
+(name, positional args, keyword args) depending on how it was constructed. When
+you construct them yourself this isn't particularly interesting, but the `call`
+objects that are in the :attr:`Mock.call_args`, :attr:`Mock.call_args_list` and
+:attr:`Mock.mock_calls` attributes can be introspected to get at the individual
+arguments they contain.
+
+The `call` objects in :attr:`Mock.call_args` and :attr:`Mock.call_args_list`
+are two-tuples of (positional args, keyword args) whereas the `call` objects
+in :attr:`Mock.mock_calls`, along with ones you construct yourself, are
+three-tuples of (name, positional args, keyword args).
+
+You can use their "tupleness" to pull out the individual arguments for more
+complex introspection and assertions. The positional arguments are a tuple
+(an empty tuple if there are no positional arguments) and the keyword
+arguments are a dictionary:
+
+.. doctest::
+
+ >>> m = MagicMock(return_value=None)
+ >>> m(1, 2, 3, arg='one', arg2='two')
+ >>> kall = m.call_args
+ >>> args, kwargs = kall
+ >>> args
+ (1, 2, 3)
+ >>> kwargs
+ {'arg2': 'two', 'arg': 'one'}
+ >>> args is kall[0]
+ True
+ >>> kwargs is kall[1]
+ True
+
+ >>> m = MagicMock()
+ >>> m.foo(4, 5, 6, arg='two', arg2='three')
+ <MagicMock name='mock.foo()' id='...'>
+ >>> kall = m.mock_calls[0]
+ >>> name, args, kwargs = kall
+ >>> name
+ 'foo'
+ >>> args
+ (4, 5, 6)
+ >>> kwargs
+ {'arg2': 'three', 'arg': 'two'}
+ >>> name is m.mock_calls[0][0]
+ True
+
+
+create_autospec
+===============
+
+.. function:: create_autospec(spec, spec_set=False, instance=False, **kwargs)
+
+ Create a mock object using another object as a spec. Attributes on the
+ mock will use the corresponding attribute on the `spec` object as their
+ spec.
+
+ Functions or methods being mocked will have their arguments checked to
+ ensure that they are called with the correct signature.
+
+ If `spec_set` is `True` then attempting to set attributes that don't exist
+ on the spec object will raise an `AttributeError`.
+
+ If a class is used as a spec then the return value of the mock (the
+ instance of the class) will have the same spec. You can use a class as the
+ spec for an instance object by passing `instance=True`. The returned mock
+ will only be callable if instances of the mock are callable.
+
+ `create_autospec` also takes arbitrary keyword arguments that are passed to
+ the constructor of the created mock.
+
+See :ref:`auto-speccing` for examples of how to use auto-speccing with
+`create_autospec` and the `autospec` argument to :func:`patch`.
+
+
+ANY
+===
+
+.. data:: ANY
+
+Sometimes you may need to make assertions about *some* of the arguments in a
+call to mock, but either not care about some of the arguments or want to pull
+them individually out of :attr:`~Mock.call_args` and make more complex
+assertions on them.
+
+To ignore certain arguments you can pass in objects that compare equal to
+*everything*. Calls to :meth:`~Mock.assert_called_with` and
+:meth:`~Mock.assert_called_once_with` will then succeed no matter what was
+passed in.
+
+.. doctest::
+
+ >>> mock = Mock(return_value=None)
+ >>> mock('foo', bar=object())
+ >>> mock.assert_called_once_with('foo', bar=ANY)
+
+`ANY` can also be used in comparisons with call lists like
+:attr:`~Mock.mock_calls`:
+
+.. doctest::
+
+ >>> m = MagicMock(return_value=None)
+ >>> m(1)
+ >>> m(1, 2)
+ >>> m(object())
+ >>> m.mock_calls == [call(1), call(1, 2), ANY]
+ True
+
+
+
+FILTER_DIR
+==========
+
+.. data:: FILTER_DIR
+
+`FILTER_DIR` is a module level variable that controls the way mock objects
+respond to `dir` (only for Python 2.6 or more recent). The default is `True`,
+which uses the filtering described below, to only show useful members. If you
+dislike this filtering, or need to switch it off for diagnostic purposes, then
+set `mock.FILTER_DIR = False`.
+
+With filtering on, `dir(some_mock)` shows only useful attributes and will
+include any dynamically created attributes that wouldn't normally be shown.
+If the mock was created with a `spec` (or `autospec` of course) then all the
+attributes from the original are shown, even if they haven't been accessed
+yet:
+
+.. doctest::
+
+ >>> dir(Mock())
+ ['assert_any_call',
+ 'assert_called_once_with',
+ 'assert_called_with',
+ 'assert_has_calls',
+ 'attach_mock',
+ ...
+ >>> import urllib2
+ >>> dir(Mock(spec=urllib2))
+ ['AbstractBasicAuthHandler',
+ 'AbstractDigestAuthHandler',
+ 'AbstractHTTPHandler',
+ 'BaseHandler',
+ ...
+
+Many of the not-very-useful (private to `Mock` rather than the thing being
+mocked) underscore and double underscore prefixed attributes have been
+filtered from the result of calling `dir` on a `Mock`. If you dislike this
+behaviour you can switch it off by setting the module level switch
+`FILTER_DIR`:
+
+.. doctest::
+
+ >>> import mock
+ >>> mock.FILTER_DIR = False
+ >>> dir(mock.Mock())
+ ['_NonCallableMock__get_return_value',
+ '_NonCallableMock__get_side_effect',
+ '_NonCallableMock__return_value_doc',
+ '_NonCallableMock__set_return_value',
+ '_NonCallableMock__set_side_effect',
+ '__call__',
+ '__class__',
+ ...
+
+Alternatively you can just use `vars(my_mock)` (instance members) and
+`dir(type(my_mock))` (type members) to bypass the filtering irrespective of
+`mock.FILTER_DIR`.
+
+
+mock_open
+=========
+
+.. function:: mock_open(mock=None, read_data=None)
+
+ A helper function to create a mock to replace the use of `open`. It works
+ for `open` called directly or used as a context manager.
+
+ The `mock` argument is the mock object to configure. If `None` (the
+ default) then a `MagicMock` will be created for you, with the API limited
+ to methods or attributes available on standard file handles.
+
+ `read_data` is a string for the `read` method of the file handle to return.
+ This is an empty string by default.
+
+Using `open` as a context manager is a great way to ensure your file handles
+are closed properly and is becoming common::
+
+ with open('/some/path', 'w') as f:
+ f.write('something')
+
+The issue is that even if you mock out the call to `open` it is the
+*returned object* that is used as a context manager (and has `__enter__` and
+`__exit__` called).
+
+Mocking context managers with a :class:`MagicMock` is common enough and fiddly
+enough that a helper function is useful.
+
+.. doctest::
+
+ >>> from mock import mock_open
+ >>> m = mock_open()
+ >>> with patch('__main__.open', m, create=True):
+ ... with open('foo', 'w') as h:
+ ... h.write('some stuff')
+ ...
+ >>> m.mock_calls
+ [call('foo', 'w'),
+ call().__enter__(),
+ call().write('some stuff'),
+ call().__exit__(None, None, None)]
+ >>> m.assert_called_once_with('foo', 'w')
+ >>> handle = m()
+ >>> handle.write.assert_called_once_with('some stuff')
+
+And for reading files:
+
+.. doctest::
+
+ >>> with patch('__main__.open', mock_open(read_data='bibble'), create=True) as m:
+ ... with open('foo') as h:
+ ... result = h.read()
+ ...
+ >>> m.assert_called_once_with('foo')
+ >>> assert result == 'bibble'
+
+
+.. _auto-speccing:
+
+Autospeccing
+============
+
+Autospeccing is based on the existing `spec` feature of mock. It limits the
+api of mocks to the api of an original object (the spec), but it is recursive
+(implemented lazily) so that attributes of mocks only have the same api as
+the attributes of the spec. In addition mocked functions / methods have the
+same call signature as the original so they raise a `TypeError` if they are
+called incorrectly.
+
+Before I explain how auto-speccing works, here's why it is needed.
+
+`Mock` is a very powerful and flexible object, but it suffers from two flaws
+when used to mock out objects from a system under test. One of these flaws is
+specific to the `Mock` api and the other is a more general problem with using
+mock objects.
+
+First the problem specific to `Mock`. `Mock` has two assert methods that are
+extremely handy: :meth:`~Mock.assert_called_with` and
+:meth:`~Mock.assert_called_once_with`.
+
+.. doctest::
+
+ >>> mock = Mock(name='Thing', return_value=None)
+ >>> mock(1, 2, 3)
+ >>> mock.assert_called_once_with(1, 2, 3)
+ >>> mock(1, 2, 3)
+ >>> mock.assert_called_once_with(1, 2, 3)
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected to be called once. Called 2 times.
+
+Because mocks auto-create attributes on demand, and allow you to call them
+with arbitrary arguments, if you misspell one of these assert methods then
+your assertion is gone:
+
+.. code-block:: pycon
+
+ >>> mock = Mock(name='Thing', return_value=None)
+ >>> mock(1, 2, 3)
+ >>> mock.assret_called_once_with(4, 5, 6)
+
+Your tests can pass silently and incorrectly because of the typo.
+
+The second issue is more general to mocking. If you refactor some of your
+code, rename members and so on, any tests for code that is still using the
+*old api* but uses mocks instead of the real objects will still pass. This
+means your tests can all pass even though your code is broken.
+
+Note that this is another reason why you need integration tests as well as
+unit tests. Testing everything in isolation is all fine and dandy, but if you
+don't test how your units are "wired together" there is still lots of room
+for bugs that tests might have caught.
+
+`mock` already provides a feature to help with this, called speccing. If you
+use a class or instance as the `spec` for a mock then you can only access
+attributes on the mock that exist on the real class:
+
+.. doctest::
+
+ >>> import urllib2
+ >>> mock = Mock(spec=urllib2.Request)
+ >>> mock.assret_called_with
+ Traceback (most recent call last):
+ ...
+ AttributeError: Mock object has no attribute 'assret_called_with'
+
+The spec only applies to the mock itself, so we still have the same issue
+with any methods on the mock:
+
+.. code-block:: pycon
+
+ >>> mock.has_data()
+ <mock.Mock object at 0x...>
+ >>> mock.has_data.assret_called_with()
+
+Auto-speccing solves this problem. You can either pass `autospec=True` to
+`patch` / `patch.object` or use the `create_autospec` function to create a
+mock with a spec. If you use the `autospec=True` argument to `patch` then the
+object that is being replaced will be used as the spec object. Because the
+speccing is done "lazily" (the spec is created as attributes on the mock are
+accessed) you can use it with very complex or deeply nested objects (like
+modules that import modules that import modules) without a big performance
+hit.
+
+Here's an example of it in use:
+
+.. doctest::
+
+ >>> import urllib2
+ >>> patcher = patch('__main__.urllib2', autospec=True)
+ >>> mock_urllib2 = patcher.start()
+ >>> urllib2 is mock_urllib2
+ True
+ >>> urllib2.Request
+ <MagicMock name='urllib2.Request' spec='Request' id='...'>
+
+You can see that `urllib2.Request` has a spec. `urllib2.Request` takes two
+arguments in the constructor (one of which is `self`). Here's what happens if
+we try to call it incorrectly:
+
+.. doctest::
+
+ >>> req = urllib2.Request()
+ Traceback (most recent call last):
+ ...
+ TypeError: <lambda>() takes at least 2 arguments (1 given)
+
+The spec also applies to instantiated classes (i.e. the return value of
+specced mocks):
+
+.. doctest::
+
+ >>> req = urllib2.Request('foo')
+ >>> req
+ <NonCallableMagicMock name='urllib2.Request()' spec='Request' id='...'>
+
+`Request` objects are not callable, so the return value of instantiating our
+mocked out `urllib2.Request` is a non-callable mock. With the spec in place
+any typos in our asserts will raise the correct error:
+
+.. doctest::
+
+ >>> req.add_header('spam', 'eggs')
+ <MagicMock name='urllib2.Request().add_header()' id='...'>
+ >>> req.add_header.assret_called_with
+ Traceback (most recent call last):
+ ...
+ AttributeError: Mock object has no attribute 'assret_called_with'
+ >>> req.add_header.assert_called_with('spam', 'eggs')
+
+In many cases you will just be able to add `autospec=True` to your existing
+`patch` calls and then be protected against bugs due to typos and api
+changes.
+
+As well as using `autospec` through `patch` there is a
+:func:`create_autospec` for creating autospecced mocks directly:
+
+.. doctest::
+
+ >>> import urllib2
+ >>> mock_urllib2 = create_autospec(urllib2)
+ >>> mock_urllib2.Request('foo', 'bar')
+ <NonCallableMagicMock name='mock.Request()' spec='Request' id='...'>
+
+This isn't without caveats and limitations however, which is why it is not
+the default behaviour. In order to know what attributes are available on the
+spec object, autospec has to introspect (access attributes) the spec. As you
+traverse attributes on the mock a corresponding traversal of the original
+object is happening under the hood. If any of your specced objects have
+properties or descriptors that can trigger code execution then you may not be
+able to use autospec. On the other hand it is much better to design your
+objects so that introspection is safe [#]_.
+
+A more serious problem is that it is common for instance attributes to be
+created in the `__init__` method and not to exist on the class at all.
+`autospec` can't know about any dynamically created attributes and restricts
+the api to visible attributes.
+
+.. doctest::
+
+ >>> class Something(object):
+ ... def __init__(self):
+ ... self.a = 33
+ ...
+ >>> with patch('__main__.Something', autospec=True):
+ ... thing = Something()
+ ... thing.a
+ ...
+ Traceback (most recent call last):
+ ...
+ AttributeError: Mock object has no attribute 'a'
+
+There are a few different ways of resolving this problem. The easiest, but
+not necessarily the least annoying, way is to simply set the required
+attributes on the mock after creation. Just because `autospec` doesn't allow
+you to fetch attributes that don't exist on the spec it doesn't prevent you
+setting them:
+
+.. doctest::
+
+ >>> with patch('__main__.Something', autospec=True):
+ ... thing = Something()
+ ... thing.a = 33
+ ...
+
+There is a more aggressive version of both `spec` and `autospec` that *does*
+prevent you setting non-existent attributes. This is useful if you want to
+ensure your code only *sets* valid attributes too, but obviously it prevents
+this particular scenario:
+
+.. doctest::
+
+ >>> with patch('__main__.Something', autospec=True, spec_set=True):
+ ... thing = Something()
+ ... thing.a = 33
+ ...
+ Traceback (most recent call last):
+ ...
+ AttributeError: Mock object has no attribute 'a'
+
+Probably the best way of solving the problem is to add class attributes as
+default values for instance members initialised in `__init__`. Note that if
+you are only setting default attributes in `__init__` then providing them via
+class attributes (shared between instances of course) is faster too. e.g.
+
+.. code-block:: python
+
+ class Something(object):
+ a = 33
+
+This brings up another issue. It is relatively common to provide a default
+value of `None` for members that will later be an object of a different type.
+`None` would be useless as a spec because it wouldn't let you access *any*
+attributes or methods on it. As `None` is *never* going to be useful as a
+spec, and probably indicates a member that will normally of some other type,
+`autospec` doesn't use a spec for members that are set to `None`. These will
+just be ordinary mocks (well - `MagicMocks`):
+
+.. doctest::
+
+ >>> class Something(object):
+ ... member = None
+ ...
+ >>> mock = create_autospec(Something)
+ >>> mock.member.foo.bar.baz()
+ <MagicMock name='mock.member.foo.bar.baz()' id='...'>
+
+If modifying your production classes to add defaults isn't to your liking
+then there are more options. One of these is simply to use an instance as the
+spec rather than the class. The other is to create a subclass of the
+production class and add the defaults to the subclass without affecting the
+production class. Both of these require you to use an alternative object as
+the spec. Thankfully `patch` supports this - you can simply pass the
+alternative object as the `autospec` argument:
+
+.. doctest::
+
+ >>> class Something(object):
+ ... def __init__(self):
+ ... self.a = 33
+ ...
+ >>> class SomethingForTest(Something):
+ ... a = 33
+ ...
+ >>> p = patch('__main__.Something', autospec=SomethingForTest)
+ >>> mock = p.start()
+ >>> mock.a
+ <NonCallableMagicMock name='Something.a' spec='int' id='...'>
+
+.. note::
+
+ An additional limitation (currently) with `autospec` is that unbound
+ methods on mocked classes *don't* take an "explicit self" as the first
+ argument - so this usage will fail with `autospec`.
+
+ .. doctest::
+
+ >>> class Foo(object):
+ ... def foo(self):
+ ... pass
+ ...
+ >>> Foo.foo(Foo())
+ >>> MockFoo = create_autospec(Foo)
+ >>> MockFoo.foo(MockFoo())
+ Traceback (most recent call last):
+ ...
+ TypeError: <lambda>() takes exactly 1 argument (2 given)
+
+ The reason is that its very hard to tell the difference between functions,
+ unbound methods and staticmethods across Python 2 & 3 and the alternative
+ implementations. This restriction may be fixed in future versions.
+
+
+------
+
+.. [#] This only applies to classes or already instantiated objects. Calling
+ a mocked class to create a mock instance *does not* create a real instance.
+ It is only attribute lookups - along with calls to `dir` - that are done. A
+ way round this problem would have been to use `getattr_static
+ <http://docs.python.org/dev/library/inspect.html#inspect.getattr_static>`_,
+ which can fetch attributes without triggering code execution. Descriptors
+ like `classmethod` and `staticmethod` *need* to be fetched correctly though,
+ so that their signatures can be mocked correctly.
diff --git a/slider-agent/src/test/python/mock/docs/index.txt b/slider-agent/src/test/python/mock/docs/index.txt
new file mode 100644
index 0000000..fe89925
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/index.txt
@@ -0,0 +1,411 @@
+====================================
+ Mock - Mocking and Testing Library
+====================================
+
+.. currentmodule:: mock
+
+:Author: `Michael Foord
+ <http://www.voidspace.org.uk/python/weblog/index.shtml>`_
+:Version: |release|
+:Date: 2012/10/07
+:Homepage: `Mock Homepage`_
+:Download: `Mock on PyPI`_
+:Documentation: `PDF Documentation
+ <http://www.voidspace.org.uk/downloads/mock-1.0.1.pdf>`_
+:License: `BSD License`_
+:Support: `Mailing list (testing-in-python@lists.idyll.org)
+ <http://lists.idyll.org/listinfo/testing-in-python>`_
+:Issue tracker: `Google code project
+ <http://code.google.com/p/mock/issues/list>`_
+
+.. _Mock Homepage: http://www.voidspace.org.uk/python/mock/
+.. _BSD License: http://www.voidspace.org.uk/python/license.shtml
+
+
+.. currentmodule:: mock
+
+.. module:: mock
+ :synopsis: Mock object and testing library.
+
+.. index:: introduction
+
+mock is a library for testing in Python. It allows you to replace parts of
+your system under test with mock objects and make assertions about how they
+have been used.
+
+mock is now part of the Python standard library, available as `unittest.mock
+<http://docs.python.org/py3k/library/unittest.mock.html#module-unittest.mock>`_
+in Python 3.3 onwards.
+
+mock provides a core :class:`Mock` class removing the need to create a host
+of stubs throughout your test suite. After performing an action, you can make
+assertions about which methods / attributes were used and arguments they were
+called with. You can also specify return values and set needed attributes in
+the normal way.
+
+Additionally, mock provides a :func:`patch` decorator that handles patching
+module and class level attributes within the scope of a test, along with
+:const:`sentinel` for creating unique objects. See the `quick guide`_ for
+some examples of how to use :class:`Mock`, :class:`MagicMock` and
+:func:`patch`.
+
+Mock is very easy to use and is designed for use with
+`unittest <http://pypi.python.org/pypi/unittest2>`_. Mock is based on
+the 'action -> assertion' pattern instead of `'record -> replay'` used by many
+mocking frameworks.
+
+mock is tested on Python versions 2.4-2.7, Python 3 plus the latest versions of
+Jython and PyPy.
+
+
+.. testsetup::
+
+ class ProductionClass(object):
+ def method(self, *args):
+ pass
+
+ module = sys.modules['module'] = ProductionClass
+ ProductionClass.ClassName1 = ProductionClass
+ ProductionClass.ClassName2 = ProductionClass
+
+
+
+API Documentation
+=================
+
+.. toctree::
+ :maxdepth: 2
+
+ mock
+ patch
+ helpers
+ sentinel
+ magicmock
+
+
+User Guide
+==========
+
+.. toctree::
+ :maxdepth: 2
+
+ getting-started
+ examples
+ compare
+ changelog
+
+
+.. index:: installing
+
+Installing
+==========
+
+The current version is |release|. Mock is stable and widely used. If you do
+find any bugs, or have suggestions for improvements / extensions
+then please contact us.
+
+* `mock on PyPI <http://pypi.python.org/pypi/mock>`_
+* `mock documentation as PDF
+ <http://www.voidspace.org.uk/downloads/mock-1.0.1.pdf>`_
+* `Google Code Home & Mercurial Repository <http://code.google.com/p/mock/>`_
+
+.. index:: repository
+.. index:: hg
+
+You can checkout the latest development version from the Google Code Mercurial
+repository with the following command:
+
+ ``hg clone https://mock.googlecode.com/hg/ mock``
+
+
+.. index:: pip
+.. index:: easy_install
+.. index:: setuptools
+
+If you have pip, setuptools or distribute you can install mock with:
+
+ | ``easy_install -U mock``
+ | ``pip install -U mock``
+
+Alternatively you can download the mock distribution from PyPI and after
+unpacking run:
+
+ ``python setup.py install``
+
+
+Quick Guide
+===========
+
+:class:`Mock` and :class:`MagicMock` objects create all attributes and
+methods as you access them and store details of how they have been used. You
+can configure them, to specify return values or limit what attributes are
+available, and then make assertions about how they have been used:
+
+.. doctest::
+
+ >>> from mock import MagicMock
+ >>> thing = ProductionClass()
+ >>> thing.method = MagicMock(return_value=3)
+ >>> thing.method(3, 4, 5, key='value')
+ 3
+ >>> thing.method.assert_called_with(3, 4, 5, key='value')
+
+:attr:`side_effect` allows you to perform side effects, including raising an
+exception when a mock is called:
+
+.. doctest::
+
+ >>> mock = Mock(side_effect=KeyError('foo'))
+ >>> mock()
+ Traceback (most recent call last):
+ ...
+ KeyError: 'foo'
+
+ >>> values = {'a': 1, 'b': 2, 'c': 3}
+ >>> def side_effect(arg):
+ ... return values[arg]
+ ...
+ >>> mock.side_effect = side_effect
+ >>> mock('a'), mock('b'), mock('c')
+ (1, 2, 3)
+ >>> mock.side_effect = [5, 4, 3, 2, 1]
+ >>> mock(), mock(), mock()
+ (5, 4, 3)
+
+Mock has many other ways you can configure it and control its behaviour. For
+example the `spec` argument configures the mock to take its specification
+from another object. Attempting to access attributes or methods on the mock
+that don't exist on the spec will fail with an `AttributeError`.
+
+The :func:`patch` decorator / context manager makes it easy to mock classes or
+objects in a module under test. The object you specify will be replaced with a
+mock (or other object) during the test and restored when the test ends:
+
+.. doctest::
+
+ >>> from mock import patch
+ >>> @patch('module.ClassName2')
+ ... @patch('module.ClassName1')
+ ... def test(MockClass1, MockClass2):
+ ... module.ClassName1()
+ ... module.ClassName2()
+
+ ... assert MockClass1 is module.ClassName1
+ ... assert MockClass2 is module.ClassName2
+ ... assert MockClass1.called
+ ... assert MockClass2.called
+ ...
+ >>> test()
+
+.. note::
+
+ When you nest patch decorators the mocks are passed in to the decorated
+ function in the same order they applied (the normal *python* order that
+ decorators are applied). This means from the bottom up, so in the example
+ above the mock for `module.ClassName1` is passed in first.
+
+ With `patch` it matters that you patch objects in the namespace where they
+ are looked up. This is normally straightforward, but for a quick guide
+ read :ref:`where to patch <where-to-patch>`.
+
+As well as a decorator `patch` can be used as a context manager in a with
+statement:
+
+.. doctest::
+
+ >>> with patch.object(ProductionClass, 'method', return_value=None) as mock_method:
+ ... thing = ProductionClass()
+ ... thing.method(1, 2, 3)
+ ...
+ >>> mock_method.assert_called_once_with(1, 2, 3)
+
+
+There is also :func:`patch.dict` for setting values in a dictionary just
+during a scope and restoring the dictionary to its original state when the test
+ends:
+
+.. doctest::
+
+ >>> foo = {'key': 'value'}
+ >>> original = foo.copy()
+ >>> with patch.dict(foo, {'newkey': 'newvalue'}, clear=True):
+ ... assert foo == {'newkey': 'newvalue'}
+ ...
+ >>> assert foo == original
+
+Mock supports the mocking of Python :ref:`magic methods <magic-methods>`. The
+easiest way of using magic methods is with the :class:`MagicMock` class. It
+allows you to do things like:
+
+.. doctest::
+
+ >>> mock = MagicMock()
+ >>> mock.__str__.return_value = 'foobarbaz'
+ >>> str(mock)
+ 'foobarbaz'
+ >>> mock.__str__.assert_called_with()
+
+Mock allows you to assign functions (or other Mock instances) to magic methods
+and they will be called appropriately. The `MagicMock` class is just a Mock
+variant that has all of the magic methods pre-created for you (well, all the
+useful ones anyway).
+
+The following is an example of using magic methods with the ordinary Mock
+class:
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> mock.__str__ = Mock(return_value='wheeeeee')
+ >>> str(mock)
+ 'wheeeeee'
+
+For ensuring that the mock objects in your tests have the same api as the
+objects they are replacing, you can use :ref:`auto-speccing <auto-speccing>`.
+Auto-speccing can be done through the `autospec` argument to patch, or the
+:func:`create_autospec` function. Auto-speccing creates mock objects that
+have the same attributes and methods as the objects they are replacing, and
+any functions and methods (including constructors) have the same call
+signature as the real object.
+
+This ensures that your mocks will fail in the same way as your production
+code if they are used incorrectly:
+
+.. doctest::
+
+ >>> from mock import create_autospec
+ >>> def function(a, b, c):
+ ... pass
+ ...
+ >>> mock_function = create_autospec(function, return_value='fishy')
+ >>> mock_function(1, 2, 3)
+ 'fishy'
+ >>> mock_function.assert_called_once_with(1, 2, 3)
+ >>> mock_function('wrong arguments')
+ Traceback (most recent call last):
+ ...
+ TypeError: <lambda>() takes exactly 3 arguments (1 given)
+
+`create_autospec` can also be used on classes, where it copies the signature of
+the `__init__` method, and on callable objects where it copies the signature of
+the `__call__` method.
+
+
+.. index:: references
+.. index:: articles
+
+References
+==========
+
+Articles, blog entries and other stuff related to testing with Mock:
+
+* `Imposing a No DB Discipline on Django unit tests
+ <https://github.com/carljm/django-testing-slides/blob/master/models/30_no_database.md>`_
+* `mock-django: tools for mocking the Django ORM and models
+ <https://github.com/dcramer/mock-django>`_
+* `PyCon 2011 Video: Testing with mock <https://blip.tv/file/4881513>`_
+* `Mock objects in Python
+ <http://noopenblockers.com/2012/01/06/mock-objects-in-python/>`_
+* `Python: Injecting Mock Objects for Powerful Testing
+ <http://blueprintforge.com/blog/2012/01/08/python-injecting-mock-objects-for-powerful-testing/>`_
+* `Python Mock: How to assert a substring of logger output
+ <http://www.michaelpollmeier.com/python-mock-how-to-assert-a-substring-of-logger-output/>`_
+* `Mocking Django <http://www.mattjmorrison.com/2011/09/mocking-django.html>`_
+* `Mocking dates and other classes that can't be modified
+ <http://williamjohnbert.com/2011/07/how-to-unit-testing-in-django-with-mocking-and-patching/>`_
+* `Mock recipes <http://konryd.blogspot.com/2010/06/mock-recipies.html>`_
+* `Mockity mock mock - some love for the mock module
+ <http://konryd.blogspot.com/2010/05/mockity-mock-mock-some-love-for-mock.html>`_
+* `Coverage and Mock (with django)
+ <http://mattsnider.com/python/mock-and-coverage/>`_
+* `Python Unit Testing with Mock <http://www.insomnihack.com/?p=194>`_
+* `Getting started with Python Mock
+ <http://myadventuresincoding.wordpress.com/2011/02/26/python-python-mock-cheat-sheet/>`_
+* `Smart Parameter Checks with mock
+ <http://tobyho.com/2011/03/24/smart-parameter-checks-in/>`_
+* `Python mock testing techniques and tools
+ <http://agiletesting.blogspot.com/2009/07/python-mock-testing-techniques-and.html>`_
+* `How To Test Django Template Tags
+ <http://techblog.ironfroggy.com/2008/10/how-to-test.html>`_
+* `A presentation on Unit Testing with Mock
+ <http://pypap.blogspot.com/2008/10/newbie-nugget-unit-testing-with-mock.html>`_
+* `Mocking with Django and Google AppEngine
+ <http://michael-a-nelson.blogspot.com/2008/09/mocking-with-django-and-google-app.html>`_
+
+
+.. index:: tests
+.. index:: unittest2
+
+Tests
+=====
+
+Mock uses `unittest2 <http://pypi.python.org/pypi/unittest2>`_ for its own
+test suite. In order to run it, use the `unit2` script that comes with
+`unittest2` module on a checkout of the source repository:
+
+ `unit2 discover`
+
+If you have `setuptools <http://pypi.python.org/pypi/distribute>`_ as well as
+unittest2 you can run:
+
+ ``python setup.py test``
+
+On Python 3.2 you can use ``unittest`` module from the standard library.
+
+ ``python3.2 -m unittest discover``
+
+.. index:: Python 3
+
+On Python 3 the tests for unicode are skipped as they are not relevant. On
+Python 2.4 tests that use the with statements are skipped as the with statement
+is invalid syntax on Python 2.4.
+
+
+.. index:: older versions
+
+Older Versions
+==============
+
+Documentation for older versions of mock:
+
+* `mock 0.8 <http://www.voidspace.org.uk/python/mock/0.8/>`_
+* `mock 0.7 <http://www.voidspace.org.uk/python/mock/0.7/>`_
+* `mock 0.6 <http://www.voidspace.org.uk/python/mock/0.6.0/>`_
+
+Docs from the in-development version of `mock` can be found at
+`mock.readthedocs.org <http://mock.readthedocs.org>`_.
+
+
+Terminology
+===========
+
+Terminology for objects used to replace other ones can be confusing. Terms
+like double, fake, mock, stub, and spy are all used with varying meanings.
+
+In `classic mock terminology
+<http://xunitpatterns.com/Mocks,%20Fakes,%20Stubs%20and%20Dummies.html>`_
+:class:`mock.Mock` is a `spy <http://xunitpatterns.com/Test%20Spy.html>`_ that
+allows for *post-mortem* examination. This is what I call the "action ->
+assertion" [#]_ pattern of testing.
+
+I'm not however a fan of this "statically typed mocking terminology"
+promulgated by `Martin Fowler
+<http://martinfowler.com/articles/mocksArentStubs.html>`_. It confuses usage
+patterns with implementation and prevents you from using natural terminology
+when discussing mocking.
+
+I much prefer duck typing, if an object used in your test suite looks like a
+mock object and quacks like a mock object then it's fine to call it a mock, no
+matter what the implementation looks like.
+
+This terminology is perhaps more useful in less capable languages where
+different usage patterns will *require* different implementations.
+`mock.Mock()` is capable of being used in most of the different roles
+described by Fowler, except (annoyingly / frustratingly / ironically) a Mock
+itself!
+
+How about a simpler definition: a "mock object" is an object used to replace a
+real one in a system under test.
+
+.. [#] This pattern is called "AAA" by some members of the testing community;
+ "Arrange - Act - Assert".
diff --git a/slider-agent/src/test/python/mock/docs/magicmock.txt b/slider-agent/src/test/python/mock/docs/magicmock.txt
new file mode 100644
index 0000000..42b2ed9
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/magicmock.txt
@@ -0,0 +1,258 @@
+
+.. currentmodule:: mock
+
+
+.. _magic-methods:
+
+Mocking Magic Methods
+=====================
+
+.. currentmodule:: mock
+
+:class:`Mock` supports mocking `magic methods
+<http://www.ironpythoninaction.com/magic-methods.html>`_. This allows mock
+objects to replace containers or other objects that implement Python
+protocols.
+
+Because magic methods are looked up differently from normal methods [#]_, this
+support has been specially implemented. This means that only specific magic
+methods are supported. The supported list includes *almost* all of them. If
+there are any missing that you need please let us know!
+
+You mock magic methods by setting the method you are interested in to a function
+or a mock instance. If you are using a function then it *must* take ``self`` as
+the first argument [#]_.
+
+.. doctest::
+
+ >>> def __str__(self):
+ ... return 'fooble'
+ ...
+ >>> mock = Mock()
+ >>> mock.__str__ = __str__
+ >>> str(mock)
+ 'fooble'
+
+ >>> mock = Mock()
+ >>> mock.__str__ = Mock()
+ >>> mock.__str__.return_value = 'fooble'
+ >>> str(mock)
+ 'fooble'
+
+ >>> mock = Mock()
+ >>> mock.__iter__ = Mock(return_value=iter([]))
+ >>> list(mock)
+ []
+
+One use case for this is for mocking objects used as context managers in a
+`with` statement:
+
+.. doctest::
+
+ >>> mock = Mock()
+ >>> mock.__enter__ = Mock(return_value='foo')
+ >>> mock.__exit__ = Mock(return_value=False)
+ >>> with mock as m:
+ ... assert m == 'foo'
+ ...
+ >>> mock.__enter__.assert_called_with()
+ >>> mock.__exit__.assert_called_with(None, None, None)
+
+Calls to magic methods do not appear in :attr:`~Mock.method_calls`, but they
+are recorded in :attr:`~Mock.mock_calls`.
+
+.. note::
+
+ If you use the `spec` keyword argument to create a mock then attempting to
+ set a magic method that isn't in the spec will raise an `AttributeError`.
+
+The full list of supported magic methods is:
+
+* ``__hash__``, ``__sizeof__``, ``__repr__`` and ``__str__``
+* ``__dir__``, ``__format__`` and ``__subclasses__``
+* ``__floor__``, ``__trunc__`` and ``__ceil__``
+* Comparisons: ``__cmp__``, ``__lt__``, ``__gt__``, ``__le__``, ``__ge__``,
+ ``__eq__`` and ``__ne__``
+* Container methods: ``__getitem__``, ``__setitem__``, ``__delitem__``,
+ ``__contains__``, ``__len__``, ``__iter__``, ``__getslice__``,
+ ``__setslice__``, ``__reversed__`` and ``__missing__``
+* Context manager: ``__enter__`` and ``__exit__``
+* Unary numeric methods: ``__neg__``, ``__pos__`` and ``__invert__``
+* The numeric methods (including right hand and in-place variants):
+ ``__add__``, ``__sub__``, ``__mul__``, ``__div__``,
+ ``__floordiv__``, ``__mod__``, ``__divmod__``, ``__lshift__``,
+ ``__rshift__``, ``__and__``, ``__xor__``, ``__or__``, and ``__pow__``
+* Numeric conversion methods: ``__complex__``, ``__int__``, ``__float__``,
+ ``__index__`` and ``__coerce__``
+* Descriptor methods: ``__get__``, ``__set__`` and ``__delete__``
+* Pickling: ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``,
+ ``__getnewargs__``, ``__getstate__`` and ``__setstate__``
+
+
+The following methods are supported in Python 2 but don't exist in Python 3:
+
+* ``__unicode__``, ``__long__``, ``__oct__``, ``__hex__`` and ``__nonzero__``
+* ``__truediv__`` and ``__rtruediv__``
+
+The following methods are supported in Python 3 but don't exist in Python 2:
+
+* ``__bool__`` and ``__next__``
+
+The following methods exist but are *not* supported as they are either in use by
+mock, can't be set dynamically, or can cause problems:
+
+* ``__getattr__``, ``__setattr__``, ``__init__`` and ``__new__``
+* ``__prepare__``, ``__instancecheck__``, ``__subclasscheck__``, ``__del__``
+
+
+
+Magic Mock
+==========
+
+There are two `MagicMock` variants: `MagicMock` and `NonCallableMagicMock`.
+
+
+.. class:: MagicMock(*args, **kw)
+
+ ``MagicMock`` is a subclass of :class:`Mock` with default implementations
+ of most of the magic methods. You can use ``MagicMock`` without having to
+ configure the magic methods yourself.
+
+ The constructor parameters have the same meaning as for :class:`Mock`.
+
+ If you use the `spec` or `spec_set` arguments then *only* magic methods
+ that exist in the spec will be created.
+
+
+.. class:: NonCallableMagicMock(*args, **kw)
+
+ A non-callable version of `MagicMock`.
+
+ The constructor parameters have the same meaning as for
+ :class:`MagicMock`, with the exception of `return_value` and
+ `side_effect` which have no meaning on a non-callable mock.
+
+The magic methods are setup with `MagicMock` objects, so you can configure them
+and use them in the usual way:
+
+.. doctest::
+
+ >>> mock = MagicMock()
+ >>> mock[3] = 'fish'
+ >>> mock.__setitem__.assert_called_with(3, 'fish')
+ >>> mock.__getitem__.return_value = 'result'
+ >>> mock[2]
+ 'result'
+
+By default many of the protocol methods are required to return objects of a
+specific type. These methods are preconfigured with a default return value, so
+that they can be used without you having to do anything if you aren't interested
+in the return value. You can still *set* the return value manually if you want
+to change the default.
+
+Methods and their defaults:
+
+* ``__lt__``: NotImplemented
+* ``__gt__``: NotImplemented
+* ``__le__``: NotImplemented
+* ``__ge__``: NotImplemented
+* ``__int__`` : 1
+* ``__contains__`` : False
+* ``__len__`` : 1
+* ``__iter__`` : iter([])
+* ``__exit__`` : False
+* ``__complex__`` : 1j
+* ``__float__`` : 1.0
+* ``__bool__`` : True
+* ``__nonzero__`` : True
+* ``__oct__`` : '1'
+* ``__hex__`` : '0x1'
+* ``__long__`` : long(1)
+* ``__index__`` : 1
+* ``__hash__`` : default hash for the mock
+* ``__str__`` : default str for the mock
+* ``__unicode__`` : default unicode for the mock
+* ``__sizeof__``: default sizeof for the mock
+
+For example:
+
+.. doctest::
+
+ >>> mock = MagicMock()
+ >>> int(mock)
+ 1
+ >>> len(mock)
+ 0
+ >>> hex(mock)
+ '0x1'
+ >>> list(mock)
+ []
+ >>> object() in mock
+ False
+
+The two equality method, `__eq__` and `__ne__`, are special (changed in
+0.7.2). They do the default equality comparison on identity, using a side
+effect, unless you change their return value to return something else:
+
+.. doctest::
+
+ >>> MagicMock() == 3
+ False
+ >>> MagicMock() != 3
+ True
+ >>> mock = MagicMock()
+ >>> mock.__eq__.return_value = True
+ >>> mock == 3
+ True
+
+In `0.8` the `__iter__` also gained special handling implemented with a
+side effect. The return value of `MagicMock.__iter__` can be any iterable
+object and isn't required to be an iterator:
+
+.. doctest::
+
+ >>> mock = MagicMock()
+ >>> mock.__iter__.return_value = ['a', 'b', 'c']
+ >>> list(mock)
+ ['a', 'b', 'c']
+ >>> list(mock)
+ ['a', 'b', 'c']
+
+If the return value *is* an iterator, then iterating over it once will consume
+it and subsequent iterations will result in an empty list:
+
+.. doctest::
+
+ >>> mock.__iter__.return_value = iter(['a', 'b', 'c'])
+ >>> list(mock)
+ ['a', 'b', 'c']
+ >>> list(mock)
+ []
+
+``MagicMock`` has all of the supported magic methods configured except for some
+of the obscure and obsolete ones. You can still set these up if you want.
+
+Magic methods that are supported but not setup by default in ``MagicMock`` are:
+
+* ``__cmp__``
+* ``__getslice__`` and ``__setslice__``
+* ``__coerce__``
+* ``__subclasses__``
+* ``__dir__``
+* ``__format__``
+* ``__get__``, ``__set__`` and ``__delete__``
+* ``__reversed__`` and ``__missing__``
+* ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, ``__getnewargs__``,
+ ``__getstate__`` and ``__setstate__``
+* ``__getformat__`` and ``__setformat__``
+
+
+
+------------
+
+.. [#] Magic methods *should* be looked up on the class rather than the
+ instance. Different versions of Python are inconsistent about applying this
+ rule. The supported protocol methods should work with all supported versions
+ of Python.
+.. [#] The function is basically hooked up to the class, but each ``Mock``
+ instance is kept isolated from the others.
diff --git a/slider-agent/src/test/python/mock/docs/mock.txt b/slider-agent/src/test/python/mock/docs/mock.txt
new file mode 100644
index 0000000..58712b2
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/mock.txt
@@ -0,0 +1,842 @@
+The Mock Class
+==============
+
+.. currentmodule:: mock
+
+.. testsetup::
+
+ class SomeClass:
+ pass
+
+
+`Mock` is a flexible mock object intended to replace the use of stubs and
+test doubles throughout your code. Mocks are callable and create attributes as
+new mocks when you access them [#]_. Accessing the same attribute will always
+return the same mock. Mocks record how you use them, allowing you to make
+assertions about what your code has done to them.
+
+:class:`MagicMock` is a subclass of `Mock` with all the magic methods
+pre-created and ready to use. There are also non-callable variants, useful
+when you are mocking out objects that aren't callable:
+:class:`NonCallableMock` and :class:`NonCallableMagicMock`
+
+The :func:`patch` decorators makes it easy to temporarily replace classes
+in a particular module with a `Mock` object. By default `patch` will create
+a `MagicMock` for you. You can specify an alternative class of `Mock` using
+the `new_callable` argument to `patch`.
+
+
+.. index:: side_effect
+.. index:: return_value
+.. index:: wraps
+.. index:: name
+.. index:: spec
+
+.. class:: Mock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, **kwargs)
+
+ Create a new `Mock` object. `Mock` takes several optional arguments
+ that specify the behaviour of the Mock object:
+
+ * `spec`: This can be either a list of strings or an existing object (a
+ class or instance) that acts as the specification for the mock object. If
+ you pass in an object then a list of strings is formed by calling dir on
+ the object (excluding unsupported magic attributes and methods).
+ Accessing any attribute not in this list will raise an `AttributeError`.
+
+ If `spec` is an object (rather than a list of strings) then
+ :attr:`__class__` returns the class of the spec object. This allows mocks
+ to pass `isinstance` tests.
+
+ * `spec_set`: A stricter variant of `spec`. If used, attempting to *set*
+ or get an attribute on the mock that isn't on the object passed as
+ `spec_set` will raise an `AttributeError`.
+
+ * `side_effect`: A function to be called whenever the Mock is called. See
+ the :attr:`~Mock.side_effect` attribute. Useful for raising exceptions or
+ dynamically changing return values. The function is called with the same
+ arguments as the mock, and unless it returns :data:`DEFAULT`, the return
+ value of this function is used as the return value.
+
+ Alternatively `side_effect` can be an exception class or instance. In
+ this case the exception will be raised when the mock is called.
+
+ If `side_effect` is an iterable then each call to the mock will return
+ the next value from the iterable. If any of the members of the iterable
+ are exceptions they will be raised instead of returned.
+
+ A `side_effect` can be cleared by setting it to `None`.
+
+ * `return_value`: The value returned when the mock is called. By default
+ this is a new Mock (created on first access). See the
+ :attr:`return_value` attribute.
+
+ * `wraps`: Item for the mock object to wrap. If `wraps` is not None then
+ calling the Mock will pass the call through to the wrapped object
+ (returning the real result and ignoring `return_value`). Attribute access
+ on the mock will return a Mock object that wraps the corresponding
+ attribute of the wrapped object (so attempting to access an attribute
+ that doesn't exist will raise an `AttributeError`).
+
+ If the mock has an explicit `return_value` set then calls are not passed
+ to the wrapped object and the `return_value` is returned instead.
+
+ * `name`: If the mock has a name then it will be used in the repr of the
+ mock. This can be useful for debugging. The name is propagated to child
+ mocks.
+
+ Mocks can also be called with arbitrary keyword arguments. These will be
+ used to set attributes on the mock after it is created. See the
+ :meth:`configure_mock` method for details.
+
+
+ .. method:: assert_called_with(*args, **kwargs)
+
+ This method is a convenient way of asserting that calls are made in a
+ particular way:
+
+ .. doctest::
+
+ >>> mock = Mock()
+ >>> mock.method(1, 2, 3, test='wow')
+ <Mock name='mock.method()' id='...'>
+ >>> mock.method.assert_called_with(1, 2, 3, test='wow')
+
+
+ .. method:: assert_called_once_with(*args, **kwargs)
+
+ Assert that the mock was called exactly once and with the specified
+ arguments.
+
+ .. doctest::
+
+ >>> mock = Mock(return_value=None)
+ >>> mock('foo', bar='baz')
+ >>> mock.assert_called_once_with('foo', bar='baz')
+ >>> mock('foo', bar='baz')
+ >>> mock.assert_called_once_with('foo', bar='baz')
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected to be called once. Called 2 times.
+
+
+ .. method:: assert_any_call(*args, **kwargs)
+
+ assert the mock has been called with the specified arguments.
+
+ The assert passes if the mock has *ever* been called, unlike
+ :meth:`assert_called_with` and :meth:`assert_called_once_with` that
+ only pass if the call is the most recent one.
+
+ .. doctest::
+
+ >>> mock = Mock(return_value=None)
+ >>> mock(1, 2, arg='thing')
+ >>> mock('some', 'thing', 'else')
+ >>> mock.assert_any_call(1, 2, arg='thing')
+
+
+ .. method:: assert_has_calls(calls, any_order=False)
+
+ assert the mock has been called with the specified calls.
+ The `mock_calls` list is checked for the calls.
+
+ If `any_order` is False (the default) then the calls must be
+ sequential. There can be extra calls before or after the
+ specified calls.
+
+ If `any_order` is True then the calls can be in any order, but
+ they must all appear in :attr:`mock_calls`.
+
+ .. doctest::
+
+ >>> mock = Mock(return_value=None)
+ >>> mock(1)
+ >>> mock(2)
+ >>> mock(3)
+ >>> mock(4)
+ >>> calls = [call(2), call(3)]
+ >>> mock.assert_has_calls(calls)
+ >>> calls = [call(4), call(2), call(3)]
+ >>> mock.assert_has_calls(calls, any_order=True)
+
+
+ .. method:: reset_mock()
+
+ The reset_mock method resets all the call attributes on a mock object:
+
+ .. doctest::
+
+ >>> mock = Mock(return_value=None)
+ >>> mock('hello')
+ >>> mock.called
+ True
+ >>> mock.reset_mock()
+ >>> mock.called
+ False
+
+ This can be useful where you want to make a series of assertions that
+ reuse the same object. Note that `reset_mock` *doesn't* clear the
+ return value, :attr:`side_effect` or any child attributes you have
+ set using normal assignment. Child mocks and the return value mock
+ (if any) are reset as well.
+
+
+ .. method:: mock_add_spec(spec, spec_set=False)
+
+ Add a spec to a mock. `spec` can either be an object or a
+ list of strings. Only attributes on the `spec` can be fetched as
+ attributes from the mock.
+
+ If `spec_set` is `True` then only attributes on the spec can be set.
+
+
+ .. method:: attach_mock(mock, attribute)
+
+ Attach a mock as an attribute of this one, replacing its name and
+ parent. Calls to the attached mock will be recorded in the
+ :attr:`method_calls` and :attr:`mock_calls` attributes of this one.
+
+
+ .. method:: configure_mock(**kwargs)
+
+ Set attributes on the mock through keyword arguments.
+
+ Attributes plus return values and side effects can be set on child
+ mocks using standard dot notation and unpacking a dictionary in the
+ method call:
+
+ .. doctest::
+
+ >>> mock = Mock()
+ >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError}
+ >>> mock.configure_mock(**attrs)
+ >>> mock.method()
+ 3
+ >>> mock.other()
+ Traceback (most recent call last):
+ ...
+ KeyError
+
+ The same thing can be achieved in the constructor call to mocks:
+
+ .. doctest::
+
+ >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError}
+ >>> mock = Mock(some_attribute='eggs', **attrs)
+ >>> mock.some_attribute
+ 'eggs'
+ >>> mock.method()
+ 3
+ >>> mock.other()
+ Traceback (most recent call last):
+ ...
+ KeyError
+
+ `configure_mock` exists to make it easier to do configuration
+ after the mock has been created.
+
+
+ .. method:: __dir__()
+
+ `Mock` objects limit the results of `dir(some_mock)` to useful results.
+ For mocks with a `spec` this includes all the permitted attributes
+ for the mock.
+
+ See :data:`FILTER_DIR` for what this filtering does, and how to
+ switch it off.
+
+
+ .. method:: _get_child_mock(**kw)
+
+ Create the child mocks for attributes and return value.
+ By default child mocks will be the same type as the parent.
+ Subclasses of Mock may want to override this to customize the way
+ child mocks are made.
+
+ For non-callable mocks the callable variant will be used (rather than
+ any custom subclass).
+
+
+ .. attribute:: called
+
+ A boolean representing whether or not the mock object has been called:
+
+ .. doctest::
+
+ >>> mock = Mock(return_value=None)
+ >>> mock.called
+ False
+ >>> mock()
+ >>> mock.called
+ True
+
+ .. attribute:: call_count
+
+ An integer telling you how many times the mock object has been called:
+
+ .. doctest::
+
+ >>> mock = Mock(return_value=None)
+ >>> mock.call_count
+ 0
+ >>> mock()
+ >>> mock()
+ >>> mock.call_count
+ 2
+
+
+ .. attribute:: return_value
+
+ Set this to configure the value returned by calling the mock:
+
+ .. doctest::
+
+ >>> mock = Mock()
+ >>> mock.return_value = 'fish'
+ >>> mock()
+ 'fish'
+
+ The default return value is a mock object and you can configure it in
+ the normal way:
+
+ .. doctest::
+
+ >>> mock = Mock()
+ >>> mock.return_value.attribute = sentinel.Attribute
+ >>> mock.return_value()
+ <Mock name='mock()()' id='...'>
+ >>> mock.return_value.assert_called_with()
+
+ `return_value` can also be set in the constructor:
+
+ .. doctest::
+
+ >>> mock = Mock(return_value=3)
+ >>> mock.return_value
+ 3
+ >>> mock()
+ 3
+
+
+ .. attribute:: side_effect
+
+ This can either be a function to be called when the mock is called,
+ or an exception (class or instance) to be raised.
+
+ If you pass in a function it will be called with same arguments as the
+ mock and unless the function returns the :data:`DEFAULT` singleton the
+ call to the mock will then return whatever the function returns. If the
+ function returns :data:`DEFAULT` then the mock will return its normal
+ value (from the :attr:`return_value`.
+
+ An example of a mock that raises an exception (to test exception
+ handling of an API):
+
+ .. doctest::
+
+ >>> mock = Mock()
+ >>> mock.side_effect = Exception('Boom!')
+ >>> mock()
+ Traceback (most recent call last):
+ ...
+ Exception: Boom!
+
+ Using `side_effect` to return a sequence of values:
+
+ .. doctest::
+
+ >>> mock = Mock()
+ >>> mock.side_effect = [3, 2, 1]
+ >>> mock(), mock(), mock()
+ (3, 2, 1)
+
+ The `side_effect` function is called with the same arguments as the
+ mock (so it is wise for it to take arbitrary args and keyword
+ arguments) and whatever it returns is used as the return value for
+ the call. The exception is if `side_effect` returns :data:`DEFAULT`,
+ in which case the normal :attr:`return_value` is used.
+
+ .. doctest::
+
+ >>> mock = Mock(return_value=3)
+ >>> def side_effect(*args, **kwargs):
+ ... return DEFAULT
+ ...
+ >>> mock.side_effect = side_effect
+ >>> mock()
+ 3
+
+ `side_effect` can be set in the constructor. Here's an example that
+ adds one to the value the mock is called with and returns it:
+
+ .. doctest::
+
+ >>> side_effect = lambda value: value + 1
+ >>> mock = Mock(side_effect=side_effect)
+ >>> mock(3)
+ 4
+ >>> mock(-8)
+ -7
+
+ Setting `side_effect` to `None` clears it:
+
+ .. doctest::
+
+ >>> from mock import Mock
+ >>> m = Mock(side_effect=KeyError, return_value=3)
+ >>> m()
+ Traceback (most recent call last):
+ ...
+ KeyError
+ >>> m.side_effect = None
+ >>> m()
+ 3
+
+
+ .. attribute:: call_args
+
+ This is either `None` (if the mock hasn't been called), or the
+ arguments that the mock was last called with. This will be in the
+ form of a tuple: the first member is any ordered arguments the mock
+ was called with (or an empty tuple) and the second member is any
+ keyword arguments (or an empty dictionary).
+
+ .. doctest::
+
+ >>> mock = Mock(return_value=None)
+ >>> print mock.call_args
+ None
+ >>> mock()
+ >>> mock.call_args
+ call()
+ >>> mock.call_args == ()
+ True
+ >>> mock(3, 4)
+ >>> mock.call_args
+ call(3, 4)
+ >>> mock.call_args == ((3, 4),)
+ True
+ >>> mock(3, 4, 5, key='fish', next='w00t!')
+ >>> mock.call_args
+ call(3, 4, 5, key='fish', next='w00t!')
+
+ `call_args`, along with members of the lists :attr:`call_args_list`,
+ :attr:`method_calls` and :attr:`mock_calls` are :data:`call` objects.
+ These are tuples, so they can be unpacked to get at the individual
+ arguments and make more complex assertions. See
+ :ref:`calls as tuples <calls-as-tuples>`.
+
+
+ .. attribute:: call_args_list
+
+ This is a list of all the calls made to the mock object in sequence
+ (so the length of the list is the number of times it has been
+ called). Before any calls have been made it is an empty list. The
+ :data:`call` object can be used for conveniently constructing lists of
+ calls to compare with `call_args_list`.
+
+ .. doctest::
+
+ >>> mock = Mock(return_value=None)
+ >>> mock()
+ >>> mock(3, 4)
+ >>> mock(key='fish', next='w00t!')
+ >>> mock.call_args_list
+ [call(), call(3, 4), call(key='fish', next='w00t!')]
+ >>> expected = [(), ((3, 4),), ({'key': 'fish', 'next': 'w00t!'},)]
+ >>> mock.call_args_list == expected
+ True
+
+ Members of `call_args_list` are :data:`call` objects. These can be
+ unpacked as tuples to get at the individual arguments. See
+ :ref:`calls as tuples <calls-as-tuples>`.
+
+
+ .. attribute:: method_calls
+
+ As well as tracking calls to themselves, mocks also track calls to
+ methods and attributes, and *their* methods and attributes:
+
+ .. doctest::
+
+ >>> mock = Mock()
+ >>> mock.method()
+ <Mock name='mock.method()' id='...'>
+ >>> mock.property.method.attribute()
+ <Mock name='mock.property.method.attribute()' id='...'>
+ >>> mock.method_calls
+ [call.method(), call.property.method.attribute()]
+
+ Members of `method_calls` are :data:`call` objects. These can be
+ unpacked as tuples to get at the individual arguments. See
+ :ref:`calls as tuples <calls-as-tuples>`.
+
+
+ .. attribute:: mock_calls
+
+ `mock_calls` records *all* calls to the mock object, its methods, magic
+ methods *and* return value mocks.
+
+ .. doctest::
+
+ >>> mock = MagicMock()
+ >>> result = mock(1, 2, 3)
+ >>> mock.first(a=3)
+ <MagicMock name='mock.first()' id='...'>
+ >>> mock.second()
+ <MagicMock name='mock.second()' id='...'>
+ >>> int(mock)
+ 1
+ >>> result(1)
+ <MagicMock name='mock()()' id='...'>
+ >>> expected = [call(1, 2, 3), call.first(a=3), call.second(),
+ ... call.__int__(), call()(1)]
+ >>> mock.mock_calls == expected
+ True
+
+ Members of `mock_calls` are :data:`call` objects. These can be
+ unpacked as tuples to get at the individual arguments. See
+ :ref:`calls as tuples <calls-as-tuples>`.
+
+
+ .. attribute:: __class__
+
+ Normally the `__class__` attribute of an object will return its type.
+ For a mock object with a `spec` `__class__` returns the spec class
+ instead. This allows mock objects to pass `isinstance` tests for the
+ object they are replacing / masquerading as:
+
+ .. doctest::
+
+ >>> mock = Mock(spec=3)
+ >>> isinstance(mock, int)
+ True
+
+ `__class__` is assignable to, this allows a mock to pass an
+ `isinstance` check without forcing you to use a spec:
+
+ .. doctest::
+
+ >>> mock = Mock()
+ >>> mock.__class__ = dict
+ >>> isinstance(mock, dict)
+ True
+
+.. class:: NonCallableMock(spec=None, wraps=None, name=None, spec_set=None, **kwargs)
+
+ A non-callable version of `Mock`. The constructor parameters have the same
+ meaning of `Mock`, with the exception of `return_value` and `side_effect`
+ which have no meaning on a non-callable mock.
+
+Mock objects that use a class or an instance as a `spec` or `spec_set` are able
+to pass `isintance` tests:
+
+.. doctest::
+
+ >>> mock = Mock(spec=SomeClass)
+ >>> isinstance(mock, SomeClass)
+ True
+ >>> mock = Mock(spec_set=SomeClass())
+ >>> isinstance(mock, SomeClass)
+ True
+
+The `Mock` classes have support for mocking magic methods. See :ref:`magic
+methods <magic-methods>` for the full details.
+
+The mock classes and the :func:`patch` decorators all take arbitrary keyword
+arguments for configuration. For the `patch` decorators the keywords are
+passed to the constructor of the mock being created. The keyword arguments
+are for configuring attributes of the mock:
+
+.. doctest::
+
+ >>> m = MagicMock(attribute=3, other='fish')
+ >>> m.attribute
+ 3
+ >>> m.other
+ 'fish'
+
+The return value and side effect of child mocks can be set in the same way,
+using dotted notation. As you can't use dotted names directly in a call you
+have to create a dictionary and unpack it using `**`:
+
+.. doctest::
+
+ >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError}
+ >>> mock = Mock(some_attribute='eggs', **attrs)
+ >>> mock.some_attribute
+ 'eggs'
+ >>> mock.method()
+ 3
+ >>> mock.other()
+ Traceback (most recent call last):
+ ...
+ KeyError
+
+
+.. class:: PropertyMock(*args, **kwargs)
+
+ A mock intended to be used as a property, or other descriptor, on a class.
+ `PropertyMock` provides `__get__` and `__set__` methods so you can specify
+ a return value when it is fetched.
+
+ Fetching a `PropertyMock` instance from an object calls the mock, with
+ no args. Setting it calls the mock with the value being set.
+
+ .. doctest::
+
+ >>> class Foo(object):
+ ... @property
+ ... def foo(self):
+ ... return 'something'
+ ... @foo.setter
+ ... def foo(self, value):
+ ... pass
+ ...
+ >>> with patch('__main__.Foo.foo', new_callable=PropertyMock) as mock_foo:
+ ... mock_foo.return_value = 'mockity-mock'
+ ... this_foo = Foo()
+ ... print this_foo.foo
+ ... this_foo.foo = 6
+ ...
+ mockity-mock
+ >>> mock_foo.mock_calls
+ [call(), call(6)]
+
+Because of the way mock attributes are stored you can't directly attach a
+`PropertyMock` to a mock object. Instead you can attach it to the mock type
+object:
+
+.. doctest::
+
+ >>> m = MagicMock()
+ >>> p = PropertyMock(return_value=3)
+ >>> type(m).foo = p
+ >>> m.foo
+ 3
+ >>> p.assert_called_once_with()
+
+
+.. index:: __call__
+.. index:: calling
+
+Calling
+=======
+
+Mock objects are callable. The call will return the value set as the
+:attr:`~Mock.return_value` attribute. The default return value is a new Mock
+object; it is created the first time the return value is accessed (either
+explicitly or by calling the Mock) - but it is stored and the same one
+returned each time.
+
+Calls made to the object will be recorded in the attributes
+like :attr:`~Mock.call_args` and :attr:`~Mock.call_args_list`.
+
+If :attr:`~Mock.side_effect` is set then it will be called after the call has
+been recorded, so if `side_effect` raises an exception the call is still
+recorded.
+
+The simplest way to make a mock raise an exception when called is to make
+:attr:`~Mock.side_effect` an exception class or instance:
+
+.. doctest::
+
+ >>> m = MagicMock(side_effect=IndexError)
+ >>> m(1, 2, 3)
+ Traceback (most recent call last):
+ ...
+ IndexError
+ >>> m.mock_calls
+ [call(1, 2, 3)]
+ >>> m.side_effect = KeyError('Bang!')
+ >>> m('two', 'three', 'four')
+ Traceback (most recent call last):
+ ...
+ KeyError: 'Bang!'
+ >>> m.mock_calls
+ [call(1, 2, 3), call('two', 'three', 'four')]
+
+If `side_effect` is a function then whatever that function returns is what
+calls to the mock return. The `side_effect` function is called with the
+same arguments as the mock. This allows you to vary the return value of the
+call dynamically, based on the input:
+
+.. doctest::
+
+ >>> def side_effect(value):
+ ... return value + 1
+ ...
+ >>> m = MagicMock(side_effect=side_effect)
+ >>> m(1)
+ 2
+ >>> m(2)
+ 3
+ >>> m.mock_calls
+ [call(1), call(2)]
+
+If you want the mock to still return the default return value (a new mock), or
+any set return value, then there are two ways of doing this. Either return
+`mock.return_value` from inside `side_effect`, or return :data:`DEFAULT`:
+
+.. doctest::
+
+ >>> m = MagicMock()
+ >>> def side_effect(*args, **kwargs):
+ ... return m.return_value
+ ...
+ >>> m.side_effect = side_effect
+ >>> m.return_value = 3
+ >>> m()
+ 3
+ >>> def side_effect(*args, **kwargs):
+ ... return DEFAULT
+ ...
+ >>> m.side_effect = side_effect
+ >>> m()
+ 3
+
+To remove a `side_effect`, and return to the default behaviour, set the
+`side_effect` to `None`:
+
+.. doctest::
+
+ >>> m = MagicMock(return_value=6)
+ >>> def side_effect(*args, **kwargs):
+ ... return 3
+ ...
+ >>> m.side_effect = side_effect
+ >>> m()
+ 3
+ >>> m.side_effect = None
+ >>> m()
+ 6
+
+The `side_effect` can also be any iterable object. Repeated calls to the mock
+will return values from the iterable (until the iterable is exhausted and
+a `StopIteration` is raised):
+
+.. doctest::
+
+ >>> m = MagicMock(side_effect=[1, 2, 3])
+ >>> m()
+ 1
+ >>> m()
+ 2
+ >>> m()
+ 3
+ >>> m()
+ Traceback (most recent call last):
+ ...
+ StopIteration
+
+If any members of the iterable are exceptions they will be raised instead of
+returned:
+
+.. doctest::
+
+ >>> iterable = (33, ValueError, 66)
+ >>> m = MagicMock(side_effect=iterable)
+ >>> m()
+ 33
+ >>> m()
+ Traceback (most recent call last):
+ ...
+ ValueError
+ >>> m()
+ 66
+
+
+.. _deleting-attributes:
+
+Deleting Attributes
+===================
+
+Mock objects create attributes on demand. This allows them to pretend to be
+objects of any type.
+
+You may want a mock object to return `False` to a `hasattr` call, or raise an
+`AttributeError` when an attribute is fetched. You can do this by providing
+an object as a `spec` for a mock, but that isn't always convenient.
+
+You "block" attributes by deleting them. Once deleted, accessing an attribute
+will raise an `AttributeError`.
+
+.. doctest::
+
+ >>> mock = MagicMock()
+ >>> hasattr(mock, 'm')
+ True
+ >>> del mock.m
+ >>> hasattr(mock, 'm')
+ False
+ >>> del mock.f
+ >>> mock.f
+ Traceback (most recent call last):
+ ...
+ AttributeError: f
+
+
+Attaching Mocks as Attributes
+=============================
+
+When you attach a mock as an attribute of another mock (or as the return
+value) it becomes a "child" of that mock. Calls to the child are recorded in
+the :attr:`~Mock.method_calls` and :attr:`~Mock.mock_calls` attributes of the
+parent. This is useful for configuring child mocks and then attaching them to
+the parent, or for attaching mocks to a parent that records all calls to the
+children and allows you to make assertions about the order of calls between
+mocks:
+
+.. doctest::
+
+ >>> parent = MagicMock()
+ >>> child1 = MagicMock(return_value=None)
+ >>> child2 = MagicMock(return_value=None)
+ >>> parent.child1 = child1
+ >>> parent.child2 = child2
+ >>> child1(1)
+ >>> child2(2)
+ >>> parent.mock_calls
+ [call.child1(1), call.child2(2)]
+
+The exception to this is if the mock has a name. This allows you to prevent
+the "parenting" if for some reason you don't want it to happen.
+
+.. doctest::
+
+ >>> mock = MagicMock()
+ >>> not_a_child = MagicMock(name='not-a-child')
+ >>> mock.attribute = not_a_child
+ >>> mock.attribute()
+ <MagicMock name='not-a-child()' id='...'>
+ >>> mock.mock_calls
+ []
+
+Mocks created for you by :func:`patch` are automatically given names. To
+attach mocks that have names to a parent you use the :meth:`~Mock.attach_mock`
+method:
+
+.. doctest::
+
+ >>> thing1 = object()
+ >>> thing2 = object()
+ >>> parent = MagicMock()
+ >>> with patch('__main__.thing1', return_value=None) as child1:
+ ... with patch('__main__.thing2', return_value=None) as child2:
+ ... parent.attach_mock(child1, 'child1')
+ ... parent.attach_mock(child2, 'child2')
+ ... child1('one')
+ ... child2('two')
+ ...
+ >>> parent.mock_calls
+ [call.child1('one'), call.child2('two')]
+
+
+-----
+
+.. [#] The only exceptions are magic methods and attributes (those that have
+ leading and trailing double underscores). Mock doesn't create these but
+ instead of raises an ``AttributeError``. This is because the interpreter
+ will often implicitly request these methods, and gets *very* confused to
+ get a new Mock object when it expects a magic method. If you need magic
+ method support see :ref:`magic methods <magic-methods>`.
diff --git a/slider-agent/src/test/python/mock/docs/patch.txt b/slider-agent/src/test/python/mock/docs/patch.txt
new file mode 100644
index 0000000..3d56264
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/patch.txt
@@ -0,0 +1,636 @@
+==================
+ Patch Decorators
+==================
+
+
+.. currentmodule:: mock
+
+.. testsetup::
+
+ class SomeClass(object):
+ static_method = None
+ class_method = None
+ attribute = None
+
+ sys.modules['package'] = package = Mock(name='package')
+ sys.modules['package.module'] = package.module
+
+ class TestCase(unittest2.TestCase):
+ def run(self):
+ result = unittest2.TestResult()
+ super(unittest2.TestCase, self).run(result)
+ assert result.wasSuccessful()
+
+.. testcleanup::
+
+ patch.TEST_PREFIX = 'test'
+
+
+The patch decorators are used for patching objects only within the scope of
+the function they decorate. They automatically handle the unpatching for you,
+even if exceptions are raised. All of these functions can also be used in with
+statements or as class decorators.
+
+
+patch
+=====
+
+.. note::
+
+ `patch` is straightforward to use. The key is to do the patching in the
+ right namespace. See the section `where to patch`_.
+
+.. function:: patch(target, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs)
+
+ `patch` acts as a function decorator, class decorator or a context
+ manager. Inside the body of the function or with statement, the `target`
+ is patched with a `new` object. When the function/with statement exits
+ the patch is undone.
+
+ If `new` is omitted, then the target is replaced with a
+ :class:`MagicMock`. If `patch` is used as a decorator and `new` is
+ omitted, the created mock is passed in as an extra argument to the
+ decorated function. If `patch` is used as a context manager the created
+ mock is returned by the context manager.
+
+ `target` should be a string in the form `'package.module.ClassName'`. The
+ `target` is imported and the specified object replaced with the `new`
+ object, so the `target` must be importable from the environment you are
+ calling `patch` from. The target is imported when the decorated function
+ is executed, not at decoration time.
+
+ The `spec` and `spec_set` keyword arguments are passed to the `MagicMock`
+ if patch is creating one for you.
+
+ In addition you can pass `spec=True` or `spec_set=True`, which causes
+ patch to pass in the object being mocked as the spec/spec_set object.
+
+ `new_callable` allows you to specify a different class, or callable object,
+ that will be called to create the `new` object. By default `MagicMock` is
+ used.
+
+ A more powerful form of `spec` is `autospec`. If you set `autospec=True`
+ then the mock with be created with a spec from the object being replaced.
+ All attributes of the mock will also have the spec of the corresponding
+ attribute of the object being replaced. Methods and functions being mocked
+ will have their arguments checked and will raise a `TypeError` if they are
+ called with the wrong signature. For mocks
+ replacing a class, their return value (the 'instance') will have the same
+ spec as the class. See the :func:`create_autospec` function and
+ :ref:`auto-speccing`.
+
+ Instead of `autospec=True` you can pass `autospec=some_object` to use an
+ arbitrary object as the spec instead of the one being replaced.
+
+ By default `patch` will fail to replace attributes that don't exist. If
+ you pass in `create=True`, and the attribute doesn't exist, patch will
+ create the attribute for you when the patched function is called, and
+ delete it again afterwards. This is useful for writing tests against
+ attributes that your production code creates at runtime. It is off by by
+ default because it can be dangerous. With it switched on you can write
+ passing tests against APIs that don't actually exist!
+
+ Patch can be used as a `TestCase` class decorator. It works by
+ decorating each test method in the class. This reduces the boilerplate
+ code when your test methods share a common patchings set. `patch` finds
+ tests by looking for method names that start with `patch.TEST_PREFIX`.
+ By default this is `test`, which matches the way `unittest` finds tests.
+ You can specify an alternative prefix by setting `patch.TEST_PREFIX`.
+
+ Patch can be used as a context manager, with the with statement. Here the
+ patching applies to the indented block after the with statement. If you
+ use "as" then the patched object will be bound to the name after the
+ "as"; very useful if `patch` is creating a mock object for you.
+
+ `patch` takes arbitrary keyword arguments. These will be passed to
+ the `Mock` (or `new_callable`) on construction.
+
+ `patch.dict(...)`, `patch.multiple(...)` and `patch.object(...)` are
+ available for alternate use-cases.
+
+`patch` as function decorator, creating the mock for you and passing it into
+the decorated function:
+
+.. doctest::
+
+ >>> @patch('__main__.SomeClass')
+ ... def function(normal_argument, mock_class):
+ ... print mock_class is SomeClass
+ ...
+ >>> function(None)
+ True
+
+
+Patching a class replaces the class with a `MagicMock` *instance*. If the
+class is instantiated in the code under test then it will be the
+:attr:`~Mock.return_value` of the mock that will be used.
+
+If the class is instantiated multiple times you could use
+:attr:`~Mock.side_effect` to return a new mock each time. Alternatively you
+can set the `return_value` to be anything you want.
+
+To configure return values on methods of *instances* on the patched class
+you must do this on the `return_value`. For example:
+
+.. doctest::
+
+ >>> class Class(object):
+ ... def method(self):
+ ... pass
+ ...
+ >>> with patch('__main__.Class') as MockClass:
+ ... instance = MockClass.return_value
+ ... instance.method.return_value = 'foo'
+ ... assert Class() is instance
+ ... assert Class().method() == 'foo'
+ ...
+
+If you use `spec` or `spec_set` and `patch` is replacing a *class*, then the
+return value of the created mock will have the same spec.
+
+.. doctest::
+
+ >>> Original = Class
+ >>> patcher = patch('__main__.Class', spec=True)
+ >>> MockClass = patcher.start()
+ >>> instance = MockClass()
+ >>> assert isinstance(instance, Original)
+ >>> patcher.stop()
+
+The `new_callable` argument is useful where you want to use an alternative
+class to the default :class:`MagicMock` for the created mock. For example, if
+you wanted a :class:`NonCallableMock` to be used:
+
+.. doctest::
+
+ >>> thing = object()
+ >>> with patch('__main__.thing', new_callable=NonCallableMock) as mock_thing:
+ ... assert thing is mock_thing
+ ... thing()
+ ...
+ Traceback (most recent call last):
+ ...
+ TypeError: 'NonCallableMock' object is not callable
+
+Another use case might be to replace an object with a `StringIO` instance:
+
+.. doctest::
+
+ >>> from StringIO import StringIO
+ >>> def foo():
+ ... print 'Something'
+ ...
+ >>> @patch('sys.stdout', new_callable=StringIO)
+ ... def test(mock_stdout):
+ ... foo()
+ ... assert mock_stdout.getvalue() == 'Something\n'
+ ...
+ >>> test()
+
+When `patch` is creating a mock for you, it is common that the first thing
+you need to do is to configure the mock. Some of that configuration can be done
+in the call to patch. Any arbitrary keywords you pass into the call will be
+used to set attributes on the created mock:
+
+.. doctest::
+
+ >>> patcher = patch('__main__.thing', first='one', second='two')
+ >>> mock_thing = patcher.start()
+ >>> mock_thing.first
+ 'one'
+ >>> mock_thing.second
+ 'two'
+
+As well as attributes on the created mock attributes, like the
+:attr:`~Mock.return_value` and :attr:`~Mock.side_effect`, of child mocks can
+also be configured. These aren't syntactically valid to pass in directly as
+keyword arguments, but a dictionary with these as keys can still be expanded
+into a `patch` call using `**`:
+
+.. doctest::
+
+ >>> config = {'method.return_value': 3, 'other.side_effect': KeyError}
+ >>> patcher = patch('__main__.thing', **config)
+ >>> mock_thing = patcher.start()
+ >>> mock_thing.method()
+ 3
+ >>> mock_thing.other()
+ Traceback (most recent call last):
+ ...
+ KeyError
+
+
+patch.object
+============
+
+.. function:: patch.object(target, attribute, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs)
+
+ patch the named member (`attribute`) on an object (`target`) with a mock
+ object.
+
+ `patch.object` can be used as a decorator, class decorator or a context
+ manager. Arguments `new`, `spec`, `create`, `spec_set`, `autospec` and
+ `new_callable` have the same meaning as for `patch`. Like `patch`,
+ `patch.object` takes arbitrary keyword arguments for configuring the mock
+ object it creates.
+
+ When used as a class decorator `patch.object` honours `patch.TEST_PREFIX`
+ for choosing which methods to wrap.
+
+You can either call `patch.object` with three arguments or two arguments. The
+three argument form takes the object to be patched, the attribute name and the
+object to replace the attribute with.
+
+When calling with the two argument form you omit the replacement object, and a
+mock is created for you and passed in as an extra argument to the decorated
+function:
+
+.. doctest::
+
+ >>> @patch.object(SomeClass, 'class_method')
+ ... def test(mock_method):
+ ... SomeClass.class_method(3)
+ ... mock_method.assert_called_with(3)
+ ...
+ >>> test()
+
+`spec`, `create` and the other arguments to `patch.object` have the same
+meaning as they do for `patch`.
+
+
+patch.dict
+==========
+
+.. function:: patch.dict(in_dict, values=(), clear=False, **kwargs)
+
+ Patch a dictionary, or dictionary like object, and restore the dictionary
+ to its original state after the test.
+
+ `in_dict` can be a dictionary or a mapping like container. If it is a
+ mapping then it must at least support getting, setting and deleting items
+ plus iterating over keys.
+
+ `in_dict` can also be a string specifying the name of the dictionary, which
+ will then be fetched by importing it.
+
+ `values` can be a dictionary of values to set in the dictionary. `values`
+ can also be an iterable of `(key, value)` pairs.
+
+ If `clear` is True then the dictionary will be cleared before the new
+ values are set.
+
+ `patch.dict` can also be called with arbitrary keyword arguments to set
+ values in the dictionary.
+
+ `patch.dict` can be used as a context manager, decorator or class
+ decorator. When used as a class decorator `patch.dict` honours
+ `patch.TEST_PREFIX` for choosing which methods to wrap.
+
+`patch.dict` can be used to add members to a dictionary, or simply let a test
+change a dictionary, and ensure the dictionary is restored when the test
+ends.
+
+.. doctest::
+
+ >>> from mock import patch
+ >>> foo = {}
+ >>> with patch.dict(foo, {'newkey': 'newvalue'}):
+ ... assert foo == {'newkey': 'newvalue'}
+ ...
+ >>> assert foo == {}
+
+ >>> import os
+ >>> with patch.dict('os.environ', {'newkey': 'newvalue'}):
+ ... print os.environ['newkey']
+ ...
+ newvalue
+ >>> assert 'newkey' not in os.environ
+
+Keywords can be used in the `patch.dict` call to set values in the dictionary:
+
+.. doctest::
+
+ >>> mymodule = MagicMock()
+ >>> mymodule.function.return_value = 'fish'
+ >>> with patch.dict('sys.modules', mymodule=mymodule):
+ ... import mymodule
+ ... mymodule.function('some', 'args')
+ ...
+ 'fish'
+
+`patch.dict` can be used with dictionary like objects that aren't actually
+dictionaries. At the very minimum they must support item getting, setting,
+deleting and either iteration or membership test. This corresponds to the
+magic methods `__getitem__`, `__setitem__`, `__delitem__` and either
+`__iter__` or `__contains__`.
+
+.. doctest::
+
+ >>> class Container(object):
+ ... def __init__(self):
+ ... self.values = {}
+ ... def __getitem__(self, name):
+ ... return self.values[name]
+ ... def __setitem__(self, name, value):
+ ... self.values[name] = value
+ ... def __delitem__(self, name):
+ ... del self.values[name]
+ ... def __iter__(self):
+ ... return iter(self.values)
+ ...
+ >>> thing = Container()
+ >>> thing['one'] = 1
+ >>> with patch.dict(thing, one=2, two=3):
+ ... assert thing['one'] == 2
+ ... assert thing['two'] == 3
+ ...
+ >>> assert thing['one'] == 1
+ >>> assert list(thing) == ['one']
+
+
+patch.multiple
+==============
+
+.. function:: patch.multiple(target, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs)
+
+ Perform multiple patches in a single call. It takes the object to be
+ patched (either as an object or a string to fetch the object by importing)
+ and keyword arguments for the patches::
+
+ with patch.multiple(settings, FIRST_PATCH='one', SECOND_PATCH='two'):
+ ...
+
+ Use :data:`DEFAULT` as the value if you want `patch.multiple` to create
+ mocks for you. In this case the created mocks are passed into a decorated
+ function by keyword, and a dictionary is returned when `patch.multiple` is
+ used as a context manager.
+
+ `patch.multiple` can be used as a decorator, class decorator or a context
+ manager. The arguments `spec`, `spec_set`, `create`, `autospec` and
+ `new_callable` have the same meaning as for `patch`. These arguments will
+ be applied to *all* patches done by `patch.multiple`.
+
+ When used as a class decorator `patch.multiple` honours `patch.TEST_PREFIX`
+ for choosing which methods to wrap.
+
+If you want `patch.multiple` to create mocks for you, then you can use
+:data:`DEFAULT` as the value. If you use `patch.multiple` as a decorator
+then the created mocks are passed into the decorated function by keyword.
+
+.. doctest::
+
+ >>> thing = object()
+ >>> other = object()
+
+ >>> @patch.multiple('__main__', thing=DEFAULT, other=DEFAULT)
+ ... def test_function(thing, other):
+ ... assert isinstance(thing, MagicMock)
+ ... assert isinstance(other, MagicMock)
+ ...
+ >>> test_function()
+
+`patch.multiple` can be nested with other `patch` decorators, but put arguments
+passed by keyword *after* any of the standard arguments created by `patch`:
+
+.. doctest::
+
+ >>> @patch('sys.exit')
+ ... @patch.multiple('__main__', thing=DEFAULT, other=DEFAULT)
+ ... def test_function(mock_exit, other, thing):
+ ... assert 'other' in repr(other)
+ ... assert 'thing' in repr(thing)
+ ... assert 'exit' in repr(mock_exit)
+ ...
+ >>> test_function()
+
+If `patch.multiple` is used as a context manager, the value returned by the
+context manger is a dictionary where created mocks are keyed by name:
+
+.. doctest::
+
+ >>> with patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) as values:
+ ... assert 'other' in repr(values['other'])
+ ... assert 'thing' in repr(values['thing'])
+ ... assert values['thing'] is thing
+ ... assert values['other'] is other
+ ...
+
+
+.. _start-and-stop:
+
+patch methods: start and stop
+=============================
+
+All the patchers have `start` and `stop` methods. These make it simpler to do
+patching in `setUp` methods or where you want to do multiple patches without
+nesting decorators or with statements.
+
+To use them call `patch`, `patch.object` or `patch.dict` as normal and keep a
+reference to the returned `patcher` object. You can then call `start` to put
+the patch in place and `stop` to undo it.
+
+If you are using `patch` to create a mock for you then it will be returned by
+the call to `patcher.start`.
+
+.. doctest::
+
+ >>> patcher = patch('package.module.ClassName')
+ >>> from package import module
+ >>> original = module.ClassName
+ >>> new_mock = patcher.start()
+ >>> assert module.ClassName is not original
+ >>> assert module.ClassName is new_mock
+ >>> patcher.stop()
+ >>> assert module.ClassName is original
+ >>> assert module.ClassName is not new_mock
+
+
+A typical use case for this might be for doing multiple patches in the `setUp`
+method of a `TestCase`:
+
+.. doctest::
+
+ >>> class MyTest(TestCase):
+ ... def setUp(self):
+ ... self.patcher1 = patch('package.module.Class1')
+ ... self.patcher2 = patch('package.module.Class2')
+ ... self.MockClass1 = self.patcher1.start()
+ ... self.MockClass2 = self.patcher2.start()
+ ...
+ ... def tearDown(self):
+ ... self.patcher1.stop()
+ ... self.patcher2.stop()
+ ...
+ ... def test_something(self):
+ ... assert package.module.Class1 is self.MockClass1
+ ... assert package.module.Class2 is self.MockClass2
+ ...
+ >>> MyTest('test_something').run()
+
+.. caution::
+
+ If you use this technique you must ensure that the patching is "undone" by
+ calling `stop`. This can be fiddlier than you might think, because if an
+ exception is raised in the setUp then tearDown is not called. `unittest2
+ <http://pypi.python.org/pypi/unittest2>`_ cleanup functions make this
+ easier.
+
+ .. doctest::
+
+ >>> class MyTest(TestCase):
+ ... def setUp(self):
+ ... patcher = patch('package.module.Class')
+ ... self.MockClass = patcher.start()
+ ... self.addCleanup(patcher.stop)
+ ...
+ ... def test_something(self):
+ ... assert package.module.Class is self.MockClass
+ ...
+ >>> MyTest('test_something').run()
+
+ As an added bonus you no longer need to keep a reference to the `patcher`
+ object.
+
+It is also possible to stop all patches which have been started by using
+`patch.stopall`.
+
+.. function:: patch.stopall
+
+ Stop all active patches. Only stops patches started with `start`.
+
+
+TEST_PREFIX
+===========
+
+All of the patchers can be used as class decorators. When used in this way
+they wrap every test method on the class. The patchers recognise methods that
+start with `test` as being test methods. This is the same way that the
+`unittest.TestLoader` finds test methods by default.
+
+It is possible that you want to use a different prefix for your tests. You can
+inform the patchers of the different prefix by setting `patch.TEST_PREFIX`:
+
+.. doctest::
+
+ >>> patch.TEST_PREFIX = 'foo'
+ >>> value = 3
+ >>>
+ >>> @patch('__main__.value', 'not three')
+ ... class Thing(object):
+ ... def foo_one(self):
+ ... print value
+ ... def foo_two(self):
+ ... print value
+ ...
+ >>>
+ >>> Thing().foo_one()
+ not three
+ >>> Thing().foo_two()
+ not three
+ >>> value
+ 3
+
+
+Nesting Patch Decorators
+========================
+
+If you want to perform multiple patches then you can simply stack up the
+decorators.
+
+You can stack up multiple patch decorators using this pattern:
+
+.. doctest::
+
+ >>> @patch.object(SomeClass, 'class_method')
+ ... @patch.object(SomeClass, 'static_method')
+ ... def test(mock1, mock2):
+ ... assert SomeClass.static_method is mock1
+ ... assert SomeClass.class_method is mock2
+ ... SomeClass.static_method('foo')
+ ... SomeClass.class_method('bar')
+ ... return mock1, mock2
+ ...
+ >>> mock1, mock2 = test()
+ >>> mock1.assert_called_once_with('foo')
+ >>> mock2.assert_called_once_with('bar')
+
+
+Note that the decorators are applied from the bottom upwards. This is the
+standard way that Python applies decorators. The order of the created mocks
+passed into your test function matches this order.
+
+Like all context-managers patches can be nested using contextlib's nested
+function; *every* patching will appear in the tuple after "as":
+
+.. doctest::
+
+ >>> from contextlib import nested
+ >>> with nested(
+ ... patch('package.module.ClassName1'),
+ ... patch('package.module.ClassName2')
+ ... ) as (MockClass1, MockClass2):
+ ... assert package.module.ClassName1 is MockClass1
+ ... assert package.module.ClassName2 is MockClass2
+ ...
+
+
+.. _where-to-patch:
+
+Where to patch
+==============
+
+`patch` works by (temporarily) changing the object that a *name* points to with
+another one. There can be many names pointing to any individual object, so
+for patching to work you must ensure that you patch the name used by the system
+under test.
+
+The basic principle is that you patch where an object is *looked up*, which
+is not necessarily the same place as where it is defined. A couple of
+examples will help to clarify this.
+
+Imagine we have a project that we want to test with the following structure::
+
+ a.py
+ -> Defines SomeClass
+
+ b.py
+ -> from a import SomeClass
+ -> some_function instantiates SomeClass
+
+Now we want to test `some_function` but we want to mock out `SomeClass` using
+`patch`. The problem is that when we import module b, which we will have to
+do then it imports `SomeClass` from module a. If we use `patch` to mock out
+`a.SomeClass` then it will have no effect on our test; module b already has a
+reference to the *real* `SomeClass` and it looks like our patching had no
+effect.
+
+The key is to patch out `SomeClass` where it is used (or where it is looked up
+). In this case `some_function` will actually look up `SomeClass` in module b,
+where we have imported it. The patching should look like:
+
+ `@patch('b.SomeClass')`
+
+However, consider the alternative scenario where instead of `from a import
+SomeClass` module b does `import a` and `some_function` uses `a.SomeClass`. Both
+of these import forms are common. In this case the class we want to patch is
+being looked up on the a module and so we have to patch `a.SomeClass` instead:
+
+ `@patch('a.SomeClass')`
+
+
+Patching Descriptors and Proxy Objects
+======================================
+
+Since version 0.6.0 both patch_ and patch.object_ have been able to correctly
+patch and restore descriptors: class methods, static methods and properties.
+You should patch these on the *class* rather than an instance.
+
+Since version 0.7.0 patch_ and patch.object_ work correctly with some objects
+that proxy attribute access, like the `django setttings object
+<http://www.voidspace.org.uk/python/weblog/arch_d7_2010_12_04.shtml#e1198>`_.
+
+.. note::
+
+ In django `import settings` and `from django.conf import settings`
+ return different objects. If you are using libraries / apps that do both you
+ may have to patch both. Grrr...
diff --git a/slider-agent/src/test/python/mock/docs/sentinel.txt b/slider-agent/src/test/python/mock/docs/sentinel.txt
new file mode 100644
index 0000000..1c5223d
--- /dev/null
+++ b/slider-agent/src/test/python/mock/docs/sentinel.txt
@@ -0,0 +1,58 @@
+==========
+ Sentinel
+==========
+
+
+.. currentmodule:: mock
+
+.. testsetup::
+
+ class ProductionClass(object):
+ def something(self):
+ return self.method()
+
+ class Test(unittest2.TestCase):
+ def testSomething(self):
+ pass
+ self = Test('testSomething')
+
+
+.. data:: sentinel
+
+ The ``sentinel`` object provides a convenient way of providing unique
+ objects for your tests.
+
+ Attributes are created on demand when you access them by name. Accessing
+ the same attribute will always return the same object. The objects
+ returned have a sensible repr so that test failure messages are readable.
+
+
+.. data:: DEFAULT
+
+ The `DEFAULT` object is a pre-created sentinel (actually
+ `sentinel.DEFAULT`). It can be used by :attr:`~Mock.side_effect`
+ functions to indicate that the normal return value should be used.
+
+
+Sentinel Example
+================
+
+Sometimes when testing you need to test that a specific object is passed as an
+argument to another method, or returned. It can be common to create named
+sentinel objects to test this. `sentinel` provides a convenient way of
+creating and testing the identity of objects like this.
+
+In this example we monkey patch `method` to return
+`sentinel.some_object`:
+
+.. doctest::
+
+ >>> real = ProductionClass()
+ >>> real.method = Mock(name="method")
+ >>> real.method.return_value = sentinel.some_object
+ >>> result = real.method()
+ >>> assert result is sentinel.some_object
+ >>> sentinel.some_object
+ sentinel.some_object
+
+
diff --git a/slider-agent/src/test/python/mock/extendmock.py b/slider-agent/src/test/python/mock/extendmock.py
new file mode 100644
index 0000000..0550d9f
--- /dev/null
+++ b/slider-agent/src/test/python/mock/extendmock.py
@@ -0,0 +1 @@
+# merged into mock.py in Mock 0.7
diff --git a/slider-agent/src/test/python/mock/mock.py b/slider-agent/src/test/python/mock/mock.py
new file mode 100644
index 0000000..2a3f869
--- /dev/null
+++ b/slider-agent/src/test/python/mock/mock.py
@@ -0,0 +1,2365 @@
+# mock.py
+# Test tools for mocking and patching.
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+
+# mock 1.0.1
+# http://www.voidspace.org.uk/python/mock/
+
+# Released subject to the BSD License
+# Please see http://www.voidspace.org.uk/python/license.shtml
+
+__all__ = (
+ 'Mock',
+ 'MagicMock',
+ 'patch',
+ 'sentinel',
+ 'DEFAULT',
+ 'ANY',
+ 'call',
+ 'create_autospec',
+ 'FILTER_DIR',
+ 'NonCallableMock',
+ 'NonCallableMagicMock',
+ 'mock_open',
+ 'PropertyMock',
+)
+
+
+__version__ = '1.0.1'
+
+
+import pprint
+import sys
+
+try:
+ import inspect
+except ImportError:
+ # for alternative platforms that
+ # may not have inspect
+ inspect = None
+
+try:
+ from functools import wraps as original_wraps
+except ImportError:
+ # Python 2.4 compatibility
+ def wraps(original):
+ def inner(f):
+ f.__name__ = original.__name__
+ f.__doc__ = original.__doc__
+ f.__module__ = original.__module__
+ wrapped = getattr(original, '__wrapped__', original)
+ f.__wrapped__ = wrapped
+ return f
+ return inner
+else:
+ if sys.version_info[:2] >= (3, 2):
+ wraps = original_wraps
+ else:
+ def wraps(func):
+ def inner(f):
+ f = original_wraps(func)(f)
+ wrapped = getattr(func, '__wrapped__', func)
+ f.__wrapped__ = wrapped
+ return f
+ return inner
+
+try:
+ unicode
+except NameError:
+ # Python 3
+ basestring = unicode = str
+
+try:
+ long
+except NameError:
+ # Python 3
+ long = int
+
+try:
+ BaseException
+except NameError:
+ # Python 2.4 compatibility
+ BaseException = Exception
+
+try:
+ next
+except NameError:
+ def next(obj):
+ return obj.next()
+
+
+BaseExceptions = (BaseException,)
+if 'java' in sys.platform:
+ # jython
+ import java
+ BaseExceptions = (BaseException, java.lang.Throwable)
+
+try:
+ _isidentifier = str.isidentifier
+except AttributeError:
+ # Python 2.X
+ import keyword
+ import re
+ regex = re.compile(r'^[a-z_][a-z0-9_]*$', re.I)
+ def _isidentifier(string):
+ if string in keyword.kwlist:
+ return False
+ return regex.match(string)
+
+
+inPy3k = sys.version_info[0] == 3
+
+# Needed to work around Python 3 bug where use of "super" interferes with
+# defining __class__ as a descriptor
+_super = super
+
+self = 'im_self'
+builtin = '__builtin__'
+if inPy3k:
+ self = '__self__'
+ builtin = 'builtins'
+
+FILTER_DIR = True
+
+
+def _is_instance_mock(obj):
+ # can't use isinstance on Mock objects because they override __class__
+ # The base class for all mocks is NonCallableMock
+ return issubclass(type(obj), NonCallableMock)
+
+
+def _is_exception(obj):
+ return (
+ isinstance(obj, BaseExceptions) or
+ isinstance(obj, ClassTypes) and issubclass(obj, BaseExceptions)
+ )
+
+
+class _slotted(object):
+ __slots__ = ['a']
+
+
+DescriptorTypes = (
+ type(_slotted.a),
+ property,
+)
+
+
+def _getsignature(func, skipfirst, instance=False):
+ if inspect is None:
+ raise ImportError('inspect module not available')
+
+ if isinstance(func, ClassTypes) and not instance:
+ try:
+ func = func.__init__
+ except AttributeError:
+ return
+ skipfirst = True
+ elif not isinstance(func, FunctionTypes):
+ # for classes where instance is True we end up here too
+ try:
+ func = func.__call__
+ except AttributeError:
+ return
+
+ if inPy3k:
+ try:
+ argspec = inspect.getfullargspec(func)
+ except TypeError:
+ # C function / method, possibly inherited object().__init__
+ return
+ regargs, varargs, varkw, defaults, kwonly, kwonlydef, ann = argspec
+ else:
+ try:
+ regargs, varargs, varkwargs, defaults = inspect.getargspec(func)
+ except TypeError:
+ # C function / method, possibly inherited object().__init__
+ return
+
+ # instance methods and classmethods need to lose the self argument
+ if getattr(func, self, None) is not None:
+ regargs = regargs[1:]
+ if skipfirst:
+ # this condition and the above one are never both True - why?
+ regargs = regargs[1:]
+
+ if inPy3k:
+ signature = inspect.formatargspec(
+ regargs, varargs, varkw, defaults,
+ kwonly, kwonlydef, ann, formatvalue=lambda value: "")
+ else:
+ signature = inspect.formatargspec(
+ regargs, varargs, varkwargs, defaults,
+ formatvalue=lambda value: "")
+ return signature[1:-1], func
+
+
+def _check_signature(func, mock, skipfirst, instance=False):
+ if not _callable(func):
+ return
+
+ result = _getsignature(func, skipfirst, instance)
+ if result is None:
+ return
+ signature, func = result
+
+ # can't use self because "self" is common as an argument name
+ # unfortunately even not in the first place
+ src = "lambda _mock_self, %s: None" % signature
+ checksig = eval(src, {})
+ _copy_func_details(func, checksig)
+ type(mock)._mock_check_sig = checksig
+
+
+def _copy_func_details(func, funcopy):
+ funcopy.__name__ = func.__name__
+ funcopy.__doc__ = func.__doc__
+ #funcopy.__dict__.update(func.__dict__)
+ funcopy.__module__ = func.__module__
+ if not inPy3k:
+ funcopy.func_defaults = func.func_defaults
+ return
+ funcopy.__defaults__ = func.__defaults__
+ funcopy.__kwdefaults__ = func.__kwdefaults__
+
+
+def _callable(obj):
+ if isinstance(obj, ClassTypes):
+ return True
+ if getattr(obj, '__call__', None) is not None:
+ return True
+ return False
+
+
+def _is_list(obj):
+ # checks for list or tuples
+ # XXXX badly named!
+ return type(obj) in (list, tuple)
+
+
+def _instance_callable(obj):
+ """Given an object, return True if the object is callable.
+ For classes, return True if instances would be callable."""
+ if not isinstance(obj, ClassTypes):
+ # already an instance
+ return getattr(obj, '__call__', None) is not None
+
+ klass = obj
+ # uses __bases__ instead of __mro__ so that we work with old style classes
+ if klass.__dict__.get('__call__') is not None:
+ return True
+
+ for base in klass.__bases__:
+ if _instance_callable(base):
+ return True
+ return False
+
+
+def _set_signature(mock, original, instance=False):
+ # creates a function with signature (*args, **kwargs) that delegates to a
+ # mock. It still does signature checking by calling a lambda with the same
+ # signature as the original.
+ if not _callable(original):
+ return
+
+ skipfirst = isinstance(original, ClassTypes)
+ result = _getsignature(original, skipfirst, instance)
+ if result is None:
+ # was a C function (e.g. object().__init__ ) that can't be mocked
+ return
+
+ signature, func = result
+
+ src = "lambda %s: None" % signature
+ checksig = eval(src, {})
+ _copy_func_details(func, checksig)
+
+ name = original.__name__
+ if not _isidentifier(name):
+ name = 'funcopy'
+ context = {'_checksig_': checksig, 'mock': mock}
+ src = """def %s(*args, **kwargs):
+ _checksig_(*args, **kwargs)
+ return mock(*args, **kwargs)""" % name
+ exec (src, context)
+ funcopy = context[name]
+ _setup_func(funcopy, mock)
+ return funcopy
+
+
+def _setup_func(funcopy, mock):
+ funcopy.mock = mock
+
+ # can't use isinstance with mocks
+ if not _is_instance_mock(mock):
+ return
+
+ def assert_called_with(*args, **kwargs):
+ return mock.assert_called_with(*args, **kwargs)
+ def assert_called_once_with(*args, **kwargs):
+ return mock.assert_called_once_with(*args, **kwargs)
+ def assert_has_calls(*args, **kwargs):
+ return mock.assert_has_calls(*args, **kwargs)
+ def assert_any_call(*args, **kwargs):
+ return mock.assert_any_call(*args, **kwargs)
+ def reset_mock():
+ funcopy.method_calls = _CallList()
+ funcopy.mock_calls = _CallList()
+ mock.reset_mock()
+ ret = funcopy.return_value
+ if _is_instance_mock(ret) and not ret is mock:
+ ret.reset_mock()
+
+ funcopy.called = False
+ funcopy.call_count = 0
+ funcopy.call_args = None
+ funcopy.call_args_list = _CallList()
+ funcopy.method_calls = _CallList()
+ funcopy.mock_calls = _CallList()
+
+ funcopy.return_value = mock.return_value
+ funcopy.side_effect = mock.side_effect
+ funcopy._mock_children = mock._mock_children
+
+ funcopy.assert_called_with = assert_called_with
+ funcopy.assert_called_once_with = assert_called_once_with
+ funcopy.assert_has_calls = assert_has_calls
+ funcopy.assert_any_call = assert_any_call
+ funcopy.reset_mock = reset_mock
+
+ mock._mock_delegate = funcopy
+
+
+def _is_magic(name):
+ return '__%s__' % name[2:-2] == name
+
+
+class _SentinelObject(object):
+ "A unique, named, sentinel object."
+ def __init__(self, name):
+ self.name = name
+
+ def __repr__(self):
+ return 'sentinel.%s' % self.name
+
+
+class _Sentinel(object):
+ """Access attributes to return a named object, usable as a sentinel."""
+ def __init__(self):
+ self._sentinels = {}
+
+ def __getattr__(self, name):
+ if name == '__bases__':
+ # Without this help(mock) raises an exception
+ raise AttributeError
+ return self._sentinels.setdefault(name, _SentinelObject(name))
+
+
+sentinel = _Sentinel()
+
+DEFAULT = sentinel.DEFAULT
+_missing = sentinel.MISSING
+_deleted = sentinel.DELETED
+
+
+class OldStyleClass:
+ pass
+ClassType = type(OldStyleClass)
+
+
+def _copy(value):
+ if type(value) in (dict, list, tuple, set):
+ return type(value)(value)
+ return value
+
+
+ClassTypes = (type,)
+if not inPy3k:
+ ClassTypes = (type, ClassType)
+
+_allowed_names = set(
+ [
+ 'return_value', '_mock_return_value', 'side_effect',
+ '_mock_side_effect', '_mock_parent', '_mock_new_parent',
+ '_mock_name', '_mock_new_name'
+ ]
+)
+
+
+def _delegating_property(name):
+ _allowed_names.add(name)
+ _the_name = '_mock_' + name
+ def _get(self, name=name, _the_name=_the_name):
+ sig = self._mock_delegate
+ if sig is None:
+ return getattr(self, _the_name)
+ return getattr(sig, name)
+ def _set(self, value, name=name, _the_name=_the_name):
+ sig = self._mock_delegate
+ if sig is None:
+ self.__dict__[_the_name] = value
+ else:
+ setattr(sig, name, value)
+
+ return property(_get, _set)
+
+
+
+class _CallList(list):
+
+ def __contains__(self, value):
+ if not isinstance(value, list):
+ return list.__contains__(self, value)
+ len_value = len(value)
+ len_self = len(self)
+ if len_value > len_self:
+ return False
+
+ for i in range(0, len_self - len_value + 1):
+ sub_list = self[i:i+len_value]
+ if sub_list == value:
+ return True
+ return False
+
+ def __repr__(self):
+ return pprint.pformat(list(self))
+
+
+def _check_and_set_parent(parent, value, name, new_name):
+ if not _is_instance_mock(value):
+ return False
+ if ((value._mock_name or value._mock_new_name) or
+ (value._mock_parent is not None) or
+ (value._mock_new_parent is not None)):
+ return False
+
+ _parent = parent
+ while _parent is not None:
+ # setting a mock (value) as a child or return value of itself
+ # should not modify the mock
+ if _parent is value:
+ return False
+ _parent = _parent._mock_new_parent
+
+ if new_name:
+ value._mock_new_parent = parent
+ value._mock_new_name = new_name
+ if name:
+ value._mock_parent = parent
+ value._mock_name = name
+ return True
+
+
+
+class Base(object):
+ _mock_return_value = DEFAULT
+ _mock_side_effect = None
+ def __init__(self, *args, **kwargs):
+ pass
+
+
+
+class NonCallableMock(Base):
+ """A non-callable version of `Mock`"""
+
+ def __new__(cls, *args, **kw):
+ # every instance has its own class
+ # so we can create magic methods on the
+ # class without stomping on other mocks
+ new = type(cls.__name__, (cls,), {'__doc__': cls.__doc__})
+ instance = object.__new__(new)
+ return instance
+
+
+ def __init__(
+ self, spec=None, wraps=None, name=None, spec_set=None,
+ parent=None, _spec_state=None, _new_name='', _new_parent=None,
+ **kwargs
+ ):
+ if _new_parent is None:
+ _new_parent = parent
+
+ __dict__ = self.__dict__
+ __dict__['_mock_parent'] = parent
+ __dict__['_mock_name'] = name
+ __dict__['_mock_new_name'] = _new_name
+ __dict__['_mock_new_parent'] = _new_parent
+
+ if spec_set is not None:
+ spec = spec_set
+ spec_set = True
+
+ self._mock_add_spec(spec, spec_set)
+
+ __dict__['_mock_children'] = {}
+ __dict__['_mock_wraps'] = wraps
+ __dict__['_mock_delegate'] = None
+
+ __dict__['_mock_called'] = False
+ __dict__['_mock_call_args'] = None
+ __dict__['_mock_call_count'] = 0
+ __dict__['_mock_call_args_list'] = _CallList()
+ __dict__['_mock_mock_calls'] = _CallList()
+
+ __dict__['method_calls'] = _CallList()
+
+ if kwargs:
+ self.configure_mock(**kwargs)
+
+ _super(NonCallableMock, self).__init__(
+ spec, wraps, name, spec_set, parent,
+ _spec_state
+ )
+
+
+ def attach_mock(self, mock, attribute):
+ """
+ Attach a mock as an attribute of this one, replacing its name and
+ parent. Calls to the attached mock will be recorded in the
+ `method_calls` and `mock_calls` attributes of this one."""
+ mock._mock_parent = None
+ mock._mock_new_parent = None
+ mock._mock_name = ''
+ mock._mock_new_name = None
+
+ setattr(self, attribute, mock)
+
+
+ def mock_add_spec(self, spec, spec_set=False):
+ """Add a spec to a mock. `spec` can either be an object or a
+ list of strings. Only attributes on the `spec` can be fetched as
+ attributes from the mock.
+
+ If `spec_set` is True then only attributes on the spec can be set."""
+ self._mock_add_spec(spec, spec_set)
+
+
+ def _mock_add_spec(self, spec, spec_set):
+ _spec_class = None
+
+ if spec is not None and not _is_list(spec):
+ if isinstance(spec, ClassTypes):
+ _spec_class = spec
+ else:
+ _spec_class = _get_class(spec)
+
+ spec = dir(spec)
+
+ __dict__ = self.__dict__
+ __dict__['_spec_class'] = _spec_class
+ __dict__['_spec_set'] = spec_set
+ __dict__['_mock_methods'] = spec
+
+
+ def __get_return_value(self):
+ ret = self._mock_return_value
+ if self._mock_delegate is not None:
+ ret = self._mock_delegate.return_value
+
+ if ret is DEFAULT:
+ ret = self._get_child_mock(
+ _new_parent=self, _new_name='()'
+ )
+ self.return_value = ret
+ return ret
+
+
+ def __set_return_value(self, value):
+ if self._mock_delegate is not None:
+ self._mock_delegate.return_value = value
+ else:
+ self._mock_return_value = value
+ _check_and_set_parent(self, value, None, '()')
+
+ __return_value_doc = "The value to be returned when the mock is called."
+ return_value = property(__get_return_value, __set_return_value,
+ __return_value_doc)
+
+
+ @property
+ def __class__(self):
+ if self._spec_class is None:
+ return type(self)
+ return self._spec_class
+
+ called = _delegating_property('called')
+ call_count = _delegating_property('call_count')
+ call_args = _delegating_property('call_args')
+ call_args_list = _delegating_property('call_args_list')
+ mock_calls = _delegating_property('mock_calls')
+
+
+ def __get_side_effect(self):
+ sig = self._mock_delegate
+ if sig is None:
+ return self._mock_side_effect
+ return sig.side_effect
+
+ def __set_side_effect(self, value):
+ value = _try_iter(value)
+ sig = self._mock_delegate
+ if sig is None:
+ self._mock_side_effect = value
+ else:
+ sig.side_effect = value
+
+ side_effect = property(__get_side_effect, __set_side_effect)
+
+
+ def reset_mock(self):
+ "Restore the mock object to its initial state."
+ self.called = False
+ self.call_args = None
+ self.call_count = 0
+ self.mock_calls = _CallList()
+ self.call_args_list = _CallList()
+ self.method_calls = _CallList()
+
+ for child in self._mock_children.values():
+ if isinstance(child, _SpecState):
+ continue
+ child.reset_mock()
+
+ ret = self._mock_return_value
+ if _is_instance_mock(ret) and ret is not self:
+ ret.reset_mock()
+
+
+ def configure_mock(self, **kwargs):
+ """Set attributes on the mock through keyword arguments.
+
+ Attributes plus return values and side effects can be set on child
+ mocks using standard dot notation and unpacking a dictionary in the
+ method call:
+
+ >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError}
+ >>> mock.configure_mock(**attrs)"""
+ for arg, val in sorted(kwargs.items(),
+ # we sort on the number of dots so that
+ # attributes are set before we set attributes on
+ # attributes
+ key=lambda entry: entry[0].count('.')):
+ args = arg.split('.')
+ final = args.pop()
+ obj = self
+ for entry in args:
+ obj = getattr(obj, entry)
+ setattr(obj, final, val)
+
+
+ def __getattr__(self, name):
+ if name == '_mock_methods':
+ raise AttributeError(name)
+ elif self._mock_methods is not None:
+ if name not in self._mock_methods or name in _all_magics:
+ raise AttributeError("Mock object has no attribute %r" % name)
+ elif _is_magic(name):
+ raise AttributeError(name)
+
+ result = self._mock_children.get(name)
+ if result is _deleted:
+ raise AttributeError(name)
+ elif result is None:
+ wraps = None
+ if self._mock_wraps is not None:
+ # XXXX should we get the attribute without triggering code
+ # execution?
+ wraps = getattr(self._mock_wraps, name)
+
+ result = self._get_child_mock(
+ parent=self, name=name, wraps=wraps, _new_name=name,
+ _new_parent=self
+ )
+ self._mock_children[name] = result
+
+ elif isinstance(result, _SpecState):
+ result = create_autospec(
+ result.spec, result.spec_set, result.instance,
+ result.parent, result.name
+ )
+ self._mock_children[name] = result
+
+ return result
+
+
+ def __repr__(self):
+ _name_list = [self._mock_new_name]
+ _parent = self._mock_new_parent
+ last = self
+
+ dot = '.'
+ if _name_list == ['()']:
+ dot = ''
+ seen = set()
+ while _parent is not None:
+ last = _parent
+
+ _name_list.append(_parent._mock_new_name + dot)
+ dot = '.'
+ if _parent._mock_new_name == '()':
+ dot = ''
+
+ _parent = _parent._mock_new_parent
+
+ # use ids here so as not to call __hash__ on the mocks
+ if id(_parent) in seen:
+ break
+ seen.add(id(_parent))
+
+ _name_list = list(reversed(_name_list))
+ _first = last._mock_name or 'mock'
+ if len(_name_list) > 1:
+ if _name_list[1] not in ('()', '().'):
+ _first += '.'
+ _name_list[0] = _first
+ name = ''.join(_name_list)
+
+ name_string = ''
+ if name not in ('mock', 'mock.'):
+ name_string = ' name=%r' % name
+
+ spec_string = ''
+ if self._spec_class is not None:
+ spec_string = ' spec=%r'
+ if self._spec_set:
+ spec_string = ' spec_set=%r'
+ spec_string = spec_string % self._spec_class.__name__
+ return "<%s%s%s id='%s'>" % (
+ type(self).__name__,
+ name_string,
+ spec_string,
+ id(self)
+ )
+
+
+ def __dir__(self):
+ """Filter the output of `dir(mock)` to only useful members.
+ XXXX
+ """
+ extras = self._mock_methods or []
+ from_type = dir(type(self))
+ from_dict = list(self.__dict__)
+
+ if FILTER_DIR:
+ from_type = [e for e in from_type if not e.startswith('_')]
+ from_dict = [e for e in from_dict if not e.startswith('_') or
+ _is_magic(e)]
+ return sorted(set(extras + from_type + from_dict +
+ list(self._mock_children)))
+
+
+ def __setattr__(self, name, value):
+ if name in _allowed_names:
+ # property setters go through here
+ return object.__setattr__(self, name, value)
+ elif (self._spec_set and self._mock_methods is not None and
+ name not in self._mock_methods and
+ name not in self.__dict__):
+ raise AttributeError("Mock object has no attribute '%s'" % name)
+ elif name in _unsupported_magics:
+ msg = 'Attempting to set unsupported magic method %r.' % name
+ raise AttributeError(msg)
+ elif name in _all_magics:
+ if self._mock_methods is not None and name not in self._mock_methods:
+ raise AttributeError("Mock object has no attribute '%s'" % name)
+
+ if not _is_instance_mock(value):
+ setattr(type(self), name, _get_method(name, value))
+ original = value
+ value = lambda *args, **kw: original(self, *args, **kw)
+ else:
+ # only set _new_name and not name so that mock_calls is tracked
+ # but not method calls
+ _check_and_set_parent(self, value, None, name)
+ setattr(type(self), name, value)
+ self._mock_children[name] = value
+ elif name == '__class__':
+ self._spec_class = value
+ return
+ else:
+ if _check_and_set_parent(self, value, name, name):
+ self._mock_children[name] = value
+ return object.__setattr__(self, name, value)
+
+
+ def __delattr__(self, name):
+ if name in _all_magics and name in type(self).__dict__:
+ delattr(type(self), name)
+ if name not in self.__dict__:
+ # for magic methods that are still MagicProxy objects and
+ # not set on the instance itself
+ return
+
+ if name in self.__dict__:
+ object.__delattr__(self, name)
+
+ obj = self._mock_children.get(name, _missing)
+ if obj is _deleted:
+ raise AttributeError(name)
+ if obj is not _missing:
+ del self._mock_children[name]
+ self._mock_children[name] = _deleted
+
+
+
+ def _format_mock_call_signature(self, args, kwargs):
+ name = self._mock_name or 'mock'
+ return _format_call_signature(name, args, kwargs)
+
+
+ def _format_mock_failure_message(self, args, kwargs):
+ message = 'Expected call: %s\nActual call: %s'
+ expected_string = self._format_mock_call_signature(args, kwargs)
+ call_args = self.call_args
+ if len(call_args) == 3:
+ call_args = call_args[1:]
+ actual_string = self._format_mock_call_signature(*call_args)
+ return message % (expected_string, actual_string)
+
+
+ def assert_called_with(_mock_self, *args, **kwargs):
+ """assert that the mock was called with the specified arguments.
+
+ Raises an AssertionError if the args and keyword args passed in are
+ different to the last call to the mock."""
+ self = _mock_self
+ if self.call_args is None:
+ expected = self._format_mock_call_signature(args, kwargs)
+ raise AssertionError('Expected call: %s\nNot called' % (expected,))
+
+ if self.call_args != (args, kwargs):
+ msg = self._format_mock_failure_message(args, kwargs)
+ raise AssertionError(msg)
+
+
+ def assert_called_once_with(_mock_self, *args, **kwargs):
+ """assert that the mock was called exactly once and with the specified
+ arguments."""
+ self = _mock_self
+ if not self.call_count == 1:
+ msg = ("Expected to be called once. Called %s times." %
+ self.call_count)
+ raise AssertionError(msg)
+ return self.assert_called_with(*args, **kwargs)
+
+
+ def assert_has_calls(self, calls, any_order=False):
+ """assert the mock has been called with the specified calls.
+ The `mock_calls` list is checked for the calls.
+
+ If `any_order` is False (the default) then the calls must be
+ sequential. There can be extra calls before or after the
+ specified calls.
+
+ If `any_order` is True then the calls can be in any order, but
+ they must all appear in `mock_calls`."""
+ if not any_order:
+ if calls not in self.mock_calls:
+ raise AssertionError(
+ 'Calls not found.\nExpected: %r\n'
+ 'Actual: %r' % (calls, self.mock_calls)
+ )
+ return
+
+ all_calls = list(self.mock_calls)
+
+ not_found = []
+ for kall in calls:
+ try:
+ all_calls.remove(kall)
+ except ValueError:
+ not_found.append(kall)
+ if not_found:
+ raise AssertionError(
+ '%r not all found in call list' % (tuple(not_found),)
+ )
+
+
+ def assert_any_call(self, *args, **kwargs):
+ """assert the mock has been called with the specified arguments.
+
+ The assert passes if the mock has *ever* been called, unlike
+ `assert_called_with` and `assert_called_once_with` that only pass if
+ the call is the most recent one."""
+ kall = call(*args, **kwargs)
+ if kall not in self.call_args_list:
+ expected_string = self._format_mock_call_signature(args, kwargs)
+ raise AssertionError(
+ '%s call not found' % expected_string
+ )
+
+
+ def _get_child_mock(self, **kw):
+ """Create the child mocks for attributes and return value.
+ By default child mocks will be the same type as the parent.
+ Subclasses of Mock may want to override this to customize the way
+ child mocks are made.
+
+ For non-callable mocks the callable variant will be used (rather than
+ any custom subclass)."""
+ _type = type(self)
+ if not issubclass(_type, CallableMixin):
+ if issubclass(_type, NonCallableMagicMock):
+ klass = MagicMock
+ elif issubclass(_type, NonCallableMock) :
+ klass = Mock
+ else:
+ klass = _type.__mro__[1]
+ return klass(**kw)
+
+
+
+def _try_iter(obj):
+ if obj is None:
+ return obj
+ if _is_exception(obj):
+ return obj
+ if _callable(obj):
+ return obj
+ try:
+ return iter(obj)
+ except TypeError:
+ # XXXX backwards compatibility
+ # but this will blow up on first call - so maybe we should fail early?
+ return obj
+
+
+
+class CallableMixin(Base):
+
+ def __init__(self, spec=None, side_effect=None, return_value=DEFAULT,
+ wraps=None, name=None, spec_set=None, parent=None,
+ _spec_state=None, _new_name='', _new_parent=None, **kwargs):
+ self.__dict__['_mock_return_value'] = return_value
+
+ _super(CallableMixin, self).__init__(
+ spec, wraps, name, spec_set, parent,
+ _spec_state, _new_name, _new_parent, **kwargs
+ )
+
+ self.side_effect = side_effect
+
+
+ def _mock_check_sig(self, *args, **kwargs):
+ # stub method that can be replaced with one with a specific signature
+ pass
+
+
+ def __call__(_mock_self, *args, **kwargs):
+ # can't use self in-case a function / method we are mocking uses self
+ # in the signature
+ _mock_self._mock_check_sig(*args, **kwargs)
+ return _mock_self._mock_call(*args, **kwargs)
+
+
+ def _mock_call(_mock_self, *args, **kwargs):
+ self = _mock_self
+ self.called = True
+ self.call_count += 1
+ self.call_args = _Call((args, kwargs), two=True)
+ self.call_args_list.append(_Call((args, kwargs), two=True))
+
+ _new_name = self._mock_new_name
+ _new_parent = self._mock_new_parent
+ self.mock_calls.append(_Call(('', args, kwargs)))
+
+ seen = set()
+ skip_next_dot = _new_name == '()'
+ do_method_calls = self._mock_parent is not None
+ name = self._mock_name
+ while _new_parent is not None:
+ this_mock_call = _Call((_new_name, args, kwargs))
+ if _new_parent._mock_new_name:
+ dot = '.'
+ if skip_next_dot:
+ dot = ''
+
+ skip_next_dot = False
+ if _new_parent._mock_new_name == '()':
+ skip_next_dot = True
+
+ _new_name = _new_parent._mock_new_name + dot + _new_name
+
+ if do_method_calls:
+ if _new_name == name:
+ this_method_call = this_mock_call
+ else:
+ this_method_call = _Call((name, args, kwargs))
+ _new_parent.method_calls.append(this_method_call)
+
+ do_method_calls = _new_parent._mock_parent is not None
+ if do_method_calls:
+ name = _new_parent._mock_name + '.' + name
+
+ _new_parent.mock_calls.append(this_mock_call)
+ _new_parent = _new_parent._mock_new_parent
+
+ # use ids here so as not to call __hash__ on the mocks
+ _new_parent_id = id(_new_parent)
+ if _new_parent_id in seen:
+ break
+ seen.add(_new_parent_id)
+
+ ret_val = DEFAULT
+ effect = self.side_effect
+ if effect is not None:
+ if _is_exception(effect):
+ raise effect
+
+ if not _callable(effect):
+ result = next(effect)
+ if _is_exception(result):
+ raise result
+ return result
+
+ ret_val = effect(*args, **kwargs)
+ if ret_val is DEFAULT:
+ ret_val = self.return_value
+
+ if (self._mock_wraps is not None and
+ self._mock_return_value is DEFAULT):
+ return self._mock_wraps(*args, **kwargs)
+ if ret_val is DEFAULT:
+ ret_val = self.return_value
+ return ret_val
+
+
+
+class Mock(CallableMixin, NonCallableMock):
+ """
+ Create a new `Mock` object. `Mock` takes several optional arguments
+ that specify the behaviour of the Mock object:
+
+ * `spec`: This can be either a list of strings or an existing object (a
+ class or instance) that acts as the specification for the mock object. If
+ you pass in an object then a list of strings is formed by calling dir on
+ the object (excluding unsupported magic attributes and methods). Accessing
+ any attribute not in this list will raise an `AttributeError`.
+
+ If `spec` is an object (rather than a list of strings) then
+ `mock.__class__` returns the class of the spec object. This allows mocks
+ to pass `isinstance` tests.
+
+ * `spec_set`: A stricter variant of `spec`. If used, attempting to *set*
+ or get an attribute on the mock that isn't on the object passed as
+ `spec_set` will raise an `AttributeError`.
+
+ * `side_effect`: A function to be called whenever the Mock is called. See
+ the `side_effect` attribute. Useful for raising exceptions or
+ dynamically changing return values. The function is called with the same
+ arguments as the mock, and unless it returns `DEFAULT`, the return
+ value of this function is used as the return value.
+
+ Alternatively `side_effect` can be an exception class or instance. In
+ this case the exception will be raised when the mock is called.
+
+ If `side_effect` is an iterable then each call to the mock will return
+ the next value from the iterable. If any of the members of the iterable
+ are exceptions they will be raised instead of returned.
+
+ * `return_value`: The value returned when the mock is called. By default
+ this is a new Mock (created on first access). See the
+ `return_value` attribute.
+
+ * `wraps`: Item for the mock object to wrap. If `wraps` is not None then
+ calling the Mock will pass the call through to the wrapped object
+ (returning the real result). Attribute access on the mock will return a
+ Mock object that wraps the corresponding attribute of the wrapped object
+ (so attempting to access an attribute that doesn't exist will raise an
+ `AttributeError`).
+
+ If the mock has an explicit `return_value` set then calls are not passed
+ to the wrapped object and the `return_value` is returned instead.
+
+ * `name`: If the mock has a name then it will be used in the repr of the
+ mock. This can be useful for debugging. The name is propagated to child
+ mocks.
+
+ Mocks can also be called with arbitrary keyword arguments. These will be
+ used to set attributes on the mock after it is created.
+ """
+
+
+
+def _dot_lookup(thing, comp, import_path):
+ try:
+ return getattr(thing, comp)
+ except AttributeError:
+ __import__(import_path)
+ return getattr(thing, comp)
+
+
+def _importer(target):
+ components = target.split('.')
+ import_path = components.pop(0)
+ thing = __import__(import_path)
+
+ for comp in components:
+ import_path += ".%s" % comp
+ thing = _dot_lookup(thing, comp, import_path)
+ return thing
+
+
+def _is_started(patcher):
+ # XXXX horrible
+ return hasattr(patcher, 'is_local')
+
+
+class _patch(object):
+
+ attribute_name = None
+ _active_patches = set()
+
+ def __init__(
+ self, getter, attribute, new, spec, create,
+ spec_set, autospec, new_callable, kwargs
+ ):
+ if new_callable is not None:
+ if new is not DEFAULT:
+ raise ValueError(
+ "Cannot use 'new' and 'new_callable' together"
+ )
+ if autospec is not None:
+ raise ValueError(
+ "Cannot use 'autospec' and 'new_callable' together"
+ )
+
+ self.getter = getter
+ self.attribute = attribute
+ self.new = new
+ self.new_callable = new_callable
+ self.spec = spec
+ self.create = create
+ self.has_local = False
+ self.spec_set = spec_set
+ self.autospec = autospec
+ self.kwargs = kwargs
+ self.additional_patchers = []
+
+
+ def copy(self):
+ patcher = _patch(
+ self.getter, self.attribute, self.new, self.spec,
+ self.create, self.spec_set,
+ self.autospec, self.new_callable, self.kwargs
+ )
+ patcher.attribute_name = self.attribute_name
+ patcher.additional_patchers = [
+ p.copy() for p in self.additional_patchers
+ ]
+ return patcher
+
+
+ def __call__(self, func):
+ if isinstance(func, ClassTypes):
+ return self.decorate_class(func)
+ return self.decorate_callable(func)
+
+
+ def decorate_class(self, klass):
+ for attr in dir(klass):
+ if not attr.startswith(patch.TEST_PREFIX):
+ continue
+
+ attr_value = getattr(klass, attr)
+ if not hasattr(attr_value, "__call__"):
+ continue
+
+ patcher = self.copy()
+ setattr(klass, attr, patcher(attr_value))
+ return klass
+
+
+ def decorate_callable(self, func):
+ if hasattr(func, 'patchings'):
+ func.patchings.append(self)
+ return func
+
+ @wraps(func)
+ def patched(*args, **keywargs):
+ # don't use a with here (backwards compatability with Python 2.4)
+ extra_args = []
+ entered_patchers = []
+
+ # can't use try...except...finally because of Python 2.4
+ # compatibility
+ exc_info = tuple()
+ try:
+ try:
+ for patching in patched.patchings:
+ arg = patching.__enter__()
+ entered_patchers.append(patching)
+ if patching.attribute_name is not None:
+ keywargs.update(arg)
+ elif patching.new is DEFAULT:
+ extra_args.append(arg)
+
+ args += tuple(extra_args)
+ return func(*args, **keywargs)
+ except:
+ if (patching not in entered_patchers and
+ _is_started(patching)):
+ # the patcher may have been started, but an exception
+ # raised whilst entering one of its additional_patchers
+ entered_patchers.append(patching)
+ # Pass the exception to __exit__
+ exc_info = sys.exc_info()
+ # re-raise the exception
+ raise
+ finally:
+ for patching in reversed(entered_patchers):
+ patching.__exit__(*exc_info)
+
+ patched.patchings = [self]
+ if hasattr(func, 'func_code'):
+ # not in Python 3
+ patched.compat_co_firstlineno = getattr(
+ func, "compat_co_firstlineno",
+ func.func_code.co_firstlineno
+ )
+ return patched
+
+
+ def get_original(self):
+ target = self.getter()
+ name = self.attribute
+
+ original = DEFAULT
+ local = False
+
+ try:
+ original = target.__dict__[name]
+ except (AttributeError, KeyError):
+ original = getattr(target, name, DEFAULT)
+ else:
+ local = True
+
+ if not self.create and original is DEFAULT:
+ raise AttributeError(
+ "%s does not have the attribute %r" % (target, name)
+ )
+ return original, local
+
+
+ def __enter__(self):
+ """Perform the patch."""
+ new, spec, spec_set = self.new, self.spec, self.spec_set
+ autospec, kwargs = self.autospec, self.kwargs
+ new_callable = self.new_callable
+ self.target = self.getter()
+
+ # normalise False to None
+ if spec is False:
+ spec = None
+ if spec_set is False:
+ spec_set = None
+ if autospec is False:
+ autospec = None
+
+ if spec is not None and autospec is not None:
+ raise TypeError("Can't specify spec and autospec")
+ if ((spec is not None or autospec is not None) and
+ spec_set not in (True, None)):
+ raise TypeError("Can't provide explicit spec_set *and* spec or autospec")
+
+ original, local = self.get_original()
+
+ if new is DEFAULT and autospec is None:
+ inherit = False
+ if spec is True:
+ # set spec to the object we are replacing
+ spec = original
+ if spec_set is True:
+ spec_set = original
+ spec = None
+ elif spec is not None:
+ if spec_set is True:
+ spec_set = spec
+ spec = None
+ elif spec_set is True:
+ spec_set = original
+
+ if spec is not None or spec_set is not None:
+ if original is DEFAULT:
+ raise TypeError("Can't use 'spec' with create=True")
+ if isinstance(original, ClassTypes):
+ # If we're patching out a class and there is a spec
+ inherit = True
+
+ Klass = MagicMock
+ _kwargs = {}
+ if new_callable is not None:
+ Klass = new_callable
+ elif spec is not None or spec_set is not None:
+ this_spec = spec
+ if spec_set is not None:
+ this_spec = spec_set
+ if _is_list(this_spec):
+ not_callable = '__call__' not in this_spec
+ else:
+ not_callable = not _callable(this_spec)
+ if not_callable:
+ Klass = NonCallableMagicMock
+
+ if spec is not None:
+ _kwargs['spec'] = spec
+ if spec_set is not None:
+ _kwargs['spec_set'] = spec_set
+
+ # add a name to mocks
+ if (isinstance(Klass, type) and
+ issubclass(Klass, NonCallableMock) and self.attribute):
+ _kwargs['name'] = self.attribute
+
+ _kwargs.update(kwargs)
+ new = Klass(**_kwargs)
+
+ if inherit and _is_instance_mock(new):
+ # we can only tell if the instance should be callable if the
+ # spec is not a list
+ this_spec = spec
+ if spec_set is not None:
+ this_spec = spec_set
+ if (not _is_list(this_spec) and not
+ _instance_callable(this_spec)):
+ Klass = NonCallableMagicMock
+
+ _kwargs.pop('name')
+ new.return_value = Klass(_new_parent=new, _new_name='()',
+ **_kwargs)
+ elif autospec is not None:
+ # spec is ignored, new *must* be default, spec_set is treated
+ # as a boolean. Should we check spec is not None and that spec_set
+ # is a bool?
+ if new is not DEFAULT:
+ raise TypeError(
+ "autospec creates the mock for you. Can't specify "
+ "autospec and new."
+ )
+ if original is DEFAULT:
+ raise TypeError("Can't use 'autospec' with create=True")
+ spec_set = bool(spec_set)
+ if autospec is True:
+ autospec = original
+
+ new = create_autospec(autospec, spec_set=spec_set,
+ _name=self.attribute, **kwargs)
+ elif kwargs:
+ # can't set keyword args when we aren't creating the mock
+ # XXXX If new is a Mock we could call new.configure_mock(**kwargs)
+ raise TypeError("Can't pass kwargs to a mock we aren't creating")
+
+ new_attr = new
+
+ self.temp_original = original
+ self.is_local = local
+ setattr(self.target, self.attribute, new_attr)
+ if self.attribute_name is not None:
+ extra_args = {}
+ if self.new is DEFAULT:
+ extra_args[self.attribute_name] = new
+ for patching in self.additional_patchers:
+ arg = patching.__enter__()
+ if patching.new is DEFAULT:
+ extra_args.update(arg)
+ return extra_args
+
+ return new
+
+
+ def __exit__(self, *exc_info):
+ """Undo the patch."""
+ if not _is_started(self):
+ raise RuntimeError('stop called on unstarted patcher')
+
+ if self.is_local and self.temp_original is not DEFAULT:
+ setattr(self.target, self.attribute, self.temp_original)
+ else:
+ delattr(self.target, self.attribute)
+ if not self.create and not hasattr(self.target, self.attribute):
+ # needed for proxy objects like django settings
+ setattr(self.target, self.attribute, self.temp_original)
+
+ del self.temp_original
+ del self.is_local
+ del self.target
+ for patcher in reversed(self.additional_patchers):
+ if _is_started(patcher):
+ patcher.__exit__(*exc_info)
+
+
+ def start(self):
+ """Activate a patch, returning any created mock."""
+ result = self.__enter__()
+ self._active_patches.add(self)
+ return result
+
+
+ def stop(self):
+ """Stop an active patch."""
+ self._active_patches.discard(self)
+ return self.__exit__()
+
+
+
+def _get_target(target):
+ try:
+ target, attribute = target.rsplit('.', 1)
+ except (TypeError, ValueError):
+ raise TypeError("Need a valid target to patch. You supplied: %r" %
+ (target,))
+ getter = lambda: _importer(target)
+ return getter, attribute
+
+
+def _patch_object(
+ target, attribute, new=DEFAULT, spec=None,
+ create=False, spec_set=None, autospec=None,
+ new_callable=None, **kwargs
+ ):
+ """
+ patch.object(target, attribute, new=DEFAULT, spec=None, create=False,
+ spec_set=None, autospec=None, new_callable=None, **kwargs)
+
+ patch the named member (`attribute`) on an object (`target`) with a mock
+ object.
+
+ `patch.object` can be used as a decorator, class decorator or a context
+ manager. Arguments `new`, `spec`, `create`, `spec_set`,
+ `autospec` and `new_callable` have the same meaning as for `patch`. Like
+ `patch`, `patch.object` takes arbitrary keyword arguments for configuring
+ the mock object it creates.
+
+ When used as a class decorator `patch.object` honours `patch.TEST_PREFIX`
+ for choosing which methods to wrap.
+ """
+ getter = lambda: target
+ return _patch(
+ getter, attribute, new, spec, create,
+ spec_set, autospec, new_callable, kwargs
+ )
+
+
+def _patch_multiple(target, spec=None, create=False, spec_set=None,
+ autospec=None, new_callable=None, **kwargs):
+ """Perform multiple patches in a single call. It takes the object to be
+ patched (either as an object or a string to fetch the object by importing)
+ and keyword arguments for the patches::
+
+ with patch.multiple(settings, FIRST_PATCH='one', SECOND_PATCH='two'):
+ ...
+
+ Use `DEFAULT` as the value if you want `patch.multiple` to create
+ mocks for you. In this case the created mocks are passed into a decorated
+ function by keyword, and a dictionary is returned when `patch.multiple` is
+ used as a context manager.
+
+ `patch.multiple` can be used as a decorator, class decorator or a context
+ manager. The arguments `spec`, `spec_set`, `create`,
+ `autospec` and `new_callable` have the same meaning as for `patch`. These
+ arguments will be applied to *all* patches done by `patch.multiple`.
+
+ When used as a class decorator `patch.multiple` honours `patch.TEST_PREFIX`
+ for choosing which methods to wrap.
+ """
+ if type(target) in (unicode, str):
+ getter = lambda: _importer(target)
+ else:
+ getter = lambda: target
+
+ if not kwargs:
+ raise ValueError(
+ 'Must supply at least one keyword argument with patch.multiple'
+ )
+ # need to wrap in a list for python 3, where items is a view
+ items = list(kwargs.items())
+ attribute, new = items[0]
+ patcher = _patch(
+ getter, attribute, new, spec, create, spec_set,
+ autospec, new_callable, {}
+ )
+ patcher.attribute_name = attribute
+ for attribute, new in items[1:]:
+ this_patcher = _patch(
+ getter, attribute, new, spec, create, spec_set,
+ autospec, new_callable, {}
+ )
+ this_patcher.attribute_name = attribute
+ patcher.additional_patchers.append(this_patcher)
+ return patcher
+
+
+def patch(
+ target, new=DEFAULT, spec=None, create=False,
+ spec_set=None, autospec=None, new_callable=None, **kwargs
+ ):
+ """
+ `patch` acts as a function decorator, class decorator or a context
+ manager. Inside the body of the function or with statement, the `target`
+ is patched with a `new` object. When the function/with statement exits
+ the patch is undone.
+
+ If `new` is omitted, then the target is replaced with a
+ `MagicMock`. If `patch` is used as a decorator and `new` is
+ omitted, the created mock is passed in as an extra argument to the
+ decorated function. If `patch` is used as a context manager the created
+ mock is returned by the context manager.
+
+ `target` should be a string in the form `'package.module.ClassName'`. The
+ `target` is imported and the specified object replaced with the `new`
+ object, so the `target` must be importable from the environment you are
+ calling `patch` from. The target is imported when the decorated function
+ is executed, not at decoration time.
+
+ The `spec` and `spec_set` keyword arguments are passed to the `MagicMock`
+ if patch is creating one for you.
+
+ In addition you can pass `spec=True` or `spec_set=True`, which causes
+ patch to pass in the object being mocked as the spec/spec_set object.
+
+ `new_callable` allows you to specify a different class, or callable object,
+ that will be called to create the `new` object. By default `MagicMock` is
+ used.
+
+ A more powerful form of `spec` is `autospec`. If you set `autospec=True`
+ then the mock with be created with a spec from the object being replaced.
+ All attributes of the mock will also have the spec of the corresponding
+ attribute of the object being replaced. Methods and functions being
+ mocked will have their arguments checked and will raise a `TypeError` if
+ they are called with the wrong signature. For mocks replacing a class,
+ their return value (the 'instance') will have the same spec as the class.
+
+ Instead of `autospec=True` you can pass `autospec=some_object` to use an
+ arbitrary object as the spec instead of the one being replaced.
+
+ By default `patch` will fail to replace attributes that don't exist. If
+ you pass in `create=True`, and the attribute doesn't exist, patch will
+ create the attribute for you when the patched function is called, and
+ delete it again afterwards. This is useful for writing tests against
+ attributes that your production code creates at runtime. It is off by by
+ default because it can be dangerous. With it switched on you can write
+ passing tests against APIs that don't actually exist!
+
+ Patch can be used as a `TestCase` class decorator. It works by
+ decorating each test method in the class. This reduces the boilerplate
+ code when your test methods share a common patchings set. `patch` finds
+ tests by looking for method names that start with `patch.TEST_PREFIX`.
+ By default this is `test`, which matches the way `unittest` finds tests.
+ You can specify an alternative prefix by setting `patch.TEST_PREFIX`.
+
+ Patch can be used as a context manager, with the with statement. Here the
+ patching applies to the indented block after the with statement. If you
+ use "as" then the patched object will be bound to the name after the
+ "as"; very useful if `patch` is creating a mock object for you.
+
+ `patch` takes arbitrary keyword arguments. These will be passed to
+ the `Mock` (or `new_callable`) on construction.
+
+ `patch.dict(...)`, `patch.multiple(...)` and `patch.object(...)` are
+ available for alternate use-cases.
+ """
+ getter, attribute = _get_target(target)
+ return _patch(
+ getter, attribute, new, spec, create,
+ spec_set, autospec, new_callable, kwargs
+ )
+
+
+class _patch_dict(object):
+ """
+ Patch a dictionary, or dictionary like object, and restore the dictionary
+ to its original state after the test.
+
+ `in_dict` can be a dictionary or a mapping like container. If it is a
+ mapping then it must at least support getting, setting and deleting items
+ plus iterating over keys.
+
+ `in_dict` can also be a string specifying the name of the dictionary, which
+ will then be fetched by importing it.
+
+ `values` can be a dictionary of values to set in the dictionary. `values`
+ can also be an iterable of `(key, value)` pairs.
+
+ If `clear` is True then the dictionary will be cleared before the new
+ values are set.
+
+ `patch.dict` can also be called with arbitrary keyword arguments to set
+ values in the dictionary::
+
+ with patch.dict('sys.modules', mymodule=Mock(), other_module=Mock()):
+ ...
+
+ `patch.dict` can be used as a context manager, decorator or class
+ decorator. When used as a class decorator `patch.dict` honours
+ `patch.TEST_PREFIX` for choosing which methods to wrap.
+ """
+
+ def __init__(self, in_dict, values=(), clear=False, **kwargs):
+ if isinstance(in_dict, basestring):
+ in_dict = _importer(in_dict)
+ self.in_dict = in_dict
+ # support any argument supported by dict(...) constructor
+ self.values = dict(values)
+ self.values.update(kwargs)
+ self.clear = clear
+ self._original = None
+
+
+ def __call__(self, f):
+ if isinstance(f, ClassTypes):
+ return self.decorate_class(f)
+ @wraps(f)
+ def _inner(*args, **kw):
+ self._patch_dict()
+ try:
+ return f(*args, **kw)
+ finally:
+ self._unpatch_dict()
+
+ return _inner
+
+
+ def decorate_class(self, klass):
+ for attr in dir(klass):
+ attr_value = getattr(klass, attr)
+ if (attr.startswith(patch.TEST_PREFIX) and
+ hasattr(attr_value, "__call__")):
+ decorator = _patch_dict(self.in_dict, self.values, self.clear)
+ decorated = decorator(attr_value)
+ setattr(klass, attr, decorated)
+ return klass
+
+
+ def __enter__(self):
+ """Patch the dict."""
+ self._patch_dict()
+
+
+ def _patch_dict(self):
+ values = self.values
+ in_dict = self.in_dict
+ clear = self.clear
+
+ try:
+ original = in_dict.copy()
+ except AttributeError:
+ # dict like object with no copy method
+ # must support iteration over keys
+ original = {}
+ for key in in_dict:
+ original[key] = in_dict[key]
+ self._original = original
+
+ if clear:
+ _clear_dict(in_dict)
+
+ try:
+ in_dict.update(values)
+ except AttributeError:
+ # dict like object with no update method
+ for key in values:
+ in_dict[key] = values[key]
+
+
+ def _unpatch_dict(self):
+ in_dict = self.in_dict
+ original = self._original
+
+ _clear_dict(in_dict)
+
+ try:
+ in_dict.update(original)
+ except AttributeError:
+ for key in original:
+ in_dict[key] = original[key]
+
+
+ def __exit__(self, *args):
+ """Unpatch the dict."""
+ self._unpatch_dict()
+ return False
+
+ start = __enter__
+ stop = __exit__
+
+
+def _clear_dict(in_dict):
+ try:
+ in_dict.clear()
+ except AttributeError:
+ keys = list(in_dict)
+ for key in keys:
+ del in_dict[key]
+
+
+def _patch_stopall():
+ """Stop all active patches."""
+ for patch in list(_patch._active_patches):
+ patch.stop()
+
+
+patch.object = _patch_object
+patch.dict = _patch_dict
+patch.multiple = _patch_multiple
+patch.stopall = _patch_stopall
+patch.TEST_PREFIX = 'test'
+
+magic_methods = (
+ "lt le gt ge eq ne "
+ "getitem setitem delitem "
+ "len contains iter "
+ "hash str sizeof "
+ "enter exit "
+ "divmod neg pos abs invert "
+ "complex int float index "
+ "trunc floor ceil "
+)
+
+numerics = "add sub mul div floordiv mod lshift rshift and xor or pow "
+inplace = ' '.join('i%s' % n for n in numerics.split())
+right = ' '.join('r%s' % n for n in numerics.split())
+extra = ''
+if inPy3k:
+ extra = 'bool next '
+else:
+ extra = 'unicode long nonzero oct hex truediv rtruediv '
+
+# not including __prepare__, __instancecheck__, __subclasscheck__
+# (as they are metaclass methods)
+# __del__ is not supported at all as it causes problems if it exists
+
+_non_defaults = set('__%s__' % method for method in [
+ 'cmp', 'getslice', 'setslice', 'coerce', 'subclasses',
+ 'format', 'get', 'set', 'delete', 'reversed',
+ 'missing', 'reduce', 'reduce_ex', 'getinitargs',
+ 'getnewargs', 'getstate', 'setstate', 'getformat',
+ 'setformat', 'repr', 'dir'
+])
+
+
+def _get_method(name, func):
+ "Turns a callable object (like a mock) into a real function"
+ def method(self, *args, **kw):
+ return func(self, *args, **kw)
+ method.__name__ = name
+ return method
+
+
+_magics = set(
+ '__%s__' % method for method in
+ ' '.join([magic_methods, numerics, inplace, right, extra]).split()
+)
+
+_all_magics = _magics | _non_defaults
+
+_unsupported_magics = set([
+ '__getattr__', '__setattr__',
+ '__init__', '__new__', '__prepare__'
+ '__instancecheck__', '__subclasscheck__',
+ '__del__'
+])
+
+_calculate_return_value = {
+ '__hash__': lambda self: object.__hash__(self),
+ '__str__': lambda self: object.__str__(self),
+ '__sizeof__': lambda self: object.__sizeof__(self),
+ '__unicode__': lambda self: unicode(object.__str__(self)),
+}
+
+_return_values = {
+ '__lt__': NotImplemented,
+ '__gt__': NotImplemented,
+ '__le__': NotImplemented,
+ '__ge__': NotImplemented,
+ '__int__': 1,
+ '__contains__': False,
+ '__len__': 0,
+ '__exit__': False,
+ '__complex__': 1j,
+ '__float__': 1.0,
+ '__bool__': True,
+ '__nonzero__': True,
+ '__oct__': '1',
+ '__hex__': '0x1',
+ '__long__': long(1),
+ '__index__': 1,
+}
+
+
+def _get_eq(self):
+ def __eq__(other):
+ ret_val = self.__eq__._mock_return_value
+ if ret_val is not DEFAULT:
+ return ret_val
+ return self is other
+ return __eq__
+
+def _get_ne(self):
+ def __ne__(other):
+ if self.__ne__._mock_return_value is not DEFAULT:
+ return DEFAULT
+ return self is not other
+ return __ne__
+
+def _get_iter(self):
+ def __iter__():
+ ret_val = self.__iter__._mock_return_value
+ if ret_val is DEFAULT:
+ return iter([])
+ # if ret_val was already an iterator, then calling iter on it should
+ # return the iterator unchanged
+ return iter(ret_val)
+ return __iter__
+
+_side_effect_methods = {
+ '__eq__': _get_eq,
+ '__ne__': _get_ne,
+ '__iter__': _get_iter,
+}
+
+
+
+def _set_return_value(mock, method, name):
+ fixed = _return_values.get(name, DEFAULT)
+ if fixed is not DEFAULT:
+ method.return_value = fixed
+ return
+
+ return_calulator = _calculate_return_value.get(name)
+ if return_calulator is not None:
+ try:
+ return_value = return_calulator(mock)
+ except AttributeError:
+ # XXXX why do we return AttributeError here?
+ # set it as a side_effect instead?
+ return_value = AttributeError(name)
+ method.return_value = return_value
+ return
+
+ side_effector = _side_effect_methods.get(name)
+ if side_effector is not None:
+ method.side_effect = side_effector(mock)
+
+
+
+class MagicMixin(object):
+ def __init__(self, *args, **kw):
+ _super(MagicMixin, self).__init__(*args, **kw)
+ self._mock_set_magics()
+
+
+ def _mock_set_magics(self):
+ these_magics = _magics
+
+ if self._mock_methods is not None:
+ these_magics = _magics.intersection(self._mock_methods)
+
+ remove_magics = set()
+ remove_magics = _magics - these_magics
+
+ for entry in remove_magics:
+ if entry in type(self).__dict__:
+ # remove unneeded magic methods
+ delattr(self, entry)
+
+ # don't overwrite existing attributes if called a second time
+ these_magics = these_magics - set(type(self).__dict__)
+
+ _type = type(self)
+ for entry in these_magics:
+ setattr(_type, entry, MagicProxy(entry, self))
+
+
+
+class NonCallableMagicMock(MagicMixin, NonCallableMock):
+ """A version of `MagicMock` that isn't callable."""
+ def mock_add_spec(self, spec, spec_set=False):
+ """Add a spec to a mock. `spec` can either be an object or a
+ list of strings. Only attributes on the `spec` can be fetched as
+ attributes from the mock.
+
+ If `spec_set` is True then only attributes on the spec can be set."""
+ self._mock_add_spec(spec, spec_set)
+ self._mock_set_magics()
+
+
+
+class MagicMock(MagicMixin, Mock):
+ """
+ MagicMock is a subclass of Mock with default implementations
+ of most of the magic methods. You can use MagicMock without having to
+ configure the magic methods yourself.
+
+ If you use the `spec` or `spec_set` arguments then *only* magic
+ methods that exist in the spec will be created.
+
+ Attributes and the return value of a `MagicMock` will also be `MagicMocks`.
+ """
+ def mock_add_spec(self, spec, spec_set=False):
+ """Add a spec to a mock. `spec` can either be an object or a
+ list of strings. Only attributes on the `spec` can be fetched as
+ attributes from the mock.
+
+ If `spec_set` is True then only attributes on the spec can be set."""
+ self._mock_add_spec(spec, spec_set)
+ self._mock_set_magics()
+
+
+
+class MagicProxy(object):
+ def __init__(self, name, parent):
+ self.name = name
+ self.parent = parent
+
+ def __call__(self, *args, **kwargs):
+ m = self.create_mock()
+ return m(*args, **kwargs)
+
+ def create_mock(self):
+ entry = self.name
+ parent = self.parent
+ m = parent._get_child_mock(name=entry, _new_name=entry,
+ _new_parent=parent)
+ setattr(parent, entry, m)
+ _set_return_value(parent, m, entry)
+ return m
+
+ def __get__(self, obj, _type=None):
+ return self.create_mock()
+
+
+
+class _ANY(object):
+ "A helper object that compares equal to everything."
+
+ def __eq__(self, other):
+ return True
+
+ def __ne__(self, other):
+ return False
+
+ def __repr__(self):
+ return '<ANY>'
+
+ANY = _ANY()
+
+
+
+def _format_call_signature(name, args, kwargs):
+ message = '%s(%%s)' % name
+ formatted_args = ''
+ args_string = ', '.join([repr(arg) for arg in args])
+ kwargs_string = ', '.join([
+ '%s=%r' % (key, value) for key, value in kwargs.items()
+ ])
+ if args_string:
+ formatted_args = args_string
+ if kwargs_string:
+ if formatted_args:
+ formatted_args += ', '
+ formatted_args += kwargs_string
+
+ return message % formatted_args
+
+
+
+class _Call(tuple):
+ """
+ A tuple for holding the results of a call to a mock, either in the form
+ `(args, kwargs)` or `(name, args, kwargs)`.
+
+ If args or kwargs are empty then a call tuple will compare equal to
+ a tuple without those values. This makes comparisons less verbose::
+
+ _Call(('name', (), {})) == ('name',)
+ _Call(('name', (1,), {})) == ('name', (1,))
+ _Call(((), {'a': 'b'})) == ({'a': 'b'},)
+
+ The `_Call` object provides a useful shortcut for comparing with call::
+
+ _Call(((1, 2), {'a': 3})) == call(1, 2, a=3)
+ _Call(('foo', (1, 2), {'a': 3})) == call.foo(1, 2, a=3)
+
+ If the _Call has no name then it will match any name.
+ """
+ def __new__(cls, value=(), name=None, parent=None, two=False,
+ from_kall=True):
+ name = ''
+ args = ()
+ kwargs = {}
+ _len = len(value)
+ if _len == 3:
+ name, args, kwargs = value
+ elif _len == 2:
+ first, second = value
+ if isinstance(first, basestring):
+ name = first
+ if isinstance(second, tuple):
+ args = second
+ else:
+ kwargs = second
+ else:
+ args, kwargs = first, second
+ elif _len == 1:
+ value, = value
+ if isinstance(value, basestring):
+ name = value
+ elif isinstance(value, tuple):
+ args = value
+ else:
+ kwargs = value
+
+ if two:
+ return tuple.__new__(cls, (args, kwargs))
+
+ return tuple.__new__(cls, (name, args, kwargs))
+
+
+ def __init__(self, value=(), name=None, parent=None, two=False,
+ from_kall=True):
+ self.name = name
+ self.parent = parent
+ self.from_kall = from_kall
+
+
+ def __eq__(self, other):
+ if other is ANY:
+ return True
+ try:
+ len_other = len(other)
+ except TypeError:
+ return False
+
+ self_name = ''
+ if len(self) == 2:
+ self_args, self_kwargs = self
+ else:
+ self_name, self_args, self_kwargs = self
+
+ other_name = ''
+ if len_other == 0:
+ other_args, other_kwargs = (), {}
+ elif len_other == 3:
+ other_name, other_args, other_kwargs = other
+ elif len_other == 1:
+ value, = other
+ if isinstance(value, tuple):
+ other_args = value
+ other_kwargs = {}
+ elif isinstance(value, basestring):
+ other_name = value
+ other_args, other_kwargs = (), {}
+ else:
+ other_args = ()
+ other_kwargs = value
+ else:
+ # len 2
+ # could be (name, args) or (name, kwargs) or (args, kwargs)
+ first, second = other
+ if isinstance(first, basestring):
+ other_name = first
+ if isinstance(second, tuple):
+ other_args, other_kwargs = second, {}
+ else:
+ other_args, other_kwargs = (), second
+ else:
+ other_args, other_kwargs = first, second
+
+ if self_name and other_name != self_name:
+ return False
+
+ # this order is important for ANY to work!
+ return (other_args, other_kwargs) == (self_args, self_kwargs)
+
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+
+ def __call__(self, *args, **kwargs):
+ if self.name is None:
+ return _Call(('', args, kwargs), name='()')
+
+ name = self.name + '()'
+ return _Call((self.name, args, kwargs), name=name, parent=self)
+
+
+ def __getattr__(self, attr):
+ if self.name is None:
+ return _Call(name=attr, from_kall=False)
+ name = '%s.%s' % (self.name, attr)
+ return _Call(name=name, parent=self, from_kall=False)
+
+
+ def __repr__(self):
+ if not self.from_kall:
+ name = self.name or 'call'
+ if name.startswith('()'):
+ name = 'call%s' % name
+ return name
+
+ if len(self) == 2:
+ name = 'call'
+ args, kwargs = self
+ else:
+ name, args, kwargs = self
+ if not name:
+ name = 'call'
+ elif not name.startswith('()'):
+ name = 'call.%s' % name
+ else:
+ name = 'call%s' % name
+ return _format_call_signature(name, args, kwargs)
+
+
+ def call_list(self):
+ """For a call object that represents multiple calls, `call_list`
+ returns a list of all the intermediate calls as well as the
+ final call."""
+ vals = []
+ thing = self
+ while thing is not None:
+ if thing.from_kall:
+ vals.append(thing)
+ thing = thing.parent
+ return _CallList(reversed(vals))
+
+
+call = _Call(from_kall=False)
+
+
+
+def create_autospec(spec, spec_set=False, instance=False, _parent=None,
+ _name=None, **kwargs):
+ """Create a mock object using another object as a spec. Attributes on the
+ mock will use the corresponding attribute on the `spec` object as their
+ spec.
+
+ Functions or methods being mocked will have their arguments checked
+ to check that they are called with the correct signature.
+
+ If `spec_set` is True then attempting to set attributes that don't exist
+ on the spec object will raise an `AttributeError`.
+
+ If a class is used as a spec then the return value of the mock (the
+ instance of the class) will have the same spec. You can use a class as the
+ spec for an instance object by passing `instance=True`. The returned mock
+ will only be callable if instances of the mock are callable.
+
+ `create_autospec` also takes arbitrary keyword arguments that are passed to
+ the constructor of the created mock."""
+ if _is_list(spec):
+ # can't pass a list instance to the mock constructor as it will be
+ # interpreted as a list of strings
+ spec = type(spec)
+
+ is_type = isinstance(spec, ClassTypes)
+
+ _kwargs = {'spec': spec}
+ if spec_set:
+ _kwargs = {'spec_set': spec}
+ elif spec is None:
+ # None we mock with a normal mock without a spec
+ _kwargs = {}
+
+ _kwargs.update(kwargs)
+
+ Klass = MagicMock
+ if type(spec) in DescriptorTypes:
+ # descriptors don't have a spec
+ # because we don't know what type they return
+ _kwargs = {}
+ elif not _callable(spec):
+ Klass = NonCallableMagicMock
+ elif is_type and instance and not _instance_callable(spec):
+ Klass = NonCallableMagicMock
+
+ _new_name = _name
+ if _parent is None:
+ # for a top level object no _new_name should be set
+ _new_name = ''
+
+ mock = Klass(parent=_parent, _new_parent=_parent, _new_name=_new_name,
+ name=_name, **_kwargs)
+
+ if isinstance(spec, FunctionTypes):
+ # should only happen at the top level because we don't
+ # recurse for functions
+ mock = _set_signature(mock, spec)
+ else:
+ _check_signature(spec, mock, is_type, instance)
+
+ if _parent is not None and not instance:
+ _parent._mock_children[_name] = mock
+
+ if is_type and not instance and 'return_value' not in kwargs:
+ mock.return_value = create_autospec(spec, spec_set, instance=True,
+ _name='()', _parent=mock)
+
+ for entry in dir(spec):
+ if _is_magic(entry):
+ # MagicMock already does the useful magic methods for us
+ continue
+
+ if isinstance(spec, FunctionTypes) and entry in FunctionAttributes:
+ # allow a mock to actually be a function
+ continue
+
+ # XXXX do we need a better way of getting attributes without
+ # triggering code execution (?) Probably not - we need the actual
+ # object to mock it so we would rather trigger a property than mock
+ # the property descriptor. Likewise we want to mock out dynamically
+ # provided attributes.
+ # XXXX what about attributes that raise exceptions other than
+ # AttributeError on being fetched?
+ # we could be resilient against it, or catch and propagate the
+ # exception when the attribute is fetched from the mock
+ try:
+ original = getattr(spec, entry)
+ except AttributeError:
+ continue
+
+ kwargs = {'spec': original}
+ if spec_set:
+ kwargs = {'spec_set': original}
+
+ if not isinstance(original, FunctionTypes):
+ new = _SpecState(original, spec_set, mock, entry, instance)
+ mock._mock_children[entry] = new
+ else:
+ parent = mock
+ if isinstance(spec, FunctionTypes):
+ parent = mock.mock
+
+ new = MagicMock(parent=parent, name=entry, _new_name=entry,
+ _new_parent=parent, **kwargs)
+ mock._mock_children[entry] = new
+ skipfirst = _must_skip(spec, entry, is_type)
+ _check_signature(original, new, skipfirst=skipfirst)
+
+ # so functions created with _set_signature become instance attributes,
+ # *plus* their underlying mock exists in _mock_children of the parent
+ # mock. Adding to _mock_children may be unnecessary where we are also
+ # setting as an instance attribute?
+ if isinstance(new, FunctionTypes):
+ setattr(mock, entry, new)
+
+ return mock
+
+
+def _must_skip(spec, entry, is_type):
+ if not isinstance(spec, ClassTypes):
+ if entry in getattr(spec, '__dict__', {}):
+ # instance attribute - shouldn't skip
+ return False
+ spec = spec.__class__
+ if not hasattr(spec, '__mro__'):
+ # old style class: can't have descriptors anyway
+ return is_type
+
+ for klass in spec.__mro__:
+ result = klass.__dict__.get(entry, DEFAULT)
+ if result is DEFAULT:
+ continue
+ if isinstance(result, (staticmethod, classmethod)):
+ return False
+ return is_type
+
+ # shouldn't get here unless function is a dynamically provided attribute
+ # XXXX untested behaviour
+ return is_type
+
+
+def _get_class(obj):
+ try:
+ return obj.__class__
+ except AttributeError:
+ # in Python 2, _sre.SRE_Pattern objects have no __class__
+ return type(obj)
+
+
+class _SpecState(object):
+
+ def __init__(self, spec, spec_set=False, parent=None,
+ name=None, ids=None, instance=False):
+ self.spec = spec
+ self.ids = ids
+ self.spec_set = spec_set
+ self.parent = parent
+ self.instance = instance
+ self.name = name
+
+
+FunctionTypes = (
+ # python function
+ type(create_autospec),
+ # instance method
+ type(ANY.__eq__),
+ # unbound method
+ type(_ANY.__eq__),
+)
+
+FunctionAttributes = set([
+ 'func_closure',
+ 'func_code',
+ 'func_defaults',
+ 'func_dict',
+ 'func_doc',
+ 'func_globals',
+ 'func_name',
+])
+
+
+file_spec = None
+
+
+def mock_open(mock=None, read_data=''):
+ """
+ A helper function to create a mock to replace the use of `open`. It works
+ for `open` called directly or used as a context manager.
+
+ The `mock` argument is the mock object to configure. If `None` (the
+ default) then a `MagicMock` will be created for you, with the API limited
+ to methods or attributes available on standard file handles.
+
+ `read_data` is a string for the `read` method of the file handle to return.
+ This is an empty string by default.
+ """
+ global file_spec
+ if file_spec is None:
+ # set on first use
+ if inPy3k:
+ import _io
+ file_spec = list(set(dir(_io.TextIOWrapper)).union(set(dir(_io.BytesIO))))
+ else:
+ file_spec = file
+
+ if mock is None:
+ mock = MagicMock(name='open', spec=open)
+
+ handle = MagicMock(spec=file_spec)
+ handle.write.return_value = None
+ handle.__enter__.return_value = handle
+ handle.read.return_value = read_data
+
+ mock.return_value = handle
+ return mock
+
+
+class PropertyMock(Mock):
+ """
+ A mock intended to be used as a property, or other descriptor, on a class.
+ `PropertyMock` provides `__get__` and `__set__` methods so you can specify
+ a return value when it is fetched.
+
+ Fetching a `PropertyMock` instance from an object calls the mock, with
+ no args. Setting it calls the mock with the value being set.
+ """
+ def _get_child_mock(self, **kwargs):
+ return MagicMock(**kwargs)
+
+ def __get__(self, obj, obj_type):
+ return self()
+ def __set__(self, obj, val):
+ self(val)
diff --git a/slider-agent/src/test/python/mock/mock.wpr b/slider-agent/src/test/python/mock/mock.wpr
new file mode 100644
index 0000000..e1ded97
--- /dev/null
+++ b/slider-agent/src/test/python/mock/mock.wpr
@@ -0,0 +1,26 @@
+#!wing
+#!version=4.0
+##################################################################
+# Wing IDE project file #
+##################################################################
+[project attributes]
+proj.directory-list = [{'dirloc': loc('.'),
+ 'excludes': [u'latex',
+ u'.hg',
+ u'.tox',
+ u'dist',
+ u'htmlcov',
+ u'extendmock.py',
+ u'__pycache__',
+ u'html',
+ u'build',
+ u'mock.egg-info',
+ u'tests/__pycache__',
+ u'.hgignore',
+ u'.hgtags'],
+ 'filter': '*',
+ 'include_hidden': 0,
+ 'recursive': 1,
+ 'watch_for_changes': 1}]
+proj.file-type = 'shared'
+testing.auto-test-file-specs = ('test*.py',)
diff --git a/slider-agent/src/test/python/mock/setup.cfg b/slider-agent/src/test/python/mock/setup.cfg
new file mode 100644
index 0000000..566eb37
--- /dev/null
+++ b/slider-agent/src/test/python/mock/setup.cfg
@@ -0,0 +1,5 @@
+[build_sphinx]
+source-dir=docs
+build-dir=html
+[sdist]
+force-manifest = 1
diff --git a/slider-agent/src/test/python/mock/setup.py b/slider-agent/src/test/python/mock/setup.py
new file mode 100644
index 0000000..a6ee625
--- /dev/null
+++ b/slider-agent/src/test/python/mock/setup.py
@@ -0,0 +1,72 @@
+#! /usr/bin/env python
+
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
+
+from mock import __version__
+
+import os
+
+
+NAME = 'mock'
+MODULES = ['mock']
+DESCRIPTION = 'A Python Mocking and Patching Library for Testing'
+
+URL = "http://www.voidspace.org.uk/python/mock/"
+
+readme = os.path.join(os.path.dirname(__file__), 'README.txt')
+LONG_DESCRIPTION = open(readme).read()
+
+CLASSIFIERS = [
+ 'Development Status :: 5 - Production/Stable',
+ 'Environment :: Console',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: BSD License',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 2.5',
+ 'Programming Language :: Python :: 2.6',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3.1',
+ 'Programming Language :: Python :: 3.2',
+ 'Programming Language :: Python :: 3.3',
+ 'Programming Language :: Python :: Implementation :: CPython',
+ 'Programming Language :: Python :: Implementation :: PyPy',
+ 'Programming Language :: Python :: Implementation :: Jython',
+ 'Operating System :: OS Independent',
+ 'Topic :: Software Development :: Libraries',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ 'Topic :: Software Development :: Testing',
+]
+
+AUTHOR = 'Michael Foord'
+AUTHOR_EMAIL = 'michael@voidspace.org.uk'
+KEYWORDS = ("testing test mock mocking unittest patching "
+ "stubs fakes doubles").split(' ')
+
+params = dict(
+ name=NAME,
+ version=__version__,
+ py_modules=MODULES,
+
+ # metadata for upload to PyPI
+ author=AUTHOR,
+ author_email=AUTHOR_EMAIL,
+ description=DESCRIPTION,
+ long_description=LONG_DESCRIPTION,
+ keywords=KEYWORDS,
+ url=URL,
+ classifiers=CLASSIFIERS,
+)
+
+try:
+ from setuptools import setup
+except ImportError:
+ from distutils.core import setup
+else:
+ params['tests_require'] = ['unittest2']
+ params['test_suite'] = 'unittest2.collector'
+
+setup(**params)
diff --git a/slider-agent/src/test/python/mock/tests/__init__.py b/slider-agent/src/test/python/mock/tests/__init__.py
new file mode 100644
index 0000000..54ddf2e
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/__init__.py
@@ -0,0 +1,3 @@
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
diff --git a/slider-agent/src/test/python/mock/tests/_testwith.py b/slider-agent/src/test/python/mock/tests/_testwith.py
new file mode 100644
index 0000000..0b54780
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/_testwith.py
@@ -0,0 +1,181 @@
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
+
+from __future__ import with_statement
+
+from tests.support import unittest2, is_instance
+
+from mock import MagicMock, Mock, patch, sentinel, mock_open, call
+
+from tests.support_with import catch_warnings, nested
+
+something = sentinel.Something
+something_else = sentinel.SomethingElse
+
+
+
+class WithTest(unittest2.TestCase):
+
+ def test_with_statement(self):
+ with patch('tests._testwith.something', sentinel.Something2):
+ self.assertEqual(something, sentinel.Something2, "unpatched")
+ self.assertEqual(something, sentinel.Something)
+
+
+ def test_with_statement_exception(self):
+ try:
+ with patch('tests._testwith.something', sentinel.Something2):
+ self.assertEqual(something, sentinel.Something2, "unpatched")
+ raise Exception('pow')
+ except Exception:
+ pass
+ else:
+ self.fail("patch swallowed exception")
+ self.assertEqual(something, sentinel.Something)
+
+
+ def test_with_statement_as(self):
+ with patch('tests._testwith.something') as mock_something:
+ self.assertEqual(something, mock_something, "unpatched")
+ self.assertTrue(is_instance(mock_something, MagicMock),
+ "patching wrong type")
+ self.assertEqual(something, sentinel.Something)
+
+
+ def test_patch_object_with_statement(self):
+ class Foo(object):
+ something = 'foo'
+ original = Foo.something
+ with patch.object(Foo, 'something'):
+ self.assertNotEqual(Foo.something, original, "unpatched")
+ self.assertEqual(Foo.something, original)
+
+
+ def test_with_statement_nested(self):
+ with catch_warnings(record=True):
+ # nested is deprecated in Python 2.7
+ with nested(patch('tests._testwith.something'),
+ patch('tests._testwith.something_else')) as (mock_something, mock_something_else):
+ self.assertEqual(something, mock_something, "unpatched")
+ self.assertEqual(something_else, mock_something_else,
+ "unpatched")
+ self.assertEqual(something, sentinel.Something)
+ self.assertEqual(something_else, sentinel.SomethingElse)
+
+
+ def test_with_statement_specified(self):
+ with patch('tests._testwith.something', sentinel.Patched) as mock_something:
+ self.assertEqual(something, mock_something, "unpatched")
+ self.assertEqual(mock_something, sentinel.Patched, "wrong patch")
+ self.assertEqual(something, sentinel.Something)
+
+
+ def testContextManagerMocking(self):
+ mock = Mock()
+ mock.__enter__ = Mock()
+ mock.__exit__ = Mock()
+ mock.__exit__.return_value = False
+
+ with mock as m:
+ self.assertEqual(m, mock.__enter__.return_value)
+ mock.__enter__.assert_called_with()
+ mock.__exit__.assert_called_with(None, None, None)
+
+
+ def test_context_manager_with_magic_mock(self):
+ mock = MagicMock()
+
+ with self.assertRaises(TypeError):
+ with mock:
+ 'foo' + 3
+ mock.__enter__.assert_called_with()
+ self.assertTrue(mock.__exit__.called)
+
+
+ def test_with_statement_same_attribute(self):
+ with patch('tests._testwith.something', sentinel.Patched) as mock_something:
+ self.assertEqual(something, mock_something, "unpatched")
+
+ with patch('tests._testwith.something') as mock_again:
+ self.assertEqual(something, mock_again, "unpatched")
+
+ self.assertEqual(something, mock_something,
+ "restored with wrong instance")
+
+ self.assertEqual(something, sentinel.Something, "not restored")
+
+
+ def test_with_statement_imbricated(self):
+ with patch('tests._testwith.something') as mock_something:
+ self.assertEqual(something, mock_something, "unpatched")
+
+ with patch('tests._testwith.something_else') as mock_something_else:
+ self.assertEqual(something_else, mock_something_else,
+ "unpatched")
+
+ self.assertEqual(something, sentinel.Something)
+ self.assertEqual(something_else, sentinel.SomethingElse)
+
+
+ def test_dict_context_manager(self):
+ foo = {}
+ with patch.dict(foo, {'a': 'b'}):
+ self.assertEqual(foo, {'a': 'b'})
+ self.assertEqual(foo, {})
+
+ with self.assertRaises(NameError):
+ with patch.dict(foo, {'a': 'b'}):
+ self.assertEqual(foo, {'a': 'b'})
+ raise NameError('Konrad')
+
+ self.assertEqual(foo, {})
+
+
+
+class TestMockOpen(unittest2.TestCase):
+
+ def test_mock_open(self):
+ mock = mock_open()
+ with patch('%s.open' % __name__, mock, create=True) as patched:
+ self.assertIs(patched, mock)
+ open('foo')
+
+ mock.assert_called_once_with('foo')
+
+
+ def test_mock_open_context_manager(self):
+ mock = mock_open()
+ handle = mock.return_value
+ with patch('%s.open' % __name__, mock, create=True):
+ with open('foo') as f:
+ f.read()
+
+ expected_calls = [call('foo'), call().__enter__(), call().read(),
+ call().__exit__(None, None, None)]
+ self.assertEqual(mock.mock_calls, expected_calls)
+ self.assertIs(f, handle)
+
+
+ def test_explicit_mock(self):
+ mock = MagicMock()
+ mock_open(mock)
+
+ with patch('%s.open' % __name__, mock, create=True) as patched:
+ self.assertIs(patched, mock)
+ open('foo')
+
+ mock.assert_called_once_with('foo')
+
+
+ def test_read_data(self):
+ mock = mock_open(read_data='foo')
+ with patch('%s.open' % __name__, mock, create=True):
+ h = open('bar')
+ result = h.read()
+
+ self.assertEqual(result, 'foo')
+
+
+if __name__ == '__main__':
+ unittest2.main()
diff --git a/slider-agent/src/test/python/mock/tests/support.py b/slider-agent/src/test/python/mock/tests/support.py
new file mode 100644
index 0000000..1b10c34
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/support.py
@@ -0,0 +1,41 @@
+import sys
+
+info = sys.version_info
+if info[:3] >= (3, 2, 0):
+ # for Python 3.2 ordinary unittest is fine
+ import unittest as unittest2
+else:
+ import unittest2
+
+
+try:
+ callable = callable
+except NameError:
+ def callable(obj):
+ return hasattr(obj, '__call__')
+
+
+inPy3k = sys.version_info[0] == 3
+with_available = sys.version_info[:2] >= (2, 5)
+
+
+def is_instance(obj, klass):
+ """Version of is_instance that doesn't access __class__"""
+ return issubclass(type(obj), klass)
+
+
+class SomeClass(object):
+ class_attribute = None
+
+ def wibble(self):
+ pass
+
+
+class X(object):
+ pass
+
+try:
+ next = next
+except NameError:
+ def next(obj):
+ return obj.next()
diff --git a/slider-agent/src/test/python/mock/tests/support_with.py b/slider-agent/src/test/python/mock/tests/support_with.py
new file mode 100644
index 0000000..fa28612
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/support_with.py
@@ -0,0 +1,93 @@
+from __future__ import with_statement
+
+import sys
+
+__all__ = ['nested', 'catch_warnings', 'examine_warnings']
+
+
+try:
+ from contextlib import nested
+except ImportError:
+ from contextlib import contextmanager
+ @contextmanager
+ def nested(*managers):
+ exits = []
+ vars = []
+ exc = (None, None, None)
+ try:
+ for mgr in managers:
+ exit = mgr.__exit__
+ enter = mgr.__enter__
+ vars.append(enter())
+ exits.append(exit)
+ yield vars
+ except:
+ exc = sys.exc_info()
+ finally:
+ while exits:
+ exit = exits.pop()
+ try:
+ if exit(*exc):
+ exc = (None, None, None)
+ except:
+ exc = sys.exc_info()
+ if exc != (None, None, None):
+ raise exc[1]
+
+# copied from Python 2.6
+try:
+ from warnings import catch_warnings
+except ImportError:
+ class catch_warnings(object):
+ def __init__(self, record=False, module=None):
+ self._record = record
+ self._module = sys.modules['warnings']
+ self._entered = False
+
+ def __repr__(self):
+ args = []
+ if self._record:
+ args.append("record=True")
+ name = type(self).__name__
+ return "%s(%s)" % (name, ", ".join(args))
+
+ def __enter__(self):
+ if self._entered:
+ raise RuntimeError("Cannot enter %r twice" % self)
+ self._entered = True
+ self._filters = self._module.filters
+ self._module.filters = self._filters[:]
+ self._showwarning = self._module.showwarning
+ if self._record:
+ log = []
+ def showwarning(*args, **kwargs):
+ log.append(WarningMessage(*args, **kwargs))
+ self._module.showwarning = showwarning
+ return log
+ else:
+ return None
+
+ def __exit__(self, *exc_info):
+ if not self._entered:
+ raise RuntimeError("Cannot exit %r without entering first" % self)
+ self._module.filters = self._filters
+ self._module.showwarning = self._showwarning
+
+ class WarningMessage(object):
+ _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
+ "line")
+ def __init__(self, message, category, filename, lineno, file=None,
+ line=None):
+ local_values = locals()
+ for attr in self._WARNING_DETAILS:
+ setattr(self, attr, local_values[attr])
+ self._category_name = None
+ if category.__name__:
+ self._category_name = category.__name__
+
+
+def examine_warnings(func):
+ def wrapper():
+ with catch_warnings(record=True) as ws:
+ func(ws)
+ return wrapper
diff --git a/slider-agent/src/test/python/mock/tests/testcallable.py b/slider-agent/src/test/python/mock/tests/testcallable.py
new file mode 100644
index 0000000..f7dcd5e
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/testcallable.py
@@ -0,0 +1,158 @@
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
+
+from tests.support import is_instance, unittest2, X, SomeClass
+
+from mock import (
+ Mock, MagicMock, NonCallableMagicMock,
+ NonCallableMock, patch, create_autospec,
+ CallableMixin
+)
+
+
+
+class TestCallable(unittest2.TestCase):
+
+ def assertNotCallable(self, mock):
+ self.assertTrue(is_instance(mock, NonCallableMagicMock))
+ self.assertFalse(is_instance(mock, CallableMixin))
+
+
+ def test_non_callable(self):
+ for mock in NonCallableMagicMock(), NonCallableMock():
+ self.assertRaises(TypeError, mock)
+ self.assertFalse(hasattr(mock, '__call__'))
+ self.assertIn(mock.__class__.__name__, repr(mock))
+
+
+ def test_heirarchy(self):
+ self.assertTrue(issubclass(MagicMock, Mock))
+ self.assertTrue(issubclass(NonCallableMagicMock, NonCallableMock))
+
+
+ def test_attributes(self):
+ one = NonCallableMock()
+ self.assertTrue(issubclass(type(one.one), Mock))
+
+ two = NonCallableMagicMock()
+ self.assertTrue(issubclass(type(two.two), MagicMock))
+
+
+ def test_subclasses(self):
+ class MockSub(Mock):
+ pass
+
+ one = MockSub()
+ self.assertTrue(issubclass(type(one.one), MockSub))
+
+ class MagicSub(MagicMock):
+ pass
+
+ two = MagicSub()
+ self.assertTrue(issubclass(type(two.two), MagicSub))
+
+
+ def test_patch_spec(self):
+ patcher = patch('%s.X' % __name__, spec=True)
+ mock = patcher.start()
+ self.addCleanup(patcher.stop)
+
+ instance = mock()
+ mock.assert_called_once_with()
+
+ self.assertNotCallable(instance)
+ self.assertRaises(TypeError, instance)
+
+
+ def test_patch_spec_set(self):
+ patcher = patch('%s.X' % __name__, spec_set=True)
+ mock = patcher.start()
+ self.addCleanup(patcher.stop)
+
+ instance = mock()
+ mock.assert_called_once_with()
+
+ self.assertNotCallable(instance)
+ self.assertRaises(TypeError, instance)
+
+
+ def test_patch_spec_instance(self):
+ patcher = patch('%s.X' % __name__, spec=X())
+ mock = patcher.start()
+ self.addCleanup(patcher.stop)
+
+ self.assertNotCallable(mock)
+ self.assertRaises(TypeError, mock)
+
+
+ def test_patch_spec_set_instance(self):
+ patcher = patch('%s.X' % __name__, spec_set=X())
+ mock = patcher.start()
+ self.addCleanup(patcher.stop)
+
+ self.assertNotCallable(mock)
+ self.assertRaises(TypeError, mock)
+
+
+ def test_patch_spec_callable_class(self):
+ class CallableX(X):
+ def __call__(self):
+ pass
+
+ class Sub(CallableX):
+ pass
+
+ class Multi(SomeClass, Sub):
+ pass
+
+ class OldStyle:
+ def __call__(self):
+ pass
+
+ class OldStyleSub(OldStyle):
+ pass
+
+ for arg in 'spec', 'spec_set':
+ for Klass in CallableX, Sub, Multi, OldStyle, OldStyleSub:
+ patcher = patch('%s.X' % __name__, **{arg: Klass})
+ mock = patcher.start()
+
+ try:
+ instance = mock()
+ mock.assert_called_once_with()
+
+ self.assertTrue(is_instance(instance, MagicMock))
+ # inherited spec
+ self.assertRaises(AttributeError, getattr, instance,
+ 'foobarbaz')
+
+ result = instance()
+ # instance is callable, result has no spec
+ instance.assert_called_once_with()
+
+ result(3, 2, 1)
+ result.assert_called_once_with(3, 2, 1)
+ result.foo(3, 2, 1)
+ result.foo.assert_called_once_with(3, 2, 1)
+ finally:
+ patcher.stop()
+
+
+ def test_create_autopsec(self):
+ mock = create_autospec(X)
+ instance = mock()
+ self.assertRaises(TypeError, instance)
+
+ mock = create_autospec(X())
+ self.assertRaises(TypeError, mock)
+
+
+ def test_create_autospec_instance(self):
+ mock = create_autospec(SomeClass, instance=True)
+
+ self.assertRaises(TypeError, mock)
+ mock.wibble()
+ mock.wibble.assert_called_once_with()
+
+ self.assertRaises(TypeError, mock.wibble, 'some', 'args')
diff --git a/slider-agent/src/test/python/mock/tests/testhelpers.py b/slider-agent/src/test/python/mock/tests/testhelpers.py
new file mode 100644
index 0000000..e788da8
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/testhelpers.py
@@ -0,0 +1,940 @@
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
+
+from tests.support import unittest2, inPy3k
+
+from mock import (
+ call, _Call, create_autospec, MagicMock,
+ Mock, ANY, _CallList, patch, PropertyMock
+)
+
+from datetime import datetime
+
+class SomeClass(object):
+ def one(self, a, b):
+ pass
+ def two(self):
+ pass
+ def three(self, a=None):
+ pass
+
+
+
+class AnyTest(unittest2.TestCase):
+
+ def test_any(self):
+ self.assertEqual(ANY, object())
+
+ mock = Mock()
+ mock(ANY)
+ mock.assert_called_with(ANY)
+
+ mock = Mock()
+ mock(foo=ANY)
+ mock.assert_called_with(foo=ANY)
+
+ def test_repr(self):
+ self.assertEqual(repr(ANY), '<ANY>')
+ self.assertEqual(str(ANY), '<ANY>')
+
+
+ def test_any_and_datetime(self):
+ mock = Mock()
+ mock(datetime.now(), foo=datetime.now())
+
+ mock.assert_called_with(ANY, foo=ANY)
+
+
+ def test_any_mock_calls_comparison_order(self):
+ mock = Mock()
+ d = datetime.now()
+ class Foo(object):
+ def __eq__(self, other):
+ return False
+ def __ne__(self, other):
+ return True
+
+ for d in datetime.now(), Foo():
+ mock.reset_mock()
+
+ mock(d, foo=d, bar=d)
+ mock.method(d, zinga=d, alpha=d)
+ mock().method(a1=d, z99=d)
+
+ expected = [
+ call(ANY, foo=ANY, bar=ANY),
+ call.method(ANY, zinga=ANY, alpha=ANY),
+ call(), call().method(a1=ANY, z99=ANY)
+ ]
+ self.assertEqual(expected, mock.mock_calls)
+ self.assertEqual(mock.mock_calls, expected)
+
+
+
+class CallTest(unittest2.TestCase):
+
+ def test_call_with_call(self):
+ kall = _Call()
+ self.assertEqual(kall, _Call())
+ self.assertEqual(kall, _Call(('',)))
+ self.assertEqual(kall, _Call(((),)))
+ self.assertEqual(kall, _Call(({},)))
+ self.assertEqual(kall, _Call(('', ())))
+ self.assertEqual(kall, _Call(('', {})))
+ self.assertEqual(kall, _Call(('', (), {})))
+ self.assertEqual(kall, _Call(('foo',)))
+ self.assertEqual(kall, _Call(('bar', ())))
+ self.assertEqual(kall, _Call(('baz', {})))
+ self.assertEqual(kall, _Call(('spam', (), {})))
+
+ kall = _Call(((1, 2, 3),))
+ self.assertEqual(kall, _Call(((1, 2, 3),)))
+ self.assertEqual(kall, _Call(('', (1, 2, 3))))
+ self.assertEqual(kall, _Call(((1, 2, 3), {})))
+ self.assertEqual(kall, _Call(('', (1, 2, 3), {})))
+
+ kall = _Call(((1, 2, 4),))
+ self.assertNotEqual(kall, _Call(('', (1, 2, 3))))
+ self.assertNotEqual(kall, _Call(('', (1, 2, 3), {})))
+
+ kall = _Call(('foo', (1, 2, 4),))
+ self.assertNotEqual(kall, _Call(('', (1, 2, 4))))
+ self.assertNotEqual(kall, _Call(('', (1, 2, 4), {})))
+ self.assertNotEqual(kall, _Call(('bar', (1, 2, 4))))
+ self.assertNotEqual(kall, _Call(('bar', (1, 2, 4), {})))
+
+ kall = _Call(({'a': 3},))
+ self.assertEqual(kall, _Call(('', (), {'a': 3})))
+ self.assertEqual(kall, _Call(('', {'a': 3})))
+ self.assertEqual(kall, _Call(((), {'a': 3})))
+ self.assertEqual(kall, _Call(({'a': 3},)))
+
+
+ def test_empty__Call(self):
+ args = _Call()
+
+ self.assertEqual(args, ())
+ self.assertEqual(args, ('foo',))
+ self.assertEqual(args, ((),))
+ self.assertEqual(args, ('foo', ()))
+ self.assertEqual(args, ('foo',(), {}))
+ self.assertEqual(args, ('foo', {}))
+ self.assertEqual(args, ({},))
+
+
+ def test_named_empty_call(self):
+ args = _Call(('foo', (), {}))
+
+ self.assertEqual(args, ('foo',))
+ self.assertEqual(args, ('foo', ()))
+ self.assertEqual(args, ('foo',(), {}))
+ self.assertEqual(args, ('foo', {}))
+
+ self.assertNotEqual(args, ((),))
+ self.assertNotEqual(args, ())
+ self.assertNotEqual(args, ({},))
+ self.assertNotEqual(args, ('bar',))
+ self.assertNotEqual(args, ('bar', ()))
+ self.assertNotEqual(args, ('bar', {}))
+
+
+ def test_call_with_args(self):
+ args = _Call(((1, 2, 3), {}))
+
+ self.assertEqual(args, ((1, 2, 3),))
+ self.assertEqual(args, ('foo', (1, 2, 3)))
+ self.assertEqual(args, ('foo', (1, 2, 3), {}))
+ self.assertEqual(args, ((1, 2, 3), {}))
+
+
+ def test_named_call_with_args(self):
+ args = _Call(('foo', (1, 2, 3), {}))
+
+ self.assertEqual(args, ('foo', (1, 2, 3)))
+ self.assertEqual(args, ('foo', (1, 2, 3), {}))
+
+ self.assertNotEqual(args, ((1, 2, 3),))
+ self.assertNotEqual(args, ((1, 2, 3), {}))
+
+
+ def test_call_with_kwargs(self):
+ args = _Call(((), dict(a=3, b=4)))
+
+ self.assertEqual(args, (dict(a=3, b=4),))
+ self.assertEqual(args, ('foo', dict(a=3, b=4)))
+ self.assertEqual(args, ('foo', (), dict(a=3, b=4)))
+ self.assertEqual(args, ((), dict(a=3, b=4)))
+
+
+ def test_named_call_with_kwargs(self):
+ args = _Call(('foo', (), dict(a=3, b=4)))
+
+ self.assertEqual(args, ('foo', dict(a=3, b=4)))
+ self.assertEqual(args, ('foo', (), dict(a=3, b=4)))
+
+ self.assertNotEqual(args, (dict(a=3, b=4),))
+ self.assertNotEqual(args, ((), dict(a=3, b=4)))
+
+
+ def test_call_with_args_call_empty_name(self):
+ args = _Call(((1, 2, 3), {}))
+ self.assertEqual(args, call(1, 2, 3))
+ self.assertEqual(call(1, 2, 3), args)
+ self.assertTrue(call(1, 2, 3) in [args])
+
+
+ def test_call_ne(self):
+ self.assertNotEqual(_Call(((1, 2, 3),)), call(1, 2))
+ self.assertFalse(_Call(((1, 2, 3),)) != call(1, 2, 3))
+ self.assertTrue(_Call(((1, 2), {})) != call(1, 2, 3))
+
+
+ def test_call_non_tuples(self):
+ kall = _Call(((1, 2, 3),))
+ for value in 1, None, self, int:
+ self.assertNotEqual(kall, value)
+ self.assertFalse(kall == value)
+
+
+ def test_repr(self):
+ self.assertEqual(repr(_Call()), 'call()')
+ self.assertEqual(repr(_Call(('foo',))), 'call.foo()')
+
+ self.assertEqual(repr(_Call(((1, 2, 3), {'a': 'b'}))),
+ "call(1, 2, 3, a='b')")
+ self.assertEqual(repr(_Call(('bar', (1, 2, 3), {'a': 'b'}))),
+ "call.bar(1, 2, 3, a='b')")
+
+ self.assertEqual(repr(call), 'call')
+ self.assertEqual(str(call), 'call')
+
+ self.assertEqual(repr(call()), 'call()')
+ self.assertEqual(repr(call(1)), 'call(1)')
+ self.assertEqual(repr(call(zz='thing')), "call(zz='thing')")
+
+ self.assertEqual(repr(call().foo), 'call().foo')
+ self.assertEqual(repr(call(1).foo.bar(a=3).bing),
+ 'call().foo.bar().bing')
+ self.assertEqual(
+ repr(call().foo(1, 2, a=3)),
+ "call().foo(1, 2, a=3)"
+ )
+ self.assertEqual(repr(call()()), "call()()")
+ self.assertEqual(repr(call(1)(2)), "call()(2)")
+ self.assertEqual(
+ repr(call()().bar().baz.beep(1)),
+ "call()().bar().baz.beep(1)"
+ )
+
+
+ def test_call(self):
+ self.assertEqual(call(), ('', (), {}))
+ self.assertEqual(call('foo', 'bar', one=3, two=4),
+ ('', ('foo', 'bar'), {'one': 3, 'two': 4}))
+
+ mock = Mock()
+ mock(1, 2, 3)
+ mock(a=3, b=6)
+ self.assertEqual(mock.call_args_list,
+ [call(1, 2, 3), call(a=3, b=6)])
+
+ def test_attribute_call(self):
+ self.assertEqual(call.foo(1), ('foo', (1,), {}))
+ self.assertEqual(call.bar.baz(fish='eggs'),
+ ('bar.baz', (), {'fish': 'eggs'}))
+
+ mock = Mock()
+ mock.foo(1, 2 ,3)
+ mock.bar.baz(a=3, b=6)
+ self.assertEqual(mock.method_calls,
+ [call.foo(1, 2, 3), call.bar.baz(a=3, b=6)])
+
+
+ def test_extended_call(self):
+ result = call(1).foo(2).bar(3, a=4)
+ self.assertEqual(result, ('().foo().bar', (3,), dict(a=4)))
+
+ mock = MagicMock()
+ mock(1, 2, a=3, b=4)
+ self.assertEqual(mock.call_args, call(1, 2, a=3, b=4))
+ self.assertNotEqual(mock.call_args, call(1, 2, 3))
+
+ self.assertEqual(mock.call_args_list, [call(1, 2, a=3, b=4)])
+ self.assertEqual(mock.mock_calls, [call(1, 2, a=3, b=4)])
+
+ mock = MagicMock()
+ mock.foo(1).bar()().baz.beep(a=6)
+
+ last_call = call.foo(1).bar()().baz.beep(a=6)
+ self.assertEqual(mock.mock_calls[-1], last_call)
+ self.assertEqual(mock.mock_calls, last_call.call_list())
+
+
+ def test_call_list(self):
+ mock = MagicMock()
+ mock(1)
+ self.assertEqual(call(1).call_list(), mock.mock_calls)
+
+ mock = MagicMock()
+ mock(1).method(2)
+ self.assertEqual(call(1).method(2).call_list(),
+ mock.mock_calls)
+
+ mock = MagicMock()
+ mock(1).method(2)(3)
+ self.assertEqual(call(1).method(2)(3).call_list(),
+ mock.mock_calls)
+
+ mock = MagicMock()
+ int(mock(1).method(2)(3).foo.bar.baz(4)(5))
+ kall = call(1).method(2)(3).foo.bar.baz(4)(5).__int__()
+ self.assertEqual(kall.call_list(), mock.mock_calls)
+
+
+ def test_call_any(self):
+ self.assertEqual(call, ANY)
+
+ m = MagicMock()
+ int(m)
+ self.assertEqual(m.mock_calls, [ANY])
+ self.assertEqual([ANY], m.mock_calls)
+
+
+ def test_two_args_call(self):
+ args = _Call(((1, 2), {'a': 3}), two=True)
+ self.assertEqual(len(args), 2)
+ self.assertEqual(args[0], (1, 2))
+ self.assertEqual(args[1], {'a': 3})
+
+ other_args = _Call(((1, 2), {'a': 3}))
+ self.assertEqual(args, other_args)
+
+
+class SpecSignatureTest(unittest2.TestCase):
+
+ def _check_someclass_mock(self, mock):
+ self.assertRaises(AttributeError, getattr, mock, 'foo')
+ mock.one(1, 2)
+ mock.one.assert_called_with(1, 2)
+ self.assertRaises(AssertionError,
+ mock.one.assert_called_with, 3, 4)
+ self.assertRaises(TypeError, mock.one, 1)
+
+ mock.two()
+ mock.two.assert_called_with()
+ self.assertRaises(AssertionError,
+ mock.two.assert_called_with, 3)
+ self.assertRaises(TypeError, mock.two, 1)
+
+ mock.three()
+ mock.three.assert_called_with()
+ self.assertRaises(AssertionError,
+ mock.three.assert_called_with, 3)
+ self.assertRaises(TypeError, mock.three, 3, 2)
+
+ mock.three(1)
+ mock.three.assert_called_with(1)
+
+ mock.three(a=1)
+ mock.three.assert_called_with(a=1)
+
+
+ def test_basic(self):
+ for spec in (SomeClass, SomeClass()):
+ mock = create_autospec(spec)
+ self._check_someclass_mock(mock)
+
+
+ def test_create_autospec_return_value(self):
+ def f():
+ pass
+ mock = create_autospec(f, return_value='foo')
+ self.assertEqual(mock(), 'foo')
+
+ class Foo(object):
+ pass
+
+ mock = create_autospec(Foo, return_value='foo')
+ self.assertEqual(mock(), 'foo')
+
+
+ def test_autospec_reset_mock(self):
+ m = create_autospec(int)
+ int(m)
+ m.reset_mock()
+ self.assertEqual(m.__int__.call_count, 0)
+
+
+ def test_mocking_unbound_methods(self):
+ class Foo(object):
+ def foo(self, foo):
+ pass
+ p = patch.object(Foo, 'foo')
+ mock_foo = p.start()
+ Foo().foo(1)
+
+ mock_foo.assert_called_with(1)
+
+
+ @unittest2.expectedFailure
+ def test_create_autospec_unbound_methods(self):
+ # see issue 128
+ class Foo(object):
+ def foo(self):
+ pass
+
+ klass = create_autospec(Foo)
+ instance = klass()
+ self.assertRaises(TypeError, instance.foo, 1)
+
+ # Note: no type checking on the "self" parameter
+ klass.foo(1)
+ klass.foo.assert_called_with(1)
+ self.assertRaises(TypeError, klass.foo)
+
+
+ def test_create_autospec_keyword_arguments(self):
+ class Foo(object):
+ a = 3
+ m = create_autospec(Foo, a='3')
+ self.assertEqual(m.a, '3')
+
+ @unittest2.skipUnless(inPy3k, "Keyword only arguments Python 3 specific")
+ def test_create_autospec_keyword_only_arguments(self):
+ func_def = "def foo(a, *, b=None):\n pass\n"
+ namespace = {}
+ exec (func_def, namespace)
+ foo = namespace['foo']
+
+ m = create_autospec(foo)
+ m(1)
+ m.assert_called_with(1)
+ self.assertRaises(TypeError, m, 1, 2)
+
+ m(2, b=3)
+ m.assert_called_with(2, b=3)
+
+ def test_function_as_instance_attribute(self):
+ obj = SomeClass()
+ def f(a):
+ pass
+ obj.f = f
+
+ mock = create_autospec(obj)
+ mock.f('bing')
+ mock.f.assert_called_with('bing')
+
+
+ def test_spec_as_list(self):
+ # because spec as a list of strings in the mock constructor means
+ # something very different we treat a list instance as the type.
+ mock = create_autospec([])
+ mock.append('foo')
+ mock.append.assert_called_with('foo')
+
+ self.assertRaises(AttributeError, getattr, mock, 'foo')
+
+ class Foo(object):
+ foo = []
+
+ mock = create_autospec(Foo)
+ mock.foo.append(3)
+ mock.foo.append.assert_called_with(3)
+ self.assertRaises(AttributeError, getattr, mock.foo, 'foo')
+
+
+ def test_attributes(self):
+ class Sub(SomeClass):
+ attr = SomeClass()
+
+ sub_mock = create_autospec(Sub)
+
+ for mock in (sub_mock, sub_mock.attr):
+ self._check_someclass_mock(mock)
+
+
+ def test_builtin_functions_types(self):
+ # we could replace builtin functions / methods with a function
+ # with *args / **kwargs signature. Using the builtin method type
+ # as a spec seems to work fairly well though.
+ class BuiltinSubclass(list):
+ def bar(self, arg):
+ pass
+ sorted = sorted
+ attr = {}
+
+ mock = create_autospec(BuiltinSubclass)
+ mock.append(3)
+ mock.append.assert_called_with(3)
+ self.assertRaises(AttributeError, getattr, mock.append, 'foo')
+
+ mock.bar('foo')
+ mock.bar.assert_called_with('foo')
+ self.assertRaises(TypeError, mock.bar, 'foo', 'bar')
+ self.assertRaises(AttributeError, getattr, mock.bar, 'foo')
+
+ mock.sorted([1, 2])
+ mock.sorted.assert_called_with([1, 2])
+ self.assertRaises(AttributeError, getattr, mock.sorted, 'foo')
+
+ mock.attr.pop(3)
+ mock.attr.pop.assert_called_with(3)
+ self.assertRaises(AttributeError, getattr, mock.attr, 'foo')
+
+
+ def test_method_calls(self):
+ class Sub(SomeClass):
+ attr = SomeClass()
+
+ mock = create_autospec(Sub)
+ mock.one(1, 2)
+ mock.two()
+ mock.three(3)
+
+ expected = [call.one(1, 2), call.two(), call.three(3)]
+ self.assertEqual(mock.method_calls, expected)
+
+ mock.attr.one(1, 2)
+ mock.attr.two()
+ mock.attr.three(3)
+
+ expected.extend(
+ [call.attr.one(1, 2), call.attr.two(), call.attr.three(3)]
+ )
+ self.assertEqual(mock.method_calls, expected)
+
+
+ def test_magic_methods(self):
+ class BuiltinSubclass(list):
+ attr = {}
+
+ mock = create_autospec(BuiltinSubclass)
+ self.assertEqual(list(mock), [])
+ self.assertRaises(TypeError, int, mock)
+ self.assertRaises(TypeError, int, mock.attr)
+ self.assertEqual(list(mock), [])
+
+ self.assertIsInstance(mock['foo'], MagicMock)
+ self.assertIsInstance(mock.attr['foo'], MagicMock)
+
+
+ def test_spec_set(self):
+ class Sub(SomeClass):
+ attr = SomeClass()
+
+ for spec in (Sub, Sub()):
+ mock = create_autospec(spec, spec_set=True)
+ self._check_someclass_mock(mock)
+
+ self.assertRaises(AttributeError, setattr, mock, 'foo', 'bar')
+ self.assertRaises(AttributeError, setattr, mock.attr, 'foo', 'bar')
+
+
+ def test_descriptors(self):
+ class Foo(object):
+ @classmethod
+ def f(cls, a, b):
+ pass
+ @staticmethod
+ def g(a, b):
+ pass
+
+ class Bar(Foo):
+ pass
+
+ class Baz(SomeClass, Bar):
+ pass
+
+ for spec in (Foo, Foo(), Bar, Bar(), Baz, Baz()):
+ mock = create_autospec(spec)
+ mock.f(1, 2)
+ mock.f.assert_called_once_with(1, 2)
+
+ mock.g(3, 4)
+ mock.g.assert_called_once_with(3, 4)
+
+
+ @unittest2.skipIf(inPy3k, "No old style classes in Python 3")
+ def test_old_style_classes(self):
+ class Foo:
+ def f(self, a, b):
+ pass
+
+ class Bar(Foo):
+ g = Foo()
+
+ for spec in (Foo, Foo(), Bar, Bar()):
+ mock = create_autospec(spec)
+ mock.f(1, 2)
+ mock.f.assert_called_once_with(1, 2)
+
+ self.assertRaises(AttributeError, getattr, mock, 'foo')
+ self.assertRaises(AttributeError, getattr, mock.f, 'foo')
+
+ mock.g.f(1, 2)
+ mock.g.f.assert_called_once_with(1, 2)
+ self.assertRaises(AttributeError, getattr, mock.g, 'foo')
+
+
+ def test_recursive(self):
+ class A(object):
+ def a(self):
+ pass
+ foo = 'foo bar baz'
+ bar = foo
+
+ A.B = A
+ mock = create_autospec(A)
+
+ mock()
+ self.assertFalse(mock.B.called)
+
+ mock.a()
+ mock.B.a()
+ self.assertEqual(mock.method_calls, [call.a(), call.B.a()])
+
+ self.assertIs(A.foo, A.bar)
+ self.assertIsNot(mock.foo, mock.bar)
+ mock.foo.lower()
+ self.assertRaises(AssertionError, mock.bar.lower.assert_called_with)
+
+
+ def test_spec_inheritance_for_classes(self):
+ class Foo(object):
+ def a(self):
+ pass
+ class Bar(object):
+ def f(self):
+ pass
+
+ class_mock = create_autospec(Foo)
+
+ self.assertIsNot(class_mock, class_mock())
+
+ for this_mock in class_mock, class_mock():
+ this_mock.a()
+ this_mock.a.assert_called_with()
+ self.assertRaises(TypeError, this_mock.a, 'foo')
+ self.assertRaises(AttributeError, getattr, this_mock, 'b')
+
+ instance_mock = create_autospec(Foo())
+ instance_mock.a()
+ instance_mock.a.assert_called_with()
+ self.assertRaises(TypeError, instance_mock.a, 'foo')
+ self.assertRaises(AttributeError, getattr, instance_mock, 'b')
+
+ # The return value isn't isn't callable
+ self.assertRaises(TypeError, instance_mock)
+
+ instance_mock.Bar.f()
+ instance_mock.Bar.f.assert_called_with()
+ self.assertRaises(AttributeError, getattr, instance_mock.Bar, 'g')
+
+ instance_mock.Bar().f()
+ instance_mock.Bar().f.assert_called_with()
+ self.assertRaises(AttributeError, getattr, instance_mock.Bar(), 'g')
+
+
+ def test_inherit(self):
+ class Foo(object):
+ a = 3
+
+ Foo.Foo = Foo
+
+ # class
+ mock = create_autospec(Foo)
+ instance = mock()
+ self.assertRaises(AttributeError, getattr, instance, 'b')
+
+ attr_instance = mock.Foo()
+ self.assertRaises(AttributeError, getattr, attr_instance, 'b')
+
+ # instance
+ mock = create_autospec(Foo())
+ self.assertRaises(AttributeError, getattr, mock, 'b')
+ self.assertRaises(TypeError, mock)
+
+ # attribute instance
+ call_result = mock.Foo()
+ self.assertRaises(AttributeError, getattr, call_result, 'b')
+
+
+ def test_builtins(self):
+ # used to fail with infinite recursion
+ create_autospec(1)
+
+ create_autospec(int)
+ create_autospec('foo')
+ create_autospec(str)
+ create_autospec({})
+ create_autospec(dict)
+ create_autospec([])
+ create_autospec(list)
+ create_autospec(set())
+ create_autospec(set)
+ create_autospec(1.0)
+ create_autospec(float)
+ create_autospec(1j)
+ create_autospec(complex)
+ create_autospec(False)
+ create_autospec(True)
+
+
+ def test_function(self):
+ def f(a, b):
+ pass
+
+ mock = create_autospec(f)
+ self.assertRaises(TypeError, mock)
+ mock(1, 2)
+ mock.assert_called_with(1, 2)
+
+ f.f = f
+ mock = create_autospec(f)
+ self.assertRaises(TypeError, mock.f)
+ mock.f(3, 4)
+ mock.f.assert_called_with(3, 4)
+
+
+ def test_skip_attributeerrors(self):
+ class Raiser(object):
+ def __get__(self, obj, type=None):
+ if obj is None:
+ raise AttributeError('Can only be accessed via an instance')
+
+ class RaiserClass(object):
+ raiser = Raiser()
+
+ @staticmethod
+ def existing(a, b):
+ return a + b
+
+ s = create_autospec(RaiserClass)
+ self.assertRaises(TypeError, lambda x: s.existing(1, 2, 3))
+ s.existing(1, 2)
+ self.assertRaises(AttributeError, lambda: s.nonexisting)
+
+ # check we can fetch the raiser attribute and it has no spec
+ obj = s.raiser
+ obj.foo, obj.bar
+
+
+ def test_signature_class(self):
+ class Foo(object):
+ def __init__(self, a, b=3):
+ pass
+
+ mock = create_autospec(Foo)
+
+ self.assertRaises(TypeError, mock)
+ mock(1)
+ mock.assert_called_once_with(1)
+
+ mock(4, 5)
+ mock.assert_called_with(4, 5)
+
+
+ @unittest2.skipIf(inPy3k, 'no old style classes in Python 3')
+ def test_signature_old_style_class(self):
+ class Foo:
+ def __init__(self, a, b=3):
+ pass
+
+ mock = create_autospec(Foo)
+
+ self.assertRaises(TypeError, mock)
+ mock(1)
+ mock.assert_called_once_with(1)
+
+ mock(4, 5)
+ mock.assert_called_with(4, 5)
+
+
+ def test_class_with_no_init(self):
+ # this used to raise an exception
+ # due to trying to get a signature from object.__init__
+ class Foo(object):
+ pass
+ create_autospec(Foo)
+
+
+ @unittest2.skipIf(inPy3k, 'no old style classes in Python 3')
+ def test_old_style_class_with_no_init(self):
+ # this used to raise an exception
+ # due to Foo.__init__ raising an AttributeError
+ class Foo:
+ pass
+ create_autospec(Foo)
+
+
+ def test_signature_callable(self):
+ class Callable(object):
+ def __init__(self):
+ pass
+ def __call__(self, a):
+ pass
+
+ mock = create_autospec(Callable)
+ mock()
+ mock.assert_called_once_with()
+ self.assertRaises(TypeError, mock, 'a')
+
+ instance = mock()
+ self.assertRaises(TypeError, instance)
+ instance(a='a')
+ instance.assert_called_once_with(a='a')
+ instance('a')
+ instance.assert_called_with('a')
+
+ mock = create_autospec(Callable())
+ mock(a='a')
+ mock.assert_called_once_with(a='a')
+ self.assertRaises(TypeError, mock)
+ mock('a')
+ mock.assert_called_with('a')
+
+
+ def test_signature_noncallable(self):
+ class NonCallable(object):
+ def __init__(self):
+ pass
+
+ mock = create_autospec(NonCallable)
+ instance = mock()
+ mock.assert_called_once_with()
+ self.assertRaises(TypeError, mock, 'a')
+ self.assertRaises(TypeError, instance)
+ self.assertRaises(TypeError, instance, 'a')
+
+ mock = create_autospec(NonCallable())
+ self.assertRaises(TypeError, mock)
+ self.assertRaises(TypeError, mock, 'a')
+
+
+ def test_create_autospec_none(self):
+ class Foo(object):
+ bar = None
+
+ mock = create_autospec(Foo)
+ none = mock.bar
+ self.assertNotIsInstance(none, type(None))
+
+ none.foo()
+ none.foo.assert_called_once_with()
+
+
+ def test_autospec_functions_with_self_in_odd_place(self):
+ class Foo(object):
+ def f(a, self):
+ pass
+
+ a = create_autospec(Foo)
+ a.f(self=10)
+ a.f.assert_called_with(self=10)
+
+
+ def test_autospec_property(self):
+ class Foo(object):
+ @property
+ def foo(self):
+ return 3
+
+ foo = create_autospec(Foo)
+ mock_property = foo.foo
+
+ # no spec on properties
+ self.assertTrue(isinstance(mock_property, MagicMock))
+ mock_property(1, 2, 3)
+ mock_property.abc(4, 5, 6)
+ mock_property.assert_called_once_with(1, 2, 3)
+ mock_property.abc.assert_called_once_with(4, 5, 6)
+
+
+ def test_autospec_slots(self):
+ class Foo(object):
+ __slots__ = ['a']
+
+ foo = create_autospec(Foo)
+ mock_slot = foo.a
+
+ # no spec on slots
+ mock_slot(1, 2, 3)
+ mock_slot.abc(4, 5, 6)
+ mock_slot.assert_called_once_with(1, 2, 3)
+ mock_slot.abc.assert_called_once_with(4, 5, 6)
+
+
+class TestCallList(unittest2.TestCase):
+
+ def test_args_list_contains_call_list(self):
+ mock = Mock()
+ self.assertIsInstance(mock.call_args_list, _CallList)
+
+ mock(1, 2)
+ mock(a=3)
+ mock(3, 4)
+ mock(b=6)
+
+ for kall in call(1, 2), call(a=3), call(3, 4), call(b=6):
+ self.assertTrue(kall in mock.call_args_list)
+
+ calls = [call(a=3), call(3, 4)]
+ self.assertTrue(calls in mock.call_args_list)
+ calls = [call(1, 2), call(a=3)]
+ self.assertTrue(calls in mock.call_args_list)
+ calls = [call(3, 4), call(b=6)]
+ self.assertTrue(calls in mock.call_args_list)
+ calls = [call(3, 4)]
+ self.assertTrue(calls in mock.call_args_list)
+
+ self.assertFalse(call('fish') in mock.call_args_list)
+ self.assertFalse([call('fish')] in mock.call_args_list)
+
+
+ def test_call_list_str(self):
+ mock = Mock()
+ mock(1, 2)
+ mock.foo(a=3)
+ mock.foo.bar().baz('fish', cat='dog')
+
+ expected = (
+ "[call(1, 2),\n"
+ " call.foo(a=3),\n"
+ " call.foo.bar(),\n"
+ " call.foo.bar().baz('fish', cat='dog')]"
+ )
+ self.assertEqual(str(mock.mock_calls), expected)
+
+
+ def test_propertymock(self):
+ p = patch('%s.SomeClass.one' % __name__, new_callable=PropertyMock)
+ mock = p.start()
+ try:
+ SomeClass.one
+ mock.assert_called_once_with()
+
+ s = SomeClass()
+ s.one
+ mock.assert_called_with()
+ self.assertEqual(mock.mock_calls, [call(), call()])
+
+ s.one = 3
+ self.assertEqual(mock.mock_calls, [call(), call(), call(3)])
+ finally:
+ p.stop()
+
+
+ def test_propertymock_returnvalue(self):
+ m = MagicMock()
+ p = PropertyMock()
+ type(m).foo = p
+
+ returned = m.foo
+ p.assert_called_once_with()
+ self.assertIsInstance(returned, MagicMock)
+ self.assertNotIsInstance(returned, PropertyMock)
+
+
+if __name__ == '__main__':
+ unittest2.main()
diff --git a/slider-agent/src/test/python/mock/tests/testmagicmethods.py b/slider-agent/src/test/python/mock/tests/testmagicmethods.py
new file mode 100644
index 0000000..ef0f16d
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/testmagicmethods.py
@@ -0,0 +1,486 @@
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
+
+from tests.support import unittest2, inPy3k
+
+try:
+ unicode
+except NameError:
+ # Python 3
+ unicode = str
+ long = int
+
+import inspect
+import sys
+from mock import Mock, MagicMock, _magics
+
+
+
+class TestMockingMagicMethods(unittest2.TestCase):
+
+ def test_deleting_magic_methods(self):
+ mock = Mock()
+ self.assertFalse(hasattr(mock, '__getitem__'))
+
+ mock.__getitem__ = Mock()
+ self.assertTrue(hasattr(mock, '__getitem__'))
+
+ del mock.__getitem__
+ self.assertFalse(hasattr(mock, '__getitem__'))
+
+
+ def test_magicmock_del(self):
+ mock = MagicMock()
+ # before using getitem
+ del mock.__getitem__
+ self.assertRaises(TypeError, lambda: mock['foo'])
+
+ mock = MagicMock()
+ # this time use it first
+ mock['foo']
+ del mock.__getitem__
+ self.assertRaises(TypeError, lambda: mock['foo'])
+
+
+ def test_magic_method_wrapping(self):
+ mock = Mock()
+ def f(self, name):
+ return self, 'fish'
+
+ mock.__getitem__ = f
+ self.assertFalse(mock.__getitem__ is f)
+ self.assertEqual(mock['foo'], (mock, 'fish'))
+ self.assertEqual(mock.__getitem__('foo'), (mock, 'fish'))
+
+ mock.__getitem__ = mock
+ self.assertTrue(mock.__getitem__ is mock)
+
+
+ def test_magic_methods_isolated_between_mocks(self):
+ mock1 = Mock()
+ mock2 = Mock()
+
+ mock1.__iter__ = Mock(return_value=iter([]))
+ self.assertEqual(list(mock1), [])
+ self.assertRaises(TypeError, lambda: list(mock2))
+
+
+ def test_repr(self):
+ mock = Mock()
+ self.assertEqual(repr(mock), "<Mock id='%s'>" % id(mock))
+ mock.__repr__ = lambda s: 'foo'
+ self.assertEqual(repr(mock), 'foo')
+
+
+ def test_str(self):
+ mock = Mock()
+ self.assertEqual(str(mock), object.__str__(mock))
+ mock.__str__ = lambda s: 'foo'
+ self.assertEqual(str(mock), 'foo')
+
+
+ @unittest2.skipIf(inPy3k, "no unicode in Python 3")
+ def test_unicode(self):
+ mock = Mock()
+ self.assertEqual(unicode(mock), unicode(str(mock)))
+
+ mock.__unicode__ = lambda s: unicode('foo')
+ self.assertEqual(unicode(mock), unicode('foo'))
+
+
+ def test_dict_methods(self):
+ mock = Mock()
+
+ self.assertRaises(TypeError, lambda: mock['foo'])
+ def _del():
+ del mock['foo']
+ def _set():
+ mock['foo'] = 3
+ self.assertRaises(TypeError, _del)
+ self.assertRaises(TypeError, _set)
+
+ _dict = {}
+ def getitem(s, name):
+ return _dict[name]
+ def setitem(s, name, value):
+ _dict[name] = value
+ def delitem(s, name):
+ del _dict[name]
+
+ mock.__setitem__ = setitem
+ mock.__getitem__ = getitem
+ mock.__delitem__ = delitem
+
+ self.assertRaises(KeyError, lambda: mock['foo'])
+ mock['foo'] = 'bar'
+ self.assertEqual(_dict, {'foo': 'bar'})
+ self.assertEqual(mock['foo'], 'bar')
+ del mock['foo']
+ self.assertEqual(_dict, {})
+
+
+ def test_numeric(self):
+ original = mock = Mock()
+ mock.value = 0
+
+ self.assertRaises(TypeError, lambda: mock + 3)
+
+ def add(self, other):
+ mock.value += other
+ return self
+ mock.__add__ = add
+ self.assertEqual(mock + 3, mock)
+ self.assertEqual(mock.value, 3)
+
+ del mock.__add__
+ def iadd(mock):
+ mock += 3
+ self.assertRaises(TypeError, iadd, mock)
+ mock.__iadd__ = add
+ mock += 6
+ self.assertEqual(mock, original)
+ self.assertEqual(mock.value, 9)
+
+ self.assertRaises(TypeError, lambda: 3 + mock)
+ mock.__radd__ = add
+ self.assertEqual(7 + mock, mock)
+ self.assertEqual(mock.value, 16)
+
+
+ @unittest2.skipIf(inPy3k, 'no truediv in Python 3')
+ def test_truediv(self):
+ mock = MagicMock()
+ mock.__truediv__.return_value = 6
+
+ context = {'mock': mock}
+ code = 'from __future__ import division\nresult = mock / 7\n'
+ exec(code, context)
+ self.assertEqual(context['result'], 6)
+
+ mock.__rtruediv__.return_value = 3
+ code = 'from __future__ import division\nresult = 2 / mock\n'
+ exec(code, context)
+ self.assertEqual(context['result'], 3)
+
+
+ @unittest2.skipIf(not inPy3k, 'truediv is available in Python 2')
+ def test_no_truediv(self):
+ self.assertRaises(
+ AttributeError, getattr, MagicMock(), '__truediv__'
+ )
+ self.assertRaises(
+ AttributeError, getattr, MagicMock(), '__rtruediv__'
+ )
+
+
+ def test_hash(self):
+ mock = Mock()
+ # test delegation
+ self.assertEqual(hash(mock), Mock.__hash__(mock))
+
+ def _hash(s):
+ return 3
+ mock.__hash__ = _hash
+ self.assertEqual(hash(mock), 3)
+
+
+ def test_nonzero(self):
+ m = Mock()
+ self.assertTrue(bool(m))
+
+ nonzero = lambda s: False
+ if not inPy3k:
+ m.__nonzero__ = nonzero
+ else:
+ m.__bool__ = nonzero
+
+ self.assertFalse(bool(m))
+
+
+ def test_comparison(self):
+ # note: this test fails with Jython 2.5.1 due to a Jython bug
+ # it is fixed in jython 2.5.2
+ if not inPy3k:
+ # incomparable in Python 3
+ self. assertEqual(Mock() < 3, object() < 3)
+ self. assertEqual(Mock() > 3, object() > 3)
+ self. assertEqual(Mock() <= 3, object() <= 3)
+ self. assertEqual(Mock() >= 3, object() >= 3)
+ else:
+ self.assertRaises(TypeError, lambda: MagicMock() < object())
+ self.assertRaises(TypeError, lambda: object() < MagicMock())
+ self.assertRaises(TypeError, lambda: MagicMock() < MagicMock())
+ self.assertRaises(TypeError, lambda: MagicMock() > object())
+ self.assertRaises(TypeError, lambda: object() > MagicMock())
+ self.assertRaises(TypeError, lambda: MagicMock() > MagicMock())
+ self.assertRaises(TypeError, lambda: MagicMock() <= object())
+ self.assertRaises(TypeError, lambda: object() <= MagicMock())
+ self.assertRaises(TypeError, lambda: MagicMock() <= MagicMock())
+ self.assertRaises(TypeError, lambda: MagicMock() >= object())
+ self.assertRaises(TypeError, lambda: object() >= MagicMock())
+ self.assertRaises(TypeError, lambda: MagicMock() >= MagicMock())
+
+ mock = Mock()
+ def comp(s, o):
+ return True
+ mock.__lt__ = mock.__gt__ = mock.__le__ = mock.__ge__ = comp
+ self. assertTrue(mock < 3)
+ self. assertTrue(mock > 3)
+ self. assertTrue(mock <= 3)
+ self. assertTrue(mock >= 3)
+
+
+ def test_equality(self):
+ for mock in Mock(), MagicMock():
+ self.assertEqual(mock == mock, True)
+ self.assertIsInstance(mock == mock, bool)
+ self.assertEqual(mock != mock, False)
+ self.assertIsInstance(mock != mock, bool)
+ self.assertEqual(mock == object(), False)
+ self.assertEqual(mock != object(), True)
+
+ def eq(self, other):
+ return other == 3
+ mock.__eq__ = eq
+ self.assertTrue(mock == 3)
+ self.assertFalse(mock == 4)
+
+ def ne(self, other):
+ return other == 3
+ mock.__ne__ = ne
+ self.assertTrue(mock != 3)
+ self.assertFalse(mock != 4)
+
+ mock = MagicMock()
+ mock.__eq__.return_value = True
+ self.assertIsInstance(mock == 3, bool)
+ self.assertEqual(mock == 3, True)
+
+ mock.__ne__.return_value = False
+ self.assertIsInstance(mock != 3, bool)
+ self.assertEqual(mock != 3, False)
+
+
+ def test_len_contains_iter(self):
+ mock = Mock()
+
+ self.assertRaises(TypeError, len, mock)
+ self.assertRaises(TypeError, iter, mock)
+ self.assertRaises(TypeError, lambda: 'foo' in mock)
+
+ mock.__len__ = lambda s: 6
+ self.assertEqual(len(mock), 6)
+
+ mock.__contains__ = lambda s, o: o == 3
+ self.assertTrue(3 in mock)
+ self.assertFalse(6 in mock)
+
+ mock.__iter__ = lambda s: iter('foobarbaz')
+ self.assertEqual(list(mock), list('foobarbaz'))
+
+
+ def test_magicmock(self):
+ mock = MagicMock()
+
+ mock.__iter__.return_value = iter([1, 2, 3])
+ self.assertEqual(list(mock), [1, 2, 3])
+
+ name = '__nonzero__'
+ other = '__bool__'
+ if inPy3k:
+ name, other = other, name
+ getattr(mock, name).return_value = False
+ self.assertFalse(hasattr(mock, other))
+ self.assertFalse(bool(mock))
+
+ for entry in _magics:
+ self.assertTrue(hasattr(mock, entry))
+ self.assertFalse(hasattr(mock, '__imaginery__'))
+
+
+ def test_magic_mock_equality(self):
+ mock = MagicMock()
+ self.assertIsInstance(mock == object(), bool)
+ self.assertIsInstance(mock != object(), bool)
+
+ self.assertEqual(mock == object(), False)
+ self.assertEqual(mock != object(), True)
+ self.assertEqual(mock == mock, True)
+ self.assertEqual(mock != mock, False)
+
+
+ def test_magicmock_defaults(self):
+ mock = MagicMock()
+ self.assertEqual(int(mock), 1)
+ self.assertEqual(complex(mock), 1j)
+ self.assertEqual(float(mock), 1.0)
+ self.assertEqual(long(mock), long(1))
+ self.assertNotIn(object(), mock)
+ self.assertEqual(len(mock), 0)
+ self.assertEqual(list(mock), [])
+ self.assertEqual(hash(mock), object.__hash__(mock))
+ self.assertEqual(str(mock), object.__str__(mock))
+ self.assertEqual(unicode(mock), object.__str__(mock))
+ self.assertIsInstance(unicode(mock), unicode)
+ self.assertTrue(bool(mock))
+ if not inPy3k:
+ self.assertEqual(oct(mock), '1')
+ else:
+ # in Python 3 oct and hex use __index__
+ # so these tests are for __index__ in py3k
+ self.assertEqual(oct(mock), '0o1')
+ self.assertEqual(hex(mock), '0x1')
+ # how to test __sizeof__ ?
+
+
+ @unittest2.skipIf(inPy3k, "no __cmp__ in Python 3")
+ def test_non_default_magic_methods(self):
+ mock = MagicMock()
+ self.assertRaises(AttributeError, lambda: mock.__cmp__)
+
+ mock = Mock()
+ mock.__cmp__ = lambda s, o: 0
+
+ self.assertEqual(mock, object())
+
+
+ def test_magic_methods_and_spec(self):
+ class Iterable(object):
+ def __iter__(self):
+ pass
+
+ mock = Mock(spec=Iterable)
+ self.assertRaises(AttributeError, lambda: mock.__iter__)
+
+ mock.__iter__ = Mock(return_value=iter([]))
+ self.assertEqual(list(mock), [])
+
+ class NonIterable(object):
+ pass
+ mock = Mock(spec=NonIterable)
+ self.assertRaises(AttributeError, lambda: mock.__iter__)
+
+ def set_int():
+ mock.__int__ = Mock(return_value=iter([]))
+ self.assertRaises(AttributeError, set_int)
+
+ mock = MagicMock(spec=Iterable)
+ self.assertEqual(list(mock), [])
+ self.assertRaises(AttributeError, set_int)
+
+
+ def test_magic_methods_and_spec_set(self):
+ class Iterable(object):
+ def __iter__(self):
+ pass
+
+ mock = Mock(spec_set=Iterable)
+ self.assertRaises(AttributeError, lambda: mock.__iter__)
+
+ mock.__iter__ = Mock(return_value=iter([]))
+ self.assertEqual(list(mock), [])
+
+ class NonIterable(object):
+ pass
+ mock = Mock(spec_set=NonIterable)
+ self.assertRaises(AttributeError, lambda: mock.__iter__)
+
+ def set_int():
+ mock.__int__ = Mock(return_value=iter([]))
+ self.assertRaises(AttributeError, set_int)
+
+ mock = MagicMock(spec_set=Iterable)
+ self.assertEqual(list(mock), [])
+ self.assertRaises(AttributeError, set_int)
+
+
+ def test_setting_unsupported_magic_method(self):
+ mock = MagicMock()
+ def set_setattr():
+ mock.__setattr__ = lambda self, name: None
+ self.assertRaisesRegexp(AttributeError,
+ "Attempting to set unsupported magic method '__setattr__'.",
+ set_setattr
+ )
+
+
+ def test_attributes_and_return_value(self):
+ mock = MagicMock()
+ attr = mock.foo
+ def _get_type(obj):
+ # the type of every mock (or magicmock) is a custom subclass
+ # so the real type is the second in the mro
+ return type(obj).__mro__[1]
+ self.assertEqual(_get_type(attr), MagicMock)
+
+ returned = mock()
+ self.assertEqual(_get_type(returned), MagicMock)
+
+
+ def test_magic_methods_are_magic_mocks(self):
+ mock = MagicMock()
+ self.assertIsInstance(mock.__getitem__, MagicMock)
+
+ mock[1][2].__getitem__.return_value = 3
+ self.assertEqual(mock[1][2][3], 3)
+
+
+ def test_magic_method_reset_mock(self):
+ mock = MagicMock()
+ str(mock)
+ self.assertTrue(mock.__str__.called)
+ mock.reset_mock()
+ self.assertFalse(mock.__str__.called)
+
+
+ @unittest2.skipUnless(sys.version_info[:2] >= (2, 6),
+ "__dir__ not available until Python 2.6 or later")
+ def test_dir(self):
+ # overriding the default implementation
+ for mock in Mock(), MagicMock():
+ def _dir(self):
+ return ['foo']
+ mock.__dir__ = _dir
+ self.assertEqual(dir(mock), ['foo'])
+
+
+ @unittest2.skipIf('PyPy' in sys.version, "This fails differently on pypy")
+ def test_bound_methods(self):
+ m = Mock()
+
+ # XXXX should this be an expected failure instead?
+
+ # this seems like it should work, but is hard to do without introducing
+ # other api inconsistencies. Failure message could be better though.
+ m.__iter__ = [3].__iter__
+ self.assertRaises(TypeError, iter, m)
+
+
+ def test_magic_method_type(self):
+ class Foo(MagicMock):
+ pass
+
+ foo = Foo()
+ self.assertIsInstance(foo.__int__, Foo)
+
+
+ def test_descriptor_from_class(self):
+ m = MagicMock()
+ type(m).__str__.return_value = 'foo'
+ self.assertEqual(str(m), 'foo')
+
+
+ def test_iterable_as_iter_return_value(self):
+ m = MagicMock()
+ m.__iter__.return_value = [1, 2, 3]
+ self.assertEqual(list(m), [1, 2, 3])
+ self.assertEqual(list(m), [1, 2, 3])
+
+ m.__iter__.return_value = iter([4, 5, 6])
+ self.assertEqual(list(m), [4, 5, 6])
+ self.assertEqual(list(m), [])
+
+
+if __name__ == '__main__':
+ unittest2.main()
diff --git a/slider-agent/src/test/python/mock/tests/testmock.py b/slider-agent/src/test/python/mock/tests/testmock.py
new file mode 100644
index 0000000..f3ceea9
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/testmock.py
@@ -0,0 +1,1351 @@
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
+
+from tests.support import (
+ callable, unittest2, inPy3k, is_instance, next
+)
+
+import copy
+import pickle
+import sys
+
+import mock
+from mock import (
+ call, DEFAULT, patch, sentinel,
+ MagicMock, Mock, NonCallableMock,
+ NonCallableMagicMock, _CallList,
+ create_autospec
+)
+
+
+try:
+ unicode
+except NameError:
+ unicode = str
+
+
+class Iter(object):
+ def __init__(self):
+ self.thing = iter(['this', 'is', 'an', 'iter'])
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ return next(self.thing)
+
+ __next__ = next
+
+
+class Subclass(MagicMock):
+ pass
+
+
+class Thing(object):
+ attribute = 6
+ foo = 'bar'
+
+
+
+class MockTest(unittest2.TestCase):
+
+ def test_all(self):
+ # if __all__ is badly defined then import * will raise an error
+ # We have to exec it because you can't import * inside a method
+ # in Python 3
+ exec("from mock import *")
+
+
+ def test_constructor(self):
+ mock = Mock()
+
+ self.assertFalse(mock.called, "called not initialised correctly")
+ self.assertEqual(mock.call_count, 0,
+ "call_count not initialised correctly")
+ self.assertTrue(is_instance(mock.return_value, Mock),
+ "return_value not initialised correctly")
+
+ self.assertEqual(mock.call_args, None,
+ "call_args not initialised correctly")
+ self.assertEqual(mock.call_args_list, [],
+ "call_args_list not initialised correctly")
+ self.assertEqual(mock.method_calls, [],
+ "method_calls not initialised correctly")
+
+ # Can't use hasattr for this test as it always returns True on a mock
+ self.assertFalse('_items' in mock.__dict__,
+ "default mock should not have '_items' attribute")
+
+ self.assertIsNone(mock._mock_parent,
+ "parent not initialised correctly")
+ self.assertIsNone(mock._mock_methods,
+ "methods not initialised correctly")
+ self.assertEqual(mock._mock_children, {},
+ "children not initialised incorrectly")
+
+
+ def test_unicode_not_broken(self):
+ # This used to raise an exception with Python 2.5 and Mock 0.4
+ unicode(Mock())
+
+
+ def test_return_value_in_constructor(self):
+ mock = Mock(return_value=None)
+ self.assertIsNone(mock.return_value,
+ "return value in constructor not honoured")
+
+
+ def test_repr(self):
+ mock = Mock(name='foo')
+ self.assertIn('foo', repr(mock))
+ self.assertIn("'%s'" % id(mock), repr(mock))
+
+ mocks = [(Mock(), 'mock'), (Mock(name='bar'), 'bar')]
+ for mock, name in mocks:
+ self.assertIn('%s.bar' % name, repr(mock.bar))
+ self.assertIn('%s.foo()' % name, repr(mock.foo()))
+ self.assertIn('%s.foo().bing' % name, repr(mock.foo().bing))
+ self.assertIn('%s()' % name, repr(mock()))
+ self.assertIn('%s()()' % name, repr(mock()()))
+ self.assertIn('%s()().foo.bar.baz().bing' % name,
+ repr(mock()().foo.bar.baz().bing))
+
+
+ def test_repr_with_spec(self):
+ class X(object):
+ pass
+
+ mock = Mock(spec=X)
+ self.assertIn(" spec='X' ", repr(mock))
+
+ mock = Mock(spec=X())
+ self.assertIn(" spec='X' ", repr(mock))
+
+ mock = Mock(spec_set=X)
+ self.assertIn(" spec_set='X' ", repr(mock))
+
+ mock = Mock(spec_set=X())
+ self.assertIn(" spec_set='X' ", repr(mock))
+
+ mock = Mock(spec=X, name='foo')
+ self.assertIn(" spec='X' ", repr(mock))
+ self.assertIn(" name='foo' ", repr(mock))
+
+ mock = Mock(name='foo')
+ self.assertNotIn("spec", repr(mock))
+
+ mock = Mock()
+ self.assertNotIn("spec", repr(mock))
+
+ mock = Mock(spec=['foo'])
+ self.assertNotIn("spec", repr(mock))
+
+
+ def test_side_effect(self):
+ mock = Mock()
+
+ def effect(*args, **kwargs):
+ raise SystemError('kablooie')
+
+ mock.side_effect = effect
+ self.assertRaises(SystemError, mock, 1, 2, fish=3)
+ mock.assert_called_with(1, 2, fish=3)
+
+ results = [1, 2, 3]
+ def effect():
+ return results.pop()
+ mock.side_effect = effect
+
+ self.assertEqual([mock(), mock(), mock()], [3, 2, 1],
+ "side effect not used correctly")
+
+ mock = Mock(side_effect=sentinel.SideEffect)
+ self.assertEqual(mock.side_effect, sentinel.SideEffect,
+ "side effect in constructor not used")
+
+ def side_effect():
+ return DEFAULT
+ mock = Mock(side_effect=side_effect, return_value=sentinel.RETURN)
+ self.assertEqual(mock(), sentinel.RETURN)
+
+
+ @unittest2.skipUnless('java' in sys.platform,
+ 'This test only applies to Jython')
+ def test_java_exception_side_effect(self):
+ import java
+ mock = Mock(side_effect=java.lang.RuntimeException("Boom!"))
+
+ # can't use assertRaises with java exceptions
+ try:
+ mock(1, 2, fish=3)
+ except java.lang.RuntimeException:
+ pass
+ else:
+ self.fail('java exception not raised')
+ mock.assert_called_with(1,2, fish=3)
+
+
+ def test_reset_mock(self):
+ parent = Mock()
+ spec = ["something"]
+ mock = Mock(name="child", parent=parent, spec=spec)
+ mock(sentinel.Something, something=sentinel.SomethingElse)
+ something = mock.something
+ mock.something()
+ mock.side_effect = sentinel.SideEffect
+ return_value = mock.return_value
+ return_value()
+
+ mock.reset_mock()
+
+ self.assertEqual(mock._mock_name, "child",
+ "name incorrectly reset")
+ self.assertEqual(mock._mock_parent, parent,
+ "parent incorrectly reset")
+ self.assertEqual(mock._mock_methods, spec,
+ "methods incorrectly reset")
+
+ self.assertFalse(mock.called, "called not reset")
+ self.assertEqual(mock.call_count, 0, "call_count not reset")
+ self.assertEqual(mock.call_args, None, "call_args not reset")
+ self.assertEqual(mock.call_args_list, [], "call_args_list not reset")
+ self.assertEqual(mock.method_calls, [],
+ "method_calls not initialised correctly: %r != %r" %
+ (mock.method_calls, []))
+ self.assertEqual(mock.mock_calls, [])
+
+ self.assertEqual(mock.side_effect, sentinel.SideEffect,
+ "side_effect incorrectly reset")
+ self.assertEqual(mock.return_value, return_value,
+ "return_value incorrectly reset")
+ self.assertFalse(return_value.called, "return value mock not reset")
+ self.assertEqual(mock._mock_children, {'something': something},
+ "children reset incorrectly")
+ self.assertEqual(mock.something, something,
+ "children incorrectly cleared")
+ self.assertFalse(mock.something.called, "child not reset")
+
+
+ def test_reset_mock_recursion(self):
+ mock = Mock()
+ mock.return_value = mock
+
+ # used to cause recursion
+ mock.reset_mock()
+
+
+ def test_call(self):
+ mock = Mock()
+ self.assertTrue(is_instance(mock.return_value, Mock),
+ "Default return_value should be a Mock")
+
+ result = mock()
+ self.assertEqual(mock(), result,
+ "different result from consecutive calls")
+ mock.reset_mock()
+
+ ret_val = mock(sentinel.Arg)
+ self.assertTrue(mock.called, "called not set")
+ self.assertEqual(mock.call_count, 1, "call_count incoreect")
+ self.assertEqual(mock.call_args, ((sentinel.Arg,), {}),
+ "call_args not set")
+ self.assertEqual(mock.call_args_list, [((sentinel.Arg,), {})],
+ "call_args_list not initialised correctly")
+
+ mock.return_value = sentinel.ReturnValue
+ ret_val = mock(sentinel.Arg, key=sentinel.KeyArg)
+ self.assertEqual(ret_val, sentinel.ReturnValue,
+ "incorrect return value")
+
+ self.assertEqual(mock.call_count, 2, "call_count incorrect")
+ self.assertEqual(mock.call_args,
+ ((sentinel.Arg,), {'key': sentinel.KeyArg}),
+ "call_args not set")
+ self.assertEqual(mock.call_args_list, [
+ ((sentinel.Arg,), {}),
+ ((sentinel.Arg,), {'key': sentinel.KeyArg})
+ ],
+ "call_args_list not set")
+
+
+ def test_call_args_comparison(self):
+ mock = Mock()
+ mock()
+ mock(sentinel.Arg)
+ mock(kw=sentinel.Kwarg)
+ mock(sentinel.Arg, kw=sentinel.Kwarg)
+ self.assertEqual(mock.call_args_list, [
+ (),
+ ((sentinel.Arg,),),
+ ({"kw": sentinel.Kwarg},),
+ ((sentinel.Arg,), {"kw": sentinel.Kwarg})
+ ])
+ self.assertEqual(mock.call_args,
+ ((sentinel.Arg,), {"kw": sentinel.Kwarg}))
+
+
+ def test_assert_called_with(self):
+ mock = Mock()
+ mock()
+
+ # Will raise an exception if it fails
+ mock.assert_called_with()
+ self.assertRaises(AssertionError, mock.assert_called_with, 1)
+
+ mock.reset_mock()
+ self.assertRaises(AssertionError, mock.assert_called_with)
+
+ mock(1, 2, 3, a='fish', b='nothing')
+ mock.assert_called_with(1, 2, 3, a='fish', b='nothing')
+
+
+ def test_assert_called_once_with(self):
+ mock = Mock()
+ mock()
+
+ # Will raise an exception if it fails
+ mock.assert_called_once_with()
+
+ mock()
+ self.assertRaises(AssertionError, mock.assert_called_once_with)
+
+ mock.reset_mock()
+ self.assertRaises(AssertionError, mock.assert_called_once_with)
+
+ mock('foo', 'bar', baz=2)
+ mock.assert_called_once_with('foo', 'bar', baz=2)
+
+ mock.reset_mock()
+ mock('foo', 'bar', baz=2)
+ self.assertRaises(
+ AssertionError,
+ lambda: mock.assert_called_once_with('bob', 'bar', baz=2)
+ )
+
+
+ def test_attribute_access_returns_mocks(self):
+ mock = Mock()
+ something = mock.something
+ self.assertTrue(is_instance(something, Mock), "attribute isn't a mock")
+ self.assertEqual(mock.something, something,
+ "different attributes returned for same name")
+
+ # Usage example
+ mock = Mock()
+ mock.something.return_value = 3
+
+ self.assertEqual(mock.something(), 3, "method returned wrong value")
+ self.assertTrue(mock.something.called,
+ "method didn't record being called")
+
+
+ def test_attributes_have_name_and_parent_set(self):
+ mock = Mock()
+ something = mock.something
+
+ self.assertEqual(something._mock_name, "something",
+ "attribute name not set correctly")
+ self.assertEqual(something._mock_parent, mock,
+ "attribute parent not set correctly")
+
+
+ def test_method_calls_recorded(self):
+ mock = Mock()
+ mock.something(3, fish=None)
+ mock.something_else.something(6, cake=sentinel.Cake)
+
+ self.assertEqual(mock.something_else.method_calls,
+ [("something", (6,), {'cake': sentinel.Cake})],
+ "method calls not recorded correctly")
+ self.assertEqual(mock.method_calls, [
+ ("something", (3,), {'fish': None}),
+ ("something_else.something", (6,), {'cake': sentinel.Cake})
+ ],
+ "method calls not recorded correctly")
+
+
+ def test_method_calls_compare_easily(self):
+ mock = Mock()
+ mock.something()
+ self.assertEqual(mock.method_calls, [('something',)])
+ self.assertEqual(mock.method_calls, [('something', (), {})])
+
+ mock = Mock()
+ mock.something('different')
+ self.assertEqual(mock.method_calls, [('something', ('different',))])
+ self.assertEqual(mock.method_calls,
+ [('something', ('different',), {})])
+
+ mock = Mock()
+ mock.something(x=1)
+ self.assertEqual(mock.method_calls, [('something', {'x': 1})])
+ self.assertEqual(mock.method_calls, [('something', (), {'x': 1})])
+
+ mock = Mock()
+ mock.something('different', some='more')
+ self.assertEqual(mock.method_calls, [
+ ('something', ('different',), {'some': 'more'})
+ ])
+
+
+ def test_only_allowed_methods_exist(self):
+ for spec in ['something'], ('something',):
+ for arg in 'spec', 'spec_set':
+ mock = Mock(**{arg: spec})
+
+ # this should be allowed
+ mock.something
+ self.assertRaisesRegexp(
+ AttributeError,
+ "Mock object has no attribute 'something_else'",
+ getattr, mock, 'something_else'
+ )
+
+
+ def test_from_spec(self):
+ class Something(object):
+ x = 3
+ __something__ = None
+ def y(self):
+ pass
+
+ def test_attributes(mock):
+ # should work
+ mock.x
+ mock.y
+ mock.__something__
+ self.assertRaisesRegexp(
+ AttributeError,
+ "Mock object has no attribute 'z'",
+ getattr, mock, 'z'
+ )
+ self.assertRaisesRegexp(
+ AttributeError,
+ "Mock object has no attribute '__foobar__'",
+ getattr, mock, '__foobar__'
+ )
+
+ test_attributes(Mock(spec=Something))
+ test_attributes(Mock(spec=Something()))
+
+
+ def test_wraps_calls(self):
+ real = Mock()
+
+ mock = Mock(wraps=real)
+ self.assertEqual(mock(), real())
+
+ real.reset_mock()
+
+ mock(1, 2, fish=3)
+ real.assert_called_with(1, 2, fish=3)
+
+
+ def test_wraps_call_with_nondefault_return_value(self):
+ real = Mock()
+
+ mock = Mock(wraps=real)
+ mock.return_value = 3
+
+ self.assertEqual(mock(), 3)
+ self.assertFalse(real.called)
+
+
+ def test_wraps_attributes(self):
+ class Real(object):
+ attribute = Mock()
+
+ real = Real()
+
+ mock = Mock(wraps=real)
+ self.assertEqual(mock.attribute(), real.attribute())
+ self.assertRaises(AttributeError, lambda: mock.fish)
+
+ self.assertNotEqual(mock.attribute, real.attribute)
+ result = mock.attribute.frog(1, 2, fish=3)
+ Real.attribute.frog.assert_called_with(1, 2, fish=3)
+ self.assertEqual(result, Real.attribute.frog())
+
+
+ def test_exceptional_side_effect(self):
+ mock = Mock(side_effect=AttributeError)
+ self.assertRaises(AttributeError, mock)
+
+ mock = Mock(side_effect=AttributeError('foo'))
+ self.assertRaises(AttributeError, mock)
+
+
+ def test_baseexceptional_side_effect(self):
+ mock = Mock(side_effect=KeyboardInterrupt)
+ self.assertRaises(KeyboardInterrupt, mock)
+
+ mock = Mock(side_effect=KeyboardInterrupt('foo'))
+ self.assertRaises(KeyboardInterrupt, mock)
+
+
+ def test_assert_called_with_message(self):
+ mock = Mock()
+ self.assertRaisesRegexp(AssertionError, 'Not called',
+ mock.assert_called_with)
+
+
+ def test__name__(self):
+ mock = Mock()
+ self.assertRaises(AttributeError, lambda: mock.__name__)
+
+ mock.__name__ = 'foo'
+ self.assertEqual(mock.__name__, 'foo')
+
+
+ def test_spec_list_subclass(self):
+ class Sub(list):
+ pass
+ mock = Mock(spec=Sub(['foo']))
+
+ mock.append(3)
+ mock.append.assert_called_with(3)
+ self.assertRaises(AttributeError, getattr, mock, 'foo')
+
+
+ def test_spec_class(self):
+ class X(object):
+ pass
+
+ mock = Mock(spec=X)
+ self.assertTrue(isinstance(mock, X))
+
+ mock = Mock(spec=X())
+ self.assertTrue(isinstance(mock, X))
+
+ self.assertIs(mock.__class__, X)
+ self.assertEqual(Mock().__class__.__name__, 'Mock')
+
+ mock = Mock(spec_set=X)
+ self.assertTrue(isinstance(mock, X))
+
+ mock = Mock(spec_set=X())
+ self.assertTrue(isinstance(mock, X))
+
+
+ def test_setting_attribute_with_spec_set(self):
+ class X(object):
+ y = 3
+
+ mock = Mock(spec=X)
+ mock.x = 'foo'
+
+ mock = Mock(spec_set=X)
+ def set_attr():
+ mock.x = 'foo'
+
+ mock.y = 'foo'
+ self.assertRaises(AttributeError, set_attr)
+
+
+ def test_copy(self):
+ current = sys.getrecursionlimit()
+ self.addCleanup(sys.setrecursionlimit, current)
+
+ # can't use sys.maxint as this doesn't exist in Python 3
+ sys.setrecursionlimit(int(10e8))
+ # this segfaults without the fix in place
+ copy.copy(Mock())
+
+
+ @unittest2.skipIf(inPy3k, "no old style classes in Python 3")
+ def test_spec_old_style_classes(self):
+ class Foo:
+ bar = 7
+
+ mock = Mock(spec=Foo)
+ mock.bar = 6
+ self.assertRaises(AttributeError, lambda: mock.foo)
+
+ mock = Mock(spec=Foo())
+ mock.bar = 6
+ self.assertRaises(AttributeError, lambda: mock.foo)
+
+
+ @unittest2.skipIf(inPy3k, "no old style classes in Python 3")
+ def test_spec_set_old_style_classes(self):
+ class Foo:
+ bar = 7
+
+ mock = Mock(spec_set=Foo)
+ mock.bar = 6
+ self.assertRaises(AttributeError, lambda: mock.foo)
+
+ def _set():
+ mock.foo = 3
+ self.assertRaises(AttributeError, _set)
+
+ mock = Mock(spec_set=Foo())
+ mock.bar = 6
+ self.assertRaises(AttributeError, lambda: mock.foo)
+
+ def _set():
+ mock.foo = 3
+ self.assertRaises(AttributeError, _set)
+
+
+ def test_subclass_with_properties(self):
+ class SubClass(Mock):
+ def _get(self):
+ return 3
+ def _set(self, value):
+ raise NameError('strange error')
+ some_attribute = property(_get, _set)
+
+ s = SubClass(spec_set=SubClass)
+ self.assertEqual(s.some_attribute, 3)
+
+ def test():
+ s.some_attribute = 3
+ self.assertRaises(NameError, test)
+
+ def test():
+ s.foo = 'bar'
+ self.assertRaises(AttributeError, test)
+
+
+ def test_setting_call(self):
+ mock = Mock()
+ def __call__(self, a):
+ return self._mock_call(a)
+
+ type(mock).__call__ = __call__
+ mock('one')
+ mock.assert_called_with('one')
+
+ self.assertRaises(TypeError, mock, 'one', 'two')
+
+
+ @unittest2.skipUnless(sys.version_info[:2] >= (2, 6),
+ "__dir__ not available until Python 2.6 or later")
+ def test_dir(self):
+ mock = Mock()
+ attrs = set(dir(mock))
+ type_attrs = set([m for m in dir(Mock) if not m.startswith('_')])
+
+ # all public attributes from the type are included
+ self.assertEqual(set(), type_attrs - attrs)
+
+ # creates these attributes
+ mock.a, mock.b
+ self.assertIn('a', dir(mock))
+ self.assertIn('b', dir(mock))
+
+ # instance attributes
+ mock.c = mock.d = None
+ self.assertIn('c', dir(mock))
+ self.assertIn('d', dir(mock))
+
+ # magic methods
+ mock.__iter__ = lambda s: iter([])
+ self.assertIn('__iter__', dir(mock))
+
+
+ @unittest2.skipUnless(sys.version_info[:2] >= (2, 6),
+ "__dir__ not available until Python 2.6 or later")
+ def test_dir_from_spec(self):
+ mock = Mock(spec=unittest2.TestCase)
+ testcase_attrs = set(dir(unittest2.TestCase))
+ attrs = set(dir(mock))
+
+ # all attributes from the spec are included
+ self.assertEqual(set(), testcase_attrs - attrs)
+
+ # shadow a sys attribute
+ mock.version = 3
+ self.assertEqual(dir(mock).count('version'), 1)
+
+
+ @unittest2.skipUnless(sys.version_info[:2] >= (2, 6),
+ "__dir__ not available until Python 2.6 or later")
+ def test_filter_dir(self):
+ patcher = patch.object(mock, 'FILTER_DIR', False)
+ patcher.start()
+ try:
+ attrs = set(dir(Mock()))
+ type_attrs = set(dir(Mock))
+
+ # ALL attributes from the type are included
+ self.assertEqual(set(), type_attrs - attrs)
+ finally:
+ patcher.stop()
+
+
+ def test_configure_mock(self):
+ mock = Mock(foo='bar')
+ self.assertEqual(mock.foo, 'bar')
+
+ mock = MagicMock(foo='bar')
+ self.assertEqual(mock.foo, 'bar')
+
+ kwargs = {'side_effect': KeyError, 'foo.bar.return_value': 33,
+ 'foo': MagicMock()}
+ mock = Mock(**kwargs)
+ self.assertRaises(KeyError, mock)
+ self.assertEqual(mock.foo.bar(), 33)
+ self.assertIsInstance(mock.foo, MagicMock)
+
+ mock = Mock()
+ mock.configure_mock(**kwargs)
+ self.assertRaises(KeyError, mock)
+ self.assertEqual(mock.foo.bar(), 33)
+ self.assertIsInstance(mock.foo, MagicMock)
+
+
+ def assertRaisesWithMsg(self, exception, message, func, *args, **kwargs):
+ # needed because assertRaisesRegex doesn't work easily with newlines
+ try:
+ func(*args, **kwargs)
+ except:
+ instance = sys.exc_info()[1]
+ self.assertIsInstance(instance, exception)
+ else:
+ self.fail('Exception %r not raised' % (exception,))
+
+ msg = str(instance)
+ self.assertEqual(msg, message)
+
+
+ def test_assert_called_with_failure_message(self):
+ mock = NonCallableMock()
+
+ expected = "mock(1, '2', 3, bar='foo')"
+ message = 'Expected call: %s\nNot called'
+ self.assertRaisesWithMsg(
+ AssertionError, message % (expected,),
+ mock.assert_called_with, 1, '2', 3, bar='foo'
+ )
+
+ mock.foo(1, '2', 3, foo='foo')
+
+
+ asserters = [
+ mock.foo.assert_called_with, mock.foo.assert_called_once_with
+ ]
+ for meth in asserters:
+ actual = "foo(1, '2', 3, foo='foo')"
+ expected = "foo(1, '2', 3, bar='foo')"
+ message = 'Expected call: %s\nActual call: %s'
+ self.assertRaisesWithMsg(
+ AssertionError, message % (expected, actual),
+ meth, 1, '2', 3, bar='foo'
+ )
+
+ # just kwargs
+ for meth in asserters:
+ actual = "foo(1, '2', 3, foo='foo')"
+ expected = "foo(bar='foo')"
+ message = 'Expected call: %s\nActual call: %s'
+ self.assertRaisesWithMsg(
+ AssertionError, message % (expected, actual),
+ meth, bar='foo'
+ )
+
+ # just args
+ for meth in asserters:
+ actual = "foo(1, '2', 3, foo='foo')"
+ expected = "foo(1, 2, 3)"
+ message = 'Expected call: %s\nActual call: %s'
+ self.assertRaisesWithMsg(
+ AssertionError, message % (expected, actual),
+ meth, 1, 2, 3
+ )
+
+ # empty
+ for meth in asserters:
+ actual = "foo(1, '2', 3, foo='foo')"
+ expected = "foo()"
+ message = 'Expected call: %s\nActual call: %s'
+ self.assertRaisesWithMsg(
+ AssertionError, message % (expected, actual), meth
+ )
+
+
+ def test_mock_calls(self):
+ mock = MagicMock()
+
+ # need to do this because MagicMock.mock_calls used to just return
+ # a MagicMock which also returned a MagicMock when __eq__ was called
+ self.assertIs(mock.mock_calls == [], True)
+
+ mock = MagicMock()
+ mock()
+ expected = [('', (), {})]
+ self.assertEqual(mock.mock_calls, expected)
+
+ mock.foo()
+ expected.append(call.foo())
+ self.assertEqual(mock.mock_calls, expected)
+ # intermediate mock_calls work too
+ self.assertEqual(mock.foo.mock_calls, [('', (), {})])
+
+ mock = MagicMock()
+ mock().foo(1, 2, 3, a=4, b=5)
+ expected = [
+ ('', (), {}), ('().foo', (1, 2, 3), dict(a=4, b=5))
+ ]
+ self.assertEqual(mock.mock_calls, expected)
+ self.assertEqual(mock.return_value.foo.mock_calls,
+ [('', (1, 2, 3), dict(a=4, b=5))])
+ self.assertEqual(mock.return_value.mock_calls,
+ [('foo', (1, 2, 3), dict(a=4, b=5))])
+
+ mock = MagicMock()
+ mock().foo.bar().baz()
+ expected = [
+ ('', (), {}), ('().foo.bar', (), {}),
+ ('().foo.bar().baz', (), {})
+ ]
+ self.assertEqual(mock.mock_calls, expected)
+ self.assertEqual(mock().mock_calls,
+ call.foo.bar().baz().call_list())
+
+ for kwargs in dict(), dict(name='bar'):
+ mock = MagicMock(**kwargs)
+ int(mock.foo)
+ expected = [('foo.__int__', (), {})]
+ self.assertEqual(mock.mock_calls, expected)
+
+ mock = MagicMock(**kwargs)
+ mock.a()()
+ expected = [('a', (), {}), ('a()', (), {})]
+ self.assertEqual(mock.mock_calls, expected)
+ self.assertEqual(mock.a().mock_calls, [call()])
+
+ mock = MagicMock(**kwargs)
+ mock(1)(2)(3)
+ self.assertEqual(mock.mock_calls, call(1)(2)(3).call_list())
+ self.assertEqual(mock().mock_calls, call(2)(3).call_list())
+ self.assertEqual(mock()().mock_calls, call(3).call_list())
+
+ mock = MagicMock(**kwargs)
+ mock(1)(2)(3).a.b.c(4)
+ self.assertEqual(mock.mock_calls,
+ call(1)(2)(3).a.b.c(4).call_list())
+ self.assertEqual(mock().mock_calls,
+ call(2)(3).a.b.c(4).call_list())
+ self.assertEqual(mock()().mock_calls,
+ call(3).a.b.c(4).call_list())
+
+ mock = MagicMock(**kwargs)
+ int(mock().foo.bar().baz())
+ last_call = ('().foo.bar().baz().__int__', (), {})
+ self.assertEqual(mock.mock_calls[-1], last_call)
+ self.assertEqual(mock().mock_calls,
+ call.foo.bar().baz().__int__().call_list())
+ self.assertEqual(mock().foo.bar().mock_calls,
+ call.baz().__int__().call_list())
+ self.assertEqual(mock().foo.bar().baz.mock_calls,
+ call().__int__().call_list())
+
+
+ def test_subclassing(self):
+ class Subclass(Mock):
+ pass
+
+ mock = Subclass()
+ self.assertIsInstance(mock.foo, Subclass)
+ self.assertIsInstance(mock(), Subclass)
+
+ class Subclass(Mock):
+ def _get_child_mock(self, **kwargs):
+ return Mock(**kwargs)
+
+ mock = Subclass()
+ self.assertNotIsInstance(mock.foo, Subclass)
+ self.assertNotIsInstance(mock(), Subclass)
+
+
+ def test_arg_lists(self):
+ mocks = [
+ Mock(),
+ MagicMock(),
+ NonCallableMock(),
+ NonCallableMagicMock()
+ ]
+
+ def assert_attrs(mock):
+ names = 'call_args_list', 'method_calls', 'mock_calls'
+ for name in names:
+ attr = getattr(mock, name)
+ self.assertIsInstance(attr, _CallList)
+ self.assertIsInstance(attr, list)
+ self.assertEqual(attr, [])
+
+ for mock in mocks:
+ assert_attrs(mock)
+
+ if callable(mock):
+ mock()
+ mock(1, 2)
+ mock(a=3)
+
+ mock.reset_mock()
+ assert_attrs(mock)
+
+ mock.foo()
+ mock.foo.bar(1, a=3)
+ mock.foo(1).bar().baz(3)
+
+ mock.reset_mock()
+ assert_attrs(mock)
+
+
+ def test_call_args_two_tuple(self):
+ mock = Mock()
+ mock(1, a=3)
+ mock(2, b=4)
+
+ self.assertEqual(len(mock.call_args), 2)
+ args, kwargs = mock.call_args
+ self.assertEqual(args, (2,))
+ self.assertEqual(kwargs, dict(b=4))
+
+ expected_list = [((1,), dict(a=3)), ((2,), dict(b=4))]
+ for expected, call_args in zip(expected_list, mock.call_args_list):
+ self.assertEqual(len(call_args), 2)
+ self.assertEqual(expected[0], call_args[0])
+ self.assertEqual(expected[1], call_args[1])
+
+
+ def test_side_effect_iterator(self):
+ mock = Mock(side_effect=iter([1, 2, 3]))
+ self.assertEqual([mock(), mock(), mock()], [1, 2, 3])
+ self.assertRaises(StopIteration, mock)
+
+ mock = MagicMock(side_effect=['a', 'b', 'c'])
+ self.assertEqual([mock(), mock(), mock()], ['a', 'b', 'c'])
+ self.assertRaises(StopIteration, mock)
+
+ mock = Mock(side_effect='ghi')
+ self.assertEqual([mock(), mock(), mock()], ['g', 'h', 'i'])
+ self.assertRaises(StopIteration, mock)
+
+ class Foo(object):
+ pass
+ mock = MagicMock(side_effect=Foo)
+ self.assertIsInstance(mock(), Foo)
+
+ mock = Mock(side_effect=Iter())
+ self.assertEqual([mock(), mock(), mock(), mock()],
+ ['this', 'is', 'an', 'iter'])
+ self.assertRaises(StopIteration, mock)
+
+
+ def test_side_effect_setting_iterator(self):
+ mock = Mock()
+ mock.side_effect = iter([1, 2, 3])
+ self.assertEqual([mock(), mock(), mock()], [1, 2, 3])
+ self.assertRaises(StopIteration, mock)
+ side_effect = mock.side_effect
+ self.assertIsInstance(side_effect, type(iter([])))
+
+ mock.side_effect = ['a', 'b', 'c']
+ self.assertEqual([mock(), mock(), mock()], ['a', 'b', 'c'])
+ self.assertRaises(StopIteration, mock)
+ side_effect = mock.side_effect
+ self.assertIsInstance(side_effect, type(iter([])))
+
+ this_iter = Iter()
+ mock.side_effect = this_iter
+ self.assertEqual([mock(), mock(), mock(), mock()],
+ ['this', 'is', 'an', 'iter'])
+ self.assertRaises(StopIteration, mock)
+ self.assertIs(mock.side_effect, this_iter)
+
+
+ def test_side_effect_iterator_exceptions(self):
+ for Klass in Mock, MagicMock:
+ iterable = (ValueError, 3, KeyError, 6)
+ m = Klass(side_effect=iterable)
+ self.assertRaises(ValueError, m)
+ self.assertEqual(m(), 3)
+ self.assertRaises(KeyError, m)
+ self.assertEqual(m(), 6)
+
+
+ def test_assert_has_calls_any_order(self):
+ mock = Mock()
+ mock(1, 2)
+ mock(a=3)
+ mock(3, 4)
+ mock(b=6)
+ mock(b=6)
+
+ kalls = [
+ call(1, 2), ({'a': 3},),
+ ((3, 4),), ((), {'a': 3}),
+ ('', (1, 2)), ('', {'a': 3}),
+ ('', (1, 2), {}), ('', (), {'a': 3})
+ ]
+ for kall in kalls:
+ mock.assert_has_calls([kall], any_order=True)
+
+ for kall in call(1, '2'), call(b=3), call(), 3, None, 'foo':
+ self.assertRaises(
+ AssertionError, mock.assert_has_calls,
+ [kall], any_order=True
+ )
+
+ kall_lists = [
+ [call(1, 2), call(b=6)],
+ [call(3, 4), call(1, 2)],
+ [call(b=6), call(b=6)],
+ ]
+
+ for kall_list in kall_lists:
+ mock.assert_has_calls(kall_list, any_order=True)
+
+ kall_lists = [
+ [call(b=6), call(b=6), call(b=6)],
+ [call(1, 2), call(1, 2)],
+ [call(3, 4), call(1, 2), call(5, 7)],
+ [call(b=6), call(3, 4), call(b=6), call(1, 2), call(b=6)],
+ ]
+ for kall_list in kall_lists:
+ self.assertRaises(
+ AssertionError, mock.assert_has_calls,
+ kall_list, any_order=True
+ )
+
+ def test_assert_has_calls(self):
+ kalls1 = [
+ call(1, 2), ({'a': 3},),
+ ((3, 4),), call(b=6),
+ ('', (1,), {'b': 6}),
+ ]
+ kalls2 = [call.foo(), call.bar(1)]
+ kalls2.extend(call.spam().baz(a=3).call_list())
+ kalls2.extend(call.bam(set(), foo={}).fish([1]).call_list())
+
+ mocks = []
+ for mock in Mock(), MagicMock():
+ mock(1, 2)
+ mock(a=3)
+ mock(3, 4)
+ mock(b=6)
+ mock(1, b=6)
+ mocks.append((mock, kalls1))
+
+ mock = Mock()
+ mock.foo()
+ mock.bar(1)
+ mock.spam().baz(a=3)
+ mock.bam(set(), foo={}).fish([1])
+ mocks.append((mock, kalls2))
+
+ for mock, kalls in mocks:
+ for i in range(len(kalls)):
+ for step in 1, 2, 3:
+ these = kalls[i:i+step]
+ mock.assert_has_calls(these)
+
+ if len(these) > 1:
+ self.assertRaises(
+ AssertionError,
+ mock.assert_has_calls,
+ list(reversed(these))
+ )
+
+
+ def test_assert_any_call(self):
+ mock = Mock()
+ mock(1, 2)
+ mock(a=3)
+ mock(1, b=6)
+
+ mock.assert_any_call(1, 2)
+ mock.assert_any_call(a=3)
+ mock.assert_any_call(1, b=6)
+
+ self.assertRaises(
+ AssertionError,
+ mock.assert_any_call
+ )
+ self.assertRaises(
+ AssertionError,
+ mock.assert_any_call,
+ 1, 3
+ )
+ self.assertRaises(
+ AssertionError,
+ mock.assert_any_call,
+ a=4
+ )
+
+
+ def test_mock_calls_create_autospec(self):
+ def f(a, b):
+ pass
+ obj = Iter()
+ obj.f = f
+
+ funcs = [
+ create_autospec(f),
+ create_autospec(obj).f
+ ]
+ for func in funcs:
+ func(1, 2)
+ func(3, 4)
+
+ self.assertEqual(
+ func.mock_calls, [call(1, 2), call(3, 4)]
+ )
+
+
+ def test_mock_add_spec(self):
+ class _One(object):
+ one = 1
+ class _Two(object):
+ two = 2
+ class Anything(object):
+ one = two = three = 'four'
+
+ klasses = [
+ Mock, MagicMock, NonCallableMock, NonCallableMagicMock
+ ]
+ for Klass in list(klasses):
+ klasses.append(lambda K=Klass: K(spec=Anything))
+ klasses.append(lambda K=Klass: K(spec_set=Anything))
+
+ for Klass in klasses:
+ for kwargs in dict(), dict(spec_set=True):
+ mock = Klass()
+ #no error
+ mock.one, mock.two, mock.three
+
+ for One, Two in [(_One, _Two), (['one'], ['two'])]:
+ for kwargs in dict(), dict(spec_set=True):
+ mock.mock_add_spec(One, **kwargs)
+
+ mock.one
+ self.assertRaises(
+ AttributeError, getattr, mock, 'two'
+ )
+ self.assertRaises(
+ AttributeError, getattr, mock, 'three'
+ )
+ if 'spec_set' in kwargs:
+ self.assertRaises(
+ AttributeError, setattr, mock, 'three', None
+ )
+
+ mock.mock_add_spec(Two, **kwargs)
+ self.assertRaises(
+ AttributeError, getattr, mock, 'one'
+ )
+ mock.two
+ self.assertRaises(
+ AttributeError, getattr, mock, 'three'
+ )
+ if 'spec_set' in kwargs:
+ self.assertRaises(
+ AttributeError, setattr, mock, 'three', None
+ )
+ # note that creating a mock, setting an instance attribute, and
+ # *then* setting a spec doesn't work. Not the intended use case
+
+
+ def test_mock_add_spec_magic_methods(self):
+ for Klass in MagicMock, NonCallableMagicMock:
+ mock = Klass()
+ int(mock)
+
+ mock.mock_add_spec(object)
+ self.assertRaises(TypeError, int, mock)
+
+ mock = Klass()
+ mock['foo']
+ mock.__int__.return_value =4
+
+ mock.mock_add_spec(int)
+ self.assertEqual(int(mock), 4)
+ self.assertRaises(TypeError, lambda: mock['foo'])
+
+
+ def test_adding_child_mock(self):
+ for Klass in NonCallableMock, Mock, MagicMock, NonCallableMagicMock:
+ mock = Klass()
+
+ mock.foo = Mock()
+ mock.foo()
+
+ self.assertEqual(mock.method_calls, [call.foo()])
+ self.assertEqual(mock.mock_calls, [call.foo()])
+
+ mock = Klass()
+ mock.bar = Mock(name='name')
+ mock.bar()
+ self.assertEqual(mock.method_calls, [])
+ self.assertEqual(mock.mock_calls, [])
+
+ # mock with an existing _new_parent but no name
+ mock = Klass()
+ mock.baz = MagicMock()()
+ mock.baz()
+ self.assertEqual(mock.method_calls, [])
+ self.assertEqual(mock.mock_calls, [])
+
+
+ def test_adding_return_value_mock(self):
+ for Klass in Mock, MagicMock:
+ mock = Klass()
+ mock.return_value = MagicMock()
+
+ mock()()
+ self.assertEqual(mock.mock_calls, [call(), call()()])
+
+
+ def test_manager_mock(self):
+ class Foo(object):
+ one = 'one'
+ two = 'two'
+ manager = Mock()
+ p1 = patch.object(Foo, 'one')
+ p2 = patch.object(Foo, 'two')
+
+ mock_one = p1.start()
+ self.addCleanup(p1.stop)
+ mock_two = p2.start()
+ self.addCleanup(p2.stop)
+
+ manager.attach_mock(mock_one, 'one')
+ manager.attach_mock(mock_two, 'two')
+
+ Foo.two()
+ Foo.one()
+
+ self.assertEqual(manager.mock_calls, [call.two(), call.one()])
+
+
+ def test_magic_methods_mock_calls(self):
+ for Klass in Mock, MagicMock:
+ m = Klass()
+ m.__int__ = Mock(return_value=3)
+ m.__float__ = MagicMock(return_value=3.0)
+ int(m)
+ float(m)
+
+ self.assertEqual(m.mock_calls, [call.__int__(), call.__float__()])
+ self.assertEqual(m.method_calls, [])
+
+
+ def test_attribute_deletion(self):
+ # this behaviour isn't *useful*, but at least it's now tested...
+ for Klass in Mock, MagicMock, NonCallableMagicMock, NonCallableMock:
+ m = Klass()
+ original = m.foo
+ m.foo = 3
+ del m.foo
+ self.assertEqual(m.foo, original)
+
+ new = m.foo = Mock()
+ del m.foo
+ self.assertEqual(m.foo, new)
+
+
+ def test_mock_parents(self):
+ for Klass in Mock, MagicMock:
+ m = Klass()
+ original_repr = repr(m)
+ m.return_value = m
+ self.assertIs(m(), m)
+ self.assertEqual(repr(m), original_repr)
+
+ m.reset_mock()
+ self.assertIs(m(), m)
+ self.assertEqual(repr(m), original_repr)
+
+ m = Klass()
+ m.b = m.a
+ self.assertIn("name='mock.a'", repr(m.b))
+ self.assertIn("name='mock.a'", repr(m.a))
+ m.reset_mock()
+ self.assertIn("name='mock.a'", repr(m.b))
+ self.assertIn("name='mock.a'", repr(m.a))
+
+ m = Klass()
+ original_repr = repr(m)
+ m.a = m()
+ m.a.return_value = m
+
+ self.assertEqual(repr(m), original_repr)
+ self.assertEqual(repr(m.a()), original_repr)
+
+
+ def test_attach_mock(self):
+ classes = Mock, MagicMock, NonCallableMagicMock, NonCallableMock
+ for Klass in classes:
+ for Klass2 in classes:
+ m = Klass()
+
+ m2 = Klass2(name='foo')
+ m.attach_mock(m2, 'bar')
+
+ self.assertIs(m.bar, m2)
+ self.assertIn("name='mock.bar'", repr(m2))
+
+ m.bar.baz(1)
+ self.assertEqual(m.mock_calls, [call.bar.baz(1)])
+ self.assertEqual(m.method_calls, [call.bar.baz(1)])
+
+
+ def test_attach_mock_return_value(self):
+ classes = Mock, MagicMock, NonCallableMagicMock, NonCallableMock
+ for Klass in Mock, MagicMock:
+ for Klass2 in classes:
+ m = Klass()
+
+ m2 = Klass2(name='foo')
+ m.attach_mock(m2, 'return_value')
+
+ self.assertIs(m(), m2)
+ self.assertIn("name='mock()'", repr(m2))
+
+ m2.foo()
+ self.assertEqual(m.mock_calls, call().foo().call_list())
+
+
+ def test_attribute_deletion(self):
+ for mock in Mock(), MagicMock():
+ self.assertTrue(hasattr(mock, 'm'))
+
+ del mock.m
+ self.assertFalse(hasattr(mock, 'm'))
+
+ del mock.f
+ self.assertFalse(hasattr(mock, 'f'))
+ self.assertRaises(AttributeError, getattr, mock, 'f')
+
+
+ def test_class_assignable(self):
+ for mock in Mock(), MagicMock():
+ self.assertNotIsInstance(mock, int)
+
+ mock.__class__ = int
+ self.assertIsInstance(mock, int)
+
+
+ @unittest2.expectedFailure
+ def test_pickle(self):
+ for Klass in (MagicMock, Mock, Subclass, NonCallableMagicMock):
+ mock = Klass(name='foo', attribute=3)
+ mock.foo(1, 2, 3)
+ data = pickle.dumps(mock)
+ new = pickle.loads(data)
+
+ new.foo.assert_called_once_with(1, 2, 3)
+ self.assertFalse(new.called)
+ self.assertTrue(is_instance(new, Klass))
+ self.assertIsInstance(new, Thing)
+ self.assertIn('name="foo"', repr(new))
+ self.assertEqual(new.attribute, 3)
+
+
+if __name__ == '__main__':
+ unittest2.main()
diff --git a/slider-agent/src/test/python/mock/tests/testpatch.py b/slider-agent/src/test/python/mock/tests/testpatch.py
new file mode 100644
index 0000000..8eb719b
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/testpatch.py
@@ -0,0 +1,1815 @@
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
+
+import os
+import sys
+
+from tests import support
+from tests.support import unittest2, inPy3k, SomeClass, is_instance, callable
+
+from mock import (
+ NonCallableMock, CallableMixin, patch, sentinel,
+ MagicMock, Mock, NonCallableMagicMock, patch, _patch,
+ DEFAULT, call, _get_target
+)
+
+builtin_string = '__builtin__'
+if inPy3k:
+ builtin_string = 'builtins'
+ unicode = str
+
+PTModule = sys.modules[__name__]
+MODNAME = '%s.PTModule' % __name__
+
+
+def _get_proxy(obj, get_only=True):
+ class Proxy(object):
+ def __getattr__(self, name):
+ return getattr(obj, name)
+ if not get_only:
+ def __setattr__(self, name, value):
+ setattr(obj, name, value)
+ def __delattr__(self, name):
+ delattr(obj, name)
+ Proxy.__setattr__ = __setattr__
+ Proxy.__delattr__ = __delattr__
+ return Proxy()
+
+
+# for use in the test
+something = sentinel.Something
+something_else = sentinel.SomethingElse
+
+
+class Foo(object):
+ def __init__(self, a):
+ pass
+ def f(self, a):
+ pass
+ def g(self):
+ pass
+ foo = 'bar'
+
+ class Bar(object):
+ def a(self):
+ pass
+
+foo_name = '%s.Foo' % __name__
+
+
+def function(a, b=Foo):
+ pass
+
+
+class Container(object):
+ def __init__(self):
+ self.values = {}
+
+ def __getitem__(self, name):
+ return self.values[name]
+
+ def __setitem__(self, name, value):
+ self.values[name] = value
+
+ def __delitem__(self, name):
+ del self.values[name]
+
+ def __iter__(self):
+ return iter(self.values)
+
+
+
+class PatchTest(unittest2.TestCase):
+
+ def assertNotCallable(self, obj, magic=True):
+ MockClass = NonCallableMagicMock
+ if not magic:
+ MockClass = NonCallableMock
+
+ self.assertRaises(TypeError, obj)
+ self.assertTrue(is_instance(obj, MockClass))
+ self.assertFalse(is_instance(obj, CallableMixin))
+
+
+ def test_single_patchobject(self):
+ class Something(object):
+ attribute = sentinel.Original
+
+ @patch.object(Something, 'attribute', sentinel.Patched)
+ def test():
+ self.assertEqual(Something.attribute, sentinel.Patched, "unpatched")
+
+ test()
+ self.assertEqual(Something.attribute, sentinel.Original,
+ "patch not restored")
+
+
+ def test_patchobject_with_none(self):
+ class Something(object):
+ attribute = sentinel.Original
+
+ @patch.object(Something, 'attribute', None)
+ def test():
+ self.assertIsNone(Something.attribute, "unpatched")
+
+ test()
+ self.assertEqual(Something.attribute, sentinel.Original,
+ "patch not restored")
+
+
+ def test_multiple_patchobject(self):
+ class Something(object):
+ attribute = sentinel.Original
+ next_attribute = sentinel.Original2
+
+ @patch.object(Something, 'attribute', sentinel.Patched)
+ @patch.object(Something, 'next_attribute', sentinel.Patched2)
+ def test():
+ self.assertEqual(Something.attribute, sentinel.Patched,
+ "unpatched")
+ self.assertEqual(Something.next_attribute, sentinel.Patched2,
+ "unpatched")
+
+ test()
+ self.assertEqual(Something.attribute, sentinel.Original,
+ "patch not restored")
+ self.assertEqual(Something.next_attribute, sentinel.Original2,
+ "patch not restored")
+
+
+ def test_object_lookup_is_quite_lazy(self):
+ global something
+ original = something
+ @patch('%s.something' % __name__, sentinel.Something2)
+ def test():
+ pass
+
+ try:
+ something = sentinel.replacement_value
+ test()
+ self.assertEqual(something, sentinel.replacement_value)
+ finally:
+ something = original
+
+
+ def test_patch(self):
+ @patch('%s.something' % __name__, sentinel.Something2)
+ def test():
+ self.assertEqual(PTModule.something, sentinel.Something2,
+ "unpatched")
+
+ test()
+ self.assertEqual(PTModule.something, sentinel.Something,
+ "patch not restored")
+
+ @patch('%s.something' % __name__, sentinel.Something2)
+ @patch('%s.something_else' % __name__, sentinel.SomethingElse)
+ def test():
+ self.assertEqual(PTModule.something, sentinel.Something2,
+ "unpatched")
+ self.assertEqual(PTModule.something_else, sentinel.SomethingElse,
+ "unpatched")
+
+ self.assertEqual(PTModule.something, sentinel.Something,
+ "patch not restored")
+ self.assertEqual(PTModule.something_else, sentinel.SomethingElse,
+ "patch not restored")
+
+ # Test the patching and restoring works a second time
+ test()
+
+ self.assertEqual(PTModule.something, sentinel.Something,
+ "patch not restored")
+ self.assertEqual(PTModule.something_else, sentinel.SomethingElse,
+ "patch not restored")
+
+ mock = Mock()
+ mock.return_value = sentinel.Handle
+ @patch('%s.open' % builtin_string, mock)
+ def test():
+ self.assertEqual(open('filename', 'r'), sentinel.Handle,
+ "open not patched")
+ test()
+ test()
+
+ self.assertNotEqual(open, mock, "patch not restored")
+
+
+ def test_patch_class_attribute(self):
+ @patch('%s.SomeClass.class_attribute' % __name__,
+ sentinel.ClassAttribute)
+ def test():
+ self.assertEqual(PTModule.SomeClass.class_attribute,
+ sentinel.ClassAttribute, "unpatched")
+ test()
+
+ self.assertIsNone(PTModule.SomeClass.class_attribute,
+ "patch not restored")
+
+
+ def test_patchobject_with_default_mock(self):
+ class Test(object):
+ something = sentinel.Original
+ something2 = sentinel.Original2
+
+ @patch.object(Test, 'something')
+ def test(mock):
+ self.assertEqual(mock, Test.something,
+ "Mock not passed into test function")
+ self.assertIsInstance(mock, MagicMock,
+ "patch with two arguments did not create a mock")
+
+ test()
+
+ @patch.object(Test, 'something')
+ @patch.object(Test, 'something2')
+ def test(this1, this2, mock1, mock2):
+ self.assertEqual(this1, sentinel.this1,
+ "Patched function didn't receive initial argument")
+ self.assertEqual(this2, sentinel.this2,
+ "Patched function didn't receive second argument")
+ self.assertEqual(mock1, Test.something2,
+ "Mock not passed into test function")
+ self.assertEqual(mock2, Test.something,
+ "Second Mock not passed into test function")
+ self.assertIsInstance(mock2, MagicMock,
+ "patch with two arguments did not create a mock")
+ self.assertIsInstance(mock2, MagicMock,
+ "patch with two arguments did not create a mock")
+
+ # A hack to test that new mocks are passed the second time
+ self.assertNotEqual(outerMock1, mock1, "unexpected value for mock1")
+ self.assertNotEqual(outerMock2, mock2, "unexpected value for mock1")
+ return mock1, mock2
+
+ outerMock1 = outerMock2 = None
+ outerMock1, outerMock2 = test(sentinel.this1, sentinel.this2)
+
+ # Test that executing a second time creates new mocks
+ test(sentinel.this1, sentinel.this2)
+
+
+ def test_patch_with_spec(self):
+ @patch('%s.SomeClass' % __name__, spec=SomeClass)
+ def test(MockSomeClass):
+ self.assertEqual(SomeClass, MockSomeClass)
+ self.assertTrue(is_instance(SomeClass.wibble, MagicMock))
+ self.assertRaises(AttributeError, lambda: SomeClass.not_wibble)
+
+ test()
+
+
+ def test_patchobject_with_spec(self):
+ @patch.object(SomeClass, 'class_attribute', spec=SomeClass)
+ def test(MockAttribute):
+ self.assertEqual(SomeClass.class_attribute, MockAttribute)
+ self.assertTrue(is_instance(SomeClass.class_attribute.wibble,
+ MagicMock))
+ self.assertRaises(AttributeError,
+ lambda: SomeClass.class_attribute.not_wibble)
+
+ test()
+
+
+ def test_patch_with_spec_as_list(self):
+ @patch('%s.SomeClass' % __name__, spec=['wibble'])
+ def test(MockSomeClass):
+ self.assertEqual(SomeClass, MockSomeClass)
+ self.assertTrue(is_instance(SomeClass.wibble, MagicMock))
+ self.assertRaises(AttributeError, lambda: SomeClass.not_wibble)
+
+ test()
+
+
+ def test_patchobject_with_spec_as_list(self):
+ @patch.object(SomeClass, 'class_attribute', spec=['wibble'])
+ def test(MockAttribute):
+ self.assertEqual(SomeClass.class_attribute, MockAttribute)
+ self.assertTrue(is_instance(SomeClass.class_attribute.wibble,
+ MagicMock))
+ self.assertRaises(AttributeError,
+ lambda: SomeClass.class_attribute.not_wibble)
+
+ test()
+
+
+ def test_nested_patch_with_spec_as_list(self):
+ # regression test for nested decorators
+ @patch('%s.open' % builtin_string)
+ @patch('%s.SomeClass' % __name__, spec=['wibble'])
+ def test(MockSomeClass, MockOpen):
+ self.assertEqual(SomeClass, MockSomeClass)
+ self.assertTrue(is_instance(SomeClass.wibble, MagicMock))
+ self.assertRaises(AttributeError, lambda: SomeClass.not_wibble)
+ test()
+
+
+ def test_patch_with_spec_as_boolean(self):
+ @patch('%s.SomeClass' % __name__, spec=True)
+ def test(MockSomeClass):
+ self.assertEqual(SomeClass, MockSomeClass)
+ # Should not raise attribute error
+ MockSomeClass.wibble
+
+ self.assertRaises(AttributeError, lambda: MockSomeClass.not_wibble)
+
+ test()
+
+
+ def test_patch_object_with_spec_as_boolean(self):
+ @patch.object(PTModule, 'SomeClass', spec=True)
+ def test(MockSomeClass):
+ self.assertEqual(SomeClass, MockSomeClass)
+ # Should not raise attribute error
+ MockSomeClass.wibble
+
+ self.assertRaises(AttributeError, lambda: MockSomeClass.not_wibble)
+
+ test()
+
+
+ def test_patch_class_acts_with_spec_is_inherited(self):
+ @patch('%s.SomeClass' % __name__, spec=True)
+ def test(MockSomeClass):
+ self.assertTrue(is_instance(MockSomeClass, MagicMock))
+ instance = MockSomeClass()
+ self.assertNotCallable(instance)
+ # Should not raise attribute error
+ instance.wibble
+
+ self.assertRaises(AttributeError, lambda: instance.not_wibble)
+
+ test()
+
+
+ def test_patch_with_create_mocks_non_existent_attributes(self):
+ @patch('%s.frooble' % builtin_string, sentinel.Frooble, create=True)
+ def test():
+ self.assertEqual(frooble, sentinel.Frooble)
+
+ test()
+ self.assertRaises(NameError, lambda: frooble)
+
+
+ def test_patchobject_with_create_mocks_non_existent_attributes(self):
+ @patch.object(SomeClass, 'frooble', sentinel.Frooble, create=True)
+ def test():
+ self.assertEqual(SomeClass.frooble, sentinel.Frooble)
+
+ test()
+ self.assertFalse(hasattr(SomeClass, 'frooble'))
+
+
+ def test_patch_wont_create_by_default(self):
+ try:
+ @patch('%s.frooble' % builtin_string, sentinel.Frooble)
+ def test():
+ self.assertEqual(frooble, sentinel.Frooble)
+
+ test()
+ except AttributeError:
+ pass
+ else:
+ self.fail('Patching non existent attributes should fail')
+
+ self.assertRaises(NameError, lambda: frooble)
+
+
+ def test_patchobject_wont_create_by_default(self):
+ try:
+ @patch.object(SomeClass, 'frooble', sentinel.Frooble)
+ def test():
+ self.fail('Patching non existent attributes should fail')
+
+ test()
+ except AttributeError:
+ pass
+ else:
+ self.fail('Patching non existent attributes should fail')
+ self.assertFalse(hasattr(SomeClass, 'frooble'))
+
+
+ def test_patch_with_static_methods(self):
+ class Foo(object):
+ @staticmethod
+ def woot():
+ return sentinel.Static
+
+ @patch.object(Foo, 'woot', staticmethod(lambda: sentinel.Patched))
+ def anonymous():
+ self.assertEqual(Foo.woot(), sentinel.Patched)
+ anonymous()
+
+ self.assertEqual(Foo.woot(), sentinel.Static)
+
+
+ def test_patch_local(self):
+ foo = sentinel.Foo
+ @patch.object(sentinel, 'Foo', 'Foo')
+ def anonymous():
+ self.assertEqual(sentinel.Foo, 'Foo')
+ anonymous()
+
+ self.assertEqual(sentinel.Foo, foo)
+
+
+ def test_patch_slots(self):
+ class Foo(object):
+ __slots__ = ('Foo',)
+
+ foo = Foo()
+ foo.Foo = sentinel.Foo
+
+ @patch.object(foo, 'Foo', 'Foo')
+ def anonymous():
+ self.assertEqual(foo.Foo, 'Foo')
+ anonymous()
+
+ self.assertEqual(foo.Foo, sentinel.Foo)
+
+
+ def test_patchobject_class_decorator(self):
+ class Something(object):
+ attribute = sentinel.Original
+
+ class Foo(object):
+ def test_method(other_self):
+ self.assertEqual(Something.attribute, sentinel.Patched,
+ "unpatched")
+ def not_test_method(other_self):
+ self.assertEqual(Something.attribute, sentinel.Original,
+ "non-test method patched")
+
+ Foo = patch.object(Something, 'attribute', sentinel.Patched)(Foo)
+
+ f = Foo()
+ f.test_method()
+ f.not_test_method()
+
+ self.assertEqual(Something.attribute, sentinel.Original,
+ "patch not restored")
+
+
+ def test_patch_class_decorator(self):
+ class Something(object):
+ attribute = sentinel.Original
+
+ class Foo(object):
+ def test_method(other_self, mock_something):
+ self.assertEqual(PTModule.something, mock_something,
+ "unpatched")
+ def not_test_method(other_self):
+ self.assertEqual(PTModule.something, sentinel.Something,
+ "non-test method patched")
+ Foo = patch('%s.something' % __name__)(Foo)
+
+ f = Foo()
+ f.test_method()
+ f.not_test_method()
+
+ self.assertEqual(Something.attribute, sentinel.Original,
+ "patch not restored")
+ self.assertEqual(PTModule.something, sentinel.Something,
+ "patch not restored")
+
+
+ def test_patchobject_twice(self):
+ class Something(object):
+ attribute = sentinel.Original
+ next_attribute = sentinel.Original2
+
+ @patch.object(Something, 'attribute', sentinel.Patched)
+ @patch.object(Something, 'attribute', sentinel.Patched)
+ def test():
+ self.assertEqual(Something.attribute, sentinel.Patched, "unpatched")
+
+ test()
+
+ self.assertEqual(Something.attribute, sentinel.Original,
+ "patch not restored")
+
+
+ def test_patch_dict(self):
+ foo = {'initial': object(), 'other': 'something'}
+ original = foo.copy()
+
+ @patch.dict(foo)
+ def test():
+ foo['a'] = 3
+ del foo['initial']
+ foo['other'] = 'something else'
+
+ test()
+
+ self.assertEqual(foo, original)
+
+ @patch.dict(foo, {'a': 'b'})
+ def test():
+ self.assertEqual(len(foo), 3)
+ self.assertEqual(foo['a'], 'b')
+
+ test()
+
+ self.assertEqual(foo, original)
+
+ @patch.dict(foo, [('a', 'b')])
+ def test():
+ self.assertEqual(len(foo), 3)
+ self.assertEqual(foo['a'], 'b')
+
+ test()
+
+ self.assertEqual(foo, original)
+
+
+ def test_patch_dict_with_container_object(self):
+ foo = Container()
+ foo['initial'] = object()
+ foo['other'] = 'something'
+
+ original = foo.values.copy()
+
+ @patch.dict(foo)
+ def test():
+ foo['a'] = 3
+ del foo['initial']
+ foo['other'] = 'something else'
+
+ test()
+
+ self.assertEqual(foo.values, original)
+
+ @patch.dict(foo, {'a': 'b'})
+ def test():
+ self.assertEqual(len(foo.values), 3)
+ self.assertEqual(foo['a'], 'b')
+
+ test()
+
+ self.assertEqual(foo.values, original)
+
+
+ def test_patch_dict_with_clear(self):
+ foo = {'initial': object(), 'other': 'something'}
+ original = foo.copy()
+
+ @patch.dict(foo, clear=True)
+ def test():
+ self.assertEqual(foo, {})
+ foo['a'] = 3
+ foo['other'] = 'something else'
+
+ test()
+
+ self.assertEqual(foo, original)
+
+ @patch.dict(foo, {'a': 'b'}, clear=True)
+ def test():
+ self.assertEqual(foo, {'a': 'b'})
+
+ test()
+
+ self.assertEqual(foo, original)
+
+ @patch.dict(foo, [('a', 'b')], clear=True)
+ def test():
+ self.assertEqual(foo, {'a': 'b'})
+
+ test()
+
+ self.assertEqual(foo, original)
+
+
+ def test_patch_dict_with_container_object_and_clear(self):
+ foo = Container()
+ foo['initial'] = object()
+ foo['other'] = 'something'
+
+ original = foo.values.copy()
+
+ @patch.dict(foo, clear=True)
+ def test():
+ self.assertEqual(foo.values, {})
+ foo['a'] = 3
+ foo['other'] = 'something else'
+
+ test()
+
+ self.assertEqual(foo.values, original)
+
+ @patch.dict(foo, {'a': 'b'}, clear=True)
+ def test():
+ self.assertEqual(foo.values, {'a': 'b'})
+
+ test()
+
+ self.assertEqual(foo.values, original)
+
+
+ def test_name_preserved(self):
+ foo = {}
+
+ @patch('%s.SomeClass' % __name__, object())
+ @patch('%s.SomeClass' % __name__, object(), autospec=True)
+ @patch.object(SomeClass, object())
+ @patch.dict(foo)
+ def some_name():
+ pass
+
+ self.assertEqual(some_name.__name__, 'some_name')
+
+
+ def test_patch_with_exception(self):
+ foo = {}
+
+ @patch.dict(foo, {'a': 'b'})
+ def test():
+ raise NameError('Konrad')
+ try:
+ test()
+ except NameError:
+ pass
+ else:
+ self.fail('NameError not raised by test')
+
+ self.assertEqual(foo, {})
+
+
+ def test_patch_dict_with_string(self):
+ @patch.dict('os.environ', {'konrad_delong': 'some value'})
+ def test():
+ self.assertIn('konrad_delong', os.environ)
+
+ test()
+
+
+ @unittest2.expectedFailure
+ def test_patch_descriptor(self):
+ # would be some effort to fix this - we could special case the
+ # builtin descriptors: classmethod, property, staticmethod
+ class Nothing(object):
+ foo = None
+
+ class Something(object):
+ foo = {}
+
+ @patch.object(Nothing, 'foo', 2)
+ @classmethod
+ def klass(cls):
+ self.assertIs(cls, Something)
+
+ @patch.object(Nothing, 'foo', 2)
+ @staticmethod
+ def static(arg):
+ return arg
+
+ @patch.dict(foo)
+ @classmethod
+ def klass_dict(cls):
+ self.assertIs(cls, Something)
+
+ @patch.dict(foo)
+ @staticmethod
+ def static_dict(arg):
+ return arg
+
+ # these will raise exceptions if patching descriptors is broken
+ self.assertEqual(Something.static('f00'), 'f00')
+ Something.klass()
+ self.assertEqual(Something.static_dict('f00'), 'f00')
+ Something.klass_dict()
+
+ something = Something()
+ self.assertEqual(something.static('f00'), 'f00')
+ something.klass()
+ self.assertEqual(something.static_dict('f00'), 'f00')
+ something.klass_dict()
+
+
+ def test_patch_spec_set(self):
+ @patch('%s.SomeClass' % __name__, spec_set=SomeClass)
+ def test(MockClass):
+ MockClass.z = 'foo'
+
+ self.assertRaises(AttributeError, test)
+
+ @patch.object(support, 'SomeClass', spec_set=SomeClass)
+ def test(MockClass):
+ MockClass.z = 'foo'
+
+ self.assertRaises(AttributeError, test)
+ @patch('%s.SomeClass' % __name__, spec_set=True)
+ def test(MockClass):
+ MockClass.z = 'foo'
+
+ self.assertRaises(AttributeError, test)
+
+ @patch.object(support, 'SomeClass', spec_set=True)
+ def test(MockClass):
+ MockClass.z = 'foo'
+
+ self.assertRaises(AttributeError, test)
+
+
+ def test_spec_set_inherit(self):
+ @patch('%s.SomeClass' % __name__, spec_set=True)
+ def test(MockClass):
+ instance = MockClass()
+ instance.z = 'foo'
+
+ self.assertRaises(AttributeError, test)
+
+
+ def test_patch_start_stop(self):
+ original = something
+ patcher = patch('%s.something' % __name__)
+ self.assertIs(something, original)
+ mock = patcher.start()
+ try:
+ self.assertIsNot(mock, original)
+ self.assertIs(something, mock)
+ finally:
+ patcher.stop()
+ self.assertIs(something, original)
+
+
+ def test_stop_without_start(self):
+ patcher = patch(foo_name, 'bar', 3)
+
+ # calling stop without start used to produce a very obscure error
+ self.assertRaises(RuntimeError, patcher.stop)
+
+
+ def test_patchobject_start_stop(self):
+ original = something
+ patcher = patch.object(PTModule, 'something', 'foo')
+ self.assertIs(something, original)
+ replaced = patcher.start()
+ try:
+ self.assertEqual(replaced, 'foo')
+ self.assertIs(something, replaced)
+ finally:
+ patcher.stop()
+ self.assertIs(something, original)
+
+
+ def test_patch_dict_start_stop(self):
+ d = {'foo': 'bar'}
+ original = d.copy()
+ patcher = patch.dict(d, [('spam', 'eggs')], clear=True)
+ self.assertEqual(d, original)
+
+ patcher.start()
+ try:
+ self.assertEqual(d, {'spam': 'eggs'})
+ finally:
+ patcher.stop()
+ self.assertEqual(d, original)
+
+
+ def test_patch_dict_class_decorator(self):
+ this = self
+ d = {'spam': 'eggs'}
+ original = d.copy()
+
+ class Test(object):
+ def test_first(self):
+ this.assertEqual(d, {'foo': 'bar'})
+ def test_second(self):
+ this.assertEqual(d, {'foo': 'bar'})
+
+ Test = patch.dict(d, {'foo': 'bar'}, clear=True)(Test)
+ self.assertEqual(d, original)
+
+ test = Test()
+
+ test.test_first()
+ self.assertEqual(d, original)
+
+ test.test_second()
+ self.assertEqual(d, original)
+
+ test = Test()
+
+ test.test_first()
+ self.assertEqual(d, original)
+
+ test.test_second()
+ self.assertEqual(d, original)
+
+
+ def test_get_only_proxy(self):
+ class Something(object):
+ foo = 'foo'
+ class SomethingElse:
+ foo = 'foo'
+
+ for thing in Something, SomethingElse, Something(), SomethingElse:
+ proxy = _get_proxy(thing)
+
+ @patch.object(proxy, 'foo', 'bar')
+ def test():
+ self.assertEqual(proxy.foo, 'bar')
+ test()
+ self.assertEqual(proxy.foo, 'foo')
+ self.assertEqual(thing.foo, 'foo')
+ self.assertNotIn('foo', proxy.__dict__)
+
+
+ def test_get_set_delete_proxy(self):
+ class Something(object):
+ foo = 'foo'
+ class SomethingElse:
+ foo = 'foo'
+
+ for thing in Something, SomethingElse, Something(), SomethingElse:
+ proxy = _get_proxy(Something, get_only=False)
+
+ @patch.object(proxy, 'foo', 'bar')
+ def test():
+ self.assertEqual(proxy.foo, 'bar')
+ test()
+ self.assertEqual(proxy.foo, 'foo')
+ self.assertEqual(thing.foo, 'foo')
+ self.assertNotIn('foo', proxy.__dict__)
+
+
+ def test_patch_keyword_args(self):
+ kwargs = {'side_effect': KeyError, 'foo.bar.return_value': 33,
+ 'foo': MagicMock()}
+
+ patcher = patch(foo_name, **kwargs)
+ mock = patcher.start()
+ patcher.stop()
+
+ self.assertRaises(KeyError, mock)
+ self.assertEqual(mock.foo.bar(), 33)
+ self.assertIsInstance(mock.foo, MagicMock)
+
+
+ def test_patch_object_keyword_args(self):
+ kwargs = {'side_effect': KeyError, 'foo.bar.return_value': 33,
+ 'foo': MagicMock()}
+
+ patcher = patch.object(Foo, 'f', **kwargs)
+ mock = patcher.start()
+ patcher.stop()
+
+ self.assertRaises(KeyError, mock)
+ self.assertEqual(mock.foo.bar(), 33)
+ self.assertIsInstance(mock.foo, MagicMock)
+
+
+ def test_patch_dict_keyword_args(self):
+ original = {'foo': 'bar'}
+ copy = original.copy()
+
+ patcher = patch.dict(original, foo=3, bar=4, baz=5)
+ patcher.start()
+
+ try:
+ self.assertEqual(original, dict(foo=3, bar=4, baz=5))
+ finally:
+ patcher.stop()
+
+ self.assertEqual(original, copy)
+
+
+ def test_autospec(self):
+ class Boo(object):
+ def __init__(self, a):
+ pass
+ def f(self, a):
+ pass
+ def g(self):
+ pass
+ foo = 'bar'
+
+ class Bar(object):
+ def a(self):
+ pass
+
+ def _test(mock):
+ mock(1)
+ mock.assert_called_with(1)
+ self.assertRaises(TypeError, mock)
+
+ def _test2(mock):
+ mock.f(1)
+ mock.f.assert_called_with(1)
+ self.assertRaises(TypeError, mock.f)
+
+ mock.g()
+ mock.g.assert_called_with()
+ self.assertRaises(TypeError, mock.g, 1)
+
+ self.assertRaises(AttributeError, getattr, mock, 'h')
+
+ mock.foo.lower()
+ mock.foo.lower.assert_called_with()
+ self.assertRaises(AttributeError, getattr, mock.foo, 'bar')
+
+ mock.Bar()
+ mock.Bar.assert_called_with()
+
+ mock.Bar.a()
+ mock.Bar.a.assert_called_with()
+ self.assertRaises(TypeError, mock.Bar.a, 1)
+
+ mock.Bar().a()
+ mock.Bar().a.assert_called_with()
+ self.assertRaises(TypeError, mock.Bar().a, 1)
+
+ self.assertRaises(AttributeError, getattr, mock.Bar, 'b')
+ self.assertRaises(AttributeError, getattr, mock.Bar(), 'b')
+
+ def function(mock):
+ _test(mock)
+ _test2(mock)
+ _test2(mock(1))
+ self.assertIs(mock, Foo)
+ return mock
+
+ test = patch(foo_name, autospec=True)(function)
+
+ mock = test()
+ self.assertIsNot(Foo, mock)
+ # test patching a second time works
+ test()
+
+ module = sys.modules[__name__]
+ test = patch.object(module, 'Foo', autospec=True)(function)
+
+ mock = test()
+ self.assertIsNot(Foo, mock)
+ # test patching a second time works
+ test()
+
+
+ def test_autospec_function(self):
+ @patch('%s.function' % __name__, autospec=True)
+ def test(mock):
+ function(1)
+ function.assert_called_with(1)
+ function(2, 3)
+ function.assert_called_with(2, 3)
+
+ self.assertRaises(TypeError, function)
+ self.assertRaises(AttributeError, getattr, function, 'foo')
+
+ test()
+
+
+ def test_autospec_keywords(self):
+ @patch('%s.function' % __name__, autospec=True,
+ return_value=3)
+ def test(mock_function):
+ #self.assertEqual(function.abc, 'foo')
+ return function(1, 2)
+
+ result = test()
+ self.assertEqual(result, 3)
+
+
+ def test_autospec_with_new(self):
+ patcher = patch('%s.function' % __name__, new=3, autospec=True)
+ self.assertRaises(TypeError, patcher.start)
+
+ module = sys.modules[__name__]
+ patcher = patch.object(module, 'function', new=3, autospec=True)
+ self.assertRaises(TypeError, patcher.start)
+
+
+ def test_autospec_with_object(self):
+ class Bar(Foo):
+ extra = []
+
+ patcher = patch(foo_name, autospec=Bar)
+ mock = patcher.start()
+ try:
+ self.assertIsInstance(mock, Bar)
+ self.assertIsInstance(mock.extra, list)
+ finally:
+ patcher.stop()
+
+
+ def test_autospec_inherits(self):
+ FooClass = Foo
+ patcher = patch(foo_name, autospec=True)
+ mock = patcher.start()
+ try:
+ self.assertIsInstance(mock, FooClass)
+ self.assertIsInstance(mock(3), FooClass)
+ finally:
+ patcher.stop()
+
+
+ def test_autospec_name(self):
+ patcher = patch(foo_name, autospec=True)
+ mock = patcher.start()
+
+ try:
+ self.assertIn(" name='Foo'", repr(mock))
+ self.assertIn(" name='Foo.f'", repr(mock.f))
+ self.assertIn(" name='Foo()'", repr(mock(None)))
+ self.assertIn(" name='Foo().f'", repr(mock(None).f))
+ finally:
+ patcher.stop()
+
+
+ def test_tracebacks(self):
+ @patch.object(Foo, 'f', object())
+ def test():
+ raise AssertionError
+ try:
+ test()
+ except:
+ err = sys.exc_info()
+
+ result = unittest2.TextTestResult(None, None, 0)
+ traceback = result._exc_info_to_string(err, self)
+ self.assertIn('raise AssertionError', traceback)
+
+
+ def test_new_callable_patch(self):
+ patcher = patch(foo_name, new_callable=NonCallableMagicMock)
+
+ m1 = patcher.start()
+ patcher.stop()
+ m2 = patcher.start()
+ patcher.stop()
+
+ self.assertIsNot(m1, m2)
+ for mock in m1, m2:
+ self.assertNotCallable(m1)
+
+
+ def test_new_callable_patch_object(self):
+ patcher = patch.object(Foo, 'f', new_callable=NonCallableMagicMock)
+
+ m1 = patcher.start()
+ patcher.stop()
+ m2 = patcher.start()
+ patcher.stop()
+
+ self.assertIsNot(m1, m2)
+ for mock in m1, m2:
+ self.assertNotCallable(m1)
+
+
+ def test_new_callable_keyword_arguments(self):
+ class Bar(object):
+ kwargs = None
+ def __init__(self, **kwargs):
+ Bar.kwargs = kwargs
+
+ patcher = patch(foo_name, new_callable=Bar, arg1=1, arg2=2)
+ m = patcher.start()
+ try:
+ self.assertIs(type(m), Bar)
+ self.assertEqual(Bar.kwargs, dict(arg1=1, arg2=2))
+ finally:
+ patcher.stop()
+
+
+ def test_new_callable_spec(self):
+ class Bar(object):
+ kwargs = None
+ def __init__(self, **kwargs):
+ Bar.kwargs = kwargs
+
+ patcher = patch(foo_name, new_callable=Bar, spec=Bar)
+ patcher.start()
+ try:
+ self.assertEqual(Bar.kwargs, dict(spec=Bar))
+ finally:
+ patcher.stop()
+
+ patcher = patch(foo_name, new_callable=Bar, spec_set=Bar)
+ patcher.start()
+ try:
+ self.assertEqual(Bar.kwargs, dict(spec_set=Bar))
+ finally:
+ patcher.stop()
+
+
+ def test_new_callable_create(self):
+ non_existent_attr = '%s.weeeee' % foo_name
+ p = patch(non_existent_attr, new_callable=NonCallableMock)
+ self.assertRaises(AttributeError, p.start)
+
+ p = patch(non_existent_attr, new_callable=NonCallableMock,
+ create=True)
+ m = p.start()
+ try:
+ self.assertNotCallable(m, magic=False)
+ finally:
+ p.stop()
+
+
+ def test_new_callable_incompatible_with_new(self):
+ self.assertRaises(
+ ValueError, patch, foo_name, new=object(), new_callable=MagicMock
+ )
+ self.assertRaises(
+ ValueError, patch.object, Foo, 'f', new=object(),
+ new_callable=MagicMock
+ )
+
+
+ def test_new_callable_incompatible_with_autospec(self):
+ self.assertRaises(
+ ValueError, patch, foo_name, new_callable=MagicMock,
+ autospec=True
+ )
+ self.assertRaises(
+ ValueError, patch.object, Foo, 'f', new_callable=MagicMock,
+ autospec=True
+ )
+
+
+ def test_new_callable_inherit_for_mocks(self):
+ class MockSub(Mock):
+ pass
+
+ MockClasses = (
+ NonCallableMock, NonCallableMagicMock, MagicMock, Mock, MockSub
+ )
+ for Klass in MockClasses:
+ for arg in 'spec', 'spec_set':
+ kwargs = {arg: True}
+ p = patch(foo_name, new_callable=Klass, **kwargs)
+ m = p.start()
+ try:
+ instance = m.return_value
+ self.assertRaises(AttributeError, getattr, instance, 'x')
+ finally:
+ p.stop()
+
+
+ def test_new_callable_inherit_non_mock(self):
+ class NotAMock(object):
+ def __init__(self, spec):
+ self.spec = spec
+
+ p = patch(foo_name, new_callable=NotAMock, spec=True)
+ m = p.start()
+ try:
+ self.assertTrue(is_instance(m, NotAMock))
+ self.assertRaises(AttributeError, getattr, m, 'return_value')
+ finally:
+ p.stop()
+
+ self.assertEqual(m.spec, Foo)
+
+
+ def test_new_callable_class_decorating(self):
+ test = self
+ original = Foo
+ class SomeTest(object):
+
+ def _test(self, mock_foo):
+ test.assertIsNot(Foo, original)
+ test.assertIs(Foo, mock_foo)
+ test.assertIsInstance(Foo, SomeClass)
+
+ def test_two(self, mock_foo):
+ self._test(mock_foo)
+ def test_one(self, mock_foo):
+ self._test(mock_foo)
+
+ SomeTest = patch(foo_name, new_callable=SomeClass)(SomeTest)
+ SomeTest().test_one()
+ SomeTest().test_two()
+ self.assertIs(Foo, original)
+
+
+ def test_patch_multiple(self):
+ original_foo = Foo
+ original_f = Foo.f
+ original_g = Foo.g
+
+ patcher1 = patch.multiple(foo_name, f=1, g=2)
+ patcher2 = patch.multiple(Foo, f=1, g=2)
+
+ for patcher in patcher1, patcher2:
+ patcher.start()
+ try:
+ self.assertIs(Foo, original_foo)
+ self.assertEqual(Foo.f, 1)
+ self.assertEqual(Foo.g, 2)
+ finally:
+ patcher.stop()
+
+ self.assertIs(Foo, original_foo)
+ self.assertEqual(Foo.f, original_f)
+ self.assertEqual(Foo.g, original_g)
+
+
+ @patch.multiple(foo_name, f=3, g=4)
+ def test():
+ self.assertIs(Foo, original_foo)
+ self.assertEqual(Foo.f, 3)
+ self.assertEqual(Foo.g, 4)
+
+ test()
+
+
+ def test_patch_multiple_no_kwargs(self):
+ self.assertRaises(ValueError, patch.multiple, foo_name)
+ self.assertRaises(ValueError, patch.multiple, Foo)
+
+
+ def test_patch_multiple_create_mocks(self):
+ original_foo = Foo
+ original_f = Foo.f
+ original_g = Foo.g
+
+ @patch.multiple(foo_name, f=DEFAULT, g=3, foo=DEFAULT)
+ def test(f, foo):
+ self.assertIs(Foo, original_foo)
+ self.assertIs(Foo.f, f)
+ self.assertEqual(Foo.g, 3)
+ self.assertIs(Foo.foo, foo)
+ self.assertTrue(is_instance(f, MagicMock))
+ self.assertTrue(is_instance(foo, MagicMock))
+
+ test()
+ self.assertEqual(Foo.f, original_f)
+ self.assertEqual(Foo.g, original_g)
+
+
+ def test_patch_multiple_create_mocks_different_order(self):
+ # bug revealed by Jython!
+ original_f = Foo.f
+ original_g = Foo.g
+
+ patcher = patch.object(Foo, 'f', 3)
+ patcher.attribute_name = 'f'
+
+ other = patch.object(Foo, 'g', DEFAULT)
+ other.attribute_name = 'g'
+ patcher.additional_patchers = [other]
+
+ @patcher
+ def test(g):
+ self.assertIs(Foo.g, g)
+ self.assertEqual(Foo.f, 3)
+
+ test()
+ self.assertEqual(Foo.f, original_f)
+ self.assertEqual(Foo.g, original_g)
+
+
+ def test_patch_multiple_stacked_decorators(self):
+ original_foo = Foo
+ original_f = Foo.f
+ original_g = Foo.g
+
+ @patch.multiple(foo_name, f=DEFAULT)
+ @patch.multiple(foo_name, foo=DEFAULT)
+ @patch(foo_name + '.g')
+ def test1(g, **kwargs):
+ _test(g, **kwargs)
+
+ @patch.multiple(foo_name, f=DEFAULT)
+ @patch(foo_name + '.g')
+ @patch.multiple(foo_name, foo=DEFAULT)
+ def test2(g, **kwargs):
+ _test(g, **kwargs)
+
+ @patch(foo_name + '.g')
+ @patch.multiple(foo_name, f=DEFAULT)
+ @patch.multiple(foo_name, foo=DEFAULT)
+ def test3(g, **kwargs):
+ _test(g, **kwargs)
+
+ def _test(g, **kwargs):
+ f = kwargs.pop('f')
+ foo = kwargs.pop('foo')
+ self.assertFalse(kwargs)
+
+ self.assertIs(Foo, original_foo)
+ self.assertIs(Foo.f, f)
+ self.assertIs(Foo.g, g)
+ self.assertIs(Foo.foo, foo)
+ self.assertTrue(is_instance(f, MagicMock))
+ self.assertTrue(is_instance(g, MagicMock))
+ self.assertTrue(is_instance(foo, MagicMock))
+
+ test1()
+ test2()
+ test3()
+ self.assertEqual(Foo.f, original_f)
+ self.assertEqual(Foo.g, original_g)
+
+
+ def test_patch_multiple_create_mocks_patcher(self):
+ original_foo = Foo
+ original_f = Foo.f
+ original_g = Foo.g
+
+ patcher = patch.multiple(foo_name, f=DEFAULT, g=3, foo=DEFAULT)
+
+ result = patcher.start()
+ try:
+ f = result['f']
+ foo = result['foo']
+ self.assertEqual(set(result), set(['f', 'foo']))
+
+ self.assertIs(Foo, original_foo)
+ self.assertIs(Foo.f, f)
+ self.assertIs(Foo.foo, foo)
+ self.assertTrue(is_instance(f, MagicMock))
+ self.assertTrue(is_instance(foo, MagicMock))
+ finally:
+ patcher.stop()
+
+ self.assertEqual(Foo.f, original_f)
+ self.assertEqual(Foo.g, original_g)
+
+
+ def test_patch_multiple_decorating_class(self):
+ test = self
+ original_foo = Foo
+ original_f = Foo.f
+ original_g = Foo.g
+
+ class SomeTest(object):
+
+ def _test(self, f, foo):
+ test.assertIs(Foo, original_foo)
+ test.assertIs(Foo.f, f)
+ test.assertEqual(Foo.g, 3)
+ test.assertIs(Foo.foo, foo)
+ test.assertTrue(is_instance(f, MagicMock))
+ test.assertTrue(is_instance(foo, MagicMock))
+
+ def test_two(self, f, foo):
+ self._test(f, foo)
+ def test_one(self, f, foo):
+ self._test(f, foo)
+
+ SomeTest = patch.multiple(
+ foo_name, f=DEFAULT, g=3, foo=DEFAULT
+ )(SomeTest)
+
+ thing = SomeTest()
+ thing.test_one()
+ thing.test_two()
+
+ self.assertEqual(Foo.f, original_f)
+ self.assertEqual(Foo.g, original_g)
+
+
+ def test_patch_multiple_create(self):
+ patcher = patch.multiple(Foo, blam='blam')
+ self.assertRaises(AttributeError, patcher.start)
+
+ patcher = patch.multiple(Foo, blam='blam', create=True)
+ patcher.start()
+ try:
+ self.assertEqual(Foo.blam, 'blam')
+ finally:
+ patcher.stop()
+
+ self.assertFalse(hasattr(Foo, 'blam'))
+
+
+ def test_patch_multiple_spec_set(self):
+ # if spec_set works then we can assume that spec and autospec also
+ # work as the underlying machinery is the same
+ patcher = patch.multiple(Foo, foo=DEFAULT, spec_set=['a', 'b'])
+ result = patcher.start()
+ try:
+ self.assertEqual(Foo.foo, result['foo'])
+ Foo.foo.a(1)
+ Foo.foo.b(2)
+ Foo.foo.a.assert_called_with(1)
+ Foo.foo.b.assert_called_with(2)
+ self.assertRaises(AttributeError, setattr, Foo.foo, 'c', None)
+ finally:
+ patcher.stop()
+
+
+ def test_patch_multiple_new_callable(self):
+ class Thing(object):
+ pass
+
+ patcher = patch.multiple(
+ Foo, f=DEFAULT, g=DEFAULT, new_callable=Thing
+ )
+ result = patcher.start()
+ try:
+ self.assertIs(Foo.f, result['f'])
+ self.assertIs(Foo.g, result['g'])
+ self.assertIsInstance(Foo.f, Thing)
+ self.assertIsInstance(Foo.g, Thing)
+ self.assertIsNot(Foo.f, Foo.g)
+ finally:
+ patcher.stop()
+
+
+ def test_nested_patch_failure(self):
+ original_f = Foo.f
+ original_g = Foo.g
+
+ @patch.object(Foo, 'g', 1)
+ @patch.object(Foo, 'missing', 1)
+ @patch.object(Foo, 'f', 1)
+ def thing1():
+ pass
+
+ @patch.object(Foo, 'missing', 1)
+ @patch.object(Foo, 'g', 1)
+ @patch.object(Foo, 'f', 1)
+ def thing2():
+ pass
+
+ @patch.object(Foo, 'g', 1)
+ @patch.object(Foo, 'f', 1)
+ @patch.object(Foo, 'missing', 1)
+ def thing3():
+ pass
+
+ for func in thing1, thing2, thing3:
+ self.assertRaises(AttributeError, func)
+ self.assertEqual(Foo.f, original_f)
+ self.assertEqual(Foo.g, original_g)
+
+
+ def test_new_callable_failure(self):
+ original_f = Foo.f
+ original_g = Foo.g
+ original_foo = Foo.foo
+
+ def crasher():
+ raise NameError('crasher')
+
+ @patch.object(Foo, 'g', 1)
+ @patch.object(Foo, 'foo', new_callable=crasher)
+ @patch.object(Foo, 'f', 1)
+ def thing1():
+ pass
+
+ @patch.object(Foo, 'foo', new_callable=crasher)
+ @patch.object(Foo, 'g', 1)
+ @patch.object(Foo, 'f', 1)
+ def thing2():
+ pass
+
+ @patch.object(Foo, 'g', 1)
+ @patch.object(Foo, 'f', 1)
+ @patch.object(Foo, 'foo', new_callable=crasher)
+ def thing3():
+ pass
+
+ for func in thing1, thing2, thing3:
+ self.assertRaises(NameError, func)
+ self.assertEqual(Foo.f, original_f)
+ self.assertEqual(Foo.g, original_g)
+ self.assertEqual(Foo.foo, original_foo)
+
+
+ def test_patch_multiple_failure(self):
+ original_f = Foo.f
+ original_g = Foo.g
+
+ patcher = patch.object(Foo, 'f', 1)
+ patcher.attribute_name = 'f'
+
+ good = patch.object(Foo, 'g', 1)
+ good.attribute_name = 'g'
+
+ bad = patch.object(Foo, 'missing', 1)
+ bad.attribute_name = 'missing'
+
+ for additionals in [good, bad], [bad, good]:
+ patcher.additional_patchers = additionals
+
+ @patcher
+ def func():
+ pass
+
+ self.assertRaises(AttributeError, func)
+ self.assertEqual(Foo.f, original_f)
+ self.assertEqual(Foo.g, original_g)
+
+
+ def test_patch_multiple_new_callable_failure(self):
+ original_f = Foo.f
+ original_g = Foo.g
+ original_foo = Foo.foo
+
+ def crasher():
+ raise NameError('crasher')
+
+ patcher = patch.object(Foo, 'f', 1)
+ patcher.attribute_name = 'f'
+
+ good = patch.object(Foo, 'g', 1)
+ good.attribute_name = 'g'
+
+ bad = patch.object(Foo, 'foo', new_callable=crasher)
+ bad.attribute_name = 'foo'
+
+ for additionals in [good, bad], [bad, good]:
+ patcher.additional_patchers = additionals
+
+ @patcher
+ def func():
+ pass
+
+ self.assertRaises(NameError, func)
+ self.assertEqual(Foo.f, original_f)
+ self.assertEqual(Foo.g, original_g)
+ self.assertEqual(Foo.foo, original_foo)
+
+
+ def test_patch_multiple_string_subclasses(self):
+ for base in (str, unicode):
+ Foo = type('Foo', (base,), {'fish': 'tasty'})
+ foo = Foo()
+ @patch.multiple(foo, fish='nearly gone')
+ def test():
+ self.assertEqual(foo.fish, 'nearly gone')
+
+ test()
+ self.assertEqual(foo.fish, 'tasty')
+
+
+ @patch('mock.patch.TEST_PREFIX', 'foo')
+ def test_patch_test_prefix(self):
+ class Foo(object):
+ thing = 'original'
+
+ def foo_one(self):
+ return self.thing
+ def foo_two(self):
+ return self.thing
+ def test_one(self):
+ return self.thing
+ def test_two(self):
+ return self.thing
+
+ Foo = patch.object(Foo, 'thing', 'changed')(Foo)
+
+ foo = Foo()
+ self.assertEqual(foo.foo_one(), 'changed')
+ self.assertEqual(foo.foo_two(), 'changed')
+ self.assertEqual(foo.test_one(), 'original')
+ self.assertEqual(foo.test_two(), 'original')
+
+
+ @patch('mock.patch.TEST_PREFIX', 'bar')
+ def test_patch_dict_test_prefix(self):
+ class Foo(object):
+ def bar_one(self):
+ return dict(the_dict)
+ def bar_two(self):
+ return dict(the_dict)
+ def test_one(self):
+ return dict(the_dict)
+ def test_two(self):
+ return dict(the_dict)
+
+ the_dict = {'key': 'original'}
+ Foo = patch.dict(the_dict, key='changed')(Foo)
+
+ foo =Foo()
+ self.assertEqual(foo.bar_one(), {'key': 'changed'})
+ self.assertEqual(foo.bar_two(), {'key': 'changed'})
+ self.assertEqual(foo.test_one(), {'key': 'original'})
+ self.assertEqual(foo.test_two(), {'key': 'original'})
+
+
+ def test_patch_with_spec_mock_repr(self):
+ for arg in ('spec', 'autospec', 'spec_set'):
+ p = patch('%s.SomeClass' % __name__, **{arg: True})
+ m = p.start()
+ try:
+ self.assertIn(" name='SomeClass'", repr(m))
+ self.assertIn(" name='SomeClass.class_attribute'",
+ repr(m.class_attribute))
+ self.assertIn(" name='SomeClass()'", repr(m()))
+ self.assertIn(" name='SomeClass().class_attribute'",
+ repr(m().class_attribute))
+ finally:
+ p.stop()
+
+
+ def test_patch_nested_autospec_repr(self):
+ p = patch('tests.support', autospec=True)
+ m = p.start()
+ try:
+ self.assertIn(" name='support.SomeClass.wibble()'",
+ repr(m.SomeClass.wibble()))
+ self.assertIn(" name='support.SomeClass().wibble()'",
+ repr(m.SomeClass().wibble()))
+ finally:
+ p.stop()
+
+
+ def test_mock_calls_with_patch(self):
+ for arg in ('spec', 'autospec', 'spec_set'):
+ p = patch('%s.SomeClass' % __name__, **{arg: True})
+ m = p.start()
+ try:
+ m.wibble()
+
+ kalls = [call.wibble()]
+ self.assertEqual(m.mock_calls, kalls)
+ self.assertEqual(m.method_calls, kalls)
+ self.assertEqual(m.wibble.mock_calls, [call()])
+
+ result = m()
+ kalls.append(call())
+ self.assertEqual(m.mock_calls, kalls)
+
+ result.wibble()
+ kalls.append(call().wibble())
+ self.assertEqual(m.mock_calls, kalls)
+
+ self.assertEqual(result.mock_calls, [call.wibble()])
+ self.assertEqual(result.wibble.mock_calls, [call()])
+ self.assertEqual(result.method_calls, [call.wibble()])
+ finally:
+ p.stop()
+
+
+ def test_patch_imports_lazily(self):
+ sys.modules.pop('squizz', None)
+
+ p1 = patch('squizz.squozz')
+ self.assertRaises(ImportError, p1.start)
+
+ squizz = Mock()
+ squizz.squozz = 6
+ sys.modules['squizz'] = squizz
+ p1 = patch('squizz.squozz')
+ squizz.squozz = 3
+ p1.start()
+ p1.stop()
+ self.assertEqual(squizz.squozz, 3)
+
+
+ def test_patch_propogrates_exc_on_exit(self):
+ class holder:
+ exc_info = None, None, None
+
+ class custom_patch(_patch):
+ def __exit__(self, etype=None, val=None, tb=None):
+ _patch.__exit__(self, etype, val, tb)
+ holder.exc_info = etype, val, tb
+ stop = __exit__
+
+ def with_custom_patch(target):
+ getter, attribute = _get_target(target)
+ return custom_patch(
+ getter, attribute, DEFAULT, None, False, None,
+ None, None, {}
+ )
+
+ @with_custom_patch('squizz.squozz')
+ def test(mock):
+ raise RuntimeError
+
+ self.assertRaises(RuntimeError, test)
+ self.assertIs(holder.exc_info[0], RuntimeError)
+ self.assertIsNotNone(holder.exc_info[1],
+ 'exception value not propgated')
+ self.assertIsNotNone(holder.exc_info[2],
+ 'exception traceback not propgated')
+
+
+ def test_create_and_specs(self):
+ for kwarg in ('spec', 'spec_set', 'autospec'):
+ p = patch('%s.doesnotexist' % __name__, create=True,
+ **{kwarg: True})
+ self.assertRaises(TypeError, p.start)
+ self.assertRaises(NameError, lambda: doesnotexist)
+
+ # check that spec with create is innocuous if the original exists
+ p = patch(MODNAME, create=True, **{kwarg: True})
+ p.start()
+ p.stop()
+
+
+ def test_multiple_specs(self):
+ original = PTModule
+ for kwarg in ('spec', 'spec_set'):
+ p = patch(MODNAME, autospec=0, **{kwarg: 0})
+ self.assertRaises(TypeError, p.start)
+ self.assertIs(PTModule, original)
+
+ for kwarg in ('spec', 'autospec'):
+ p = patch(MODNAME, spec_set=0, **{kwarg: 0})
+ self.assertRaises(TypeError, p.start)
+ self.assertIs(PTModule, original)
+
+ for kwarg in ('spec_set', 'autospec'):
+ p = patch(MODNAME, spec=0, **{kwarg: 0})
+ self.assertRaises(TypeError, p.start)
+ self.assertIs(PTModule, original)
+
+
+ def test_specs_false_instead_of_none(self):
+ p = patch(MODNAME, spec=False, spec_set=False, autospec=False)
+ mock = p.start()
+ try:
+ # no spec should have been set, so attribute access should not fail
+ mock.does_not_exist
+ mock.does_not_exist = 3
+ finally:
+ p.stop()
+
+
+ def test_falsey_spec(self):
+ for kwarg in ('spec', 'autospec', 'spec_set'):
+ p = patch(MODNAME, **{kwarg: 0})
+ m = p.start()
+ try:
+ self.assertRaises(AttributeError, getattr, m, 'doesnotexit')
+ finally:
+ p.stop()
+
+
+ def test_spec_set_true(self):
+ for kwarg in ('spec', 'autospec'):
+ p = patch(MODNAME, spec_set=True, **{kwarg: True})
+ m = p.start()
+ try:
+ self.assertRaises(AttributeError, setattr, m,
+ 'doesnotexist', 'something')
+ self.assertRaises(AttributeError, getattr, m, 'doesnotexist')
+ finally:
+ p.stop()
+
+
+ def test_callable_spec_as_list(self):
+ spec = ('__call__',)
+ p = patch(MODNAME, spec=spec)
+ m = p.start()
+ try:
+ self.assertTrue(callable(m))
+ finally:
+ p.stop()
+
+
+ def test_not_callable_spec_as_list(self):
+ spec = ('foo', 'bar')
+ p = patch(MODNAME, spec=spec)
+ m = p.start()
+ try:
+ self.assertFalse(callable(m))
+ finally:
+ p.stop()
+
+
+ def test_patch_stopall(self):
+ unlink = os.unlink
+ chdir = os.chdir
+ path = os.path
+ patch('os.unlink', something).start()
+ patch('os.chdir', something_else).start()
+
+ @patch('os.path')
+ def patched(mock_path):
+ patch.stopall()
+ self.assertIs(os.path, mock_path)
+ self.assertIs(os.unlink, unlink)
+ self.assertIs(os.chdir, chdir)
+
+ patched()
+ self.assertIs(os.path, path)
+
+
+ def test_wrapped_patch(self):
+ decorated = patch('sys.modules')(function)
+ self.assertIs(decorated.__wrapped__, function)
+
+
+ def test_wrapped_several_times_patch(self):
+ decorated = patch('sys.modules')(function)
+ decorated = patch('sys.modules')(decorated)
+ self.assertIs(decorated.__wrapped__, function)
+
+
+ def test_wrapped_patch_object(self):
+ decorated = patch.object(sys, 'modules')(function)
+ self.assertIs(decorated.__wrapped__, function)
+
+
+ def test_wrapped_patch_dict(self):
+ decorated = patch.dict('sys.modules')(function)
+ self.assertIs(decorated.__wrapped__, function)
+
+
+ def test_wrapped_patch_multiple(self):
+ decorated = patch.multiple('sys', modules={})(function)
+ self.assertIs(decorated.__wrapped__, function)
+
+
+if __name__ == '__main__':
+ unittest2.main()
diff --git a/slider-agent/src/test/python/mock/tests/testsentinel.py b/slider-agent/src/test/python/mock/tests/testsentinel.py
new file mode 100644
index 0000000..981171a
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/testsentinel.py
@@ -0,0 +1,33 @@
+# Copyright (C) 2007-2012 Michael Foord & the mock team
+# E-mail: fuzzyman AT voidspace DOT org DOT uk
+# http://www.voidspace.org.uk/python/mock/
+
+from tests.support import unittest2
+
+from mock import sentinel, DEFAULT
+
+
+class SentinelTest(unittest2.TestCase):
+
+ def testSentinels(self):
+ self.assertEqual(sentinel.whatever, sentinel.whatever,
+ 'sentinel not stored')
+ self.assertNotEqual(sentinel.whatever, sentinel.whateverelse,
+ 'sentinel should be unique')
+
+
+ def testSentinelName(self):
+ self.assertEqual(str(sentinel.whatever), 'sentinel.whatever',
+ 'sentinel name incorrect')
+
+
+ def testDEFAULT(self):
+ self.assertTrue(DEFAULT is sentinel.DEFAULT)
+
+ def testBases(self):
+ # If this doesn't raise an AttributeError then help(mock) is broken
+ self.assertRaises(AttributeError, lambda: sentinel.__bases__)
+
+
+if __name__ == '__main__':
+ unittest2.main()
diff --git a/slider-agent/src/test/python/mock/tests/testwith.py b/slider-agent/src/test/python/mock/tests/testwith.py
new file mode 100644
index 0000000..34529eb
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tests/testwith.py
@@ -0,0 +1,16 @@
+import sys
+
+if sys.version_info[:2] >= (2, 5):
+ from tests._testwith import *
+else:
+ from tests.support import unittest2
+
+ class TestWith(unittest2.TestCase):
+
+ @unittest2.skip('tests using with statement skipped on Python 2.4')
+ def testWith(self):
+ pass
+
+
+if __name__ == '__main__':
+ unittest2.main()
diff --git a/slider-agent/src/test/python/mock/tox.ini b/slider-agent/src/test/python/mock/tox.ini
new file mode 100644
index 0000000..58e29d2
--- /dev/null
+++ b/slider-agent/src/test/python/mock/tox.ini
@@ -0,0 +1,40 @@
+[tox]
+envlist = py25,py26,py27,py31,pypy,py32,py33,jython
+
+[testenv]
+deps=unittest2
+commands={envbindir}/unit2 discover []
+
+[testenv:py26]
+commands=
+ {envbindir}/unit2 discover []
+ {envbindir}/sphinx-build -E -b doctest docs html
+ {envbindir}/sphinx-build -E docs html
+deps =
+ unittest2
+ sphinx
+
+[testenv:py27]
+commands=
+ {envbindir}/unit2 discover []
+ {envbindir}/sphinx-build -E -b doctest docs html
+deps =
+ unittest2
+ sphinx
+
+[testenv:py31]
+deps =
+ unittest2py3k
+
+[testenv:py32]
+commands=
+ {envbindir}/python -m unittest discover []
+deps =
+
+[testenv:py33]
+commands=
+ {envbindir}/python -m unittest discover []
+deps =
+
+# note for jython. Execute in tests directory:
+# rm `find . -name '*$py.class'`
\ No newline at end of file
diff --git a/slider-agent/src/test/python/mock/unittest.cfg b/slider-agent/src/test/python/mock/unittest.cfg
new file mode 100644
index 0000000..b2d6f67
--- /dev/null
+++ b/slider-agent/src/test/python/mock/unittest.cfg
@@ -0,0 +1,95 @@
+
+[unittest]
+plugins =
+ unittest2.plugins.debugger
+ unittest2.plugins.checker
+ unittest2.plugins.doctestloader
+ unittest2.plugins.matchregexp
+ unittest2.plugins.moduleloading
+ unittest2.plugins.testcoverage
+ unittest2.plugins.growl
+ unittest2.plugins.filtertests
+ unittest2.plugins.junitxml
+ unittest2.plugins.timed
+ unittest2.plugins.counttests
+ unittest2.plugins.logchannels
+
+excluded-plugins =
+
+# 0, 1 or 2 (default is 1)
+# quiet, normal or verbose
+# can be overriden at command line
+verbosity = normal
+
+# true or false
+# even if false can be switched on at command line
+catch =
+buffer =
+failfast =
+
+
+[matchregexp]
+always-on = False
+full-path = True
+
+[debugger]
+always-on = False
+errors-only = True
+
+[coverage]
+always-on = False
+config =
+report-html = False
+# only used if report-html is false
+annotate = False
+# defaults to './htmlcov/'
+html-directory =
+# if unset will output to console
+text-file =
+branch = False
+timid = False
+cover-pylib = False
+exclude-lines =
+ # Have to re-enable the standard pragma
+ pragma: no cover
+
+ # Don't complain about missing debug-only code:
+ def __repr__
+ if self\.debug
+
+ # Don't complain if tests don't hit defensive assertion code:
+ raise AssertionError
+ raise NotImplementedError
+
+ # Don't complain if non-runnable code isn't run:
+ if 0:
+ if __name__ == .__main__.
+
+ignore-errors = False
+modules =
+
+[growl]
+always-on = False
+
+[doctest]
+always-on = False
+
+[module-loading]
+always-on = False
+
+[checker]
+always-on = False
+pep8 = False
+pyflakes = True
+
+[junit-xml]
+always-on = False
+path = junit.xml
+
+[timed]
+always-on = True
+threshold = 0.01
+
+[count]
+always-on = True
+enhanced = False
diff --git a/slider-agent/src/test/python/resource_management/TestContentSources.py b/slider-agent/src/test/python/resource_management/TestContentSources.py
new file mode 100644
index 0000000..2527f30
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestContentSources.py
@@ -0,0 +1,343 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+
+from resource_management.core import Environment
+from resource_management.core.system import System
+from resource_management.core.source import StaticFile
+from resource_management.core.source import DownloadSource
+from resource_management.core.source import Template
+from resource_management.core.source import InlineTemplate
+
+from jinja2 import UndefinedError, TemplateNotFound
+import urllib2
+import os
+
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestContentSources(TestCase):
+
+ @patch("__builtin__.open")
+ @patch.object(os.path, "join")
+ def test_static_file_absolute_path(self, join_mock, open_mock):
+ """
+ Testing StaticFile source with absolute path
+ """
+ file_mock = MagicMock(name = 'file_mock')
+ file_mock.__enter__.return_value = file_mock
+ file_mock.read.return_value = 'content'
+ open_mock.return_value = file_mock
+
+ with Environment("/base") as env:
+ static_file = StaticFile("/absolute/path/file")
+ content = static_file.get_content()
+
+ self.assertEqual('content', content)
+ self.assertEqual(file_mock.read.call_count, 1)
+ self.assertEqual(join_mock.call_count, 0)
+
+
+ @patch("__builtin__.open")
+ @patch.object(os.path, "join")
+ def test_static_file_relative_path(self, join_mock, open_mock):
+ """
+ Testing StaticFile source with relative path
+ """
+ file_mock = MagicMock(name = 'file_mock')
+ file_mock.__enter__.return_value = file_mock
+ file_mock.read.return_value = 'content'
+ open_mock.return_value = file_mock
+
+ with Environment("/base") as env:
+ static_file = StaticFile("relative/path/file")
+ content = static_file.get_content()
+
+ self.assertEqual('content', content)
+ self.assertEqual(file_mock.read.call_count, 1)
+ self.assertEqual(join_mock.call_count, 1)
+ join_mock.assert_called_with('/base', 'files', 'relative/path/file')
+ self.assertEqual(open_mock.call_count, 1)
+
+ @patch.object(os, "makedirs")
+ @patch.object(os.path, "exists")
+ def test_download_source_init_existent_download_directory(self, exists_mock, makedirs_mock):
+ """
+ Testing DownloadSource without cache with existent download directory
+ """
+ exists_mock.return_value = True
+
+ with Environment("/base") as env:
+ static_file = DownloadSource("http://download/source")
+
+ self.assertEqual(makedirs_mock.call_count, 0)
+ self.assertEqual(exists_mock.call_count, 1)
+ pass
+
+
+ @patch.object(os, "makedirs")
+ @patch.object(os.path, "exists")
+ def test_download_source_init_nonexistent_download_directory(self, exists_mock, makedirs_mock):
+ """
+ Testing DownloadSource without cache with non-existent download directory
+ """
+ exists_mock.return_value = False
+
+ with Environment("/base") as env:
+ static_file = DownloadSource("http://download/source")
+
+ self.assertEqual(makedirs_mock.call_count, 1)
+ makedirs_mock.assert_called_with("/var/tmp/downloads")
+ self.assertEqual(exists_mock.call_count, 1)
+ pass
+
+ @patch.object(urllib2, "urlopen")
+ @patch.object(os, "makedirs")
+ @patch.object(os.path, "exists")
+ def test_download_source_get_content_nocache(self, exists_mock, makedirs_mock, urlopen_mock):
+ """
+ Testing DownloadSource.get_content without cache
+ """
+ exists_mock.return_value = True
+ web_file_mock = MagicMock()
+ web_file_mock.read.return_value = 'web_content'
+ urlopen_mock.return_value = web_file_mock
+
+ with Environment("/base") as env:
+ download_source = DownloadSource("http://download/source", cache=False)
+ content = download_source.get_content()
+
+ self.assertEqual('web_content', content)
+ self.assertEqual(urlopen_mock.call_count, 1)
+ urlopen_mock.assert_called_with('http://download/source')
+ self.assertEqual(web_file_mock.read.call_count, 1)
+
+ @patch.object(urllib2, "urlopen")
+ @patch.object(os, "makedirs")
+ @patch.object(os.path, "exists")
+ def test_download_source_get_content_cache_new(self, exists_mock, makedirs_mock, urlopen_mock):
+ """
+ Testing DownloadSource.get_content with cache on non-cached resource
+ """
+ exists_mock.side_effect = [True, False]
+ web_file_mock = MagicMock()
+ web_file_mock.read.return_value = 'web_content'
+ urlopen_mock.return_value = web_file_mock
+
+ with Environment("/base") as env:
+ download_source = DownloadSource("http://download/source", cache=True)
+ content = download_source.get_content()
+
+ self.assertEqual('web_content', content)
+ self.assertEqual(urlopen_mock.call_count, 1)
+ urlopen_mock.assert_called_with('http://download/source')
+ self.assertEqual(web_file_mock.read.call_count, 1)
+
+ @patch("__builtin__.open")
+ @patch.object(os, "makedirs")
+ @patch.object(os.path, "exists")
+ def test_download_source_get_content_cache_existent(self, exists_mock, makedirs_mock, open_mock):
+ """
+ Testing DownloadSource.get_content with cache on cached resource
+ """
+ exists_mock.side_effect = [True, True, False]
+
+ file_mock = MagicMock(name = 'file_mock')
+ file_mock.__enter__.return_value = file_mock
+ file_mock.read.return_value = 'cached_content'
+ open_mock.return_value = file_mock
+
+
+ with Environment("/base") as env:
+ download_source = DownloadSource("http://download/source", cache=True)
+ content = download_source.get_content()
+
+ self.assertEqual('cached_content', content)
+ self.assertEqual(open_mock.call_count, 1)
+ open_mock.assert_called_with('/var/tmp/downloads/source')
+ self.assertEqual(file_mock.read.call_count, 1)
+
+ @patch.object(urllib2, "urlopen")
+ @patch("__builtin__.open")
+ @patch.object(os, "makedirs")
+ @patch.object(os.path, "exists")
+ def test_download_source_get_content_cache_existent_md5_match(self, exists_mock, makedirs_mock, open_mock,
+ urlopen_mock):
+ """
+ Testing DownloadSource.get_content with cache on cached resource with md5 check
+ """
+ exists_mock.side_effect = [True, True, False]
+
+ file_mock = MagicMock(name = 'file_mock')
+ file_mock.__enter__.return_value = file_mock
+ file_mock.read.return_value = 'cached_content'
+ open_mock.return_value = file_mock
+
+
+ with Environment("/base") as env:
+ download_source = DownloadSource("http://download/source", cache=True)
+ content = download_source.get_content()
+
+ self.assertEqual('cached_content', content)
+ self.assertEqual(open_mock.call_count, 1)
+ open_mock.assert_called_with('/var/tmp/downloads/source')
+ self.assertEqual(file_mock.read.call_count, 1)
+ self.assertEqual(urlopen_mock.call_count, 0)
+
+ @patch.object(urllib2, "urlopen")
+ @patch("__builtin__.open")
+ @patch.object(os, "makedirs")
+ @patch.object(os.path, "exists")
+ def test_download_source_get_content_cache_existent_md5_unmatch(self, exists_mock, makedirs_mock, open_mock,
+ urlopen_mock):
+ """
+ Testing DownloadSource.get_content with cache on cached resource with md5 check
+ """
+ exists_mock.side_effect = [True, True, False]
+ fake_md5 = "144c9defac04969c7bfad8efaa8ea194"
+ file_mock = MagicMock(name = 'file_mock')
+ file_mock.__enter__.return_value = file_mock
+ file_mock.read.return_value = 'cached_content'
+ open_mock.return_value = file_mock
+ web_file_mock = MagicMock()
+ web_file_mock.read.return_value = 'web_content'
+ urlopen_mock.return_value = web_file_mock
+
+ with Environment("/base") as env:
+ download_source = DownloadSource("http://download/source", cache=True, md5sum=fake_md5)
+ content = download_source.get_content()
+
+ self.assertEqual('web_content', content)
+ self.assertEqual(open_mock.call_count, 2)
+ open_mock.assert_once_called('/var/tmp/downloads/source', 'w')
+ open_mock.assert_once_called('/var/tmp/downloads/source')
+ self.assertEqual(file_mock.read.call_count, 1)
+ self.assertEqual(urlopen_mock.call_count, 1)
+ urlopen_mock.assert_called_with('http://download/source')
+
+ @patch("__builtin__.open")
+ @patch.object(os.path, "getmtime")
+ @patch.object(os.path, "exists")
+ def test_template_loader(self, exists_mock, getmtime_mock, open_mock):
+ """
+ Testing template loader on existent file
+ """
+ exists_mock.return_value = True
+ getmtime_mock.return_value = 10
+ file_mock = MagicMock(name = 'file_mock')
+ file_mock.__enter__.return_value = file_mock
+ file_mock.read.return_value = 'template content'
+ open_mock.return_value = file_mock
+
+ with Environment("/base") as env:
+ template = Template("test.j2")
+
+ self.assertEqual(open_mock.call_count, 1)
+ open_mock.assert_called_with('/base/templates/test.j2', 'rb')
+ self.assertEqual(getmtime_mock.call_count, 1)
+ getmtime_mock.assert_called_with('/base/templates/test.j2')
+
+ @patch.object(os.path, "exists")
+ def test_template_loader_fail(self, exists_mock):
+ """
+ Testing template loader on non-existent file
+ """
+ exists_mock.return_value = False
+
+ try:
+ with Environment("/base") as env:
+ template = Template("test.j2")
+ self.fail("Template should fail with nonexistent template file")
+ except TemplateNotFound:
+ pass
+
+
+
+ @patch("__builtin__.open")
+ @patch.object(os.path, "getmtime")
+ @patch.object(os.path, "exists")
+ def test_template_loader_absolute_path(self, exists_mock, getmtime_mock, open_mock):
+ """
+ Testing template loader with absolute file-path
+ """
+ exists_mock.return_value = True
+ getmtime_mock.return_value = 10
+ file_mock = MagicMock(name = 'file_mock')
+ file_mock.__enter__.return_value = file_mock
+ file_mock.read.return_value = 'template content'
+ open_mock.return_value = file_mock
+
+ with Environment("/base") as env:
+ template = Template("/absolute/path/test.j2")
+
+ self.assertEqual(open_mock.call_count, 1)
+ open_mock.assert_called_with('/absolute/path/test.j2', 'rb')
+ self.assertEqual(getmtime_mock.call_count, 1)
+ getmtime_mock.assert_called_with('/absolute/path/test.j2')
+
+ @patch("__builtin__.open")
+ @patch.object(os.path, "getmtime")
+ @patch.object(os.path, "exists")
+ def test_template_loader_arguments(self, exists_mock, getmtime_mock, open_mock):
+ """
+ Testing template loader additional arguments in template and absolute file-path
+ """
+ exists_mock.return_value = True
+ getmtime_mock.return_value = 10
+ file_mock = MagicMock(name = 'file_mock')
+ file_mock.__enter__.return_value = file_mock
+ file_mock.read.return_value = '{{test_arg1}} template content'
+ open_mock.return_value = file_mock
+
+ with Environment("/base") as env:
+ template = Template("/absolute/path/test.j2", [], test_arg1 = "test")
+ content = template.get_content()
+ self.assertEqual(open_mock.call_count, 1)
+
+ self.assertEqual(u'test template content\n', content)
+ open_mock.assert_called_with('/absolute/path/test.j2', 'rb')
+ self.assertEqual(getmtime_mock.call_count, 1)
+ getmtime_mock.assert_called_with('/absolute/path/test.j2')
+
+ def test_inline_template(self):
+ """
+ Testing InlineTemplate
+ """
+ with Environment("/base") as env:
+ template = InlineTemplate("{{test_arg1}} template content", [], test_arg1 = "test")
+ content = template.get_content()
+
+ self.assertEqual(u'test template content\n', content)
+
+ def test_template_imports(self):
+ """
+ Testing Template additional imports
+ """
+ try:
+ with Environment("/base") as env:
+ template = InlineTemplate("{{test_arg1}} template content {{os.path.join(path[0],path[1])}}", [], test_arg1 = "test", path = ["/one","two"])
+ content = template.get_content()
+ self.fail("Template.get_content should fail when evaluating unknown import")
+ except UndefinedError:
+ pass
+ with Environment("/base") as env:
+ template = InlineTemplate("{{test_arg1}} template content {{os.path.join(path[0],path[1])}}", [os], test_arg1 = "test", path = ["/one","two"])
+ content = template.get_content()
+ self.assertEqual(u'test template content /one/two\n', content)
diff --git a/slider-agent/src/test/python/resource_management/TestCopyFromLocal.py b/slider-agent/src/test/python/resource_management/TestCopyFromLocal.py
new file mode 100644
index 0000000..7653b24
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestCopyFromLocal.py
@@ -0,0 +1,65 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch
+from resource_management import *
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestCopyFromLocal(TestCase):
+
+ @patch("resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider")
+ def test_run_default_args(self, execute_hadoop_mock):
+ with Environment() as env:
+ CopyFromLocal('/user/testdir/*.files',
+ owner='user1',
+ dest_dir='/apps/test/',
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
+ )
+ self.assertEqual(execute_hadoop_mock.call_count, 2)
+ call_arg_list = execute_hadoop_mock.call_args_list
+ self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
+ call_arg_list[0][0][0].command)
+ self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+ call_arg_list[0][0][0].arguments)
+ self.assertEquals('fs -chown user1 /apps/test/*.files', call_arg_list[1][0][0].command)
+ self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
+
+
+ @patch("resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider")
+ def test_run_with_chmod(self, execute_hadoop_mock):
+ with Environment() as env:
+ CopyFromLocal('/user/testdir/*.files',
+ mode=0655,
+ owner='user1',
+ group='hdfs',
+ dest_dir='/apps/test/',
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
+ )
+ self.assertEqual(execute_hadoop_mock.call_count, 3)
+ call_arg_list = execute_hadoop_mock.call_args_list
+ self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
+ call_arg_list[0][0][0].command)
+ self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+ call_arg_list[0][0][0].arguments)
+ self.assertEquals('fs -chown user1:hdfs /apps/test/*.files', call_arg_list[1][0][0].command)
+ self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
+
+
diff --git a/slider-agent/src/test/python/resource_management/TestDirectoryResource.py b/slider-agent/src/test/python/resource_management/TestDirectoryResource.py
new file mode 100644
index 0000000..866486e
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestDirectoryResource.py
@@ -0,0 +1,176 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+import os
+import shutil
+from resource_management.core.system import System
+from resource_management.core import Environment, Fail
+from resource_management.core.resources import Directory
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestFileResource(TestCase):
+
+ @patch.object(os.path, "exists")
+ @patch.object(os, "makedirs")
+ @patch.object(os.path, "isdir")
+ @patch.object(os, "stat")
+ @patch.object(os,"chmod")
+ @patch.object(os,"chown")
+ @patch("resource_management.core.providers.system._coerce_uid")
+ @patch("resource_management.core.providers.system._coerce_gid")
+ def test_create_directory_recursive(self, _coerce_gid_mock, _coerce_uid_mock,
+ os_chown_mock, os_chmod_mock, os_stat_mock,
+ isdir_mock, os_makedirs_mock,
+ os_path_exists_mock):
+ os_path_exists_mock.return_value = False
+ isdir_mock.return_value = True
+ _coerce_uid_mock.return_value = 66
+ _coerce_gid_mock.return_value = 77
+ os_stat_mock.return_value = type("", (), dict(st_mode=0755, st_uid=0, st_gid=0))()
+
+ with Environment('/') as env:
+ Directory('/a/b/c/d',
+ action='create',
+ mode=0777,
+ owner="hdfs",
+ group="hadoop",
+ recursive=True
+ )
+
+ os_makedirs_mock.assert_called_with('/a/b/c/d', 0777)
+ os_chmod_mock.assert_called_with('/a/b/c/d', 0777)
+ os_chown_mock.assert_any_call('/a/b/c/d', 66, -1)
+ os_chown_mock.assert_any_call('/a/b/c/d', -1, 77)
+
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "dirname")
+ @patch.object(os.path, "isdir")
+ @patch.object(os, "mkdir")
+ @patch.object(os, "stat")
+ @patch.object(os,"chmod")
+ @patch.object(os,"chown")
+ @patch("resource_management.core.providers.system._coerce_uid")
+ @patch("resource_management.core.providers.system._coerce_gid")
+ def test_create_directory_not_recursive(self, _coerce_gid_mock, _coerce_uid_mock,
+ os_chown_mock, os_chmod_mock, os_stat_mock,
+ mkdir_mock, isdir_mock, os_dirname_mock,
+ os_path_exists_mock):
+ os_path_exists_mock.return_value = False
+ os_dirname_mock.return_value = "/a/b/c"
+ isdir_mock.return_value = True
+ _coerce_uid_mock.return_value = 66
+ _coerce_gid_mock.return_value = 77
+ os_stat_mock.return_value = type("", (), dict(st_mode=0755, st_uid=0, st_gid=0))()
+
+ with Environment('/') as env:
+ Directory('/a/b/c/d',
+ action='create',
+ mode=0777,
+ owner="hdfs",
+ group="hadoop"
+ )
+
+ mkdir_mock.assert_called_with('/a/b/c/d', 0777)
+ os_chmod_mock.assert_called_with('/a/b/c/d', 0777)
+ os_chown_mock.assert_any_call('/a/b/c/d', 66, -1)
+ os_chown_mock.assert_any_call('/a/b/c/d', -1, 77)
+
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "dirname")
+ @patch.object(os.path, "isdir")
+ def test_create_directory_failed_no_parent(self, isdir_mock, os_dirname_mock,
+ os_path_exists_mock):
+ os_path_exists_mock.return_value = False
+ os_dirname_mock.return_value = "/a/b/c"
+ isdir_mock.return_value = False
+
+
+ try:
+ with Environment('/') as env:
+ Directory('/a/b/c/d',
+ action='create',
+ mode=0777,
+ owner="hdfs",
+ group="hadoop"
+ )
+ self.fail("Must fail because parent directory /a/b/c doesn't exist")
+ except Fail as e:
+ self.assertEqual("Applying Directory['/a/b/c/d'] failed, parent directory /a/b/c doesn't exist",
+ str(e))
+
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ def test_create_directory_path_is_file_or_line(self, isdir_mock, os_path_exists_mock):
+ os_path_exists_mock.return_value = True
+ isdir_mock.return_value = False
+
+ try:
+ with Environment('/') as env:
+ Directory('/a/b/c/d',
+ action='create',
+ mode=0777,
+ owner="hdfs",
+ group="hadoop"
+ )
+ self.fail("Must fail because file /a/b/c/d already exists")
+ except Fail as e:
+ self.assertEqual("Applying Directory['/a/b/c/d'] failed, file /a/b/c/d already exists",
+ str(e))
+
+ @patch.object(shutil, "rmtree")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ def test_delete_directory(self, isdir_mock, os_path_exists_mock, rmtree_mock):
+ os_path_exists_mock.return_value = True
+ isdir_mock.return_value = True
+
+ with Environment('/') as env:
+ Directory('/a/b/c/d',
+ action='delete'
+ )
+
+ rmtree_mock.assert_called_with('/a/b/c/d')
+
+ @patch.object(os.path, "exists")
+ def test_delete_noexisting_directory(self, os_path_exists_mock):
+ os_path_exists_mock.return_value = False
+
+ with Environment('/') as env:
+ Directory('/a/b/c/d',
+ action='delete'
+ )
+
+ @patch.object(shutil, "rmtree")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ def test_delete_directory_with_path_to_file(self, isdir_mock, os_path_exists_mock, rmtree_mock):
+ os_path_exists_mock.return_value = True
+ isdir_mock.return_value = False
+
+ try:
+ with Environment('/') as env:
+ Directory('/a/b/c/d',
+ action='delete'
+ )
+ self.fail("Must fail because /a/b/c/d is not a directory")
+ except Fail as e:
+ self.assertEqual("Applying Directory['/a/b/c/d'] failed, /a/b/c/d is not a directory",
+ str(e))
\ No newline at end of file
diff --git a/slider-agent/src/test/python/resource_management/TestExecuteHadoopResource.py b/slider-agent/src/test/python/resource_management/TestExecuteHadoopResource.py
new file mode 100644
index 0000000..e357390
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestExecuteHadoopResource.py
@@ -0,0 +1,207 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch
+from resource_management import *
+from resource_management.libraries.resources.execute_hadoop\
+ import ExecuteHadoop
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestExecuteHadoopResource(TestCase):
+ @patch("resource_management.core.providers.system.ExecuteProvider")
+ def test_run_default_args(self, execute_mock):
+ '''
+ Test if default arguments are passed to Execute
+ '''
+ with Environment() as env:
+ ExecuteHadoop("command",
+ conf_dir="conf_dir",
+ user="user",
+ logoutput=True,
+ )
+ self.assertEqual(execute_mock.call_count, 1)
+ self.assertEqual(execute_mock.call_args[0][0].command,'hadoop --config conf_dir command')
+ self.assertEqual(execute_mock.call_args[0][0].arguments,
+ {'logoutput': True, 'tries': 1, 'user': 'user', 'try_sleep': 0})
+
+
+ @patch("resource_management.core.providers.system.ExecuteProvider")
+ def test_run_unknown_conf(self, execute_mock):
+ '''
+ Test when UnknownConfiguration passed
+ '''
+ with Environment() as env:
+ ExecuteHadoop("command",
+ kinit_path_local=UnknownConfiguration(name="kinit_path_local"),
+ conf_dir="conf_dir",
+ user="user",
+ keytab=UnknownConfiguration(name="keytab"),
+ security_enabled=False,
+ principal=UnknownConfiguration(name="principal")
+ )
+ self.assertEqual(execute_mock.call_count, 1)
+ self.assertEqual(execute_mock.call_args[0][0].command,'hadoop --config conf_dir command')
+ self.assertEqual(execute_mock.call_args[0][0].arguments,
+ {'logoutput': False, 'tries': 1, 'user': 'user', 'try_sleep': 0})
+
+
+ @patch("resource_management.core.providers.system.ExecuteProvider")
+ def test_run_defined_args(self, execute_mock):
+ '''
+ Test if defined arguments are passed to Execute
+ '''
+ with Environment("/") as env:
+ ExecuteHadoop("command",
+ action="run",
+ kinit_path_local="path",
+ conf_dir="conf_dir",
+ user="user",
+ tries=2,
+ keytab="keytab",
+ security_enabled=False,
+ kinit_override=False,
+ try_sleep=2,
+ logoutput=True,
+ principal="principal"
+ )
+ self.assertEqual(execute_mock.call_count, 1)
+ self.assertEqual(execute_mock.call_args[0][0].command,'hadoop --config conf_dir command')
+ self.assertEqual(execute_mock.call_args[0][0].arguments,
+ {'logoutput': True, 'tries': 2, 'user': 'user', 'try_sleep': 2})
+
+
+ @patch("resource_management.core.providers.system.ExecuteProvider")
+ def test_run_command_list(self, execute_mock):
+ '''
+ Test for "command" passed as List
+ '''
+ with Environment("/") as env:
+ ExecuteHadoop(["command1","command2"],
+ action="run",
+ kinit_path_local="path",
+ conf_dir="conf_dir",
+ user="user",
+ keytab="keytab"
+ )
+ self.assertEqual(execute_mock.call_count, 2)
+ self.assertEqual(execute_mock.call_args_list[0][0][0].command,
+ 'hadoop --config conf_dir command1')
+ self.assertEqual(execute_mock.call_args_list[1][0][0].command,
+ 'hadoop --config conf_dir command2')
+ self.assertEqual(execute_mock.call_args_list[0][0][0].arguments,
+ {'logoutput': False, 'tries': 1, 'user': 'user', 'try_sleep': 0})
+ self.assertEqual(execute_mock.call_args_list[1][0][0].arguments,
+ {'logoutput': False, 'tries': 1, 'user': 'user', 'try_sleep': 0})
+
+
+ @patch("resource_management.core.providers.system.ExecuteProvider")
+ def test_run_command_tuple(self, execute_mock):
+ '''
+ Test for "command" passed as Tuple
+ '''
+ with Environment("/") as env:
+ ExecuteHadoop(("command1","command2","command3"),
+ action="run",
+ kinit_path_local="path",
+ conf_dir="conf_dir",
+ user="user",
+ keytab="keytab"
+ )
+ self.assertEqual(execute_mock.call_count, 1)
+ self.assertEqual(execute_mock.call_args[0][0].command,
+ 'hadoop --config conf_dir command1 command2 command3')
+
+
+ @patch("resource_management.core.providers.system.ExecuteProvider")
+ def test_run_secured(self, execute_mock):
+ '''
+ Test security_enabled=True behaviour
+ '''
+ with Environment("/") as env:
+ ExecuteHadoop("command",
+ action="run",
+ kinit_path_local="path",
+ conf_dir="conf_dir",
+ user="user",
+ tries=1,
+ keytab="keytab",
+ security_enabled=True,
+ kinit_override=False,
+ try_sleep=0,
+ logoutput=True
+ )
+ self.assertEqual(execute_mock.call_count, 2)
+ self.assertEqual(str(execute_mock.call_args_list[0][0][0]),
+ "Execute['path -kt keytab user']")
+ self.assertEqual(execute_mock.call_args_list[0][0][0].command,
+ 'path -kt keytab user')
+ self.assertEqual(execute_mock.call_args_list[0][0][0].arguments,
+ {'path': ['/bin'], 'user': 'user'})
+ self.assertEqual(execute_mock.call_args_list[1][0][0].command,
+ 'hadoop --config conf_dir command')
+ self.assertEqual(execute_mock.call_args_list[1][0][0].arguments,
+ {'logoutput': True, 'tries': 1, 'user': 'user', 'try_sleep': 0})
+
+
+ @patch("resource_management.core.providers.system.ExecuteProvider")
+ def test_run_secured_kinit_override(self, execute_mock):
+ '''
+ Test security_enabled=True and kinit_override=True behaviour
+ '''
+ with Environment("/") as env:
+ ExecuteHadoop("command",
+ action="run",
+ kinit_path_local="path",
+ conf_dir="conf_dir",
+ user="user",
+ tries=1,
+ keytab="keytab",
+ security_enabled=True,
+ kinit_override=True,
+ try_sleep=0,
+ logoutput=True
+ )
+ self.assertEqual(execute_mock.call_count, 1)
+ self.assertEqual(execute_mock.call_args_list[0][0][0].command,
+ 'hadoop --config conf_dir command')
+
+
+ @patch("resource_management.core.providers.system.ExecuteProvider")
+ def test_run_secured_principal(self, execute_mock):
+ '''
+ Test with "principal" argument
+ '''
+ with Environment("/") as env:
+ ExecuteHadoop("command",
+ action="run",
+ kinit_path_local="path",
+ conf_dir="conf_dir",
+ user="user",
+ tries=1,
+ keytab="keytab",
+ security_enabled=True,
+ kinit_override=False,
+ try_sleep=0,
+ logoutput=True,
+ principal="principal")
+ self.assertEqual(execute_mock.call_count, 2)
+ self.assertEqual(execute_mock.call_args_list[0][0][0].command,
+ 'path -kt keytab principal')
+ self.assertEqual(execute_mock.call_args_list[1][0][0].command,
+ 'hadoop --config conf_dir command')
\ No newline at end of file
diff --git a/slider-agent/src/test/python/resource_management/TestExecuteResource.py b/slider-agent/src/test/python/resource_management/TestExecuteResource.py
new file mode 100644
index 0000000..f0a4539
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestExecuteResource.py
@@ -0,0 +1,224 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock, call
+
+from resource_management.core.system import System
+from resource_management.core.resources.system import Execute
+from resource_management.core.environment import Environment
+
+import subprocess
+import logging
+import os
+from resource_management import Fail
+import grp
+import pwd
+
+
+@patch.object(System, "os_family", new='redhat')
+class TestExecuteResource(TestCase):
+ @patch.object(logging.Logger, "info")
+ @patch.object(subprocess, "Popen")
+ def test_attribute_logoutput(self, popen_mock, info_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ subproc_mock.communicate.side_effect = [["1"], ["2"]]
+ popen_mock.return_value = subproc_mock
+
+ with Environment("/") as env:
+ Execute('echo "1"',
+ logoutput=True)
+ Execute('echo "2"',
+ logoutput=False)
+
+ info_mock.assert_called('1')
+ self.assertTrue("call('2')" not in str(info_mock.mock_calls))
+
+ @patch('subprocess.Popen.communicate')
+ @patch('subprocess.Popen')
+ def test_attribute_wait(self, popen_mock, proc_communicate_mock):
+ with Environment("/") as env:
+ Execute('echo "1"',
+ wait_for_finish=False)
+ Execute('echo "2"',
+ wait_for_finish=False)
+
+ self.assertTrue(popen_mock.called, 'subprocess.Popen should have been called!')
+ self.assertFalse(proc_communicate_mock.called, 'proc.communicate should not have been called!')
+
+ @patch.object(os.path, "exists")
+ @patch.object(subprocess, "Popen")
+ def test_attribute_creates(self, popen_mock, exists_mock):
+ exists_mock.return_value = True
+
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ subproc_mock.communicate.side_effect = [["1"]]
+ popen_mock.return_value = subproc_mock
+
+ with Environment("/") as env:
+ Execute('echo "1"',
+ creates="/must/be/created")
+
+ exists_mock.assert_called_with("/must/be/created")
+ self.assertEqual(subproc_mock.call_count, 0)
+
+ @patch.object(subprocess, "Popen")
+ def test_attribute_path(self, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ subproc_mock.communicate.side_effect = [["1"]]
+ popen_mock.return_value = subproc_mock
+
+ with Environment("/") as env:
+ execute_resource = Execute('echo "1"',
+ path=["/test/one", "test/two"]
+ )
+ self.assertEqual(execute_resource.environment["PATH"], '/test/one:test/two')
+
+ @patch('time.sleep')
+ @patch.object(subprocess, "Popen")
+ def test_attribute_try_sleep_tries(self, popen_mock, time_mock):
+ expected_call = "call('Retrying after %d seconds. Reason: %s', 1, 'Fail')"
+
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ subproc_mock.communicate.side_effect = [Fail("Fail"), ["1"]]
+ popen_mock.return_value = subproc_mock
+
+ with Environment("/") as env:
+ Execute('echo "1"',
+ tries=2,
+ try_sleep=10
+ )
+ pass
+
+ time_mock.assert_called_once_with(10)
+
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_group(self, getpwnam_mock):
+ def error(argument):
+ self.assertEqual(argument, "test_user")
+ raise KeyError("fail")
+
+ getpwnam_mock.side_effect = error
+ try:
+ with Environment("/") as env:
+ Execute('echo "1"',
+ user="test_user",
+ )
+ except Fail as e:
+ pass
+
+ @patch.object(grp, "getgrnam")
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_group(self, getpwnam_mock, getgrnam_mock):
+ def error(argument):
+ self.assertEqual(argument, "test_group")
+ raise KeyError("fail")
+
+ getpwnam_mock.side_effect = 1
+ getgrnam_mock.side_effect = error
+ try:
+ with Environment("/") as env:
+ Execute('echo "1"',
+ group="test_group",
+ )
+ except Fail as e:
+ pass
+
+ @patch.object(subprocess, "Popen")
+ def test_attribute_environment(self, popen_mock):
+ expected_dict = {"JAVA_HOME": "/test/java/home"}
+
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ subproc_mock.communicate.side_effect = [["1"]]
+ popen_mock.return_value = subproc_mock
+
+ with Environment("/") as env:
+ Execute('echo "1"',
+ environment=expected_dict
+ )
+
+ self.assertEqual(popen_mock.call_args_list[0][1]["env"], expected_dict)
+ pass
+
+ @patch.object(subprocess, "Popen")
+ def test_attribute_cwd(self, popen_mock):
+ expected_cwd = "/test/work/directory"
+
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ subproc_mock.communicate.side_effect = [["1"]]
+ popen_mock.return_value = subproc_mock
+
+ with Environment("/") as env:
+ Execute('echo "1"',
+ cwd=expected_cwd
+ )
+
+ self.assertEqual(popen_mock.call_args_list[0][1]["cwd"], expected_cwd)
+
+ @patch.object(subprocess, "Popen")
+ def test_attribute_command_escaping(self, popen_mock):
+ expected_command0 = "arg1 arg2 'quoted arg'"
+ expected_command1 = "arg1 arg2 'command \"arg\"'"
+ expected_command2 = 'arg1 arg2 \'command \'"\'"\'arg\'"\'"\'\''
+ expected_command3 = "arg1 arg2 'echo `ls /root`'"
+ expected_command4 = "arg1 arg2 '$ROOT'"
+ expected_command5 = "arg1 arg2 '`ls /root`'"
+
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+
+ with Environment("/") as env:
+ Execute(('arg1', 'arg2', 'quoted arg'),
+ )
+ Execute(('arg1', 'arg2', 'command "arg"'),
+ )
+ Execute(('arg1', 'arg2', "command 'arg'"),
+ )
+ Execute(('arg1', 'arg2', "echo `ls /root`"),
+ )
+ Execute(('arg1', 'arg2', "$ROOT"),
+ )
+ Execute(('arg1', 'arg2', "`ls /root`"),
+ )
+
+ self.assertEqual(popen_mock.call_args_list[0][0][0][3], expected_command0)
+ self.assertEqual(popen_mock.call_args_list[1][0][0][3], expected_command1)
+ self.assertEqual(popen_mock.call_args_list[2][0][0][3], expected_command2)
+ self.assertEqual(popen_mock.call_args_list[3][0][0][3], expected_command3)
+ self.assertEqual(popen_mock.call_args_list[4][0][0][3], expected_command4)
+ self.assertEqual(popen_mock.call_args_list[5][0][0][3], expected_command5)
+
+ @patch.object(subprocess, "Popen")
+ def test_attribute_command_one_line(self, popen_mock):
+ expected_command = "rm -rf /somedir"
+
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+
+ with Environment("/") as env:
+ Execute(expected_command)
+
+ self.assertEqual(popen_mock.call_args_list[0][0][0][3], expected_command)
diff --git a/slider-agent/src/test/python/resource_management/TestFileResource.py b/slider-agent/src/test/python/resource_management/TestFileResource.py
new file mode 100644
index 0000000..7da0dbd
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestFileResource.py
@@ -0,0 +1,332 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+import os
+import sys
+from resource_management.core import Environment, Fail
+from resource_management.core.resources import File
+from resource_management.core.system import System
+import resource_management.core.providers.system
+import resource_management
+
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestFileResource(TestCase):
+ @patch.object(os.path, "dirname")
+ @patch.object(os.path, "isdir")
+ def test_action_create_dir_exist(self, isdir_mock, dirname_mock):
+ """
+ Tests if 'create' action fails when path is existent directory
+ """
+ isdir_mock.side_effect = [True, False]
+ try:
+ with Environment('/') as env:
+ File('/existent_directory',
+ action='create',
+ mode=0777,
+ content='file-content'
+ )
+
+ self.fail("Must fail when directory with name 'path' exist")
+ except Fail as e:
+ self.assertEqual("Applying File['/existent_directory'] failed, directory with name /existent_directory exists",
+ str(e))
+ self.assertFalse(dirname_mock.called)
+
+ @patch.object(os.path, "dirname")
+ @patch.object(os.path, "isdir")
+ def test_action_create_parent_dir_non_exist(self, isdir_mock, dirname_mock):
+ """
+ Tests if 'create' action fails when parent directory of path
+ doesn't exist
+ """
+ isdir_mock.side_effect = [False, False]
+ dirname_mock.return_value = "/non_existent_directory"
+ try:
+ with Environment('/') as env:
+ File('/non_existent_directory/file',
+ action='create',
+ mode=0777,
+ content='file-content'
+ )
+
+ self.fail('Must fail on non existent parent directory')
+ except Fail as e:
+ self.assertEqual(
+ "Applying File['/non_existent_directory/file'] failed, parent directory /non_existent_directory doesn't exist",
+ str(e))
+ self.assertTrue(dirname_mock.called)
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ def test_action_create_non_existent_file(self, isdir_mock, exists_mock, open_mock, ensure_mock):
+ """
+ Tests if 'create' action create new non existent file and write proper data
+ """
+ isdir_mock.side_effect = [False, True]
+ exists_mock.return_value = False
+ new_file = MagicMock()
+ open_mock.return_value = new_file
+ with Environment('/') as env:
+ File('/directory/file',
+ action='create',
+ mode=0777,
+ content='file-content'
+ )
+
+
+ open_mock.assert_called_with('/directory/file', 'wb')
+ new_file.__enter__().write.assert_called_with('file-content')
+ self.assertEqual(open_mock.call_count, 1)
+ ensure_mock.assert_called()
+
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ def test_action_create_replace(self, isdir_mock, exists_mock, open_mock, ensure_mock):
+ """
+ Tests if 'create' action rewrite existent file with new data
+ """
+ isdir_mock.side_effect = [False, True]
+ old_file, new_file = MagicMock(), MagicMock()
+ open_mock.side_effect = [old_file, new_file]
+ old_file.read.return_value = 'old-content'
+ exists_mock.return_value = True
+
+ with Environment('/') as env:
+ File('/directory/file',
+ action='create',
+ mode=0777,
+ backup=False,
+ content='new-content'
+ )
+
+
+ old_file.read.assert_called()
+ new_file.__enter__().write.assert_called_with('new-content')
+ ensure_mock.assert_called()
+ self.assertEqual(open_mock.call_count, 2)
+ open_mock.assert_any_call('/directory/file', 'rb')
+ open_mock.assert_any_call('/directory/file', 'wb')
+
+
+ @patch.object(os, "unlink")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ def test_action_delete_is_directory(self, isdir_mock, exist_mock, unlink_mock):
+ """
+ Tests if 'delete' action fails when path is directory
+ """
+ isdir_mock.return_value = True
+
+ try:
+ with Environment('/') as env:
+ File('/directory/file',
+ action='delete',
+ mode=0777,
+ backup=False,
+ content='new-content'
+ )
+
+ self.fail("Should fail when deleting directory")
+ except Fail:
+ pass
+
+ self.assertEqual(isdir_mock.call_count, 1)
+ self.assertEqual(exist_mock.call_count, 0)
+ self.assertEqual(unlink_mock.call_count, 0)
+
+ @patch.object(os, "unlink")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ def test_action_delete(self, isdir_mock, exist_mock, unlink_mock):
+ """
+ Tests if 'delete' action removes file
+ """
+ isdir_mock.return_value = False
+
+ with Environment('/') as env:
+ File('/directory/file',
+ action='delete',
+ mode=0777,
+ backup=False,
+ content='new-content'
+ )
+
+
+ self.assertEqual(isdir_mock.call_count, 1)
+ self.assertEqual(exist_mock.call_count, 1)
+ self.assertEqual(unlink_mock.call_count, 1)
+
+
+ @patch.object(os.path, "isdir")
+ def test_attribute_path(self, isdir_mock):
+ """
+ Tests 'path' attribute
+ """
+ isdir_mock.side_effect = [True, False]
+
+ try:
+ with Environment('/') as env:
+ File('/existent_directory',
+ action='create',
+ mode=0777,
+ content='file-content'
+ )
+
+ self.fail("Must fail when directory with name 'path' exist")
+ except Fail as e:
+ pass
+
+ isdir_mock.assert_called_with('/existent_directory')
+
+ @patch.object(resource_management.core.Environment, "backup_file")
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ def test_attribute_backup(self, isdir_mock, exists_mock, open_mock, ensure_mock, backup_file_mock):
+ """
+ Tests 'backup' attribute
+ """
+ isdir_mock.side_effect = [False, True, False, True]
+ open_mock.return_value = MagicMock()
+ exists_mock.return_value = True
+
+ with Environment('/') as env:
+ File('/directory/file',
+ action='create',
+ mode=0777,
+ backup=False,
+ content='new-content'
+ )
+
+
+ self.assertEqual(backup_file_mock.call_count, 0)
+
+ with Environment('/') as env:
+ File('/directory/file',
+ action='create',
+ mode=0777,
+ backup=True,
+ content='new-content'
+ )
+
+
+ self.assertEqual(backup_file_mock.call_count, 1)
+ backup_file_mock.assert_called_with('/directory/file')
+
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ def test_attribute_replace(self, isdir_mock, exists_mock, open_mock, ensure_mock):
+ """
+ Tests 'replace' attribute
+ """
+ isdir_mock.side_effect = [False, True]
+ old_file, new_file = MagicMock(), MagicMock()
+ open_mock.side_effect = [old_file, new_file]
+ old_file.read.return_value = 'old-content'
+ exists_mock.return_value = True
+
+ with Environment('/') as env:
+ File('/directory/file',
+ action='create',
+ mode=0777,
+ backup=False,
+ content='new-content',
+ replace=False
+ )
+
+
+ old_file.read.assert_called()
+ self.assertEqual(new_file.__enter__().write.call_count, 0)
+ ensure_mock.assert_called()
+ self.assertEqual(open_mock.call_count, 0)
+
+
+ @patch("resource_management.core.providers.system._coerce_uid")
+ @patch("resource_management.core.providers.system._coerce_gid")
+ @patch.object(os, "chown")
+ @patch.object(os, "chmod")
+ @patch.object(os, "stat")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ def test_ensure_metadata(self, isdir_mock, exists_mock, open_mock, stat_mock, chmod_mock, chown_mock, gid_mock,
+ uid_mock):
+ """
+ Tests if _ensure_metadata changes owner, usergroup and permissions of file to proper values
+ """
+ isdir_mock.side_effect = [False, True, False, True]
+ exists_mock.return_value = False
+
+ class stat():
+ def __init__(self):
+ self.st_mode = 0666
+ self.st_uid = 1
+ self.st_gid = 1
+
+ stat_mock.return_value = stat()
+ gid_mock.return_value = 0
+ uid_mock.return_value = 0
+
+ with Environment('/') as env:
+ File('/directory/file',
+ action='create',
+ mode=0777,
+ content='file-content',
+ owner='root',
+ group='hdfs'
+ )
+
+
+ open_mock.assert_called_with('/directory/file', 'wb')
+ self.assertEqual(open_mock.call_count, 1)
+ stat_mock.assert_called_with('/directory/file')
+ self.assertEqual(chmod_mock.call_count, 1)
+ self.assertEqual(chown_mock.call_count, 2)
+ gid_mock.assert_called_once_with('hdfs')
+ uid_mock.assert_called_once_with('root')
+
+ chmod_mock.reset_mock()
+ chown_mock.reset_mock()
+ gid_mock.return_value = 1
+ uid_mock.return_value = 1
+
+ with Environment('/') as env:
+ File('/directory/file',
+ action='create',
+ mode=0777,
+ content='file-content',
+ owner='root',
+ group='hdfs'
+ )
+
+
+ self.assertEqual(chmod_mock.call_count, 1)
+ self.assertEqual(chown_mock.call_count, 0)
diff --git a/slider-agent/src/test/python/resource_management/TestGroupResource.py b/slider-agent/src/test/python/resource_management/TestGroupResource.py
new file mode 100644
index 0000000..ac25073
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestGroupResource.py
@@ -0,0 +1,138 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+from resource_management.core import Environment, Fail
+from resource_management.core.resources import Group
+from resource_management.core.system import System
+
+import subprocess
+import grp
+
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestGroupResource(TestCase):
+
+ @patch.object(grp, "getgrnam")
+ @patch.object(subprocess, "Popen")
+ def test_action_create_nonexistent(self, popen_mock, getgrnam_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getgrnam_mock.side_effect = KeyError()
+
+ with Environment('/') as env:
+ Group('hadoop',
+ action='create',
+ password='secure'
+ )
+
+
+ self.assertEqual(popen_mock.call_count, 1)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupadd -p secure hadoop'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ getgrnam_mock.assert_called_with('hadoop')
+
+
+ @patch.object(grp, "getgrnam")
+ @patch.object(subprocess, "Popen")
+ def test_action_create_existent(self, popen_mock, getgrnam_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getgrnam_mock.return_value = "mapred"
+
+ with Environment('/') as env:
+ Group('mapred',
+ action='create',
+ gid=2,
+ password='secure'
+ )
+
+
+ self.assertEqual(popen_mock.call_count, 1)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupmod -p secure -g 2 mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ getgrnam_mock.assert_called_with('mapred')
+
+
+ @patch.object(grp, "getgrnam")
+ @patch.object(subprocess, "Popen")
+ def test_action_create_fail(self, popen_mock, getgrnam_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 1
+ popen_mock.return_value = subproc_mock
+ getgrnam_mock.return_value = "mapred"
+
+ try:
+ with Environment('/') as env:
+ Group('mapred',
+ action='create',
+ gid=2,
+ password='secure'
+ )
+
+ self.fail("Action 'create' should fail when checked_call fails")
+ except Fail:
+ pass
+ self.assertEqual(popen_mock.call_count, 1)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupmod -p secure -g 2 mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ getgrnam_mock.assert_called_with('mapred')
+
+
+ @patch.object(grp, "getgrnam")
+ @patch.object(subprocess, "Popen")
+ def test_action_remove(self, popen_mock, getgrnam_mock):
+
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getgrnam_mock.return_value = "mapred"
+
+ with Environment('/') as env:
+ Group('mapred',
+ action='remove'
+ )
+
+
+ self.assertEqual(popen_mock.call_count, 1)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ getgrnam_mock.assert_called_with('mapred')
+
+
+ @patch.object(grp, "getgrnam")
+ @patch.object(subprocess, "Popen")
+ def test_action_remove_fail(self, popen_mock, getgrnam_mock):
+
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 1
+ popen_mock.return_value = subproc_mock
+ getgrnam_mock.return_value = "mapred"
+
+ try:
+ with Environment('/') as env:
+ Group('mapred',
+ action='remove'
+ )
+
+ self.fail("Action 'delete' should fail when checked_call fails")
+ except Fail:
+ pass
+
+ self.assertEqual(popen_mock.call_count, 1)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ getgrnam_mock.assert_called_with('mapred')
diff --git a/slider-agent/src/test/python/resource_management/TestLinkResource.py b/slider-agent/src/test/python/resource_management/TestLinkResource.py
new file mode 100644
index 0000000..87af645
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestLinkResource.py
@@ -0,0 +1,148 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+
+from resource_management.core import Environment, Fail
+from resource_management.core.system import System
+from resource_management.core.resources.system import Link
+
+import os
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestLinkResource(TestCase):
+
+ @patch.object(os.path, "realpath")
+ @patch.object(os.path, "lexists")
+ @patch.object(os.path, "islink")
+ @patch.object(os, "unlink")
+ @patch.object(os, "symlink")
+ def test_action_create_relink(self, symlink_mock, unlink_mock,
+ islink_mock, lexists_mock,
+ realmock):
+ lexists_mock.return_value = True
+ realmock.return_value = "/old_to_link_path"
+ islink_mock.return_value = True
+ with Environment('/') as env:
+ Link("/some_path",
+ to = "/a/b/link_to_path"
+ )
+
+ unlink_mock.assert_called_with("/some_path")
+ symlink_mock.assert_called_with("/a/b/link_to_path", "/some_path")
+
+ @patch.object(os.path, "realpath")
+ @patch.object(os.path, "lexists")
+ @patch.object(os.path, "islink")
+ def test_action_create_failed_due_to_file_exists(self, islink_mock,
+ lexists_mock, realmock):
+ lexists_mock.return_value = True
+ realmock.return_value = "/old_to_link_path"
+ islink_mock.return_value = False
+ with Environment('/') as env:
+ try:
+ Link("/some_path",
+ to = "/a/b/link_to_path"
+ )
+
+ self.fail("Must fail when directory or file with name /some_path exist")
+ except Fail as e:
+ self.assertEqual("LinkProvider[Link['/some_path']] trying to create a symlink with the same name as an existing file or directory",
+ str(e))
+
+ @patch.object(os.path, "lexists")
+ @patch.object(os, "symlink")
+ def test_action_create_symlink_clean_create(self, symlink_mock, lexists_mock):
+ lexists_mock.return_value = False
+
+ with Environment('/') as env:
+ Link("/some_path",
+ to = "/a/b/link_to_path"
+ )
+
+ symlink_mock.assert_called_with("/a/b/link_to_path", "/some_path")
+
+ @patch.object(os.path, "isdir")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "lexists")
+ @patch.object(os, "link")
+ def test_action_create_hardlink_clean_create(self, link_mock, lexists_mock,
+ exists_mock, isdir_mock):
+ lexists_mock.return_value = False
+ exists_mock.return_value = True
+ isdir_mock.return_value = False
+
+ with Environment('/') as env:
+ Link("/some_path",
+ hard = True,
+ to = "/a/b/link_to_path"
+ )
+
+ link_mock.assert_called_with("/a/b/link_to_path", "/some_path")
+
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "lexists")
+ def test_action_create_hardlink_target_doesnt_exist(self, lexists_mock,
+ exists_mock):
+ lexists_mock.return_value = False
+ exists_mock.return_value = False
+
+ with Environment('/') as env:
+ try:
+ Link("/some_path",
+ hard = True,
+ to = "/a/b/link_to_path"
+ )
+ self.fail("Must fail when target directory do doenst exist")
+ except Fail as e:
+ self.assertEqual("Failed to apply Link['/some_path'], linking to nonexistent location /a/b/link_to_path",
+ str(e))
+
+ @patch.object(os.path, "isdir")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "lexists")
+ def test_action_create_hardlink_target_is_dir(self, lexists_mock,
+ exists_mock, isdir_mock):
+ lexists_mock.return_value = False
+ exists_mock.return_value = True
+ isdir_mock = True
+
+ with Environment('/') as env:
+ try:
+ Link("/some_path",
+ hard = True,
+ to = "/a/b/link_to_path"
+ )
+ self.fail("Must fail when hardlinking to directory")
+ except Fail as e:
+ self.assertEqual("Failed to apply Link['/some_path'], cannot create hard link to a directory (/a/b/link_to_path)",
+ str(e))
+
+ @patch.object(os, "unlink")
+ @patch.object(os.path, "exists")
+ def test_action_delete(self, exists_mock, unlink_mock):
+ exists_mock.return_value = True
+
+ with Environment('/') as env:
+ Link("/some_path",
+ action = "delete"
+ )
+ unlink_mock.assert_called_with("/some_path")
+
+
diff --git a/slider-agent/src/test/python/resource_management/TestMonitorWebserverResource.py b/slider-agent/src/test/python/resource_management/TestMonitorWebserverResource.py
new file mode 100644
index 0000000..533ecaa
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestMonitorWebserverResource.py
@@ -0,0 +1,69 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+from resource_management import *
+from resource_management.libraries.providers.monitor_webserver\
+ import MonitorWebserverProvider
+from resource_management.libraries.resources.monitor_webserver\
+ import MonitorWebserver
+
+
+class TestMonitorWebserverResource(TestCase):
+ @patch.object(System, "os_family", new='redhat')
+ def test_setup_redhat(self):
+ with Environment(test_mode=True) as env:
+ MonitorWebserverProvider(MonitorWebserver("start")).action_start()
+ defined_resources = env.resource_list
+ expected_resources = "[MonitorWebserver['start'], " \
+ "Execute['grep -E 'KeepAlive (On|Off)' /etc/httpd/conf/httpd.conf" \
+ " && sed -i 's/KeepAlive Off/KeepAlive On/' /etc/httpd/conf/httpd.conf" \
+ " || echo 'KeepAlive On' >> /etc/httpd/conf/httpd.conf']," \
+ " Execute['/etc/init.d/httpd start']]"
+ self.assertEqual(str(defined_resources), expected_resources)
+
+ @patch.object(System, "os_family", new='suse')
+ def test_setup_suse(self):
+ with Environment(test_mode=True) as env:
+ MonitorWebserverProvider(MonitorWebserver("start")).action_start()
+ defined_resources = env.resource_list
+ expected_resources = "[MonitorWebserver['start'], " \
+ "Execute['grep -E 'KeepAlive (On|Off)' /etc/apache2/httpd.conf " \
+ "&& sed -i 's/KeepAlive Off/KeepAlive On/' /etc/apache2/httpd.conf " \
+ "|| echo 'KeepAlive On' >> /etc/apache2/httpd.conf']," \
+ " Execute['/etc/init.d/apache2 start']]"
+ self.assertEqual(str(defined_resources), expected_resources)
+
+ @patch.object(System, "os_family", new='redhat')
+ def test_stop_redhat(self):
+ with Environment(test_mode=True) as env:
+ MonitorWebserverProvider(MonitorWebserver("stop")).action_stop()
+ defined_resources = env.resource_list
+ expected_resources = "[MonitorWebserver['stop'], " \
+ "Execute['/etc/init.d/httpd stop']]"
+ self.assertEqual(str(defined_resources), expected_resources)
+
+ @patch.object(System, "os_family", new='suse')
+ def test_stop_suse(self):
+ with Environment(test_mode=True) as env:
+ MonitorWebserverProvider(MonitorWebserver("stop")).action_stop()
+ defined_resources = env.resource_list
+ expected_resources = "[MonitorWebserver['stop'], " \
+ "Execute['/etc/init.d/apache2 stop']]"
+ self.assertEqual(str(defined_resources), expected_resources)
diff --git a/slider-agent/src/test/python/resource_management/TestPackageResource.py b/slider-agent/src/test/python/resource_management/TestPackageResource.py
new file mode 100644
index 0000000..02aa3fb
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestPackageResource.py
@@ -0,0 +1,110 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+
+from resource_management.core import Environment, Fail
+from resource_management.core.system import System
+from resource_management.core.resources import Package
+
+from resource_management.core import shell
+
+class TestPackageResource(TestCase):
+
+ @patch.object(shell, "call")
+ @patch.object(shell, "checked_call")
+ @patch.object(System, "os_family", new = 'debian')
+ def test_action_install_debian(self, shell_mock, call_mock):
+ call_mock.return_value= (1, None)
+ with Environment('/') as env:
+ Package("some_package",
+ )
+ call_mock.assert_called_with('dpkg --get-selections some_package | grep -v deinstall')
+ shell_mock.assert_called_with("/usr/bin/apt-get --assume-yes install some_package")
+
+
+ @patch.object(shell, "call")
+ @patch.object(shell, "checked_call")
+ @patch.object(System, "os_family", new = 'redhat')
+ def test_action_install_rhel(self, shell_mock, call_mock):
+ call_mock.return_value= (1, None)
+ with Environment('/') as env:
+ Package("some_package",
+ )
+ call_mock.assert_called_with('rpm -q --quiet some_package')
+ shell_mock.assert_called_with("/usr/bin/yum -d 0 -e 0 -y install some_package")
+
+ @patch.object(shell, "call")
+ @patch.object(shell, "checked_call")
+ @patch.object(System, "os_family", new = 'suse')
+ def test_action_install_suse(self, shell_mock, call_mock):
+ call_mock.return_value= (1, None)
+ with Environment('/') as env:
+ Package("some_package",
+ )
+ call_mock.assert_called_with('rpm -q --quiet some_package')
+ shell_mock.assert_called_with("/usr/bin/zypper --quiet install --auto-agree-with-licenses --no-confirm some_package")
+
+ @patch.object(shell, "call", new = MagicMock(return_value=(0, None)))
+ @patch.object(shell, "checked_call")
+ @patch.object(System, "os_family", new = 'redhat')
+ def test_action_install_existent_rhel(self, shell_mock):
+ with Environment('/') as env:
+ Package("some_package",
+ )
+ self.assertFalse(shell_mock.mock_calls)
+
+ @patch.object(shell, "call", new = MagicMock(return_value=(0, None)))
+ @patch.object(shell, "checked_call")
+ @patch.object(System, "os_family", new = 'suse')
+ def test_action_install_existent_suse(self, shell_mock):
+ with Environment('/') as env:
+ Package("some_package",
+ )
+ self.assertFalse(shell_mock.mock_calls)
+
+ @patch.object(shell, "call", new = MagicMock(return_value=(0, None)))
+ @patch.object(shell, "checked_call")
+ @patch.object(System, "os_family", new = 'redhat')
+ def test_action_remove_rhel(self, shell_mock):
+ with Environment('/') as env:
+ Package("some_package",
+ action = "remove"
+ )
+ shell_mock.assert_called_with("/usr/bin/yum -d 0 -e 0 -y erase some_package")
+
+ @patch.object(shell, "call", new = MagicMock(return_value=(0, None)))
+ @patch.object(shell, "checked_call")
+ @patch.object(System, "os_family", new = 'suse')
+ def test_action_remove_suse(self, shell_mock):
+ with Environment('/') as env:
+ Package("some_package",
+ action = "remove"
+ )
+ shell_mock.assert_called_with("/usr/bin/zypper --quiet remove --no-confirm some_package")
+
+ @patch.object(shell, "call", new = MagicMock(return_value=(1, None)))
+ @patch.object(shell, "checked_call")
+ @patch.object(System, "os_family", new = 'redhat')
+ def test_action_install_version_attr(self, shell_mock):
+ with Environment('/') as env:
+ Package("some_package",
+ version = "3.5.0"
+ )
+ shell_mock.assert_called_with("/usr/bin/yum -d 0 -e 0 -y install some_package-3.5.0")
diff --git a/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py b/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py
new file mode 100644
index 0000000..79aef58
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+import os
+import time
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+from resource_management.core import Environment
+from resource_management.core.system import System
+from resource_management.libraries import PropertiesFile
+
+@patch.object(System, "os_family", new='redhat')
+class TestPropertiesFIleResource(TestCase):
+ """
+ PropertiesFile="resource_management.libraries.providers.properties_file.PropertiesFileProvider"
+ Testing PropertiesFile(PropertiesFileProvider) with different 'properties dictionary'
+ """
+
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ @patch.object(time, "asctime")
+ def test_action_create_empty_properties_without_dir(self,
+ time_asctime_mock,
+ os_path_isdir_mock,
+ os_path_exists_mock,
+ open_mock,
+ ensure_mock):
+ """
+ Tests if 'action_create' - creates new non existent file and write proper data
+ 1) properties={}
+ 2) dir=None
+ """
+ os_path_isdir_mock.side_effect = [False, True]
+ os_path_exists_mock.return_value = False
+ time_asctime_mock.return_value = 'Today is Wednesday'
+
+ result_file = MagicMock()
+ open_mock.return_value = result_file
+
+ with Environment('/') as env:
+ PropertiesFile('/somewhere_in_system/one_file.properties',
+ dir=None,
+ properties={}
+ )
+
+ open_mock.assert_called_with('/somewhere_in_system/one_file.properties', 'wb')
+ result_file.__enter__().write.assert_called_with( u'# Generated by Apache Slider. Today is Wednesday\n \n \n')
+ self.assertEqual(open_mock.call_count, 1)
+ ensure_mock.assert_called()
+
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ @patch.object(time, "asctime")
+ def test_action_create_empty_properties_with_dir(self,
+ time_asctime_mock,
+ os_path_isdir_mock,
+ os_path_exists_mock,
+ open_mock,
+ ensure_mock):
+ """
+ Tests if 'action_create' - creates new non existent file and write proper data
+ 1) properties={}
+ 2) dir='Some directory that exist '
+ """
+ os_path_isdir_mock.side_effect = [False, True]
+ os_path_exists_mock.return_value = False
+ time_asctime_mock.return_value = 'Some other day'
+
+ result_file = MagicMock()
+ open_mock.return_value = result_file
+
+ with Environment('/') as env:
+ PropertiesFile('file.txt',
+ dir="/dir/and/dir",
+ properties={},
+ )
+
+ open_mock.assert_called_with('/dir/and/dir/file.txt', 'wb')
+ result_file.__enter__().write.assert_called_with(u'# Generated by Apache Slider. Some other day\n \n \n')
+ self.assertEqual(open_mock.call_count, 1)
+ ensure_mock.assert_called()
+
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ @patch.object(time, "asctime")
+ def test_action_create_properties_simple(self,
+ time_asctime_mock,
+ os_path_isdir_mock,
+ os_path_exists_mock,
+ open_mock,
+ ensure_mock):
+ """
+ Tests if 'action_create' - creates new non existent file and write proper data
+ 1) properties={"Some property":"Some value"}
+ 2) dir=None
+ """
+
+ os_path_isdir_mock.side_effect = [False, True]
+ os_path_exists_mock.return_value = False
+ time_asctime_mock.return_value = 777
+
+ result_file = MagicMock()
+ open_mock.return_value = result_file
+
+ with Environment('/') as env:
+ PropertiesFile('/dir/new_file',
+ properties={'property1': 'value1'},
+ )
+
+ open_mock.assert_called_with('/dir/new_file',
+ 'wb')
+ result_file.__enter__().write.assert_called_with(u'# Generated by Apache Slider. 777\n \nproperty1=value1\n \n')
+ self.assertEqual(open_mock.call_count, 1)
+ ensure_mock.assert_called()
+
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ @patch.object(time, "asctime")
+ def test_action_create_properties_with_metacharacters(self,
+ time_asctime_mock,
+ os_path_isdir_mock,
+ os_path_exists_mock,
+ open_mock,
+ ensure_mock):
+ """
+ Tests if 'action_create' - creates new non existent file and write proper data
+ 1) properties={"":"", "Some property":"Metacharacters: -%{} ${a.a}/"}
+ 2) dir=None
+ """
+ os_path_isdir_mock.side_effect = [False, True]
+ os_path_exists_mock.return_value = False
+ time_asctime_mock.return_value = 777
+
+ result_file = MagicMock()
+ open_mock.return_value = result_file
+
+ with Environment('/') as env:
+ PropertiesFile('/dir/new_file',
+ properties={"": "",
+ "prop.1": "'.'yyyy-MM-dd-HH",
+ "prop.3": "%d{ISO8601} %5p %c{1}:%L - %m%n",
+ "prop.2": "INFO, openjpa",
+ "prop.4": "${oozie.log.dir}/oozie.log",
+ "prop.empty": "",
+ },
+ )
+
+ open_mock.assert_called_with('/dir/new_file','wb')
+ result_file.__enter__().write.assert_called_with(u"# Generated by Apache Slider. 777\n \n=\nprop.1='.'yyyy-MM-dd-HH\nprop.2=INFO, openjpa\nprop.3=%d{ISO8601} %5p %c{1}:%L - %m%n\nprop.4=${oozie.log.dir}/oozie.log\nprop.empty=\n \n")
+ self.assertEqual(open_mock.call_count, 1)
+ ensure_mock.assert_called()
+
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ @patch.object(time, "asctime")
+ def test_action_create_properties_rewrite_content(self,
+ time_asctime_mock,
+ os_path_isdir_mock,
+ os_path_exists_mock,
+ open_mock,
+ ensure_mock):
+ """
+ Tests if 'action_create' - rewrite file that exist
+ 1) properties={"Some property":"Some value"}
+ 2) dir="Some dir"
+ """
+ os_path_isdir_mock.side_effect = [False, True]
+ os_path_exists_mock.return_value = True
+ time_asctime_mock.return_value = 777
+
+ result_file = MagicMock()
+ result_file.read.return_value = 'old-content'
+ open_mock.return_value = result_file
+
+ with Environment('/') as env:
+ PropertiesFile('new_file',
+ dir='/dir1',
+ properties={'property_1': 'value1'},
+ )
+
+ result_file.read.assert_called()
+ open_mock.assert_called_with('/dir1/new_file', 'wb')
+ result_file.__enter__().write.assert_called_with(u'# Generated by Apache Slider. 777\n \nproperty_1=value1\n \n')
+ self.assertEqual(open_mock.call_count, 2)
+ ensure_mock.assert_called()
diff --git a/slider-agent/src/test/python/resource_management/TestRepositoryResource.py b/slider-agent/src/test/python/resource_management/TestRepositoryResource.py
new file mode 100644
index 0000000..1aaa111
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestRepositoryResource.py
@@ -0,0 +1,217 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import tempfile
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+
+from resource_management import *
+
+
+class TestRepositoryResource(TestCase):
+ @patch.object(System, "os_family", new='redhat')
+ @patch("resource_management.libraries.providers.repository.File")
+ def test_create_repo_redhat(self, file_mock):
+ with Environment('/') as env:
+ Repository('hadoop',
+ base_url='http://download.base_url.org/rpm/',
+ mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
+ repo_file_name='Repository')
+
+ self.assertTrue('hadoop' in env.resources['Repository'])
+ defined_arguments = env.resources['Repository']['hadoop'].arguments
+ expected_arguments = {'base_url': 'http://download.base_url.org/rpm/',
+ 'mirror_list': 'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
+ 'repo_file_name': 'Repository'}
+
+ self.assertEqual(defined_arguments, expected_arguments)
+ self.assertEqual(file_mock.call_args[0][0], '/etc/yum.repos.d/Repository.repo')
+
+ template_item = file_mock.call_args[1]['content']
+ template = str(template_item.name)
+ expected_arguments.update({'repo_id': 'hadoop'})
+
+ self.assertEqual(expected_arguments, template_item.context._dict)
+ self.assertEqual("""[{{repo_id}}]
+name={{repo_file_name}}
+{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}
+path=/
+enabled=1
+gpgcheck=0""", template)
+
+
+ @patch.object(System, "os_family", new='suse')
+ @patch("resource_management.libraries.providers.repository.File")
+ def test_create_repo_suse(self, file_mock):
+ with Environment('/') as env:
+ Repository('hadoop',
+ base_url='http://download.base_url.org/rpm/',
+ mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
+ repo_file_name='Repository')
+
+ self.assertTrue('hadoop' in env.resources['Repository'])
+ defined_arguments = env.resources['Repository']['hadoop'].arguments
+ expected_arguments = {'base_url': 'http://download.base_url.org/rpm/',
+ 'mirror_list': 'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
+ 'repo_file_name': 'Repository'}
+
+ self.assertEqual(defined_arguments, expected_arguments)
+ self.assertEqual(file_mock.call_args[0][0], '/etc/zypp/repos.d/Repository.repo')
+
+ template_item = file_mock.call_args[1]['content']
+ template = str(template_item.name)
+ expected_arguments.update({'repo_id': 'hadoop'})
+
+ self.assertEqual(expected_arguments, template_item.context._dict)
+ self.assertEqual("""[{{repo_id}}]
+name={{repo_file_name}}
+{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}
+path=/
+enabled=1
+gpgcheck=0""", template)
+
+
+ @patch.object(tempfile, "NamedTemporaryFile")
+ @patch("resource_management.libraries.providers.repository.Execute")
+ @patch("resource_management.libraries.providers.repository.File")
+ @patch("os.path.isfile", new=MagicMock(return_value=True))
+ @patch("filecmp.cmp", new=MagicMock(return_value=False))
+ @patch.object(System, "os_release_name", new='precise')
+ @patch.object(System, "os_family", new='debian')
+ def test_create_repo_debian_repo_exists(self, file_mock, execute_mock, tempfile_mock):
+ tempfile_mock.return_value = MagicMock(spec=file)
+ tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt"
+
+ with Environment('/') as env:
+ Repository('HDP',
+ base_url='http://download.base_url.org/rpm/',
+ repo_file_name='HDP',
+ components = ['a','b','c']
+ )
+
+ template_item = file_mock.call_args_list[0]
+ template_name = template_item[0][0]
+ template_content = template_item[1]['content'].get_content()
+
+ self.assertEquals(template_name, '/tmp/1.txt')
+ self.assertEquals(template_content, 'deb http://download.base_url.org/rpm/ precise a b c\n')
+
+ copy_item = str(file_mock.call_args_list[1])
+ self.assertEqual(copy_item, "call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))")
+
+ execute_command_item = execute_mock.call_args_list[0][0][0]
+ self.assertEqual(execute_command_item, 'apt-get update -o Dir::Etc::sourcelist="sources.list.d/HDP.list" -o APT::Get::List-Cleanup="0"')
+
+ @patch.object(tempfile, "NamedTemporaryFile")
+ @patch("resource_management.libraries.providers.repository.Execute")
+ @patch("resource_management.libraries.providers.repository.File")
+ @patch("os.path.isfile", new=MagicMock(return_value=True))
+ @patch("filecmp.cmp", new=MagicMock(return_value=True))
+ @patch.object(System, "os_release_name", new='precise')
+ @patch.object(System, "os_family", new='debian')
+ def test_create_repo_debian_doesnt_repo_exist(self, file_mock, execute_mock, tempfile_mock):
+ tempfile_mock.return_value = MagicMock(spec=file)
+ tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt"
+
+ with Environment('/') as env:
+ Repository('HDP',
+ base_url='http://download.base_url.org/rpm/',
+ repo_file_name='HDP',
+ components = ['a','b','c']
+ )
+
+ template_item = file_mock.call_args_list[0]
+ template_name = template_item[0][0]
+ template_content = template_item[1]['content'].get_content()
+
+ self.assertEquals(template_name, '/tmp/1.txt')
+ self.assertEquals(template_content, 'deb http://download.base_url.org/rpm/ precise a b c\n')
+
+ self.assertEqual(file_mock.call_count, 1)
+ self.assertEqual(execute_mock.call_count, 0)
+
+
+ @patch("os.path.isfile", new=MagicMock(return_value=True))
+ @patch.object(System, "os_family", new='debian')
+ @patch("resource_management.libraries.providers.repository.Execute")
+ @patch("resource_management.libraries.providers.repository.File")
+ def test_remove_repo_debian_repo_exist(self, file_mock, execute_mock):
+ with Environment('/') as env:
+ Repository('HDP',
+ action = "remove",
+ repo_file_name='HDP'
+ )
+
+ self.assertEqual(str(file_mock.call_args), "call('/etc/apt/sources.list.d/HDP.list', action='delete')")
+ self.assertEqual(execute_mock.call_args[0][0], 'apt-get update -o Dir::Etc::sourcelist="sources.list.d/HDP.list" -o APT::Get::List-Cleanup="0"')
+
+ @patch("os.path.isfile", new=MagicMock(return_value=False))
+ @patch.object(System, "os_family", new='debian')
+ @patch("resource_management.libraries.providers.repository.Execute")
+ @patch("resource_management.libraries.providers.repository.File")
+ def test_remove_repo_debian_repo_doenst_exist(self, file_mock, execute_mock):
+ with Environment('/') as env:
+ Repository('HDP',
+ action = "remove",
+ repo_file_name='HDP'
+ )
+
+ self.assertEqual(file_mock.call_count, 0)
+ self.assertEqual(execute_mock.call_count, 0)
+
+ @patch.object(System, "os_family", new='redhat')
+ @patch("resource_management.libraries.providers.repository.File")
+ def test_remove_repo_redhat(self, file_mock):
+ with Environment('/') as env:
+ Repository('hadoop',
+ action='remove',
+ base_url='http://download.base_url.org/rpm/',
+ mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
+ repo_file_name='Repository')
+
+ self.assertTrue('hadoop' in env.resources['Repository'])
+ defined_arguments = env.resources['Repository']['hadoop'].arguments
+ expected_arguments = {'action': ['remove'],
+ 'base_url': 'http://download.base_url.org/rpm/',
+ 'mirror_list': 'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
+ 'repo_file_name': 'Repository'}
+ self.assertEqual(defined_arguments, expected_arguments)
+ self.assertEqual(file_mock.call_args[1]['action'], 'delete')
+ self.assertEqual(file_mock.call_args[0][0], '/etc/yum.repos.d/Repository.repo')
+
+
+ @patch.object(System, "os_family", new='suse')
+ @patch("resource_management.libraries.providers.repository.File")
+ def test_remove_repo_suse(self, file_mock):
+ with Environment('/') as env:
+ Repository('hadoop',
+ action='remove',
+ base_url='http://download.base_url.org/rpm/',
+ mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
+ repo_file_name='Repository')
+
+ self.assertTrue('hadoop' in env.resources['Repository'])
+ defined_arguments = env.resources['Repository']['hadoop'].arguments
+ expected_arguments = {'action': ['remove'],
+ 'base_url': 'http://download.base_url.org/rpm/',
+ 'mirror_list': 'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
+ 'repo_file_name': 'Repository'}
+ self.assertEqual(defined_arguments, expected_arguments)
+ self.assertEqual(file_mock.call_args[1]['action'], 'delete')
+ self.assertEqual(file_mock.call_args[0][0], '/etc/zypp/repos.d/Repository.repo')
diff --git a/slider-agent/src/test/python/resource_management/TestScript.py b/slider-agent/src/test/python/resource_management/TestScript.py
new file mode 100644
index 0000000..c260b97
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestScript.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import ConfigParser
+import os
+
+import pprint
+
+from unittest import TestCase
+import threading
+import tempfile
+import time
+from threading import Thread
+
+
+import StringIO
+import sys, logging, pprint
+from agent import AgentException
+from resource_management.libraries.script import Script
+from resource_management.core.environment import Environment
+from mock.mock import MagicMock, patch
+
+class TestScript(TestCase):
+
+ def setUp(self):
+ # disable stdout
+ out = StringIO.StringIO()
+ sys.stdout = out
+
+
+
+ @patch("resource_management.core.providers.package.PackageProvider")
+ def test_install_packages(self, package_provider_mock):
+ no_packages_config = {
+ 'hostLevelParams' : {
+ 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"
+ }
+ }
+ empty_config = {
+ 'hostLevelParams' : {
+ 'package_list' : '',
+ 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"
+ }
+ }
+ dummy_config = {
+ 'hostLevelParams' : {
+ 'package_list' : "[{\"type\":\"rpm\",\"name\":\"hbase\"},"
+ "{\"type\":\"rpm\",\"name\":\"yet-another-package\"}]",
+ 'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]",
+ 'service_repo_info' : "[{\"mirrorsList\":\"abc\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"
+ }
+ }
+
+ # Testing config without any keys
+ with Environment(".", test_mode=True) as env:
+ script = Script()
+ Script.config = no_packages_config
+ script.install_packages(env)
+ resource_dump = pprint.pformat(env.resource_list)
+ self.assertEquals(resource_dump, "[]")
+
+ # Testing empty package list
+ with Environment(".", test_mode=True) as env:
+ script = Script()
+ Script.config = empty_config
+ script.install_packages(env)
+ resource_dump = pprint.pformat(env.resource_list)
+ self.assertEquals(resource_dump, "[]")
+
+ # Testing installing of a list of packages
+ with Environment(".", test_mode=True) as env:
+ Script.config = dummy_config
+ script.install_packages("env")
+ resource_dump = pprint.pformat(env.resource_list)
+ self.assertEqual(resource_dump, "[Repository['HDP-2.0._'],\n Repository['HDP-2.0._'],\n Package['hbase'],\n Package['yet-another-package']]")
+
+ @patch("__builtin__.open")
+ def test_structured_out(self, open_mock):
+ script = Script()
+ script.stroutfile = ''
+
+ self.assertEqual(Script.structuredOut, {})
+
+ script.put_structured_out({"1": "1"})
+ self.assertEqual(Script.structuredOut, {"1": "1"})
+ self.assertTrue(open_mock.called)
+
+ script.put_structured_out({"2": "2"})
+ self.assertEqual(open_mock.call_count, 2)
+ self.assertEqual(Script.structuredOut, {"1": "1", "2": "2"})
+
+ #Overriding
+ script.put_structured_out({"1": "3"})
+ self.assertEqual(open_mock.call_count, 3)
+ self.assertEqual(Script.structuredOut, {"1": "3", "2": "2"})
+
+ def tearDown(self):
+ # enable stdout
+ sys.stdout = sys.__stdout__
+
+
diff --git a/slider-agent/src/test/python/resource_management/TestServiceResource.py b/slider-agent/src/test/python/resource_management/TestServiceResource.py
new file mode 100644
index 0000000..cf92278
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestServiceResource.py
@@ -0,0 +1,117 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+from resource_management import *
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestServiceResource(TestCase):
+
+ @patch("resource_management.core.shell.call")
+ def test_action_start(self,shell_mock):
+ return_values = [(-1, None),(0,None)]
+ shell_mock.side_effect = return_values
+ with Environment('/') as env:
+ Service('some_service', action="start")
+ self.assertEqual(shell_mock.call_count,2)
+ self.assertEqual(shell_mock.call_args_list[0][0][0],['/etc/init.d/some_service','status'])
+ self.assertEqual(shell_mock.call_args_list[1][0][0],['/etc/init.d/some_service','start'])
+
+ @patch("resource_management.core.shell.call")
+ def test_action_stop(self,shell_mock):
+ return_values = [(0, None),(0,None)]
+ shell_mock.side_effect = return_values
+ with Environment('/') as env:
+ Service('some_service', action="stop")
+ self.assertEqual(shell_mock.call_count,2)
+ self.assertEqual(shell_mock.call_args_list[0][0][0],['/etc/init.d/some_service','status'])
+ self.assertEqual(shell_mock.call_args_list[1][0][0],['/etc/init.d/some_service','stop'])
+
+ @patch("resource_management.core.shell.call")
+ def test_action_restart(self,shell_mock):
+ return_values = [(0, None),(0,None)]
+ shell_mock.side_effect = return_values
+ with Environment('/') as env:
+ Service('some_service', action="restart")
+ self.assertEqual(shell_mock.call_count,2)
+ self.assertEqual(shell_mock.call_args_list[0][0][0],['/etc/init.d/some_service','status'])
+ self.assertEqual(shell_mock.call_args_list[1][0][0],['/etc/init.d/some_service','restart'])
+
+ @patch("resource_management.core.shell.call")
+ def test_action_start_running(self,shell_mock):
+ return_values = [(0, None)]
+ shell_mock.side_effect = return_values
+ with Environment('/') as env:
+ Service('some_service', action="start")
+ self.assertEqual(shell_mock.call_count,1)
+ self.assertEqual(shell_mock.call_args_list[0][0][0],['/etc/init.d/some_service','status'])
+
+ @patch("resource_management.core.shell.call")
+ def test_action_stop_stopped(self,shell_mock):
+ return_values = [(1, None)]
+ shell_mock.side_effect = return_values
+ with Environment('/') as env:
+ Service('some_service', action="stop")
+ self.assertEqual(shell_mock.call_count,1)
+ self.assertEqual(shell_mock.call_args_list[0][0][0],['/etc/init.d/some_service','status'])
+
+ @patch("resource_management.core.shell.call")
+ def test_action_restart_stopped(self,shell_mock):
+ return_values = [(1, None),(0,None)]
+ shell_mock.side_effect = return_values
+ with Environment('/') as env:
+ Service('some_service', action="restart")
+ self.assertEqual(shell_mock.call_count,2)
+ self.assertEqual(shell_mock.call_args_list[0][0][0],['/etc/init.d/some_service','status'])
+ self.assertEqual(shell_mock.call_args_list[1][0][0],['/etc/init.d/some_service','start'])
+
+ @patch("resource_management.core.shell.call")
+ def test_action_reload(self,shell_mock):
+ return_values = [(0, None),(0,None)]
+ shell_mock.side_effect = return_values
+ with Environment('/') as env:
+ Service('some_service', action="reload")
+ self.assertEqual(shell_mock.call_count,2)
+ self.assertEqual(shell_mock.call_args_list[0][0][0],['/etc/init.d/some_service','status'])
+ self.assertEqual(shell_mock.call_args_list[1][0][0],['/etc/init.d/some_service','reload'])
+
+ @patch("resource_management.core.shell.call")
+ def test_action_reload_stopped(self,shell_mock):
+ return_values = [(1, None),(0,None)]
+ shell_mock.side_effect = return_values
+ with Environment('/') as env:
+ Service('some_service', action="reload")
+ self.assertEqual(shell_mock.call_count,2)
+ self.assertEqual(shell_mock.call_args_list[0][0][0],['/etc/init.d/some_service','status'])
+ self.assertEqual(shell_mock.call_args_list[1][0][0],['/etc/init.d/some_service','start'])
+
+ @patch("resource_management.core.shell.call")
+ def test_action_nothing(self,shell_mock):
+ shell_mock.return_value = (0,0)
+ with Environment('/') as env:
+ Service('some_service', action="nothing")
+ self.assertEqual(shell_mock.call_count,0)
+
+ def test_action_not_existing(self):
+ try:
+ with Environment('/') as env:
+ Service('some_service', action="not_existing_action")
+ self.fail("Service should fail with nonexistent action")
+ except Fail as e:
+ self.assertEqual("ServiceProvider[Service['some_service']] does not implement action not_existing_action",str(e))
diff --git a/slider-agent/src/test/python/resource_management/TestSubstituteVars.py b/slider-agent/src/test/python/resource_management/TestSubstituteVars.py
new file mode 100644
index 0000000..b3623cd
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestSubstituteVars.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from unittest import TestCase, main
+from resource_management.libraries.functions.substitute_vars import substitute_vars
+
+import StringIO, sys
+
+class TestSubstituteVars(TestCase):
+ def setUp(self):
+ # disable stdout
+ out = StringIO.StringIO()
+ sys.stdout = out
+
+ def test_substitute_vars(self):
+ raw_config = {
+ 'val.intvar' : '42',
+ 'pass.intvar' : '${val.intvar}',
+ 'fail.unknown' : 'a${unknown}b',
+ 'fail.empty' : '${}',
+ 'fail.space' : '${ my.int}',
+ 'val.0' : 'will_fail',
+ 'fail.digit' : '${val.0}',
+ 'val.file' : 'hello',
+ 'val.suffix' : '.txt',
+ 'pass.seq.depth' : '${val.file}${val.suffix}${pass.intvar}',
+ 'fail.seq.depth.a' : '${val.file}${unknown}${pass.intvar}',
+ 'fail.seq.depth.b' : '${val.file}${fail.seq.depth.a}${pass.intvar}',
+ 'val.name' : 'val.intvar',
+ 'pass.name.as.param' : '${${val.name}}',
+ 'fail.inf.loop' : '${fail.inf.loop}'
+ }
+ expected_config = {
+ 'val.intvar' : '42',
+ 'pass.intvar' : '42',
+ 'fail.unknown' : 'a${unknown}b',
+ 'fail.empty' : '${}',
+ 'fail.space' : '${ my.int}',
+ 'val.0' : 'will_fail',
+ 'fail.digit' : '${val.0}',
+ 'val.file' : 'hello',
+ 'val.suffix' : '.txt',
+ 'pass.seq.depth' : 'hello.txt42',
+ 'fail.seq.depth.a' : 'hello${unknown}${pass.intvar}',
+ 'fail.seq.depth.b' : 'hellohello${unknown}${pass.intvar}${pass.intvar}',
+ 'val.name' : 'val.intvar',
+ 'pass.name.as.param' : '42',
+ 'fail.inf.loop' : '${fail.inf.loop}'
+ }
+
+ for key in raw_config.keys():
+ actual_value = substitute_vars(raw_config[key], raw_config)
+ expected_value = expected_config[key]
+
+ self.assertEqual(actual_value, expected_value)
+
+ def tearDown(self):
+ # enable stdout
+ sys.stdout = sys.__stdout__
diff --git a/slider-agent/src/test/python/resource_management/TestTemplateConfigResource.py b/slider-agent/src/test/python/resource_management/TestTemplateConfigResource.py
new file mode 100644
index 0000000..2808e63
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestTemplateConfigResource.py
@@ -0,0 +1,58 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch
+from resource_management import *
+from resource_management.libraries.resources.template_config \
+ import TemplateConfig
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestTemplateConfigResource(TestCase):
+
+ @patch("resource_management.libraries.providers.template_config.Template")
+ @patch("resource_management.libraries.providers.template_config.File")
+ def test_create_template_wo_tag(self, file_mock, template_mock):
+ with Environment() as env:
+ TemplateConfig("path",
+ action="create",
+ mode=0755,
+ owner="owner",
+ group="group",
+ extra_imports=["extra_imports"]
+ )
+ defined_arguments = env.resources['TemplateConfig']['path'].arguments
+ expected_arguments = {'group': 'group', 'extra_imports': ['extra_imports'], 'action': ['create'], 'mode': 0755, 'owner': 'owner'}
+ self.assertEqual(defined_arguments,expected_arguments)
+ self.assertEqual(file_mock.call_args[0][0],'path')
+ call_args = file_mock.call_args[1].copy()
+ del call_args['content']
+ self.assertEqual(call_args,{'owner': 'owner', 'group': 'group', 'mode': 0755})
+ self.assertEqual(template_mock.call_args[0][0],'path.j2')
+ self.assertEqual(template_mock.call_args[1],{'extra_imports': ['extra_imports']})
+
+
+ @patch("resource_management.libraries.providers.template_config.Template")
+ @patch("resource_management.core.providers.system.FileProvider")
+ def test_create_template_with_tag(self, file_mock, template_mock):
+ with Environment("/") as env:
+ TemplateConfig("path",
+ action="create",
+ template_tag="template_tag"
+ )
+ self.assertEqual(template_mock.call_args[0][0],'path-template_tag.j2')
\ No newline at end of file
diff --git a/slider-agent/src/test/python/resource_management/TestUserResource.py b/slider-agent/src/test/python/resource_management/TestUserResource.py
new file mode 100644
index 0000000..859b111
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestUserResource.py
@@ -0,0 +1,192 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+
+from resource_management.core import Environment, Fail
+from resource_management.core.system import System
+from resource_management.core.resources import User
+import pwd
+import subprocess
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestUserResource(TestCase):
+
+ @patch.object(subprocess, "Popen")
+ @patch.object(pwd, "getpwnam")
+ def test_action_create_nonexistent(self, getpwnam_mock, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getpwnam_mock.return_value = None
+ with Environment('/') as env:
+ user = User("mapred", action = "create")
+
+
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'useradd -m -s /bin/bash mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ self.assertEqual(popen_mock.call_count, 1)
+
+ @patch.object(subprocess, "Popen")
+ @patch.object(pwd, "getpwnam")
+ def test_action_create_existent(self, getpwnam_mock, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getpwnam_mock.return_value = 1
+
+ with Environment('/') as env:
+ user = User("mapred", action = "create")
+
+
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'usermod -s /bin/bash mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ self.assertEqual(popen_mock.call_count, 1)
+
+ @patch.object(subprocess, "Popen")
+ @patch.object(pwd, "getpwnam")
+ def test_action_delete(self, getpwnam_mock, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getpwnam_mock.return_value = 1
+
+ with Environment('/') as env:
+ user = User("mapred", action = "remove")
+
+
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'userdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ self.assertEqual(popen_mock.call_count, 1)
+
+ @patch.object(subprocess, "Popen")
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_comment(self, getpwnam_mock, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getpwnam_mock.return_value = 1
+
+ with Environment('/') as env:
+ user = User("mapred",
+ action = "create",
+ comment = "testComment")
+
+
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'usermod -c testComment -s /bin/bash mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ self.assertEqual(popen_mock.call_count, 1)
+
+ @patch.object(subprocess, "Popen")
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_home(self, getpwnam_mock, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getpwnam_mock.return_value = 1
+
+ with Environment('/') as env:
+ user = User("mapred",
+ action = "create",
+ home = "/test/home")
+
+
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'usermod -s /bin/bash -d /test/home mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ self.assertEqual(popen_mock.call_count, 1)
+
+ @patch.object(subprocess, "Popen")
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_password(self, getpwnam_mock, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getpwnam_mock.return_value = 1
+
+ with Environment('/') as env:
+ user = User("mapred",
+ action = "create",
+ password = "secure")
+
+
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'usermod -s /bin/bash -p secure mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ self.assertEqual(popen_mock.call_count, 1)
+
+ @patch.object(subprocess, "Popen")
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_shell(self, getpwnam_mock, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getpwnam_mock.return_value = 1
+
+ with Environment('/') as env:
+ user = User("mapred",
+ action = "create",
+ shell = "/bin/sh")
+
+
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'usermod -s /bin/sh mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ self.assertEqual(popen_mock.call_count, 1)
+
+ @patch.object(subprocess, "Popen")
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_uid(self, getpwnam_mock, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getpwnam_mock.return_value = 1
+
+ with Environment('/') as env:
+ user = User("mapred",
+ action = "create",
+ uid = "1")
+
+
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'usermod -s /bin/bash -u 1 mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ self.assertEqual(popen_mock.call_count, 1)
+
+ @patch.object(subprocess, "Popen")
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_gid(self, getpwnam_mock, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getpwnam_mock.return_value = 1
+
+ with Environment('/') as env:
+ user = User("mapred",
+ action = "create",
+ gid = "1")
+
+
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'usermod -s /bin/bash -g 1 mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ self.assertEqual(popen_mock.call_count, 1)
+
+ @patch.object(subprocess, "Popen")
+ @patch.object(pwd, "getpwnam")
+ def test_attribute_groups(self, getpwnam_mock, popen_mock):
+ subproc_mock = MagicMock()
+ subproc_mock.returncode = 0
+ popen_mock.return_value = subproc_mock
+ getpwnam_mock.return_value = 1
+
+ with Environment('/') as env:
+ user = User("mapred",
+ action = "create",
+ groups = ['1','2','3'])
+
+
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'usermod -G 1,2,3 -s /bin/bash mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ self.assertEqual(popen_mock.call_count, 1)
diff --git a/slider-agent/src/test/python/resource_management/TestXmlConfigResource.py b/slider-agent/src/test/python/resource_management/TestXmlConfigResource.py
new file mode 100644
index 0000000..3a79dba
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestXmlConfigResource.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+import os
+import time
+from unittest import TestCase
+from mock.mock import patch, MagicMock
+from resource_management.core import Environment
+from resource_management.core.system import System
+from resource_management.libraries import XmlConfig
+
+
+@patch.object(System, "os_family", new='redhat')
+class TestXmlConfigResource(TestCase):
+ """
+ XmlConfig="resource_management.libraries.providers.xml_config.XmlConfigProvider",
+ Testing XmlConfig(XmlConfigProvider) with different 'resource configurations'
+ """
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ @patch.object(time, "asctime")
+ def test_action_create_empty_xml_config(self,
+ time_asctime_mock,
+ os_path_isdir_mock,
+ os_path_exists_mock,
+ open_mock,
+ ensure_mock):
+ """
+ Tests if 'create' action - creates new non existent xml file and write proper data
+ where configurations={}
+ """
+ os_path_isdir_mock.side_effect = [False, True]
+ os_path_exists_mock.return_value = False
+ time_asctime_mock.return_value = 'Wed 2014-02'
+
+ result_file = MagicMock()
+ open_mock.return_value = result_file
+
+ with Environment('/') as env:
+ XmlConfig('file.xml',
+ conf_dir='/dir/conf',
+ configurations={}
+ )
+
+ open_mock.assert_called_with('/dir/conf/file.xml', 'wb')
+ result_file.__enter__().write.assert_called_with(u'<!--Wed 2014-02-->\n <configuration>\n \n </configuration>\n')
+
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ @patch.object(time, "asctime")
+ def test_action_create_simple_xml_config(self,
+ time_asctime_mock,
+ os_path_isdir_mock,
+ os_path_exists_mock,
+ open_mock,
+ ensure_mock):
+ """
+ Tests if 'create' action - creates new non existent xml file and write proper data
+ where configurations={"Some conf":"Some value"}
+ """
+ os_path_isdir_mock.side_effect = [False, True]
+ os_path_exists_mock.return_value = False
+ time_asctime_mock.return_value = 'Wed 2014-02'
+
+ result_file = MagicMock()
+ open_mock.return_value = result_file
+
+ with Environment('/') as env:
+ XmlConfig('file.xml',
+ conf_dir='/dir/conf',
+ configurations={'property1': 'value1'}
+ )
+
+ open_mock.assert_called_with('/dir/conf/file.xml', 'wb')
+ result_file.__enter__().write.assert_called_with(u'<!--Wed 2014-02-->\n <configuration>\n \n <property>\n <name>property1</name>\n <value>value1</value>\n </property>\n \n </configuration>\n')
+
+
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("__builtin__.open")
+ @patch.object(os.path, "exists")
+ @patch.object(os.path, "isdir")
+ @patch.object(time, "asctime")
+ def test_action_create_xml_config_with_metacharacters(self,
+ time_asctime_mock,
+ os_path_isdir_mock,
+ os_path_exists_mock,
+ open_mock,
+ ensure_mock):
+ """
+ Tests if 'create' action - creates new non existent xml file and write proper data
+ where configurations={"Some conf":"Some metacharacters"}
+ """
+ os_path_isdir_mock.side_effect = [False, True]
+ os_path_exists_mock.return_value = False
+ time_asctime_mock.return_value = 'Wed 2014-02'
+
+ result_file = MagicMock()
+ open_mock.return_value = result_file
+
+ with Environment('/') as env:
+ XmlConfig('file.xml',
+ conf_dir='/dir/conf',
+ configurations={"": "",
+ "prop.1": "'.'yyyy-MM-dd-HH",
+ "prop.3": "%d{ISO8601} %5p %c{1}:%L - %m%n",
+ "prop.2": "INFO, openjpa",
+ "prop.4": "${oozie.log.dir}/oozie.log",
+ "prop.empty": "",
+ },
+ )
+
+ open_mock.assert_called_with('/dir/conf/file.xml', 'wb')
+ result_file.__enter__().write.assert_called_with(u'<!--Wed 2014-02-->\n <configuration>\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>prop.empty</name>\n <value></value>\n </property>\n \n <property>\n <name>prop.3</name>\n <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>\n </property>\n \n <property>\n <name>prop.2</name>\n <value>INFO, openjpa</value>\n </property>\n \n <property>\n <name>prop.1</name>\n <value>'.'yyyy-MM-dd-HH</value>\n </property>\n \n <property>\n <name>prop.4</name>\n <value>${oozie.log.dir}/oozie.log</value>\n </property>\n \n </configuration>\n')
diff --git a/slider-agent/src/test/python/unitTests.py b/slider-agent/src/test/python/unitTests.py
new file mode 100644
index 0000000..0d822fd
--- /dev/null
+++ b/slider-agent/src/test/python/unitTests.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import unittest
+import doctest
+from os.path import dirname, split, isdir
+import logging.handlers
+import logging
+from random import shuffle
+
+LOG_FILE_NAME='tests.log'
+SELECTED_PREFIX = "_"
+PY_EXT='.py'
+ignoredDirs = ["mock"]
+
+class TestAgent(unittest.TestSuite):
+ def run(self, result):
+ run = unittest.TestSuite.run
+ run(self, result)
+ return result
+
+
+def parent_dir(path):
+ if isdir(path):
+ if path.endswith(os.sep):
+ path = os.path.dirname(path)
+ parent_dir = os.path.dirname(path)
+ else:
+ parent_dir = os.path.dirname(os.path.dirname(path))
+
+ return parent_dir
+
+
+def all_tests_suite():
+ src_dir = os.getcwd()
+ files_list = []
+ for directory in os.listdir(src_dir):
+ if os.path.isdir(directory) and not directory in ignoredDirs:
+ files_list += os.listdir(src_dir + os.sep + directory)
+ shuffle(files_list)
+ tests_list = []
+
+ logger.info('------------------------TESTS LIST:-------------------------------------')
+ # If test with special name exists, run only this test
+ selected_test = None
+ for file_name in files_list:
+ if file_name.endswith(PY_EXT) and not file_name == __file__ and file_name.startswith(SELECTED_PREFIX):
+ logger.info("Running only selected test " + str(file_name))
+ selected_test = file_name
+ if selected_test is not None:
+ tests_list.append(selected_test.replace(PY_EXT, ''))
+ else:
+ for file_name in files_list:
+ if file_name.endswith(PY_EXT) and not file_name == __file__:
+ logger.info(file_name)
+ tests_list.append(file_name.replace(PY_EXT, ''))
+ logger.info('------------------------------------------------------------------------')
+
+ suite = unittest.TestLoader().loadTestsFromNames(tests_list)
+ return TestAgent([suite])
+
+def main():
+
+ logger.info('------------------------------------------------------------------------')
+ logger.info('PYTHON AGENT TESTS')
+ logger.info('------------------------------------------------------------------------')
+ runner = unittest.TextTestRunner(verbosity=2, stream=sys.stdout)
+ suite = all_tests_suite()
+ status = runner.run(suite).wasSuccessful()
+
+ if not status:
+ logger.error('-----------------------------------------------------------------------')
+ logger.error('Python unit tests failed')
+ logger.error('Find detailed logs in ' + path)
+ logger.error('-----------------------------------------------------------------------')
+ exit(1)
+ else:
+ logger.info('------------------------------------------------------------------------')
+ logger.info('Python unit tests finished succesfully')
+ logger.info('------------------------------------------------------------------------')
+
+if __name__ == '__main__':
+ import os
+ import sys
+ import io
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python')
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python' + os.sep + 'agent')
+ logger = logging.getLogger()
+ logger.setLevel(logging.INFO)
+ formatter = logging.Formatter("[%(levelname)s] %(message)s")
+ src_dir = os.getcwd()
+ target_dir = parent_dir(parent_dir(parent_dir(src_dir))) + os.sep + 'target'
+ if not os.path.exists(target_dir):
+ os.mkdir(target_dir)
+ path = target_dir + os.sep + LOG_FILE_NAME
+ file=open(path, "w")
+ consoleLog = logging.StreamHandler(file)
+ consoleLog.setFormatter(formatter)
+ logger.addHandler(consoleLog)
+ main()
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
new file mode 100644
index 0000000..10f2594
--- /dev/null
+++ b/slider-core/pom.xml
@@ -0,0 +1,679 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-core</artifactId>
+ <name>Slider Core</name>
+ <version>0.23.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <description>Core Slider Module and minicluster tests
+ </description>
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+
+ <!-- resources are filtered for dynamic updates. This gets build info in-->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
+ <plugins>
+
+ <!--read in a build.properties file if defined-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <version>${maven.properties.version}</version>
+ <executions>
+ <execution>
+ <phase>initialize</phase>
+ <goals>
+ <goal>read-project-properties</goal>
+ </goals>
+ <configuration>
+ <quiet>true</quiet>
+ <files>
+ <file>build.properties</file>
+ <file>../build.properties</file>
+ </files>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler-plugin.version}</version>
+ <configuration>
+ <compilerId>groovy-eclipse-compiler</compilerId>
+ <!-- set verbose to be true if you want lots of uninteresting messages -->
+ <!-- <verbose>true</verbose> -->
+ <source>${project.java.src.version}</source>
+ <target>${project.java.src.version}</target>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>${groovy-eclipse-compiler.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-batch</artifactId>
+ <version>${groovy-eclipse-batch.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${maven-jar-plugin.version}</version>
+ <!-- The configuration of the plugin -->
+ <configuration>
+ <!-- Configuration of the archiver -->
+ <archive>
+ <manifestEntries>
+ <mode>development</mode>
+ <url>${project.url}</url>
+ </manifestEntries>
+ <!-- Manifest specific configuration -->
+ <manifest>
+ </manifest>
+ </archive>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ <configuration>
+ <reuseForks>${test.reuseForks}</reuseForks>
+ <forkMode>${test.forkMode}</forkMode>
+ <forkCount>1</forkCount>
+ <forkedProcessTimeoutInSeconds>${test.forkedProcessTimeoutInSeconds}
+ </forkedProcessTimeoutInSeconds>
+ <threadCount>1</threadCount>
+ <argLine>${test.argLine}</argLine>
+ <failIfNoTests>${test.failIfNoTests}</failIfNoTests>
+ <redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
+ <systemPropertyVariables>
+ <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
+ <java.awt.headless>true</java.awt.headless>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/Test*.java</include>
+ </includes>
+ <excludes>
+ <exclude>**/Test*$*.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-maven-plugin</artifactId>
+ <version>${avro.version}</version>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>schema</goal>
+ </goals>
+ <configuration>
+ <sourceDirectory>${project.basedir}/src/main/avro/
+ </sourceDirectory>
+ <outputDirectory>${project.build.directory}/generated-sources/java
+ </outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <version>${hadoop.version}</version>
+
+ <executions>
+ <execution>
+ <id>version-info</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>version-info</goal>
+ </goals>
+ <configuration>
+ <source>
+ <directory>${basedir}/src/main</directory>
+ <includes>
+ <include>java/**/*.java</include>
+ <include>proto/**/*.proto</include>
+ </includes>
+ </source>
+ </configuration>
+ </execution>
+ <execution>
+ <id>compile-protoc</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>protoc</protocCommand>
+ <imports>
+ <param>${basedir}/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/main/proto</directory>
+ <includes>
+ <include>SliderClusterMessages.proto</include>
+ <include>SliderClusterProtocol.proto</include>
+ </includes>
+ </source>
+ <output>${project.build.directory}/generated-sources/java</output>
+ </configuration>
+ </execution>
+ <!--
+ <execution>
+ <id>compile-test-protoc</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
+ <imports>
+ <param>${basedir}/src/test/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/test/proto</directory>
+ <includes>
+ <include>test.proto</include>
+ <include>test_rpc_service.proto</include>
+ </includes>
+ </source>
+ <output>${project.build.directory}/generated-test-sources/java
+ </output>
+ </configuration>
+ </execution>
+ -->
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>${maven-site-plugin.version}</version>
+
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-module-markdown</artifactId>
+ <version>${maven-doxia-module-markdown.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+ <!--
+ see http://mojo.codehaus.org/buildnumber-maven-plugin/usage.html
+ -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>buildnumber-maven-plugin</artifactId>
+ <version>${buildnumber-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <phase>validate</phase>
+ <goals>
+ <goal>create</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <!-- disabled to allow releases with local binaries-->
+ <doCheck>false</doCheck>
+ <!-- skips any update-->
+ <doUpdate>false</doUpdate>
+ <!-- uses shorter Git checksums-->
+ <shortRevisionLength>10</shortRevisionLength>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>src/test/python/agent.ini</exclude>
+ <exclude>src/test/python/version</exclude>
+ <exclude>src/main/resources/webapps/slideram/.keep</exclude>
+ <exclude>src/main/resources/webapps/static/yarn.dt.plugins.js</exclude>
+ <!-- jQuery DataTables files (BSD license) -->
+ <exclude>src/main/resources/webapps/static/dt-1.9.4/**</exclude>
+ <!-- jQuery (MIT license) -->
+ <exclude>src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js</exclude>
+ <!-- jQuery UI (MIT license) -->
+ <exclude>src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js</exclude>
+ <exclude>src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css</exclude>
+ <!-- jQuery jsTree (MIT license) -->
+ <exclude>src/main/resources/webapps/static/jt/jquery.jstree.js</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <reporting>
+
+ </reporting>
+
+ <dependencies>
+
+
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <type>pom</type>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!--
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo</artifactId>
+ <version>${accumulo.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ -->
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-trace</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-digester</groupId>
+ <artifactId>commons-digester</artifactId>
+ </dependency>
+
+ <!-- ======================================================== -->
+ <!-- service registry -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-x-discovery</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-x-discovery-server</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+
+ <!-- ======================================================== -->
+ <!-- Jersey and webapp support -->
+ <!-- ======================================================== -->
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-guice</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>1.8.5</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>3.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-core</artifactId>
+ <version>1.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-reflect</artifactId>
+ <version>1.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-easymock</artifactId>
+ <version>1.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>1.5</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+
+ <profiles>
+
+
+ <!--
+ a test run, currently hard-coded for stevel's secure
+ VM cluster
+ -->
+ <profile>
+ <id>testrun</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ <configuration>
+ <executable>java</executable>
+ <arguments>
+ <argument>-Xms512m</argument>
+ <argument>-Xmx512m</argument>
+ <argument>-classpath</argument>
+ <classpath/>
+ <argument>org.apache.slider.Slider</argument>
+ <argument>list</argument>
+ </arguments>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <profile>
+ <id>list</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>java</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <mainClass>org.apache.slider.Slider</mainClass>
+ <arguments>
+ <argument>list</argument>
+ </arguments>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ </profiles>
+
+</project>
diff --git a/slider-core/src/assembly/executable-jar.xml b/slider-core/src/assembly/executable-jar.xml
new file mode 100644
index 0000000..23383c8
--- /dev/null
+++ b/slider-core/src/assembly/executable-jar.xml
@@ -0,0 +1,47 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>distribution</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.directory}</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>*.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+ <dependencySets>
+ <dependencySet>
+ <scope>runtime</scope>
+ <outputDirectory>/lib</outputDirectory>
+ <!-- dont copy JAR into /lib-->
+ <useProjectArtifact>false</useProjectArtifact>
+ <!-- <includeBaseDirectory>false</includeBaseDirectory>-->
+ <unpack>false</unpack>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
diff --git a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
new file mode 100644
index 0000000..0380664
--- /dev/null
+++ b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+[
+
+ {
+ "type": "record",
+ "name": "NodeEntryRecord",
+ "namespace": "org.apache.slider.server.avro",
+ "fields": [
+ {
+ "name": "host",
+ "type": "string"
+ },
+ {
+ "name": "role",
+ "type": "int"
+ },
+ {
+ "name": "active",
+ "type": "boolean"
+ },
+ {
+ "name": "last_used",
+ "type": "long"
+ }
+ ]
+ },
+
+ {
+ "type": "record",
+ "name": "RoleHistoryHeader",
+ "namespace": "org.apache.slider.server.avro",
+ "fields": [
+ {
+ "name": "version",
+ "type": "int"
+ },
+ {
+ "name": "saved",
+ "type": "long"
+ },
+ {
+ "name": "savedx",
+ "type": "string"
+ },
+ {
+ "name": "savedate",
+ "type": "string",
+ "default": ""
+ },
+ {
+ "name": "roles",
+ "type": "int"
+ }
+ ]
+ },
+ {
+ "type": "record",
+ "name": "RoleHistoryFooter",
+ "namespace": "org.apache.slider.server.avro",
+ "fields": [
+ {
+ "name": "count",
+ "type": "long"
+ }
+ ]
+ },
+
+ {
+ "type": "record",
+ "name": "RoleHistoryRecord",
+ "namespace": "org.apache.slider.server.avro",
+ "fields": [
+ {
+ "name": "entry",
+ "type": [
+ "org.apache.slider.server.avro.NodeEntryRecord",
+ "org.apache.slider.server.avro.RoleHistoryHeader",
+ "org.apache.slider.server.avro.RoleHistoryFooter"
+ ]
+ }
+ ]
+ }
+
+]
diff --git a/slider-core/src/main/java/org/apache/slider/Slider.java b/slider-core/src/main/java/org/apache/slider/Slider.java
new file mode 100644
index 0000000..5fc8618
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/Slider.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider;
+
+import org.apache.slider.client.SliderClient;
+import org.apache.slider.core.main.ServiceLauncher;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * This is just the entry point class
+ */
+public class Slider extends SliderClient {
+
+
+ public static final String SERVICE_CLASSNAME = "org.apache.slider.Slider";
+
+ /**
+ * This is the main entry point for the service launcher.
+ * @param args command line arguments.
+ */
+ public static void main(String[] args) {
+
+ //turn the args to a list
+ List<String> argsList = Arrays.asList(args);
+ //create a new list, as the ArrayList type doesn't push() on an insert
+ List<String> extendedArgs = new ArrayList<String>(argsList);
+ //insert the service name
+ extendedArgs.add(0, SERVICE_CLASSNAME);
+ //now have the service launcher do its work
+ ServiceLauncher.serviceMain(extendedArgs);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
new file mode 100644
index 0000000..d875d66
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
@@ -0,0 +1,760 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.SliderProviderFactory;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_HOME;
+import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH;
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
+
+/**
+ * Represents a cluster specification; designed to be sendable over the wire
+ * and persisted in JSON by way of Jackson.
+ *
+ * When used in cluster status operations the <code>info</code>
+ * and <code>statistics</code> maps contain information about the cluster.
+ *
+ * As a wire format it is less efficient in both xfer and ser/deser than
+ * a binary format, but by having one unified format for wire and persistence,
+ * the code paths are simplified.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+
+public class ClusterDescription implements Cloneable {
+ protected static final Logger
+ log = LoggerFactory.getLogger(ClusterDescription.class);
+
+ private static final String UTF_8 = "UTF-8";
+
+ /**
+ * version counter
+ */
+ public String version = "1.0";
+
+ /**
+ * Name of the cluster
+ */
+ public String name;
+
+ /**
+ * Type of cluster
+ */
+ public String type = SliderProviderFactory.DEFAULT_CLUSTER_TYPE;
+
+ /**
+ * State of the cluster
+ */
+ public int state;
+
+ /*
+ State list for both clusters and nodes in them. Ordered so that destroyed follows
+ stopped.
+
+ Some of the states are only used for recording
+ the persistent state of the cluster and are not
+ seen in node descriptions
+ */
+
+ /**
+ * Specification is incomplete & cannot
+ * be used: {@value}
+ */
+ public static final int STATE_INCOMPLETE = 0;
+
+ /**
+ * Spec has been submitted: {@value}
+ */
+ public static final int STATE_SUBMITTED = 1;
+ /**
+ * Cluster created: {@value}
+ */
+ public static final int STATE_CREATED = 2;
+ /**
+ * Live: {@value}
+ */
+ public static final int STATE_LIVE = 3;
+ /**
+ * Stopped
+ */
+ public static final int STATE_STOPPED = 4;
+ /**
+ * destroyed
+ */
+ public static final int STATE_DESTROYED = 5;
+
+ /**
+ * When was the cluster specification created?
+ * This is not the time a cluster was thawed; that will
+ * be in the <code>info</code> section.
+ */
+ public long createTime;
+
+ /**
+ * When was the cluster specification last updated
+ */
+ public long updateTime;
+
+ /**
+ * URL path to the original configuration
+ * files; these are re-read when
+ * restoring a cluster
+ */
+
+ public String originConfigurationPath;
+
+ /**
+ * URL path to the generated configuration
+ */
+ public String generatedConfigurationPath;
+
+ /**
+ * This is where the data goes
+ */
+ public String dataPath;
+
+ /**
+ * cluster-specific options -to control both
+ * the Slider AM and the application that it deploys
+ */
+ public Map<String, String> options =
+ new HashMap<String, String>();
+
+ /**
+ * cluster information
+ * This is only valid when querying the cluster status.
+ */
+ public Map<String, String> info =
+ new HashMap<String, String>();
+
+ /**
+ * Statistics. This is only relevant when querying the cluster status
+ */
+ public Map<String, Map<String, Integer>> statistics =
+ new HashMap<String, Map<String, Integer>>();
+
+ /**
+ * Instances: role->count
+ */
+ public Map<String, List<String>> instances =
+ new HashMap<String, List<String>>();
+
+ /**
+ * Role options,
+ * role -> option -> value
+ */
+ public Map<String, Map<String, String>> roles =
+ new HashMap<String, Map<String, String>>();
+
+
+ /**
+ * List of key-value pairs to add to a client config to set up the client
+ */
+ public Map<String, String> clientProperties =
+ new HashMap<String, String>();
+
+ /**
+ * Status information
+ */
+ public Map<String, Object> status;
+
+
+ /**
+ * Creator.
+ */
+ public ClusterDescription() {
+ }
+
+
+ @Override
+ public String toString() {
+ try {
+ return toJsonString();
+ } catch (Exception e) {
+ log.debug("Failed to convert CD to JSON ", e);
+ return super.toString();
+ }
+ }
+
+ /**
+ * Shallow clone
+ * @return a shallow clone
+ * @throws CloneNotSupportedException
+ */
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return super.clone();
+ }
+
+ /**
+ * A deep clone of the spec. This is done inefficiently with a ser/derser
+ * @return the cluster description
+ */
+ public ClusterDescription deepClone() {
+ try {
+ return fromJson(toJsonString());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ /**
+ * Save a cluster description to a hadoop filesystem
+ * @param fs filesystem
+ * @param path path
+ * @param overwrite should any existing file be overwritten
+ * @throws IOException IO exception
+ */
+ public void save(FileSystem fs, Path path, boolean overwrite) throws
+ IOException {
+ FSDataOutputStream dataOutputStream = fs.create(path, overwrite);
+ writeJsonAsBytes(dataOutputStream);
+ }
+
+ /**
+ * Save a cluster description to the local filesystem
+ * @param file file
+ * @throws IOException IO excpetion
+ */
+ public void save(File file) throws IOException {
+ log.debug("Saving to {}", file.getAbsolutePath());
+ if (!file.getParentFile().mkdirs()) {
+ log.warn("Failed to mkdirs for " + file.getParentFile());
+ }
+ DataOutputStream dataOutputStream = new DataOutputStream(new FileOutputStream(file));
+ writeJsonAsBytes(dataOutputStream);
+ }
+
+ /**
+ * Write the json as bytes -then close the file
+ * @param dataOutputStream an outout stream that will always be closed
+ * @throws IOException any failure
+ */
+ private void writeJsonAsBytes(DataOutputStream dataOutputStream) throws
+ IOException {
+ try {
+ String json = toJsonString();
+ byte[] b = json.getBytes(UTF_8);
+ dataOutputStream.write(b);
+ } finally {
+ dataOutputStream.close();
+ }
+ }
+
+ /**
+ * Load from the filesystem
+ * @param fs filesystem
+ * @param path path
+ * @return a loaded CD
+ * @throws IOException IO problems
+ */
+ public static ClusterDescription load(FileSystem fs, Path path)
+ throws IOException, JsonParseException, JsonMappingException {
+ FileStatus status = fs.getFileStatus(path);
+ byte[] b = new byte[(int) status.getLen()];
+ FSDataInputStream dataInputStream = fs.open(path);
+ int count = dataInputStream.read(b);
+ String json = new String(b, 0, count, UTF_8);
+ return fromJson(json);
+ }
+
+ /**
+ * Make a deep copy of the class
+ * @param source source
+ * @return the copy
+ */
+ public static ClusterDescription copy(ClusterDescription source) {
+ //currently the copy is done by a generate/save. Inefficient but it goes
+ //down the tree nicely
+ try {
+ return fromJson(source.toJsonString());
+ } catch (IOException e) {
+ throw new RuntimeException("ClusterDescription copy failed " + e, e);
+ }
+ }
+
+ /**
+ * Convert to a JSON string
+ * @return a JSON string description
+ * @throws IOException Problems mapping/writing the object
+ */
+ public String toJsonString() throws IOException,
+ JsonGenerationException,
+ JsonMappingException {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ return mapper.writeValueAsString(this);
+ }
+
+ /**
+ * Convert from JSON
+ * @param json input
+ * @return the parsed JSON
+ * @throws IOException IO
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public static ClusterDescription fromJson(String json)
+ throws IOException, JsonParseException, JsonMappingException {
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ return mapper.readValue(json, ClusterDescription.class);
+ } catch (IOException e) {
+ log.error("Exception while parsing json : " + e + "\n" + json, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from input stream
+ * @param is input stream of cluster description
+ * @return the parsed JSON
+ * @throws IOException IO
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public static ClusterDescription fromStream(InputStream is)
+ throws IOException, JsonParseException, JsonMappingException {
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ return mapper.readValue(is, ClusterDescription.class);
+ } catch (IOException e) {
+ log.error("Exception while parsing input stream : " + e, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from a JSON file
+ * @param jsonFile input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public static ClusterDescription fromFile(File jsonFile)
+ throws IOException, JsonParseException, JsonMappingException {
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ return mapper.readValue(jsonFile, ClusterDescription.class);
+ } catch (IOException e) {
+ log.error("Exception while parsing json file {}: {}" , jsonFile, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Set a cluster option: a key val pair in the options {} section
+ * @param key key option name
+ * @param val value option value
+ */
+ public void setOption(String key, String val) {
+ options.put(key, val);
+ }
+
+ /**
+ * Set a cluster option if it is unset. If it is already set,
+ * in the Cluster Description, it is left alone
+ * @param key key key to query/set
+ * @param val value value
+ */
+
+ public void setOptionifUnset(String key, String val) {
+ if (options.get(key) == null) {
+ options.put(key, val);
+ }
+ }
+
+ /**
+ * Set an integer option -it's converted to a string before saving
+ * @param option option name
+ * @param val integer value
+ */
+ public void setOption(String option, int val) {
+ setOption(option, Integer.toString(val));
+ }
+
+ /**
+ * Set a boolean option
+ * @param option option name
+ * @param val bool value
+ */
+ public void setOption(String option, boolean val) {
+ setOption(option, Boolean.toString(val));
+ }
+
+ /**
+ * Get a cluster option or value
+ *
+ * @param key
+ * @param defVal
+ * @return
+ */
+ public String getOption(String key, String defVal) {
+ String val = options.get(key);
+ return val != null ? val : defVal;
+ }
+
+ /**
+ * Get a cluster option or value
+ *
+ * @param key
+ * @return the value
+ * @throws BadConfigException if the option is missing
+ */
+ public String getMandatoryOption(String key) throws BadConfigException {
+ String val = options.get(key);
+ if (val == null) {
+ throw new BadConfigException("Missing option " + key);
+ }
+ return val;
+ }
+
+ /**
+ * Get an integer option; use {@link Integer#decode(String)} so as to take hex
+ * oct and bin values too.
+ *
+ * @param option option name
+ * @param defVal default value
+ * @return parsed value
+ * @throws NumberFormatException if the role could not be parsed.
+ */
+ public int getOptionInt(String option, int defVal) {
+ String val = getOption(option, Integer.toString(defVal));
+ return Integer.decode(val);
+ }
+
+ /**
+ * Verify that an option is set: that is defined AND non-empty
+ * @param key
+ * @throws BadConfigException
+ */
+ public void verifyOptionSet(String key) throws BadConfigException {
+ if (SliderUtils.isUnset(getOption(key, null))) {
+ throw new BadConfigException("Unset cluster option %s", key);
+ }
+ }
+
+ /**
+ * Get an option as a boolean. Note that {@link Boolean#valueOf(String)}
+ * is used for parsing -its policy of what is true vs false applies.
+ * @param option name
+ * @param defVal default
+ * @return the option.
+ */
+ public boolean getOptionBool(String option, boolean defVal) {
+ return Boolean.valueOf(getOption(option, Boolean.toString(defVal)));
+ }
+
+ /**
+ * Get a role option
+ * @param role role to get from
+ * @param option option name
+ * @param defVal default value
+ * @return resolved value
+ */
+ public String getRoleOpt(String role, String option, String defVal) {
+ Map<String, String> roleopts = getRole(role);
+ if (roleopts == null) {
+ return defVal;
+ }
+ String val = roleopts.get(option);
+ return val != null ? val : defVal;
+ }
+
+ /**
+ * Get a mandatory role option
+ * @param role role to get from
+ * @param option option name
+ * @return resolved value
+ * @throws BadConfigException if the option is not defined
+ */
+ public String getMandatoryRoleOpt(String role, String option) throws
+ BadConfigException {
+ Map<String, String> roleopts = getRole(role);
+ if (roleopts == null) {
+ throw new BadConfigException("Missing role %s ", role);
+ }
+ String val = roleopts.get(option);
+ if (val == null) {
+ throw new BadConfigException("Missing option '%s' in role %s ", option,
+ role);
+ }
+ return val;
+ }
+ /**
+ * Get a mandatory integer role option
+ * @param role role to get from
+ * @param option option name
+ * @return resolved value
+ * @throws BadConfigException if the option is not defined
+ */
+ public int getMandatoryRoleOptInt(String role, String option) throws
+ BadConfigException {
+ getMandatoryRoleOpt(role, option);
+ return getRoleOptInt(role, option, 0);
+ }
+
+ /**
+ * look up a role and return its options
+ * @param role role
+ * @return role mapping or null
+ */
+ public Map<String, String> getRole(String role) {
+ return roles.get(role);
+ }
+
+ /**
+ * Get a role -adding it to the roleopts map if
+ * none with that name exists
+ * @param role role
+ * @return role mapping
+ */
+ public Map<String, String> getOrAddRole(String role) {
+ Map<String, String> map = getRole(role);
+ if (map == null) {
+ map = new HashMap<String, String>();
+ }
+ roles.put(role, map);
+ return map;
+ }
+
+ /*
+ * return the Set of role names
+ */
+ @JsonIgnore
+ public Set<String> getRoleNames() {
+ return new HashSet<String>(roles.keySet());
+ }
+
+ /**
+ * Get a role whose presence is mandatory
+ * @param role role name
+ * @return the mapping
+ * @throws BadConfigException if the role is not there
+ */
+ public Map<String, String> getMandatoryRole(String role) throws
+ BadConfigException {
+ Map<String, String> roleOptions = getRole(role);
+ if (roleOptions == null) {
+ throw new BadConfigException("Missing role " + role);
+ }
+ return roleOptions;
+ }
+
+ /**
+ * Get a role opt; use {@link Integer#decode(String)} so as to take hex
+ * oct and bin values too.
+ *
+ * @param role role to get from
+ * @param option option name
+ * @param defVal default value
+ * @return parsed value
+ * @throws NumberFormatException if the role could not be parsed.
+ */
+ public int getRoleOptInt(String role, String option, int defVal) {
+ String val = getRoleOpt(role, option, Integer.toString(defVal));
+ return Integer.decode(val);
+ }
+
+ /**
+ * Set a role option, creating the role if necessary
+ * @param role role name
+ * @param option option name
+ * @param val value
+ */
+ public void setRoleOpt(String role, String option, String val) {
+ Map<String, String> roleopts = getOrAddRole(role);
+ roleopts.put(option, val);
+ }
+
+ /**
+ * Set an integer role option, creating the role if necessary
+ * @param role role name
+ * @param option option name
+ * @param val integer value
+ */
+ public void setRoleOpt(String role, String option, int val) {
+ setRoleOpt(role, option, Integer.toString(val));
+ }
+
+ /**
+ * Get the value of a role requirement (cores, RAM, etc).
+ * These are returned as integers, but there is special handling of the
+ * string {@link ResourceKeys#YARN_RESOURCE_MAX}, which triggers
+ * the return of the maximum value.
+ * @param role role to get from
+ * @param option option name
+ * @param defVal default value
+ * @param maxVal value to return if the max val is requested
+ * @return parsed value
+ * @throws NumberFormatException if the role could not be parsed.
+ */
+ public int getRoleResourceRequirement(String role, String option, int defVal, int maxVal) {
+ String val = getRoleOpt(role, option, Integer.toString(defVal));
+ Integer intVal;
+ if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
+ intVal = maxVal;
+ } else {
+ intVal = Integer.decode(val);
+ }
+ return intVal;
+ }
+
+
+ /**
+ * Set the time for an information (human, machine) timestamp pair of fields.
+ * The human time is the time in millis converted via the {@link Date} class.
+ * @param keyHumanTime name of human time key
+ * @param keyMachineTime name of machine time
+ * @param time timestamp
+ */
+
+ public void setInfoTime(String keyHumanTime, String keyMachineTime, long time) {
+ SliderUtils.setInfoTime(info, keyHumanTime, keyMachineTime, time);
+ }
+
+ /**
+ * Set an information string. This is content that is only valid in status
+ * reports.
+ * @param key key
+ * @param value string value
+ */
+ @JsonIgnore
+ public void setInfo(String key, String value) {
+ info.put(key, value);
+ }
+
+ /**
+ * Get an information string. This is content that is only valid in status
+ * reports.
+ * @param key key
+ * @return the value or null
+ */
+ @JsonIgnore
+ public String getInfo(String key) {
+ return info.get(key);
+ }
+
+ /**
+ * Get an information string. This is content that is only valid in status
+ * reports.
+ * @param key key
+ * @return the value or null
+ */
+ @JsonIgnore
+ public boolean getInfoBool(String key) {
+ String val = info.get(key);
+ if (val != null) {
+ return Boolean.valueOf(val);
+ }
+ return false;
+ }
+
+ @JsonIgnore
+ public String getZkHosts() throws BadConfigException {
+ return getMandatoryOption(ZOOKEEPER_QUORUM);
+ }
+
+ /**
+ * Set the hosts for the ZK quorum
+ * @param zkHosts a comma separated list of hosts
+ */
+ @JsonIgnore
+ public void setZkHosts(String zkHosts) {
+ setOption(ZOOKEEPER_QUORUM, zkHosts);
+ }
+
+ @JsonIgnore
+ public String getZkPath() throws BadConfigException {
+ return getMandatoryOption(ZOOKEEPER_PATH);
+ }
+
+ @JsonIgnore
+ public void setZkPath(String zkPath) {
+ setOption(ZOOKEEPER_PATH, zkPath);
+ }
+
+ /**
+ * HBase home: if non-empty defines where a copy of HBase is preinstalled
+ */
+ @JsonIgnore
+ public String getApplicationHome() {
+ return getOption(INTERNAL_APPLICATION_HOME, "");
+ }
+
+ @JsonIgnore
+ public void setApplicationHome(String applicationHome) {
+ setOption(INTERNAL_APPLICATION_HOME, applicationHome);
+ }
+
+ /**
+ * The path in HDFS where the HBase image is
+ */
+ @JsonIgnore
+ public String getImagePath() {
+ return getOption(INTERNAL_APPLICATION_IMAGE_PATH, "");
+ }
+
+ /**
+ * Set the path in HDFS where the HBase image is
+ */
+ @JsonIgnore
+ public void setImagePath(String imagePath) {
+ setOption(INTERNAL_APPLICATION_IMAGE_PATH, imagePath);
+ }
+
+ /**
+ * Query for the image path being set (non null/non empty)
+ * @return true if there is a path in the image path option
+ */
+ @JsonIgnore
+ public boolean isImagePathSet() {
+ return SliderUtils.isSet(getImagePath());
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionKeys.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionKeys.java
new file mode 100644
index 0000000..5b7a92a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionKeys.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+public class ClusterDescriptionKeys {
+
+ public static final String KEY_CLUSTER_LIVE = "live";
+ public static final String KEY_CLUSTER_FAILED = "failed";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
new file mode 100644
index 0000000..7e73a92
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.SliderProviderFactory;
+
+import java.util.Map;
+
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
+
+/**
+ * Operations on Cluster Descriptions
+ */
+public class ClusterDescriptionOperations {
+
+
+ public static ClusterDescription buildFromInstanceDefinition(AggregateConf aggregateConf) throws
+ BadConfigException {
+
+ ClusterDescription cd = new ClusterDescription();
+
+ aggregateConf.resolve();
+
+ //options are a merge of all globals
+ Map<String, String> options = cd.options;
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(options,
+ aggregateConf.getInternal().global);
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(options,
+ aggregateConf.getAppConf().global);
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(options,
+ aggregateConf.getResources().global);
+
+ //roles are the role values merged in the same order
+ mergeInComponentMap(cd, aggregateConf.getInternal());
+ mergeInComponentMap(cd, aggregateConf.getAppConf());
+ mergeInComponentMap(cd, aggregateConf.getResources());
+
+ //now add the extra bits
+ cd.state = ClusterDescription.STATE_LIVE;
+ MapOperations internalOptions =
+ aggregateConf.getInternalOperations().getGlobalOptions();
+ MapOperations appOptions =
+ aggregateConf.getAppConfOperations().getGlobalOptions();
+
+ cd.type = internalOptions.getOption(OptionKeys.INTERNAL_PROVIDER_NAME,
+ SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
+
+ cd.dataPath = internalOptions.get(OptionKeys.INTERNAL_DATA_DIR_PATH);
+ cd.name = internalOptions.get(OptionKeys.APPLICATION_NAME);
+ cd.originConfigurationPath = internalOptions.get(OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH);
+ cd.generatedConfigurationPath = internalOptions.get(OptionKeys.INTERNAL_GENERATED_CONF_PATH);
+ cd.setImagePath(internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH));
+ cd.setApplicationHome(internalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME));
+ cd.setZkPath(appOptions.get(ZOOKEEPER_PATH));
+ cd.setZkHosts(appOptions.get(ZOOKEEPER_QUORUM));
+
+ return cd;
+ }
+
+ private static void mergeInComponentMap(ClusterDescription cd,
+ ConfTree confTree
+ ) {
+
+ Map<String, Map<String, String>> components = confTree.components;
+ for (Map.Entry<String, Map<String, String>> compEntry : components.entrySet()) {
+ String name = compEntry.getKey();
+ Map<String, String> destRole = cd.getOrAddRole(name);
+ Map<String, String> sourceComponent = compEntry.getValue();
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(destRole, sourceComponent);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java b/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
new file mode 100644
index 0000000..94371eb
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.api.proto.Messages;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Describe a specific node in the cluster
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL )
+public class ClusterNode {
+ protected static final Logger
+ LOG = LoggerFactory.getLogger(ClusterDescription.class);
+
+ @JsonIgnore
+ public ContainerId containerId;
+
+ /**
+ * server name
+ */
+ public String name;
+
+
+ /**
+ * UUID of container used in Slider RPC to refer to instances
+ */
+ public String id;
+
+ public String role;
+
+ public int roleId;
+
+ public long createTime;
+ public long startTime;
+ /**
+ * flag set when it is released, to know if it has
+ * already been targeted for termination
+ */
+ public boolean released;
+ public String host;
+ public String hostUrl;
+
+
+ /**
+ * state from {@link ClusterDescription}
+ */
+ public int state;
+
+ /**
+ * Exit code: only valid if the state >= STOPPED
+ */
+ public int exitCode;
+
+ /**
+ * what was the command executed?
+ */
+ public String command;
+
+ /**
+ * Any diagnostics
+ */
+ public String diagnostics;
+
+ /**
+ * What is the tail output from the executed process (or [] if not started
+ * or the log cannot be picked up
+ */
+ public String[] output;
+
+ /**
+ * Any environment details
+ */
+ public String[] environment;
+
+ /**
+ * server-side ctor takes the container ID and builds the name from it
+ * @param containerId container ID
+ */
+ public ClusterNode(ContainerId containerId) {
+ this.containerId = containerId;
+ this.name = containerId.toString();
+ }
+
+ /**
+ * ctor for deserialization
+ */
+ public ClusterNode() {
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append(name).append(": ");
+ builder.append(state).append("\n");
+ builder.append("state: ").append(state).append("\n");
+ builder.append("role: ").append(role).append("\n");
+ append(builder, "host", host);
+ append(builder, "hostURL", hostUrl);
+ append(builder, "command", command);
+ if (output != null) {
+ for (String line : output) {
+ builder.append(line).append("\n");
+ }
+ }
+ append(builder, "diagnostics", diagnostics);
+ return builder.toString();
+ }
+
+ private void append(StringBuilder builder, String name, Object val) {
+ if (val != null) {
+ builder.append(name).append(val.toString()).append("\n");
+ }
+ }
+
+ /**
+ * Convert to a JSON string
+ * @return a JSON string description
+ * @throws IOException Problems mapping/writing the object
+ */
+ public String toJsonString() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.writeValueAsString(this);
+ }
+
+
+ /**
+ * Convert from JSON
+ * @param json input
+ * @return the parsed JSON
+ * @throws IOException IO
+ */
+ public static ClusterNode fromJson(String json)
+ throws IOException, JsonParseException, JsonMappingException {
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ return mapper.readValue(json, ClusterNode.class);
+ } catch (IOException e) {
+ LOG.error("Exception while parsing json : " + e + "\n" + json, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Build from a protobuf response
+ * @param message
+ * @return
+ */
+ public static ClusterNode fromProtobuf(Messages.RoleInstanceState message) {
+ ClusterNode node = new ClusterNode();
+ node.name = message.getName();
+ node.command = message.getCommand();
+ node.diagnostics = message.getDiagnostics();
+ String[] arr;
+ int environmentCount = message.getEnvironmentCount();
+ if (environmentCount > 0) {
+ arr = new String[environmentCount];
+ node.environment = message.getEnvironmentList().toArray(arr);
+ }
+ node.exitCode = message.getExitCode();
+ int outputCount = message.getOutputCount();
+ if (outputCount > 0) {
+ arr = new String[outputCount];
+ node.output = message.getOutputList().toArray(arr);
+ }
+ node.role = message.getRole();
+ node.roleId = message.getRoleId();
+ node.state = message.getState();
+ node.host = message.getHost();
+// node.hostUrl = message.getHostURL();
+ node.createTime = message.getCreateTime();
+ node.startTime = message.getStartTime();
+ node.released = message.getReleased();
+ return node;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java b/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
new file mode 100644
index 0000000..048fefa
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+/**
+ * Keys for entries in the <code>options</code> section
+ * of a cluster description.
+ */
+public interface OptionKeys {
+
+ /**
+ * Home dir of the app: {@value}
+ * If set, implies there is a home dir to use
+ */
+ String INTERNAL_APPLICATION_HOME = "internal.application.home";
+
+ /**
+ * Path to an image file containing the app: {@value}
+ */
+ String INTERNAL_APPLICATION_IMAGE_PATH = "internal.application.image.path";
+
+ /**
+ * Time in milliseconds to wait after forking any in-AM
+ * process before attempting to start up the containers: {@value}
+ *
+ * A shorter value brings the cluster up faster, but means that if the
+ * in AM process fails (due to a bad configuration), then time
+ * is wasted starting containers on a cluster that isn't going to come
+ * up
+ */
+ String INTERNAL_CONTAINER_STARTUP_DELAY = "internal.container.startup.delay";
+
+ /**
+ * Time in milliseconds to wait after forking any in-AM
+ * process before attempting to start up the containers: {@value}
+ *
+ * A shorter value brings the cluster up faster, but means that if the
+ * in AM process fails (due to a bad configuration), then time
+ * is wasted starting containers on a cluster that isn't going to come
+ * up
+ */
+ String APPLICATION_TYPE = "application.type";
+
+ /**
+ * Time in milliseconds to wait after forking any in-AM
+ * process before attempting to start up the containers: {@value}
+ *
+ * A shorter value brings the cluster up faster, but means that if the
+ * in AM process fails (due to a bad configuration), then time
+ * is wasted starting containers on a cluster that isn't going to come
+ * up
+ */
+ String APPLICATION_NAME = "application.name";
+
+ /**
+ * Time in milliseconds before a container is considered long-lived.
+ * Shortlived containers are interpreted as a problem with the role
+ * and/or the host: {@value}
+ */
+ String INTERNAL_CONTAINER_FAILURE_SHORTLIFE = "internal.container.failure.shortlife";
+
+ /**
+ * Default short life threshold: {@value}
+ */
+ int DEFAULT_CONTAINER_FAILURE_SHORTLIFE = 60;
+
+ /**
+ * maximum number of failed containers (in a single role)
+ * before the cluster is deemed to have failed {@value}
+ */
+ String INTERNAL_CONTAINER_FAILURE_THRESHOLD = "internal.container.failure.threshold";
+
+ /**
+ * Default failure threshold: {@value}
+ */
+ int DEFAULT_CONTAINER_FAILURE_THRESHOLD = 5;
+
+ /**
+ * delay for container startup:{@value}
+ */
+ int DEFAULT_CONTAINER_STARTUP_DELAY = 5000;
+
+ /**
+ * Version of the app: {@value}
+ */
+ String KEYTAB_LOCATION = "internal.keytab.location";
+
+ /**
+ * Prefix for site.xml options: {@value}
+ */
+ String SITE_XML_PREFIX = "site.";
+
+ /**
+ * internal temp directory: {@value}
+ */
+ String INTERNAL_AM_TMP_DIR = "internal.tmp.dir";
+
+ /**
+ * where a snapshot of the original conf dir is: {@value}
+ */
+ String INTERNAL_SNAPSHOT_CONF_PATH = "internal.snapshot.conf.path";
+
+ /**
+ * where a snapshot of the original conf dir is: {@value}
+ */
+ String INTERNAL_GENERATED_CONF_PATH = "internal.generated.conf.path";
+
+ /**
+ * where a snapshot of the original conf dir is: {@value}
+ */
+ String INTERNAL_PROVIDER_NAME = "internal.provider.name";
+
+
+ /**
+ * where a snapshot of the original conf dir is: {@value}
+ */
+ String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path";
+
+ /**
+ * Zookeeper quorum host list: {@value}
+ */
+ String ZOOKEEPER_QUORUM = "zookeeper.quorum";
+ String ZOOKEEPER_HOSTS = "zookeeper.hosts";
+ String ZOOKEEPER_PORT = "zookeeper.port";
+
+ /**
+ * Zookeeper path value (string): {@value}
+ */
+ String ZOOKEEPER_PATH = "zookeeper.path";
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
new file mode 100644
index 0000000..164e91e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+/**
+ * These are the keys valid in resource options
+ */
+public interface ResourceKeys {
+
+
+ /**
+ * #of instances of a component
+ *
+ */
+ String COMPONENT_INSTANCES = "component.instances";
+
+ /**
+ * Amount of memory to ask YARN for in MB.
+ * <i>Important:</i> this may be a hard limit on the
+ * amount of RAM that the service can use
+ * {@value}
+ */
+ String YARN_MEMORY = "yarn.memory";
+
+ /** {@value} */
+ int DEF_YARN_MEMORY = 256;
+
+ /**
+ * Number of cores/virtual cores to ask YARN for
+ * {@value}
+ */
+ String YARN_CORES = "yarn.vcores";
+
+ /** {@value} */
+ int DEF_YARN_CORES = 1;
+
+ /**
+ * Constant to indicate that the requirements of a YARN resource limit
+ * (cores, memory, ...) should be set to the maximum allowed by
+ * the queue into which the YARN container requests are placed.
+ */
+ String YARN_RESOURCE_MAX = "max";
+
+ /**
+ * Mandatory property for all roles
+ * 1. this must be defined.
+ * 2. this must be >= 1
+ * 3. this must not match any other role priority in the cluster.
+ */
+ String COMPONENT_PRIORITY = "role.priority";
+
+ /**
+ * placement policy
+ */
+ String COMPONENT_PLACEMENT_POLICY = "component.placement.policy";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
new file mode 100644
index 0000000..0f0fb8c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+/**
+ * Standard options for roles
+ */
+public interface RoleKeys {
+
+
+ /**
+ * The name of a role: {@value}
+ */
+ String ROLE_NAME = "role.name";
+
+ /**
+ * Status report: number actually granted : {@value}
+ */
+ String ROLE_ACTUAL_INSTANCES = "role.actual.instances";
+
+ /**
+ * Status report: number currently requested: {@value}
+ */
+ String ROLE_REQUESTED_INSTANCES = "role.requested.instances";
+
+ /**
+ * Status report: number currently being released: {@value}
+ */
+ String ROLE_RELEASING_INSTANCES = "role.releasing.instances";
+
+ /**
+ * Status report: number currently being released: {@value}
+ */
+ String ROLE_FAILED_INSTANCES = "role.failed.instances";
+
+ /**
+ * Status report: number currently being released: {@value}
+ */
+ String ROLE_FAILED_STARTING_INSTANCES = "role.failed.starting.instances";
+
+ /**
+ * Extra arguments (non-JVM) to use when starting this role
+ */
+ String ROLE_ADDITIONAL_ARGS = "role.additional.args";
+
+ /**
+ * JVM heap size for Java applications in MB. Only relevant for Java applications.
+ * This MUST be less than or equal to the {@link ResourceKeys#YARN_MEMORY} option
+ * {@value}
+ */
+ String JVM_HEAP = "jvm.heapsize";
+
+ /*
+ * GC options for Java applications.
+ */
+ String GC_OPTS = "gc.opts";
+
+ /**
+ * JVM options other than heap size. Only relevant for Java applications.
+ * {@value}
+ */
+ String JVM_OPTS = "jvm.opts";
+
+
+ /**
+ * All keys w/ env. are converted into env variables and passed down
+ */
+ String ENV_PREFIX = "env.";
+
+
+ /**
+ * Default no. of cores in the AM {@value}
+ */
+ int DEFAULT_AM_V_CORES = 1;
+
+ /**
+ * The default memory of the AM: {@value}
+ */
+ int DEFAULT_AM_MEMORY = 1024;
+
+ /**
+ * The default heap of the AM: {@value}
+ */
+ String DEFAULT_AM_HEAP = "512M";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
new file mode 100644
index 0000000..2f198c5
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License") throws IOException, YarnException; you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.common.SliderXmlConfKeys;
+
+import java.io.IOException;
+
+/**
+ * Cluster protocol. This can currently act as a versioned IPC
+ * endpoint or be relayed via protobuf
+ */
+@KerberosInfo(
+ serverPrincipal = SliderXmlConfKeys.KEY_KERBEROS_PRINCIPAL)
+public interface SliderClusterProtocol extends VersionedProtocol {
+ public static final long versionID = 0x01;
+
+ /**
+ * Stop the cluster
+ */
+
+
+ Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request) throws
+ IOException, YarnException;
+
+
+ /**
+ * Flex the cluster.
+ */
+ Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) throws IOException,
+ YarnException;
+
+
+ /**
+ * Get the current cluster status
+ */
+ Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(Messages.GetJSONClusterStatusRequestProto request) throws IOException, YarnException;
+
+
+ /**
+ * List all running nodes in a role
+ */
+ Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request) throws IOException, YarnException;
+
+
+ /**
+ * Get the details on a node
+ */
+ Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request) throws IOException, YarnException;
+
+ /**
+ * Get the
+ * details on a list of nodes.
+ * Unknown nodes are not returned
+ * <i>Important: the order of the results are undefined</i>
+ */
+ Messages.GetClusterNodesResponseProto getClusterNodes(Messages.GetClusterNodesRequestProto request) throws IOException, YarnException;
+
+ /**
+ * Echo back the submitted text (after logging it).
+ * Useful for adding information to the log, and for testing round trip
+ * operations of the protocol
+ * @param request request
+ * @return response
+ * @throws IOException
+ * @throws YarnException
+ */
+ Messages.EchoResponseProto echo(Messages.EchoRequestProto request) throws IOException, YarnException;
+
+ /**
+ * Kill an identified container
+ * @param request request containing the container to kill
+ * @return the response
+ * @throws IOException
+ * @throws YarnException
+ */
+ Messages.KillContainerResponseProto killContainer(Messages.KillContainerRequestProto request) throws IOException, YarnException;
+
+ /**
+ * AM to commit suicide. If the Hadoop halt entry point has not been disabled,
+ * this will fail rather than return with a response.
+ * @param request request
+ * @return response (this is not the expected outcome)
+ * @throws IOException
+ * @throws YarnException
+ */
+ Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request) throws
+ IOException,
+ YarnException;
+
+ /**
+ * Get the instance definition
+ */
+ Messages.GetInstanceDefinitionResponseProto getInstanceDefinition(
+ Messages.GetInstanceDefinitionRequestProto request)
+ throws IOException, YarnException;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java b/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
new file mode 100644
index 0000000..709c137
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+/**
+ * Contains status and statistics keys
+ */
+public interface StatusKeys {
+
+ String STATISTICS_CONTAINERS_ACTIVE_REQUESTS = "containers.active.requests";
+ String STATISTICS_CONTAINERS_COMPLETED = "containers.completed";
+ String STATISTICS_CONTAINERS_DESIRED = "containers.desired";
+ String STATISTICS_CONTAINERS_FAILED = "containers.failed";
+ String STATISTICS_CONTAINERS_LIVE = "containers.live";
+ String STATISTICS_CONTAINERS_REQUESTED = "containers.requested";
+ String STATISTICS_CONTAINERS_STARTED = "containers.start.started";
+ String STATISTICS_CONTAINERS_START_FAILED =
+ "containers.start.failed";
+ String STATISTICS_CONTAINERS_SURPLUS =
+ "containers.surplus";
+ String STATISTICS_CONTAINERS_UNKNOWN_COMPLETED =
+ "containers.unknown.completed";
+ /**
+ * No of containers provided on AM restart
+ */
+ String INFO_CONTAINERS_AM_RESTART = "containers.at.am-restart";
+
+ String INFO_CREATE_TIME_MILLIS = "create.time.millis";
+ String INFO_CREATE_TIME_HUMAN = "create.time";
+ String INFO_LIVE_TIME_MILLIS = "live.time.millis";
+ String INFO_LIVE_TIME_HUMAN = "live.time";
+ String INFO_FLEX_TIME_MILLIS = "flex.time.millis";
+ String INFO_FLEX_TIME_HUMAN = "flex.time";
+
+ String INFO_MASTER_ADDRESS = "info.master.address";
+
+ /**
+ * System time in millis when the status report was generated
+ */
+ String INFO_STATUS_TIME_MILLIS = "status.time.millis";
+
+ /**
+ * System time in human form when the status report was generated
+ */
+ String INFO_STATUS_TIME_HUMAN = "status.time";
+
+ String INFO_AM_APP_ID = "info.am.app.id";
+ String INFO_AM_ATTEMPT_ID = "info.am.attempt.id";
+ String INFO_AM_CONTAINER_ID = "info.am.container.id";
+ String INFO_AM_HOSTNAME = "info.am.hostname";
+ String INFO_AM_RPC_PORT = "info.am.rpc.port";
+ String INFO_AM_WEB_PORT = "info.am.web.port";
+ String INFO_AM_WEB_URL = "info.am.web.url";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
new file mode 100644
index 0000000..b89077a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -0,0 +1,2039 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.slider.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.common.Constants;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionAMSuicideArgs;
+import org.apache.slider.common.params.ActionCreateArgs;
+import org.apache.slider.common.params.ActionEchoArgs;
+import org.apache.slider.common.params.ActionFlexArgs;
+import org.apache.slider.common.params.ActionFreezeArgs;
+import org.apache.slider.common.params.ActionGetConfArgs;
+import org.apache.slider.common.params.ActionKillContainerArgs;
+import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionStatusArgs;
+import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.params.ClientArgs;
+import org.apache.slider.common.params.LaunchArgsAccessor;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.Duration;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.common.tools.SliderVersionInfo;
+import org.apache.slider.core.build.InstanceBuilder;
+import org.apache.slider.core.build.InstanceIO;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
+import org.apache.slider.core.exceptions.WaitTimeoutException;
+import org.apache.slider.core.launch.AppMasterLauncher;
+import org.apache.slider.core.launch.ClasspathConstructor;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.core.launch.LaunchedApplication;
+import org.apache.slider.core.launch.RunningApplication;
+import org.apache.slider.core.main.RunService;
+import org.apache.slider.core.persist.ConfPersister;
+import org.apache.slider.core.persist.LockAcquireFailedException;
+import org.apache.slider.core.registry.YARNRegistryClient;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.core.registry.zk.ZKPathBuilder;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.SliderProviderFactory;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.providers.slideram.SliderAMClientProvider;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.rpc.RpcBinder;
+import org.apache.slider.server.services.curator.CuratorServiceInstance;
+import org.apache.slider.server.services.curator.RegistryBinderService;
+import org.apache.slider.server.services.docstore.utility.AbstractSliderLaunchedService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Client service for Slider
+ */
+
+public class SliderClient extends AbstractSliderLaunchedService implements RunService,
+ SliderExitCodes,
+ SliderKeys,
+ ErrorStrings {
+ private static final Logger log = LoggerFactory.getLogger(SliderClient.class);
+
+ private ClientArgs serviceArgs;
+ public ApplicationId applicationId;
+
+ private String deployedClusterName;
+ /**
+ * Cluster opaerations against the deployed cluster -will be null
+ * if no bonding has yet taken place
+ */
+ private SliderClusterOperations sliderClusterOperations;
+
+ private SliderFileSystem sliderFileSystem;
+
+ /**
+ * Yarn client service
+ */
+ private SliderYarnClientImpl yarnClient;
+ private YARNRegistryClient YARNRegistryClient;
+ private AggregateConf launchedInstanceDefinition;
+ private RegistryBinderService<ServiceInstanceData> registry;
+
+ /**
+ * Constructor
+ */
+ public SliderClient() {
+ super("Slider Client");
+ }
+
+ @Override
+ public Configuration bindArgs(Configuration config, String... args) throws Exception {
+ config = super.bindArgs(config, args);
+ serviceArgs = new ClientArgs(args);
+ serviceArgs.parse();
+ // yarn-ify
+ YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
+ return SliderUtils.patchConfiguration(yarnConfiguration);
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ Configuration clientConf = SliderUtils.loadClientConfigurationResource();
+ ConfigHelper.mergeConfigurations(conf, clientConf, CLIENT_RESOURCE);
+ serviceArgs.applyDefinitions(conf);
+ serviceArgs.applyFileSystemURL(conf);
+ // init security with our conf
+ if (SliderUtils.isHadoopClusterSecure(conf)) {
+ SliderUtils.forceLogin();
+ SliderUtils.initProcessSecurity(conf);
+ }
+ //create the YARN client
+ yarnClient = new SliderYarnClientImpl();
+ addService(yarnClient);
+
+ super.serviceInit(conf);
+
+ //here the superclass is inited; getConfig returns a non-null value
+ sliderFileSystem = new SliderFileSystem(getConfig());
+ YARNRegistryClient =
+ new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+ }
+
+ /**
+ * this is where the work is done.
+ * @return the exit code
+ * @throws Throwable anything that went wrong
+ */
+ @Override
+ public int runService() throws Throwable {
+
+ // choose the action
+ String action = serviceArgs.getAction();
+ int exitCode = EXIT_SUCCESS;
+ String clusterName = serviceArgs.getClusterName();
+ // actions
+ if (SliderActions.ACTION_BUILD.equals(action)) {
+ exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs());
+ } else if (SliderActions.ACTION_CREATE.equals(action)) {
+ exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs());
+ } else if (SliderActions.ACTION_FREEZE.equals(action)) {
+ exitCode = actionFreeze(clusterName,
+ serviceArgs.getActionFreezeArgs());
+ } else if (SliderActions.ACTION_THAW.equals(action)) {
+ exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
+ } else if (SliderActions.ACTION_DESTROY.equals(action)) {
+ exitCode = actionDestroy(clusterName);
+ } else if (SliderActions.ACTION_EXISTS.equals(action)) {
+ exitCode = actionExists(clusterName,
+ serviceArgs.getActionExistsArgs().live);
+ } else if (SliderActions.ACTION_FLEX.equals(action)) {
+ exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
+ } else if (SliderActions.ACTION_GETCONF.equals(action)) {
+ exitCode = actionGetConf(clusterName, serviceArgs.getActionGetConfArgs());
+ } else if (SliderActions.ACTION_HELP.equals(action) ||
+ SliderActions.ACTION_USAGE.equals(action)) {
+ log.info(serviceArgs.usage());
+
+ } else if (SliderActions.ACTION_KILL_CONTAINER.equals(action)) {
+ exitCode = actionKillContainer(clusterName,
+ serviceArgs.getActionKillContainerArgs());
+
+ } else if (SliderActions.ACTION_AM_SUICIDE.equals(action)) {
+ exitCode = actionAmSuicide(clusterName,
+ serviceArgs.getActionAMSuicideArgs());
+
+ } else if (SliderActions.ACTION_LIST.equals(action)) {
+ exitCode = actionList(clusterName);
+ } else if (SliderActions.ACTION_REGISTRY.equals(action)) {
+ exitCode = actionRegistry(
+ serviceArgs.getActionRegistryArgs());
+ } else if (SliderActions.ACTION_STATUS.equals(action)) {
+ exitCode = actionStatus(clusterName,
+ serviceArgs.getActionStatusArgs());
+ } else if (SliderActions.ACTION_VERSION.equals(action)) {
+
+ exitCode = actionVersion();
+ } else {
+ throw new SliderException(EXIT_UNIMPLEMENTED,
+ "Unimplemented: " + action);
+ }
+
+ return exitCode;
+ }
+
+
+ /**
+ * Destroy a cluster. There's two race conditions here
+ * #1 the cluster is started between verifying that there are no live
+ * clusters of that name.
+ */
+ public int actionDestroy(String clustername) throws YarnException,
+ IOException {
+ // verify that a live cluster isn't there
+ SliderUtils.validateClusterName(clustername);
+ //no=op, it is now mandatory.
+ verifyBindingsDefined();
+ verifyNoLiveClusters(clustername);
+
+ // create the directory path
+ Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
+ // delete the directory;
+ boolean exists = sliderFileSystem.getFileSystem().exists(clusterDirectory);
+ if (exists) {
+ log.info("Application Instance {} found at {}: destroying", clustername, clusterDirectory);
+ } else {
+ log.info("Application Instance {} already destroyed", clustername);
+ }
+ boolean deleted =
+ sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
+ if (!deleted) {
+ log.warn("Filesystem returned false from delete() operation");
+ }
+
+ List<ApplicationReport> instances = findAllLiveInstances(clustername);
+ // detect any race leading to cluster creation during the check/destroy process
+ // and report a problem.
+ if (!instances.isEmpty()) {
+ throw new SliderException(EXIT_APPLICATION_IN_USE,
+ clustername + ": "
+ + E_DESTROY_CREATE_RACE_CONDITION
+ + " :" +
+ instances.get(0));
+ }
+ log.info("Destroyed cluster {}", clustername);
+ return EXIT_SUCCESS;
+ }
+
+ /**
+ * AM to commit an asynchronous suicide
+ */
+ public int actionAmSuicide(String clustername,
+ ActionAMSuicideArgs args) throws
+ YarnException,
+ IOException {
+ SliderClusterOperations clusterOperations =
+ createClusterOperations(clustername);
+ clusterOperations.amSuicide(args.message, args.exitcode, args.waittime);
+ return EXIT_SUCCESS;
+ }
+
+ /**
+ * Get the provider for this cluster
+ * @param provider the name of the provider
+ * @return the provider instance
+ * @throws SliderException problems building the provider
+ */
+ private AbstractClientProvider createClientProvider(String provider)
+ throws SliderException {
+ SliderProviderFactory factory =
+ SliderProviderFactory.createSliderProviderFactory(provider);
+ return factory.createClientProvider();
+ }
+
+ /**
+ * Create the cluster -saving the arguments to a specification file first
+ * @param clustername cluster name
+ * @return the status code
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ public int actionCreate(String clustername, ActionCreateArgs createArgs) throws
+ YarnException,
+ IOException {
+
+ actionBuild(clustername, createArgs);
+ return startCluster(clustername, createArgs);
+ }
+
+ /**
+ * Build up the cluster specification/directory
+ *
+ * @param clustername cluster name
+ * @param buildInfo the arguments needed to build the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ public int actionBuild(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo) throws
+ YarnException,
+ IOException {
+
+ buildInstanceDefinition(clustername, buildInfo);
+ return EXIT_SUCCESS;
+ }
+
+
+ /**
+ * Build up the AggregateConfiguration for an application instance then
+ * persists it
+ * @param clustername name of the cluster
+ * @param buildInfo the arguments needed to build the cluster
+ * @throws YarnException
+ * @throws IOException
+ */
+
+ public void buildInstanceDefinition(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo)
+ throws YarnException, IOException {
+ // verify that a live cluster isn't there
+ SliderUtils.validateClusterName(clustername);
+ verifyBindingsDefined();
+ verifyNoLiveClusters(clustername);
+
+ Configuration conf = getConfig();
+ String registryQuorum = lookupZKQuorum();
+
+ Path appconfdir = buildInfo.getConfdir();
+ // Provider
+ String providerName = buildInfo.getProvider();
+ requireArgumentSet(Arguments.ARG_PROVIDER, providerName);
+ log.debug("Provider is {}", providerName);
+ SliderAMClientProvider sliderAM = new SliderAMClientProvider(conf);
+ AbstractClientProvider provider =
+ createClientProvider(providerName);
+ InstanceBuilder builder =
+ new InstanceBuilder(sliderFileSystem,
+ getConfig(),
+ clustername);
+
+
+
+
+ AggregateConf instanceDefinition = new AggregateConf();
+ ConfTreeOperations appConf = instanceDefinition.getAppConfOperations();
+ ConfTreeOperations resources = instanceDefinition.getResourceOperations();
+ ConfTreeOperations internal = instanceDefinition.getInternalOperations();
+ //initial definition is set by the providers
+ sliderAM.prepareInstanceConfiguration(instanceDefinition);
+ provider.prepareInstanceConfiguration(instanceDefinition);
+
+ //load in any specified on the command line
+ if (buildInfo.resources != null) {
+ try {
+ resources.mergeFile(buildInfo.resources);
+
+ } catch (IOException e) {
+ throw new BadConfigException(e,
+ "incorrect argument to %s: \"%s\" : %s ",
+ Arguments.ARG_RESOURCES,
+ buildInfo.resources,
+ e.toString());
+ }
+ }
+ if (buildInfo.template != null) {
+ try {
+ appConf.mergeFile(buildInfo.template);
+ } catch (IOException e) {
+ throw new BadConfigException(e,
+ "incorrect argument to %s: \"%s\" : %s ",
+ Arguments.ARG_TEMPLATE,
+ buildInfo.template,
+ e.toString());
+ }
+ }
+
+ //get the command line options
+ ConfTree cmdLineAppOptions = buildInfo.buildAppOptionsConfTree();
+ ConfTree cmdLineResourceOptions = buildInfo.buildResourceOptionsConfTree();
+
+ appConf.merge(cmdLineAppOptions);
+
+ // put the role counts into the resources file
+ Map<String, String> argsRoleMap = buildInfo.getComponentMap();
+ for (Map.Entry<String, String> roleEntry : argsRoleMap.entrySet()) {
+ String count = roleEntry.getValue();
+ String key = roleEntry.getKey();
+ log.debug("{} => {}", key, count);
+ resources.getOrAddComponent(key)
+ .put(ResourceKeys.COMPONENT_INSTANCES, count);
+ }
+
+ //all CLI role options
+ Map<String, Map<String, String>> appOptionMap =
+ buildInfo.getCompOptionMap();
+ appConf.mergeComponents(appOptionMap);
+
+ //internal picks up core. values only
+ internal.propagateGlobalKeys(appConf, "slider.");
+ internal.propagateGlobalKeys(appConf, "internal.");
+
+ //copy over role. and yarn. values ONLY to the resources
+ if (PROPAGATE_RESOURCE_OPTION) {
+ resources.propagateGlobalKeys(appConf, "component.");
+ resources.propagateGlobalKeys(appConf, "role.");
+ resources.propagateGlobalKeys(appConf, "yarn.");
+ resources.mergeComponentsPrefix(appOptionMap, "component.", true);
+ resources.mergeComponentsPrefix(appOptionMap, "yarn.", true);
+ resources.mergeComponentsPrefix(appOptionMap, "role.", true);
+ }
+
+ // resource component args
+ appConf.merge(cmdLineResourceOptions);
+ resources.mergeComponents(buildInfo.getResourceCompOptionMap());
+
+ builder.init(provider.getName(), instanceDefinition);
+ builder.propagateFilename();
+ builder.propagatePrincipals();
+ builder.setImageDetails(buildInfo.getImage(), buildInfo.getAppHomeDir());
+
+
+ String quorum = buildInfo.getZKhosts();
+ if (SliderUtils.isUnset(quorum)) {
+ quorum = registryQuorum;
+ }
+ if (isUnset(quorum)) {
+ throw new BadConfigException("No Zookeeper quorum defined");
+ }
+ ZKPathBuilder zkPaths = new ZKPathBuilder(getAppName(),
+ getUsername(),
+ clustername,
+ registryQuorum,
+ quorum);
+ String zookeeperRoot = buildInfo.getAppZKPath();
+
+ if (isSet(zookeeperRoot)) {
+ zkPaths.setAppPath(zookeeperRoot);
+
+ }
+ builder.addZKBinding(zkPaths);
+
+ //then propagate any package URI
+ if (buildInfo.packageURI != null) {
+ appConf.set(AgentKeys.PACKAGE_PATH, buildInfo.packageURI);
+ }
+
+ // provider to validate what there is
+ try {
+ sliderAM.validateInstanceDefinition(builder.getInstanceDescription());
+ provider.validateInstanceDefinition(builder.getInstanceDescription());
+ } catch (SliderException e) {
+ //problem, reject it
+ log.info("Error {} validating application instance definition ", e.toString());
+ log.debug("Error {} validating application instance definition ", e);
+ log.info(instanceDefinition.toString());
+ throw e;
+ }
+ try {
+ builder.persist(appconfdir);
+ } catch (LockAcquireFailedException e) {
+ log.warn("Failed to get a Lock on {} : {}", builder, e);
+ throw new BadClusterStateException("Failed to save " + clustername
+ + ": " + e);
+ }
+
+ }
+
+ public FsPermission getClusterDirectoryPermissions(Configuration conf) {
+ String clusterDirPermsOct =
+ conf.get(CLUSTER_DIRECTORY_PERMISSIONS,
+ DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
+ return new FsPermission(clusterDirPermsOct);
+ }
+
+ /**
+ * Verify that the Resource MAnager is configured, if not fail
+ * with a useful error message
+ * @throws BadCommandArgumentsException the exception raised on an invalid config
+ */
+ public void verifyBindingsDefined() throws BadCommandArgumentsException {
+ InetSocketAddress rmAddr = SliderUtils.getRmAddress(getConfig());
+ if (!SliderUtils.isAddressDefined(rmAddr)) {
+ throw new BadCommandArgumentsException(
+ "No valid Resource Manager address provided in the argument "
+ + Arguments.ARG_MANAGER
+ + " or the configuration property "
+ + YarnConfiguration.RM_ADDRESS
+ + " value :" + rmAddr);
+ }
+
+ }
+
+ /**
+ * Load and start a cluster specification.
+ * This assumes that all validation of args and cluster state
+ * have already taken place
+ *
+ * @param clustername name of the cluster.
+ * @param launchArgs launch arguments
+ * @return the exit code
+ * @throws YarnException
+ * @throws IOException
+ */
+ private int startCluster(String clustername,
+ LaunchArgsAccessor launchArgs) throws
+ YarnException,
+ IOException {
+ Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
+ AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
+ clustername,
+ clusterDirectory);
+
+ LaunchedApplication launchedApplication =
+ launchApplication(clustername, clusterDirectory, instanceDefinition,
+ serviceArgs.isDebug());
+ applicationId = launchedApplication.getApplicationId();
+
+ return waitForAppAccepted(launchedApplication, launchArgs.getWaittime());
+ }
+
+ /**
+ * Load the instance definition. It is not resolved at this point
+ * @param name
+ * @param clusterDirectory cluster dir
+ * @return the loaded configuration
+ * @throws IOException
+ * @throws SliderException
+ * @throws UnknownApplicationInstanceException if the file is not found
+ */
+ public AggregateConf loadInstanceDefinitionUnresolved(String name,
+ Path clusterDirectory) throws
+ IOException,
+ SliderException {
+
+ try {
+ AggregateConf definition =
+ InstanceIO.loadInstanceDefinitionUnresolved(sliderFileSystem,
+ clusterDirectory);
+ return definition;
+ } catch (FileNotFoundException e) {
+ throw UnknownApplicationInstanceException.unknownInstance(name, e);
+ }
+ }
+ /**
+ * Load the instance definition.
+ * @param name
+ * @param resolved flag to indicate the cluster should be resolved
+ * @return the loaded configuration
+ * @throws IOException
+ * @throws SliderException
+ * @throws UnknownApplicationInstanceException if the file is not found
+ */
+ public AggregateConf loadInstanceDefinition(String name, boolean resolved) throws
+ IOException,
+ SliderException {
+
+ Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name);
+ AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
+ name,
+ clusterDirectory);
+ if (resolved) {
+ instanceDefinition.resolve();
+ }
+ return instanceDefinition;
+
+ }
+
+
+
+
+ /**
+ *
+ * @param clustername
+ * @param clusterDirectory
+ * @param instanceDefinition
+ * @param debugAM
+ * @return the launched application
+ * @throws YarnException
+ * @throws IOException
+ */
+ public LaunchedApplication launchApplication(String clustername,
+ Path clusterDirectory,
+ AggregateConf instanceDefinition,
+ boolean debugAM)
+ throws YarnException, IOException {
+
+
+ deployedClusterName = clustername;
+ SliderUtils.validateClusterName(clustername);
+ verifyNoLiveClusters(clustername);
+ Configuration config = getConfig();
+ lookupZKQuorum();
+ boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config);
+ //create the Slider AM provider -this helps set up the AM
+ SliderAMClientProvider sliderAM = new SliderAMClientProvider(config);
+
+ instanceDefinition.resolve();
+ launchedInstanceDefinition = instanceDefinition;
+
+ ConfTreeOperations internalOperations =
+ instanceDefinition.getInternalOperations();
+ MapOperations internalOptions = internalOperations.getGlobalOptions();
+ ConfTreeOperations resourceOperations =
+ instanceDefinition.getResourceOperations();
+ ConfTreeOperations appOperations =
+ instanceDefinition.getAppConfOperations();
+ Path generatedConfDirPath =
+ createPathThatMustExist(internalOptions.getMandatoryOption(
+ OptionKeys.INTERNAL_GENERATED_CONF_PATH));
+ Path snapshotConfPath =
+ createPathThatMustExist(internalOptions.getMandatoryOption(
+ OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH));
+
+
+ // cluster Provider
+ AbstractClientProvider provider = createClientProvider(
+ internalOptions.getMandatoryOption(
+ OptionKeys.INTERNAL_PROVIDER_NAME));
+ // make sure the conf dir is valid;
+
+ // now build up the image path
+ // TODO: consider supporting apps that don't have an image path
+ Path imagePath =
+ SliderUtils.extractImagePath(sliderFileSystem, internalOptions);
+ if (log.isDebugEnabled()) {
+ log.debug(instanceDefinition.toString());
+ }
+ MapOperations sliderAMResourceComponent =
+ resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM);
+ AppMasterLauncher amLauncher = new AppMasterLauncher(clustername,
+ SliderKeys.APP_TYPE,
+ config,
+ sliderFileSystem,
+ yarnClient,
+ clusterSecure,
+ sliderAMResourceComponent);
+
+ ApplicationId appId = amLauncher.getApplicationId();
+ // set the application name;
+ amLauncher.setKeepContainersOverRestarts(true);
+
+ amLauncher.setMaxAppAttempts(config.getInt(KEY_AM_RESTART_LIMIT,
+ DEFAULT_AM_RESTART_LIMIT));
+
+ sliderFileSystem.purgeAppInstanceTempFiles(clustername);
+ Path tempPath = sliderFileSystem.createAppInstanceTempPath(
+ clustername,
+ appId.toString() + "/am");
+ String libdir = "lib";
+ Path libPath = new Path(tempPath, libdir);
+ sliderFileSystem.getFileSystem().mkdirs(libPath);
+ log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem.toString(),
+ tempPath, libPath);
+ // set local resources for the application master
+ // local files or archives as needed
+ // In this scenario, the jar file for the application master is part of the local resources
+ Map<String, LocalResource> localResources = amLauncher.getLocalResources();
+ // conf directory setup
+ Path remoteConfPath = null;
+ String relativeConfDir = null;
+ String confdirProp =
+ System.getProperty(SliderKeys.PROPERTY_CONF_DIR);
+ if (confdirProp == null || confdirProp.isEmpty()) {
+ log.debug("No local configuration directory provided as system property");
+ } else {
+ File confDir = new File(confdirProp);
+ if (!confDir.exists()) {
+ throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND,
+ confDir);
+ }
+ Path localConfDirPath = SliderUtils.createLocalPath(confDir);
+ log.debug("Copying AM configuration data from {}", localConfDirPath);
+ remoteConfPath = new Path(clusterDirectory,
+ SliderKeys.SUBMITTED_CONF_DIR);
+ SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath,
+ null);
+ }
+ // the assumption here is that minimr cluster => this is a test run
+ // and the classpath can look after itself
+
+ boolean usingMiniMRCluster = getUsingMiniMRCluster();
+ if (!usingMiniMRCluster) {
+
+ log.debug("Destination is not a MiniYARNCluster -copying full classpath");
+
+ // insert conf dir first
+ if (remoteConfPath != null) {
+ relativeConfDir = SliderKeys.SUBMITTED_CONF_DIR;
+ Map<String, LocalResource> submittedConfDir =
+ sliderFileSystem.submitDirectory(remoteConfPath,
+ relativeConfDir);
+ SliderUtils.mergeMaps(localResources, submittedConfDir);
+ }
+ }
+ // build up the configuration
+ // IMPORTANT: it is only after this call that site configurations
+ // will be valid.
+
+ propagatePrincipals(config, instanceDefinition);
+ Configuration clientConfExtras = new Configuration(false);
+ // then build up the generated path.
+ FsPermission clusterPerms = getClusterDirectoryPermissions(config);
+ SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath,
+ clusterPerms);
+
+
+ // add AM and provider specific artifacts to the resource map
+ Map<String, LocalResource> providerResources;
+ // standard AM resources
+ sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem,
+ config,
+ amLauncher,
+ instanceDefinition,
+ snapshotConfPath,
+ generatedConfDirPath,
+ clientConfExtras,
+ libdir,
+ tempPath,
+ usingMiniMRCluster);
+ //add provider-specific resources
+ provider.prepareAMAndConfigForLaunch(sliderFileSystem,
+ config,
+ amLauncher,
+ instanceDefinition,
+ snapshotConfPath,
+ generatedConfDirPath,
+ clientConfExtras,
+ libdir,
+ tempPath,
+ usingMiniMRCluster);
+
+ // now that the site config is fully generated, the provider gets
+ // to do a quick review of them.
+ log.debug("Preflight validation of cluster configuration");
+
+
+ sliderAM.preflightValidateClusterConfiguration(sliderFileSystem,
+ clustername,
+ config,
+ instanceDefinition,
+ clusterDirectory,
+ generatedConfDirPath,
+ clusterSecure
+ );
+
+ provider.preflightValidateClusterConfiguration(sliderFileSystem,
+ clustername,
+ config,
+ instanceDefinition,
+ clusterDirectory,
+ generatedConfDirPath,
+ clusterSecure
+ );
+
+
+ // now add the image if it was set
+ if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) {
+ log.debug("Registered image path {}", imagePath);
+ }
+
+
+ // build the environment
+ amLauncher.putEnv(
+ SliderUtils.buildEnvMap(sliderAMResourceComponent));
+ ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir,
+ libdir,
+ getConfig(),
+ usingMiniMRCluster);
+ amLauncher.setEnv("CLASSPATH",
+ classpath.buildClasspath());
+ if (log.isDebugEnabled()) {
+ log.debug("AM classpath={}", classpath);
+ log.debug("Environment Map:\n{}",
+ SliderUtils.stringifyMap(amLauncher.getEnv()));
+ log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath));
+ }
+
+ // rm address
+
+ InetSocketAddress rmSchedulerAddress = null;
+ try {
+ rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config);
+ } catch (IllegalArgumentException e) {
+ throw new BadConfigException("%s Address invalid: %s",
+ YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ config.get(
+ YarnConfiguration.RM_SCHEDULER_ADDRESS)
+ );
+
+ }
+ String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress);
+
+ CommandLineBuilder commandLine = new CommandLineBuilder();
+ commandLine.addJavaBinary();
+ // insert any JVM options);
+ sliderAM.addJVMOptions(instanceDefinition, commandLine);
+ // enable asserts if the text option is set
+ commandLine.enableJavaAssertions();
+ // add the AM sevice entry point
+ commandLine.add(SliderAppMaster.SERVICE_CLASSNAME);
+
+ // create action and the cluster name
+ commandLine.add(SliderActions.ACTION_CREATE, clustername);
+
+ // debug
+ if (debugAM) {
+ commandLine.add(Arguments.ARG_DEBUG);
+ }
+
+ // set the cluster directory path
+ commandLine.add(Arguments.ARG_CLUSTER_URI, clusterDirectory.toUri());
+
+ if (!isUnset(rmAddr)) {
+ commandLine.add(Arguments.ARG_RM_ADDR, rmAddr);
+ }
+
+ if (serviceArgs.getFilesystemURL() != null) {
+ commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemURL());
+ }
+
+ addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
+ DEFAULT_REGISTRY_PATH);
+ addMandatoryConfOptionToCLI(commandLine, config, REGISTRY_ZK_QUORUM);
+
+ if (clusterSecure) {
+ // if the cluster is secure, make sure that
+ // the relevant security settings go over
+ addConfOptionToCLI(commandLine, config, KEY_SECURITY_ENABLED);
+ addConfOptionToCLI(commandLine,
+ config,
+ DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ }
+ // write out the path output
+ commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
+
+ String cmdStr = commandLine.build();
+ log.info("Completed setting up app master command {}", cmdStr);
+
+ amLauncher.addCommandLine(commandLine);
+
+ // the Slider AM gets to configure the AM requirements, not the custom provider
+ sliderAM.prepareAMResourceRequirements(sliderAMResourceComponent,
+ amLauncher.getResource());
+
+
+ // Set the priority for the application master
+
+ int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY,
+ DEFAULT_YARN_QUEUE_PRIORITY);
+
+
+ amLauncher.setPriority(amPriority);
+
+ // Set the queue to which this application is to be submitted in the RM
+ // Queue for App master
+ String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE);
+
+ amLauncher.setQueue(amQueue);
+
+ // Submit the application to the applications manager
+ // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
+ // Ignore the response as either a valid response object is returned on success
+ // or an exception thrown to denote some form of a failure
+
+
+ // submit the application
+ LaunchedApplication launchedApplication = amLauncher.submitApplication();
+ return launchedApplication;
+ }
+
+
+ /**
+ * Wait for the launched app to be accepted
+ * @param waittime time in millis
+ * @return exit code
+ * @throws YarnException
+ * @throws IOException
+ */
+ public int waitForAppAccepted(LaunchedApplication launchedApplication,
+ int waittime) throws
+ YarnException,
+ IOException {
+ assert launchedApplication != null;
+ int exitCode;
+ // wait for the submit state to be reached
+ ApplicationReport report = launchedApplication.monitorAppToState(
+ YarnApplicationState.ACCEPTED,
+ new Duration(Constants.ACCEPT_TIME));
+
+
+ // may have failed, so check that
+ if (SliderUtils.hasAppFinished(report)) {
+ exitCode = buildExitCode(report);
+ } else {
+ // exit unless there is a wait
+ exitCode = EXIT_SUCCESS;
+
+ if (waittime != 0) {
+ // waiting for state to change
+ Duration duration = new Duration(waittime * 1000);
+ duration.start();
+ report = launchedApplication.monitorAppToState(
+ YarnApplicationState.RUNNING, duration);
+ if (report != null &&
+ report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
+ exitCode = EXIT_SUCCESS;
+ } else {
+
+ launchedApplication.kill("");
+ exitCode = buildExitCode(report);
+ }
+ }
+ }
+ return exitCode;
+ }
+
+
+ /**
+ * Propagate any critical principals from the current site config down to the HBase one.
+ * @param clusterSpec cluster spec
+ * @param config config to read from
+ */
+ private void propagatePrincipals(ClusterDescription clusterSpec,
+ Configuration config) {
+ String dfsPrincipal = config.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ if (dfsPrincipal != null) {
+ String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
+ DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY;
+ clusterSpec.setOptionifUnset(siteDfsPrincipal, dfsPrincipal);
+ }
+ }
+
+
+ /**
+ * Propagate any critical principals from the current site config down to the HBase one.
+ * @param config config to read from
+ * @param clusterSpec cluster spec
+ */
+ private void propagatePrincipals(Configuration config,
+ AggregateConf clusterSpec) {
+ String dfsPrincipal = config.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ if (dfsPrincipal != null) {
+ String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
+ DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY;
+ clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset(
+ siteDfsPrincipal,
+ dfsPrincipal);
+ }
+ }
+
+
+ private boolean addConfOptionToCLI(CommandLineBuilder cmdLine,
+ Configuration conf,
+ String key) {
+ String val = conf.get(key);
+ if (val != null) {
+ cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private void addConfOptionToCLI(CommandLineBuilder cmdLine,
+ Configuration conf,
+ String key, String defVal) {
+ String val = conf.get(key, defVal);
+ cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
+ }
+
+ private void addMandatoryConfOptionToCLI(CommandLineBuilder cmdLine,
+ Configuration conf,
+ String key) throws BadConfigException {
+ if (!addConfOptionToCLI(cmdLine, conf, key)) {
+ throw new BadConfigException("Missing configuration option: " + key);
+ }
+ }
+
+ /**
+ * Create a path that must exist in the cluster fs
+ * @param uri uri to create
+ * @return the path
+ * @throws FileNotFoundException if the path does not exist
+ */
+ public Path createPathThatMustExist(String uri) throws
+ SliderException,
+ IOException {
+ return sliderFileSystem.createPathThatMustExist(uri);
+ }
+
+ /**
+ * verify that a live cluster isn't there
+ * @param clustername cluster name
+ * @throws SliderException with exit code EXIT_CLUSTER_LIVE
+ * if a cluster of that name is either live or starting up.
+ */
+ public void verifyNoLiveClusters(String clustername) throws
+ IOException,
+ YarnException {
+ List<ApplicationReport> existing = findAllLiveInstances(clustername);
+
+ if (!existing.isEmpty()) {
+ throw new SliderException(EXIT_APPLICATION_IN_USE,
+ clustername + ": " + E_CLUSTER_RUNNING + " :" +
+ existing.get(0));
+ }
+ }
+
+ public String getUsername() throws IOException {
+ return UserGroupInformation.getCurrentUser().getShortUserName();
+ }
+
+ /**
+ * Get the name of any deployed cluster
+ * @return the cluster name
+ */
+ public String getDeployedClusterName() {
+ return deployedClusterName;
+ }
+
+ @VisibleForTesting
+ public void setDeployedClusterName(String deployedClusterName) {
+ this.deployedClusterName = deployedClusterName;
+ }
+
+ /**
+ * ask if the client is using a mini MR cluster
+ * @return true if they are
+ */
+ private boolean getUsingMiniMRCluster() {
+ return getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER,
+ false);
+ }
+
+ /**
+ * Get the application name used in the zookeeper root paths
+ * @return an application-specific path in ZK
+ */
+ private String getAppName() {
+ return "slider";
+ }
+
+ /**
+ * Wait for the app to start running (or go past that state)
+ * @param duration time to wait
+ * @return the app report; null if the duration turned out
+ * @throws YarnException YARN or app issues
+ * @throws IOException IO problems
+ */
+ @VisibleForTesting
+ public ApplicationReport monitorAppToRunning(Duration duration)
+ throws YarnException, IOException {
+ return monitorAppToState(YarnApplicationState.RUNNING, duration);
+ }
+
+ /**
+ * Build an exit code for an application Id and its report.
+ * If the report parameter is null, the app is killed
+ * @param appId app
+ * @param report report
+ * @return the exit code
+ */
+ private int buildExitCode(ApplicationReport report) throws
+ IOException,
+ YarnException {
+ if (null == report) {
+ forceKillApplication("Reached client specified timeout for application");
+ return EXIT_TIMED_OUT;
+ }
+
+ YarnApplicationState state = report.getYarnApplicationState();
+ FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
+ switch (state) {
+ case FINISHED:
+ if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
+ log.info("Application has completed successfully");
+ return EXIT_SUCCESS;
+ } else {
+ log.info("Application finished unsuccessfully." +
+ "YarnState = {}, DSFinalStatus = {} Breaking monitoring loop",
+ state, dsStatus);
+ return EXIT_YARN_SERVICE_FINISHED_WITH_ERROR;
+ }
+
+ case KILLED:
+ log.info("Application did not finish. YarnState={}, DSFinalStatus={}",
+ state, dsStatus);
+ return EXIT_YARN_SERVICE_KILLED;
+
+ case FAILED:
+ log.info("Application Failed. YarnState={}, DSFinalStatus={}", state,
+ dsStatus);
+ return EXIT_YARN_SERVICE_FAILED;
+ default:
+ //not in any of these states
+ return EXIT_SUCCESS;
+ }
+ }
+
+ /**
+ * Monitor the submitted application for reaching the requested state.
+ * Will also report if the app reaches a later state (failed, killed, etc)
+ * Kill application if duration!= null & time expires.
+ * Prerequisite: the applicatin was launched.
+ * @param desiredState desired state.
+ * @param duration how long to wait -must be more than 0
+ * @return the application report -null on a timeout
+ * @throws YarnException
+ * @throws IOException
+ */
+ @VisibleForTesting
+ public ApplicationReport monitorAppToState(
+ YarnApplicationState desiredState,
+ Duration duration)
+ throws YarnException, IOException {
+ LaunchedApplication launchedApplication =
+ new LaunchedApplication(applicationId, yarnClient);
+ return launchedApplication.monitorAppToState(desiredState, duration);
+ }
+
+ /**
+ * Get the report of a this application
+ * @return the app report or null if it could not be found.
+ * @throws IOException
+ * @throws YarnException
+ */
+ public ApplicationReport getApplicationReport() throws
+ IOException,
+ YarnException {
+ return getApplicationReport(applicationId);
+ }
+
+ /**
+ * Kill the submitted application by sending a call to the ASM
+ * @throws YarnException
+ * @throws IOException
+ */
+ public boolean forceKillApplication(String reason)
+ throws YarnException, IOException {
+ if (applicationId != null) {
+ new LaunchedApplication(applicationId, yarnClient).forceKill(reason);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * List Slider instances belonging to a specific user
+ * @param user user: "" means all users
+ * @return a possibly empty list of Slider AMs
+ */
+ @VisibleForTesting
+ public List<ApplicationReport> listSliderInstances(String user)
+ throws YarnException, IOException {
+ return YARNRegistryClient.listInstances();
+ }
+
+ /**
+ * Implement the list action: list all nodes
+ * @return exit code of 0 if a list was created
+ */
+ @VisibleForTesting
+ public int actionList(String clustername) throws IOException, YarnException {
+ verifyBindingsDefined();
+
+ String user = UserGroupInformation.getCurrentUser().getUserName();
+ List<ApplicationReport> instances = listSliderInstances(user);
+
+ if (isUnset(clustername)) {
+ log.info("Instances for {}: {}",
+ (user != null ? user : "all users"),
+ instances.size());
+ for (ApplicationReport report : instances) {
+ logAppReport(report);
+ }
+ return EXIT_SUCCESS;
+ } else {
+ SliderUtils.validateClusterName(clustername);
+ log.debug("Listing cluster named {}", clustername);
+ ApplicationReport report =
+ findClusterInInstanceList(instances, clustername);
+ if (report != null) {
+ logAppReport(report);
+ return EXIT_SUCCESS;
+ } else {
+ throw unknownClusterException(clustername);
+ }
+ }
+ }
+
+ /**
+ * Log the application report at INFO
+ * @param report report to log
+ */
+ public void logAppReport(ApplicationReport report) {
+ log.info(SliderUtils.appReportToString(report, "\n"));
+ }
+
+ /**
+ * Implement the islive action: probe for a cluster of the given name existing
+ * @return exit code
+ */
+ @VisibleForTesting
+ public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException {
+ verifyBindingsDefined();
+ SliderUtils.validateClusterName(name);
+ log.debug("actionFlex({})", name);
+ Map<String, Integer> roleInstances = new HashMap<String, Integer>();
+ Map<String, String> roleMap = args.getComponentMap();
+ for (Map.Entry<String, String> roleEntry : roleMap.entrySet()) {
+ String key = roleEntry.getKey();
+ String val = roleEntry.getValue();
+ try {
+ roleInstances.put(key, Integer.valueOf(val));
+ } catch (NumberFormatException e) {
+ throw new BadCommandArgumentsException("Requested count of role %s" +
+ " is not a number: \"%s\"",
+ key, val);
+ }
+ }
+ return flex(name, roleInstances);
+ }
+
+ /**
+ * Test for a cluster existing probe for a cluster of the given name existing
+ * in the filesystem. If the live param is set, it must be a live cluster
+ * @return exit code
+ */
+ @VisibleForTesting
+ public int actionExists(String name, boolean live) throws YarnException, IOException {
+ verifyBindingsDefined();
+ SliderUtils.validateClusterName(name);
+ log.debug("actionExists({}, {})", name, live);
+
+ //initial probe for a cluster in the filesystem
+ Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name);
+ if (!sliderFileSystem.getFileSystem().exists(clusterDirectory)) {
+ throw unknownClusterException(name);
+ }
+
+ //test for liveness if desired
+
+ if (live) {
+ ApplicationReport instance = findInstance(name);
+ if (instance == null) {
+ log.info("cluster {} not running", name);
+ return EXIT_FALSE;
+ } else {
+ // the app exists, but it may be in a terminated state
+ SliderUtils.OnDemandReportStringifier report =
+ new SliderUtils.OnDemandReportStringifier(instance);
+ YarnApplicationState state =
+ instance.getYarnApplicationState();
+ if (state.ordinal() >= YarnApplicationState.FINISHED.ordinal()) {
+ //cluster in the list of apps but not running
+ log.info("Cluster {} found but is in state {}", name, state);
+ log.debug("State {}", report);
+ return EXIT_FALSE;
+ }
+ log.info("Cluster {} is running:\n{}", name, report);
+ }
+ } else {
+ log.info("Cluster {} exists but is not running", name);
+
+ }
+ return EXIT_SUCCESS;
+ }
+
+
+ /**
+ * Kill a specific container of the cluster
+ * @param name cluster name
+ * @param args arguments
+ * @return exit code
+ * @throws YarnException
+ * @throws IOException
+ */
+ public int actionKillContainer(String name,
+ ActionKillContainerArgs args) throws
+ YarnException,
+ IOException {
+ String id = args.id;
+ if (SliderUtils.isUnset(id)) {
+ throw new BadCommandArgumentsException("Missing container id");
+ }
+ log.info("killingContainer {}:{}", name, id);
+ SliderClusterOperations clusterOps =
+ new SliderClusterOperations(bondToCluster(name));
+ try {
+ clusterOps.killContainer(id);
+ } catch (NoSuchNodeException e) {
+ throw new BadClusterStateException("Container %s not found in cluster %s",
+ id, name);
+ }
+ return EXIT_SUCCESS;
+ }
+
+ /**
+ * Echo operation (not currently wired up to command line)
+ * @param name cluster name
+ * @param args arguments
+ * @return the echoed text
+ * @throws YarnException
+ * @throws IOException
+ */
+ public String actionEcho(String name, ActionEchoArgs args) throws
+ YarnException,
+ IOException {
+ String message = args.message;
+ if (message == null) {
+ throw new BadCommandArgumentsException("missing message");
+ }
+ SliderClusterOperations clusterOps =
+ new SliderClusterOperations(bondToCluster(name));
+ return clusterOps.echo(message);
+ }
+
+ /**
+ * Get at the service registry operations
+ * @return registry client -valid after the service is inited.
+ */
+ public YARNRegistryClient getYARNRegistryClient() {
+ return YARNRegistryClient;
+ }
+
+ /**
+ * Find an instance of an application belonging to the current user
+ * @param appname application name
+ * @return the app report or null if none is found
+ * @throws YarnException YARN issues
+ * @throws IOException IO problems
+ */
+ private ApplicationReport findInstance(String appname) throws
+ YarnException,
+ IOException {
+ return YARNRegistryClient.findInstance(appname);
+ }
+
+ private RunningApplication findApplication(String appname) throws
+ YarnException,
+ IOException {
+ ApplicationReport applicationReport = findInstance(appname);
+ return applicationReport != null ? new RunningApplication(yarnClient, applicationReport): null;
+
+ }
+
+ /**
+ * find all live instances of a specific app -if there is >1 in the cluster,
+ * this returns them all. State should be running or less
+ * @param appname application name
+ * @return the list of all matching application instances
+ */
+ private List<ApplicationReport> findAllLiveInstances(String appname)
+ throws YarnException, IOException {
+
+ return YARNRegistryClient.findAllLiveInstances(appname);
+ }
+
+
+ public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
+ String appname) {
+ return yarnClient.findClusterInInstanceList(instances, appname);
+ }
+
+ /**
+ * Connect to a Slider AM
+ * @param app application report providing the details on the application
+ * @return an instance
+ * @throws YarnException
+ * @throws IOException
+ */
+ private SliderClusterProtocol connect(ApplicationReport app) throws
+ YarnException,
+ IOException {
+
+ try {
+ return RpcBinder.getProxy(getConfig(),
+ yarnClient.getRmClient(),
+ app,
+ Constants.CONNECT_TIMEOUT,
+ Constants.RPC_TIMEOUT);
+ } catch (InterruptedException e) {
+ throw new SliderException(SliderExitCodes.EXIT_TIMED_OUT,
+ e,
+ "Interrupted waiting for communications with the Slider AM");
+ }
+ }
+
+ /**
+ * Status operation
+ *
+ * @param clustername cluster name
+ * @param statusArgs status arguments
+ * @return 0 -for success, else an exception is thrown
+ * @throws YarnException
+ * @throws IOException
+ */
+ @VisibleForTesting
+ public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws
+ YarnException,
+ IOException {
+ verifyBindingsDefined();
+ SliderUtils.validateClusterName(clustername);
+ String outfile = statusArgs.getOutput();
+ ClusterDescription status = getClusterDescription(clustername);
+ String text = status.toJsonString();
+ if (outfile == null) {
+ log.info(text);
+ } else {
+ status.save(new File(outfile).getAbsoluteFile());
+ }
+ return EXIT_SUCCESS;
+ }
+
+ /**
+ * Version Details
+ * @return exit code
+ */
+ public int actionVersion() {
+ SliderVersionInfo.loadAndPrintVersionInfo(log);
+ return EXIT_SUCCESS;
+ }
+
+ /**
+ * Freeze the cluster
+ *
+ * @param clustername cluster name
+ * @param freezeArgs arguments to the freeze
+ * @return EXIT_SUCCESS if the cluster was not running by the end of the operation
+ */
+ public int actionFreeze(String clustername,
+ ActionFreezeArgs freezeArgs) throws
+ YarnException,
+ IOException {
+ verifyBindingsDefined();
+ SliderUtils.validateClusterName(clustername);
+ int waittime = freezeArgs.getWaittime();
+ String text = freezeArgs.message;
+ boolean forcekill = freezeArgs.force;
+ log.debug("actionFreeze({}, reason={}, wait={}, force={})", clustername,
+ text,
+ waittime,
+ forcekill);
+
+ //is this actually a known cluster?
+ sliderFileSystem.locateInstanceDefinition(clustername);
+ ApplicationReport app = findInstance(clustername);
+ if (app == null) {
+ // exit early
+ log.info("Cluster {} not running", clustername);
+ // not an error to freeze a frozen cluster
+ return EXIT_SUCCESS;
+ }
+ log.debug("App to freeze was found: {}:\n{}", clustername,
+ new SliderUtils.OnDemandReportStringifier(app));
+ if (app.getYarnApplicationState().ordinal() >=
+ YarnApplicationState.FINISHED.ordinal()) {
+ log.info("Cluster {} is a terminated state {}", clustername,
+ app.getYarnApplicationState());
+ return EXIT_SUCCESS;
+ }
+ LaunchedApplication application = new LaunchedApplication(yarnClient, app);
+ applicationId = application.getApplicationId();
+
+
+ if (forcekill) {
+ //escalating to forced kill
+ application.kill("Forced freeze of " + clustername +
+ ": " + text);
+ } else {
+ try {
+ SliderClusterProtocol appMaster = connect(app);
+ Messages.StopClusterRequestProto r =
+ Messages.StopClusterRequestProto
+ .newBuilder()
+ .setMessage(text)
+ .build();
+ appMaster.stopCluster(r);
+
+ log.debug("Cluster stop command issued");
+
+ } catch (YarnException e) {
+ log.warn("Exception while trying to terminate {}: {}", clustername, e);
+ return EXIT_FALSE;
+ } catch (IOException e) {
+ log.warn("Exception while trying to terminate {}: {}", clustername, e);
+ return EXIT_FALSE;
+ }
+ }
+
+ //wait for completion. We don't currently return an exception during this process
+ //as the stop operation has been issued, this is just YARN.
+ try {
+ if (waittime > 0) {
+ ApplicationReport applicationReport =
+ application.monitorAppToState(YarnApplicationState.FINISHED,
+ new Duration(waittime * 1000));
+ if (applicationReport == null) {
+ log.info("application did not shut down in time");
+ return EXIT_FALSE;
+ }
+ }
+ } catch (YarnException e) {
+ log.warn("Exception while waiting for the cluster {} to shut down: {}",
+ clustername, e);
+ } catch (IOException e) {
+ log.warn("Exception while waiting for the cluster {} to shut down: {}",
+ clustername, e);
+ }
+
+ return EXIT_SUCCESS;
+ }
+
+ /*
+ * Creates a site conf with entries from clientProperties of ClusterStatus
+ * @param desc ClusterDescription, can be null
+ * @param clustername, can be null
+ * @return site conf
+ */
+ public Configuration getSiteConf(ClusterDescription desc, String clustername)
+ throws YarnException, IOException {
+ if (desc == null) {
+ desc = getClusterDescription();
+ }
+ if (clustername == null) {
+ clustername = getDeployedClusterName();
+ }
+ String description = "Slider Application Instance " + clustername;
+
+ Configuration siteConf = new Configuration(false);
+ for (String key : desc.clientProperties.keySet()) {
+ siteConf.set(key, desc.clientProperties.get(key), description);
+ }
+ return siteConf;
+ }
+
+
+ /**
+ * get the cluster configuration
+ * @param clustername cluster name
+ * @return the cluster name
+ */
+
+ @SuppressWarnings(
+ {"UseOfSystemOutOrSystemErr", "IOResourceOpenedButNotSafelyClosed"})
+ public int actionGetConf(String clustername, ActionGetConfArgs confArgs) throws
+ YarnException,
+ IOException {
+ File outfile = null;
+
+ if (confArgs.getOutput() != null) {
+ outfile = new File(confArgs.getOutput());
+ }
+
+ String format = confArgs.getFormat();
+ verifyBindingsDefined();
+ SliderUtils.validateClusterName(clustername);
+ ClusterDescription status = getClusterDescription(clustername);
+ Writer writer;
+ boolean toPrint;
+ if (outfile != null) {
+ writer = new FileWriter(outfile);
+ toPrint = false;
+ } else {
+ writer = new StringWriter();
+ toPrint = true;
+ }
+ try {
+ String description = "Slider Application Instance " + clustername;
+ if (format.equals(Arguments.FORMAT_XML)) {
+ Configuration siteConf = getSiteConf(status, clustername);
+ siteConf.writeXml(writer);
+ } else if (format.equals(Arguments.FORMAT_PROPERTIES)) {
+ Properties props = new Properties();
+ props.putAll(status.clientProperties);
+ props.store(writer, description);
+ } else {
+ throw new BadCommandArgumentsException("Unknown format: " + format);
+ }
+ } finally {
+ // data is written.
+ // close the file
+ writer.close();
+ }
+ // then, if this is not a file write, print it
+ if (toPrint) {
+ // not logged
+ System.err.println(writer.toString());
+ }
+ return EXIT_SUCCESS;
+ }
+
+ /**
+ * Restore a cluster
+ */
+ public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException {
+ SliderUtils.validateClusterName(clustername);
+ // see if it is actually running and bail out;
+ verifyBindingsDefined();
+ verifyNoLiveClusters(clustername);
+
+
+ //start the cluster
+ return startCluster(clustername, thaw);
+ }
+
+ /**
+ * Implement flexing
+ * @param clustername name of the cluster
+ * @param roleInstances map of new role instances
+ * @return EXIT_SUCCESS if the #of nodes in a live cluster changed
+ * @throws YarnException
+ * @throws IOException
+ */
+ public int flex(String clustername,
+ Map<String, Integer> roleInstances) throws
+ YarnException,
+ IOException {
+ verifyBindingsDefined();
+ SliderUtils.validateClusterName(clustername);
+ Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
+ AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
+ clustername,
+ clusterDirectory);
+
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+ for (Map.Entry<String, Integer> entry : roleInstances.entrySet()) {
+ String role = entry.getKey();
+ int count = entry.getValue();
+ if (count < 0) {
+ throw new BadCommandArgumentsException("Requested number of " + role
+ + " instances is out of range");
+ }
+ resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES,
+ Integer.toString(count));
+
+
+ log.debug("Flexed cluster specification ( {} -> {}) : \n{}",
+ role,
+ count,
+ resources);
+ }
+ int exitCode = EXIT_FALSE;
+ // save the specification
+ try {
+ InstanceIO.updateInstanceDefinition(sliderFileSystem, clusterDirectory,instanceDefinition);
+ } catch (LockAcquireFailedException e) {
+ // lock failure
+ log.debug("Failed to lock dir {}", clusterDirectory, e);
+ log.warn("Failed to save new resource definition to {} : {}", clusterDirectory,
+ e.toString());
+
+
+ }
+
+ // now see if it is actually running and tell it about the update if it is
+ ApplicationReport instance = findInstance(clustername);
+ if (instance != null) {
+ log.info("Flexing running cluster");
+ SliderClusterProtocol appMaster = connect(instance);
+ SliderClusterOperations clusterOps = new SliderClusterOperations(appMaster);
+ if (clusterOps.flex(instanceDefinition.getResources())) {
+ log.info("Cluster size updated");
+ exitCode = EXIT_SUCCESS;
+ } else {
+ log.info("Requested size is the same as current size: no change");
+ }
+ } else {
+ log.info("No running instance to update");
+ }
+ return exitCode;
+ }
+
+
+ /**
+ * Load the persistent cluster description
+ * @param clustername name of the cluster
+ * @return the description in the filesystem
+ * @throws IOException any problems loading -including a missing file
+ */
+ @VisibleForTesting
+ public AggregateConf loadPersistedClusterDescription(String clustername) throws
+ IOException,
+ SliderException,
+ LockAcquireFailedException {
+ Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
+ ConfPersister persister = new ConfPersister(sliderFileSystem, clusterDirectory);
+ AggregateConf instanceDescription = new AggregateConf();
+ persister.load(instanceDescription);
+ return instanceDescription;
+ }
+
+ /**
+ * Connect to a live cluster and get its current state
+ * @param clustername the cluster name
+ * @return its description
+ */
+ @VisibleForTesting
+ public ClusterDescription getClusterDescription(String clustername) throws
+ YarnException,
+ IOException {
+ SliderClusterOperations clusterOperations =
+ createClusterOperations(clustername);
+ return clusterOperations.getClusterDescription();
+ }
+
+ /**
+ * Connect to the cluster and get its current state
+ * @return its description
+ */
+ @VisibleForTesting
+ public ClusterDescription getClusterDescription() throws
+ YarnException,
+ IOException {
+ return getClusterDescription(getDeployedClusterName());
+ }
+
+ /**
+ * List all node UUIDs in a role
+ * @param role role name or "" for all
+ * @return an array of UUID strings
+ * @throws IOException
+ * @throws YarnException
+ */
+ @VisibleForTesting
+ public String[] listNodeUUIDsByRole(String role) throws
+ IOException,
+ YarnException {
+ return createClusterOperations()
+ .listNodeUUIDsByRole(role);
+ }
+
+ /**
+ * List all nodes in a role. This is a double round trip: once to list
+ * the nodes in a role, another to get their details
+ * @param role
+ * @return an array of ContainerNode instances
+ * @throws IOException
+ * @throws YarnException
+ */
+ @VisibleForTesting
+ public List<ClusterNode> listClusterNodesInRole(String role) throws
+ IOException,
+ YarnException {
+ return createClusterOperations().listClusterNodesInRole(role);
+ }
+
+ /**
+ * Get the details on a list of uuids
+ * @param uuids
+ * @return a possibly empty list of node details
+ * @throws IOException
+ * @throws YarnException
+ */
+ @VisibleForTesting
+ public List<ClusterNode> listClusterNodes(String[] uuids) throws
+ IOException,
+ YarnException {
+
+ if (uuids.length == 0) {
+ // short cut on an empty list
+ return new LinkedList<ClusterNode>();
+ }
+ return createClusterOperations().listClusterNodes(uuids);
+ }
+
+ /**
+ * Get a node from the AM
+ * @param uuid uuid of node
+ * @return deserialized node
+ * @throws IOException IO problems
+ * @throws NoSuchNodeException if the node isn't found
+ */
+ @VisibleForTesting
+ public ClusterNode getNode(String uuid) throws IOException, YarnException {
+ return createClusterOperations().getNode(uuid);
+ }
+
+ /**
+ * Get the instance definition from the far end
+ */
+ @VisibleForTesting
+ public AggregateConf getLiveInstanceDefinition() throws IOException, YarnException {
+ return createClusterOperations().getInstanceDefinition();
+ }
+
+ /**
+ * Bond to a running cluster
+ * @param clustername cluster name
+ * @return the AM RPC client
+ * @throws SliderException if the cluster is unkown
+ */
+ private SliderClusterProtocol bondToCluster(String clustername) throws
+ YarnException,
+ IOException {
+ verifyBindingsDefined();
+ if (clustername == null) {
+ throw unknownClusterException("(undefined)");
+ }
+ ApplicationReport instance = findInstance(clustername);
+ if (null == instance) {
+ throw unknownClusterException(clustername);
+ }
+ return connect(instance);
+ }
+
+ /**
+ * Create a cluster operations instance against a given cluster
+ * @param clustername cluster name
+ * @return a bonded cluster operations instance
+ * @throws YarnException YARN issues
+ * @throws IOException IO problems
+ */
+ private SliderClusterOperations createClusterOperations(String clustername) throws
+ YarnException,
+ IOException {
+ SliderClusterProtocol sliderAM = bondToCluster(clustername);
+ return new SliderClusterOperations(sliderAM);
+ }
+
+ /**
+ * Create a cluster operations instance against the active cluster
+ * -returning any previous created one if held.
+ * @return a bonded cluster operations instance
+ * @throws YarnException YARN issues
+ * @throws IOException IO problems
+ */
+ public SliderClusterOperations createClusterOperations() throws
+ YarnException,
+ IOException {
+ if (sliderClusterOperations == null) {
+ sliderClusterOperations =
+ createClusterOperations(getDeployedClusterName());
+ }
+ return sliderClusterOperations;
+ }
+
+ /**
+ * Wait for an instance of a named role to be live (or past it in the lifecycle)
+ * @param role role to look for
+ * @param timeout time to wait
+ * @return the state. If still in CREATED, the cluster didn't come up
+ * in the time period. If LIVE, all is well. If >LIVE, it has shut for a reason
+ * @throws IOException IO
+ * @throws SliderException Slider
+ * @throws WaitTimeoutException if the wait timed out
+ */
+ @VisibleForTesting
+ public int waitForRoleInstanceLive(String role, long timeout)
+ throws WaitTimeoutException, IOException, YarnException {
+ return createClusterOperations().waitForRoleInstanceLive(role, timeout);
+ }
+
+ /**
+ * Generate an exception for an unknown cluster
+ * @param clustername cluster name
+ * @return an exception with text and a relevant exit code
+ */
+ public UnknownApplicationInstanceException unknownClusterException(String clustername) {
+ return UnknownApplicationInstanceException.unknownInstance(clustername);
+ }
+
+ @Override
+ public String toString() {
+ return "Slider Client in state " + getServiceState()
+ + " and Slider Application Instance " + deployedClusterName;
+ }
+
+ /**
+ * Get all YARN applications
+ * @return a possibly empty list
+ * @throws YarnException
+ * @throws IOException
+ */
+ @VisibleForTesting
+ public List<ApplicationReport> getApplications() throws YarnException, IOException {
+ return yarnClient.getApplications();
+ }
+
+ @VisibleForTesting
+ public ApplicationReport getApplicationReport(ApplicationId appId)
+ throws YarnException, IOException {
+ return new LaunchedApplication(appId, yarnClient).getApplicationReport();
+
+ }
+
+ /**
+ * The configuration used for deployment (after resolution)
+ * @return
+ */
+ @VisibleForTesting
+ public AggregateConf getLaunchedInstanceDefinition() {
+ return launchedInstanceDefinition;
+ }
+
+
+ /**
+ * Status operation
+ *
+ * @param registryArgs registry Arguments
+ * @throws YarnException
+ * @throws IOException
+ */
+ @VisibleForTesting
+ public int actionRegistry(ActionRegistryArgs registryArgs) throws
+ YarnException,
+ IOException {
+ maybeStartRegistry();
+ List<CuratorServiceInstance<ServiceInstanceData>> instances =
+ registry.listInstances(SliderKeys.APP_TYPE);
+
+ for (CuratorServiceInstance<ServiceInstanceData> instance : instances) {
+ log.info("{} at http://{}:{}/", instance.id, instance.address,
+ instance.port);
+ }
+ return EXIT_SUCCESS;
+ }
+
+ /**
+ * List names in the registry
+ * @return
+ * @throws IOException
+ * @throws YarnException
+ */
+ public Collection<String> listRegistryNames() throws IOException, YarnException {
+ Collection<String> names;
+ verifyBindingsDefined();
+
+ return getRegistry().queryForNames();
+ }
+
+ /**
+ * List instances in the registry
+ * @return
+ * @throws IOException
+ * @throws YarnException
+ */
+ public List<CuratorServiceInstance<ServiceInstanceData>> listRegistryInstances()
+ throws IOException, YarnException {
+ maybeStartRegistry();
+ return registry.listInstances(SliderKeys.APP_TYPE);
+ }
+
+ /**
+ * List instances in the registry
+ * @return
+ * @throws IOException
+ * @throws YarnException
+ */
+ public List<String> listRegistryInstanceIDs() throws
+ IOException,
+ YarnException {
+ try {
+ maybeStartRegistry();
+ return registry.instanceIDs(SliderKeys.APP_TYPE);
+ } catch (IOException e) {
+ throw e;
+ } catch (YarnException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Start the registry if it is not there yet
+ * @return the registry service
+ * @throws SliderException
+ * @throws IOException
+ */
+ private synchronized RegistryBinderService<ServiceInstanceData> maybeStartRegistry() throws
+ SliderException,
+ IOException {
+
+ if (registry == null) {
+ registry = startRegistrationService();
+ }
+ return registry;
+ }
+
+ /**
+ * Get the registry binding. As this may start the registry, it can take time
+ * and fail
+ * @return registry the registry service
+ * @throws SliderException slider-specific failures
+ * @throws IOException other failures
+ */
+ @VisibleForTesting
+
+ public RegistryBinderService<ServiceInstanceData> getRegistry() throws
+ SliderException,
+ IOException {
+ return maybeStartRegistry();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java b/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
new file mode 100644
index 0000000..9e1f568
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.common.tools.Duration;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.WaitTimeoutException;
+import org.apache.slider.core.persist.ConfTreeSerDeser;
+import org.codehaus.jackson.JsonParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Cluster operations at a slightly higher level than the RPC code
+ */
+public class SliderClusterOperations {
+ protected static final Logger
+ log = LoggerFactory.getLogger(SliderClusterOperations.class);
+
+ private final SliderClusterProtocol appMaster;
+
+ public SliderClusterOperations(SliderClusterProtocol appMaster) {
+ this.appMaster = appMaster;
+ }
+
+ /**
+ * Get a node from the AM
+ * @param appMaster AM
+ * @param uuid uuid of node
+ * @return deserialized node
+ * @throws IOException IO problems
+ * @throws NoSuchNodeException if the node isn't found
+ */
+ public ClusterNode getNode(String uuid)
+ throws IOException, NoSuchNodeException, YarnException {
+ Messages.GetNodeRequestProto req =
+ Messages.GetNodeRequestProto.newBuilder().setUuid(uuid).build();
+ Messages.GetNodeResponseProto node = appMaster.getNode(req);
+ return ClusterNode.fromProtobuf(node.getClusterNode());
+ }
+
+ public List<ClusterNode> convertNodeWireToClusterNodes(List<Messages.RoleInstanceState> nodes)
+ throws IOException {
+ List<ClusterNode> nodeList = new ArrayList<ClusterNode>(nodes.size());
+ for (Messages.RoleInstanceState node : nodes) {
+ nodeList.add(ClusterNode.fromProtobuf(node));
+ }
+ return nodeList;
+ }
+
+ /**
+ * Echo text (debug action)
+ * @param text text
+ * @return the text, echoed back
+ * @throws YarnException
+ * @throws IOException
+ */
+ public String echo(String text) throws
+ YarnException,
+ IOException {
+ Messages.EchoRequestProto.Builder builder =
+ Messages.EchoRequestProto.newBuilder();
+ builder.setText(text);
+ Messages.EchoRequestProto req =
+ builder.build();
+ Messages.EchoResponseProto response =
+ appMaster.echo(req);
+ return response.getText();
+ }
+
+
+ /**
+ * Connect to a live cluster and get its current state
+ * @return its description
+ */
+ public ClusterDescription getClusterDescription()
+ throws YarnException, IOException {
+
+ Messages.GetJSONClusterStatusRequestProto req =
+ Messages.GetJSONClusterStatusRequestProto.newBuilder().build();
+ Messages.GetJSONClusterStatusResponseProto resp =
+ appMaster.getJSONClusterStatus(req);
+ String statusJson = resp.getClusterSpec();
+ try {
+ return ClusterDescription.fromJson(statusJson);
+ } catch (JsonParseException e) {
+ log.error(
+ "Exception " + e + " parsing:\n" + statusJson,
+ e);
+ throw e;
+ }
+ }
+
+ public AggregateConf getInstanceDefinition()
+ throws YarnException, IOException {
+ Messages.GetInstanceDefinitionRequestProto.Builder builder =
+ Messages.GetInstanceDefinitionRequestProto.newBuilder();
+
+ Messages.GetInstanceDefinitionRequestProto request = builder.build();
+ Messages.GetInstanceDefinitionResponseProto response =
+ appMaster.getInstanceDefinition(request);
+
+ ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
+
+ ConfTree internal = confTreeSerDeser.fromJson(response.getInternal());
+ ConfTree resources = confTreeSerDeser.fromJson(response.getResources());
+ ConfTree app = confTreeSerDeser.fromJson(response.getApplication());
+ AggregateConf instanceDefinition =
+ new AggregateConf(resources, app, internal);
+ return instanceDefinition;
+ }
+ /**
+ * Kill a container
+ * @param id container ID
+ * @return a success flag
+ * @throws YarnException
+ * @throws IOException
+ */
+ public boolean killContainer(String id) throws
+ YarnException,
+ IOException {
+ Messages.KillContainerRequestProto.Builder builder =
+ Messages.KillContainerRequestProto.newBuilder();
+ builder.setId(id);
+ Messages.KillContainerRequestProto req =
+ builder.build();
+ Messages.KillContainerResponseProto response =
+ appMaster.killContainer(req);
+ return response.getSuccess();
+ }
+
+ /**
+ * List all node UUIDs in a role
+ * @param role role name or "" for all
+ * @return an array of UUID strings
+ * @throws IOException
+ * @throws YarnException
+ */
+ public String[] listNodeUUIDsByRole(String role) throws
+ IOException,
+ YarnException {
+ Collection<String> uuidList = innerListNodeUUIDSByRole(role);
+ String[] uuids = new String[uuidList.size()];
+ return uuidList.toArray(uuids);
+ }
+
+ public List<String> innerListNodeUUIDSByRole(String role) throws
+ IOException,
+ YarnException {
+ Messages.ListNodeUUIDsByRoleRequestProto req =
+ Messages.ListNodeUUIDsByRoleRequestProto
+ .newBuilder()
+ .setRole(role)
+ .build();
+ Messages.ListNodeUUIDsByRoleResponseProto resp =
+ appMaster.listNodeUUIDsByRole(req);
+ return resp.getUuidList();
+ }
+
+ /**
+ * List all nodes in a role. This is a double round trip: once to list
+ * the nodes in a role, another to get their details
+ * @param role
+ * @return an array of ContainerNode instances
+ * @throws IOException
+ * @throws YarnException
+ */
+ public List<ClusterNode> listClusterNodesInRole(String role) throws
+ IOException,
+ YarnException {
+
+ Collection<String> uuidList = innerListNodeUUIDSByRole(role);
+ Messages.GetClusterNodesRequestProto req =
+ Messages.GetClusterNodesRequestProto
+ .newBuilder()
+ .addAllUuid(uuidList)
+ .build();
+ Messages.GetClusterNodesResponseProto resp = appMaster.getClusterNodes(req);
+ return convertNodeWireToClusterNodes(resp.getClusterNodeList());
+ }
+
+ /**
+ * Get the details on a list of uuids
+ * @param uuids
+ * @return a possibly empty list of node details
+ * @throws IOException
+ * @throws YarnException
+ */
+ @VisibleForTesting
+ public List<ClusterNode> listClusterNodes(String[] uuids) throws
+ IOException,
+ YarnException {
+
+ Messages.GetClusterNodesRequestProto req =
+ Messages.GetClusterNodesRequestProto
+ .newBuilder()
+ .addAllUuid(Arrays.asList(uuids))
+ .build();
+ Messages.GetClusterNodesResponseProto resp = appMaster.getClusterNodes(req);
+ return convertNodeWireToClusterNodes(resp.getClusterNodeList());
+ }
+
+ /**
+ * Wait for an instance of a named role to be live (or past it in the lifecycle)
+ * @param clustername cluster
+ * @param role role to look for
+ * @param timeout time to wait
+ * @return the state. If still in CREATED, the cluster didn't come up
+ * in the time period. If LIVE, all is well. If >LIVE, it has shut for a reason
+ * @throws IOException IO
+ * @throws SliderException Slider
+ * @throws WaitTimeoutException if the wait timed out
+ */
+ @VisibleForTesting
+ public int waitForRoleInstanceLive(String role, long timeout)
+ throws WaitTimeoutException, IOException, YarnException {
+ Duration duration = new Duration(timeout).start();
+ boolean live = false;
+ int state = ClusterDescription.STATE_CREATED;
+
+ log.info("Waiting {} millis for a live node in role {}", timeout, role);
+ while (!live) {
+ // see if there is a node in that role yet
+ List<String> uuids = innerListNodeUUIDSByRole(role);
+ String[] containers = uuids.toArray(new String[uuids.size()]);
+ int roleCount = containers.length;
+ ClusterNode roleInstance = null;
+ if (roleCount != 0) {
+
+ // if there is, get the node
+ roleInstance = getNode(containers[0]);
+ if (roleInstance != null) {
+ state = roleInstance.state;
+ live = state >= ClusterDescription.STATE_LIVE;
+ }
+ }
+ if (!live) {
+ if (duration.getLimitExceeded()) {
+ throw new WaitTimeoutException(
+ String.format("Timeout after %d millis" +
+ " waiting for a live instance of type %s; " +
+ "instances found %d %s",
+ timeout, role, roleCount,
+ (roleInstance != null
+ ? (" instance -\n" + roleInstance.toString())
+ : "")
+ ));
+ } else {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignored) {
+ // ignored
+ }
+ }
+ }
+ }
+ return state;
+ }
+
+ public boolean flex(ConfTree resources) throws IOException, YarnException {
+ Messages.FlexClusterRequestProto request =
+ Messages.FlexClusterRequestProto.newBuilder()
+ .setClusterSpec(resources.toJson())
+ .build();
+ Messages.FlexClusterResponseProto response =
+ appMaster.flexCluster(request);
+ return response.getResponse();
+ }
+
+
+ /**
+ * Commit (possibly delayed) AM suicide
+ *
+ * @param signal exit code
+ * @param text text text to log
+ * @param delay delay in millis
+ * @throws YarnException
+ * @throws IOException
+ */
+ public void amSuicide(String text, int signal, int delay) throws
+ YarnException,
+ IOException {
+ Messages.AMSuicideRequestProto.Builder builder =
+ Messages.AMSuicideRequestProto.newBuilder();
+ builder.setText(text);
+ builder.setSignal(signal);
+ builder.setDelay(delay);
+ Messages.AMSuicideRequestProto req =
+ builder.build();
+ Messages.AMSuicideResponseProto response =
+ appMaster.amSuicide(req);
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
new file mode 100644
index 0000000..26130ee
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.Duration;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A class that extends visibility to some of the YarnClientImpl
+ * members and data structures, and factors out pure-YARN operations
+ * from the slider entry point service
+ */
+public class SliderYarnClientImpl extends YarnClientImpl {
+ protected static final Logger
+ log = LoggerFactory.getLogger(SliderYarnClientImpl.class);
+
+ /**
+ * Get the RM Client RPC interface
+ * @return an RPC interface valid after initialization and authentication
+ */
+ public ApplicationClientProtocol getRmClient() {
+ return rmClient;
+ }
+
+
+ /**
+ * List Slider instances belonging to a specific user
+ * @param user user: "" means all users
+ * @return a possibly empty list of Slider AMs
+ */
+ public List<ApplicationReport> listInstances(String user)
+ throws YarnException, IOException {
+ Set<String> types = new HashSet<String>(1);
+ types.add(SliderKeys.APP_TYPE);
+ List<ApplicationReport> allApps = getApplications(types);
+ List<ApplicationReport> results = new ArrayList<ApplicationReport>();
+ for (ApplicationReport report : allApps) {
+ if (user == null || user.equals(report.getUser())) {
+ results.add(report);
+ }
+ }
+ return results;
+ }
+
+
+ /**
+ * find all instances of a specific app -if there is >1 in the cluster,
+ * this returns them all
+ * @param user user
+ * @param appname application name
+ * @return the list of all matching application instances
+ */
+ @VisibleForTesting
+ public List<ApplicationReport> findAllInstances(String user,
+ String appname) throws
+ IOException,
+ YarnException {
+ List<ApplicationReport> instances = listInstances(user);
+ List<ApplicationReport> results =
+ new ArrayList<ApplicationReport>(instances.size());
+ for (ApplicationReport report : instances) {
+ if (report.getName().equals(appname)) {
+ results.add(report);
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Helper method to determine if a cluster application is running -or
+ * is earlier in the lifecycle
+ * @param app application report
+ * @return true if the application is considered live
+ */
+ public boolean isApplicationLive(ApplicationReport app) {
+ return app.getYarnApplicationState().ordinal() <=
+ YarnApplicationState.RUNNING.ordinal();
+ }
+
+
+ /**
+ * Kill a running application
+ * @param applicationId
+ * @return the response
+ * @throws YarnException YARN problems
+ * @throws IOException IO problems
+ */
+ public KillApplicationResponse killRunningApplication(ApplicationId applicationId,
+ String reason) throws
+ YarnException,
+ IOException {
+ log.info("Killing application {} - {}", applicationId.getClusterTimestamp(),
+ reason);
+ KillApplicationRequest request =
+ Records.newRecord(KillApplicationRequest.class);
+ request.setApplicationId(applicationId);
+ return getRmClient().forceKillApplication(request);
+ }
+
+ private String getUsername() throws IOException {
+ return UserGroupInformation.getCurrentUser().getShortUserName();
+ }
+ /**
+ * Force kill a yarn application by ID. No niceities here
+ */
+ public void emergencyForceKill(String applicationId) throws
+ YarnException,
+ IOException {
+
+
+ if ("all".equals(applicationId)) {
+ // user wants all instances killed
+ String user = getUsername();
+ log.info("Killing all applications belonging to {}", user);
+ Collection<ApplicationReport> instances = listInstances(user);
+ for (ApplicationReport instance : instances) {
+ if (isApplicationLive(instance)) {
+ ApplicationId appId = instance.getApplicationId();
+ log.info("Killing Application {}", appId);
+
+ killRunningApplication(appId, "forced kill");
+ }
+ }
+ } else {
+ ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
+
+ log.info("Killing Application {}", applicationId);
+
+ killRunningApplication(appId, "forced kill");
+ }
+ }
+
+ /**
+ * Monitor the submitted application for reaching the requested state.
+ * Will also report if the app reaches a later state (failed, killed, etc)
+ * Kill application if duration!= null & time expires.
+ * @param appId Application Id of application to be monitored
+ * @param duration how long to wait -must be more than 0
+ * @param desiredState desired state.
+ * @return the application report -null on a timeout
+ * @throws YarnException
+ * @throws IOException
+ */
+ public ApplicationReport monitorAppToState(
+ ApplicationId appId, YarnApplicationState desiredState, Duration duration)
+ throws YarnException, IOException {
+
+ if (appId == null) {
+ throw new BadCommandArgumentsException("null application ID");
+ }
+ if (duration.limit <= 0) {
+ throw new BadCommandArgumentsException("Invalid monitoring duration");
+ }
+ log.debug("Waiting {} millis for app to reach state {} ",
+ duration.limit,
+ desiredState);
+ duration.start();
+ while (true) {
+
+ // Get application report for the appId we are interested in
+
+ ApplicationReport r = getApplicationReport(appId);
+
+ log.debug("queried status is\n{}",
+ new SliderUtils.OnDemandReportStringifier(r));
+
+ YarnApplicationState state = r.getYarnApplicationState();
+ if (state.ordinal() >= desiredState.ordinal()) {
+ log.debug("App in desired state (or higher) :{}", state);
+ return r;
+ }
+ if (duration.getLimitExceeded()) {
+ log.debug(
+ "Wait limit of {} millis to get to state {}, exceeded; app status\n {}",
+ duration.limit,
+ desiredState,
+ new SliderUtils.OnDemandReportStringifier(r));
+ return null;
+ }
+
+ // sleep 1s.
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignored) {
+ log.debug("Thread sleep in monitoring loop interrupted");
+ }
+ }
+ }
+
+ /**
+ * find all live instances of a specific app -if there is >1 in the cluster,
+ * this returns them all. State should be running or less
+ * @param user user
+ * @param appname application name
+ * @return the list of all matching application instances
+ */
+ public List<ApplicationReport> findAllLiveInstances(String user,
+ String appname) throws
+ YarnException,
+ IOException {
+ List<ApplicationReport> instances = listInstances(user);
+ List<ApplicationReport> results =
+ new ArrayList<ApplicationReport>(instances.size());
+ for (ApplicationReport app : instances) {
+ if (app.getName().equals(appname)
+ && isApplicationLive(app)) {
+ results.add(app);
+ }
+ }
+ return results;
+ }
+
+ public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
+ String appname) {
+ ApplicationReport found = null;
+ ApplicationReport foundAndLive = null;
+ for (ApplicationReport app : instances) {
+ if (app.getName().equals(appname)) {
+ found = app;
+ if (isApplicationLive(app)) {
+ foundAndLive = app;
+ }
+ }
+ }
+ if (foundAndLive != null) {
+ found = foundAndLive;
+ }
+ return found;
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/Constants.java b/slider-core/src/main/java/org/apache/slider/common/Constants.java
new file mode 100644
index 0000000..2fe0250
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/Constants.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common;
+
+public class Constants {
+ public static final int ACCEPT_TIME = 60000;
+ public static final int CONNECT_TIMEOUT = 10000;
+ public static final int RPC_TIMEOUT = 15000;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java b/slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
new file mode 100644
index 0000000..17e2e48
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common;
+
+import org.apache.slider.core.main.LauncherExitCodes;
+
+public interface SliderExitCodes extends LauncherExitCodes {
+
+ /**
+ * starting point for exit codes; not an exception itself
+ */
+ int _EXIT_CODE_BASE = 64;
+
+ /**
+ * internal error: {@value}
+ */
+ int EXIT_INTERNAL_ERROR = _EXIT_CODE_BASE;
+
+ /**
+ * Unimplemented feature: {@value}
+ */
+ int EXIT_UNIMPLEMENTED = 65;
+
+ /**
+ * service entered the failed state: {@value}
+ */
+ int EXIT_YARN_SERVICE_FAILED = 66;
+
+ /**
+ * service was killed: {@value}
+ */
+ int EXIT_YARN_SERVICE_KILLED = 67;
+
+ /**
+ * timeout on monitoring client: {@value}
+ */
+ int EXIT_TIMED_OUT = 68;
+
+ /**
+ * service finished with an error: {@value}
+ */
+ int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 69;
+
+ /**
+ * the application instance is unknown: {@value}
+ */
+ int EXIT_UNKNOWN_INSTANCE = 70;
+
+ /**
+ * the application instance is in the wrong state for that operation: {@value}
+ */
+ int EXIT_BAD_STATE = 71;
+
+ /**
+ * A spawned master process failed
+ */
+ int EXIT_PROCESS_FAILED = 72;
+
+ /**
+ * The cluster failed -too many containers were
+ * failing or some other threshold was reached
+ */
+ int EXIT_DEPLOYMENT_FAILED = 73;
+
+ /**
+ * The application is live -and the requested operation
+ * does not work if the cluster is running
+ */
+ int EXIT_APPLICATION_IN_USE = 74;
+
+ /**
+ * There already is an application instance of that name
+ * when an attempt is made to create a new instance
+ */
+ int EXIT_INSTANCE_EXISTS = 75;
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
new file mode 100644
index 0000000..c1114a9
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common;
+
+
+/**
+ * Keys and various constants for Slider
+ */
+public interface SliderKeys extends SliderXmlConfKeys {
+
+
+ String COMPONENT_AM = "slider-appmaster";
+
+ /**
+ * Slider role is "special"
+ */
+ int ROLE_AM_PRIORITY_INDEX = 0;
+
+
+ /**
+ * The path under which cluster and temp data are stored
+ * {@value}
+ */
+ String SLIDER_BASE_DIRECTORY = ".slider";
+
+ /**
+ * name of the relative path to expaned an image into: {@value}.
+ * The title of this path is to help people understand it when
+ * they see it in their error messages
+ */
+ String LOCAL_TARBALL_INSTALL_SUBDIR = "expandedarchive";
+
+
+ /**
+ * Application type for YARN {@value}
+ */
+ String APP_TYPE = "org.apache.slider";
+
+ /**
+ * JVM arg to force IPv4 {@value}
+ */
+ String JVM_ENABLE_ASSERTIONS = "-ea";
+
+ /**
+ * JVM arg enable JVM system/runtime {@value}
+ */
+ String JVM_ENABLE_SYSTEM_ASSERTIONS = "-esa";
+
+ /**
+ * JVM arg to force IPv4 {@value}
+ */
+ String JVM_FORCE_IPV4 = "-Djava.net.preferIPv4Stack=true";
+
+ /**
+ * JVM arg to go headless {@value}
+ */
+
+ String JVM_JAVA_HEADLESS = "-Djava.awt.headless=true";
+
+ /**
+ * This is the name of the dir/subdir containing
+ * the hbase conf that is propagated via YARN
+ * {@value}
+ */
+ String PROPAGATED_CONF_DIR_NAME = "propagatedconf";
+ String INFRA_DIR_NAME = "infra";
+ String GENERATED_CONF_DIR_NAME = "generated";
+ String SNAPSHOT_CONF_DIR_NAME = "snapshot";
+ String DATA_DIR_NAME = "database";
+ String HISTORY_DIR_NAME = "history";
+ String HISTORY_FILENAME_SUFFIX = "json";
+ String HISTORY_FILENAME_PREFIX = "rolehistory-";
+
+ /**
+ * Filename pattern is required to save in strict temporal order.
+ * Important: older files must sort less-than newer files when using
+ * case-sensitive name sort.
+ */
+ String HISTORY_FILENAME_CREATION_PATTERN = HISTORY_FILENAME_PREFIX +"%016x."+
+ HISTORY_FILENAME_SUFFIX;
+ /**
+ * The posix regexp used to locate this
+ */
+ String HISTORY_FILENAME_MATCH_PATTERN = HISTORY_FILENAME_PREFIX +"[0-9a-f]+\\."+
+ HISTORY_FILENAME_SUFFIX;
+ /**
+ * The posix regexp used to locate this
+ */
+ String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+
+ HISTORY_FILENAME_SUFFIX;
+ /**
+ * XML resource listing the standard Slider providers
+ * {@value}
+ */
+ String SLIDER_XML = "org/apache/slider/slider.xml";
+
+ String CLUSTER_DIRECTORY = "cluster";
+
+ /**
+ * JVM property to define the slider configuration directory;
+ * this is set by the slider script: {@value}
+ */
+ String PROPERTY_CONF_DIR = "slider.confdir";
+
+ /**
+ * name of generated dir for this conf: {@value}
+ */
+ String SUBMITTED_CONF_DIR = "confdir";
+
+ /**
+ * name of the Slider client resource
+ * loaded when the service is loaded.
+ */
+ String CLIENT_RESOURCE = "slider-client.xml";
+
+ /**
+ * The name of the resource to put on the classpath
+ * This only goes up on a real cluster, not a test run.
+ */
+ String SERVER_RESOURCE = "slider-server.xml";
+
+ String TMP_LOGDIR_PREFIX = "/tmp/slider-";
+ String TMP_DIR_PREFIX = "tmp";
+
+ String SLIDER_JAR = "slider.jar";
+ String JCOMMANDER_JAR = "jcommander.jar";
+ String GSON_JAR = "gson.jar";
+
+ String DEFAULT_JVM_HEAP = "256M";
+ int DEFAULT_YARN_MEMORY = 256;
+ String STDOUT_AM = "slider-out.txt";
+ String STDERR_AM = "slider-err.txt";
+ String DEFAULT_GC_OPTS = "";
+
+ String HADOOP_USER_NAME = "HADOOP_USER_NAME";
+ String HADOOP_PROXY_USER = "HADOOP_PROXY_USER";
+
+ boolean PROPAGATE_RESOURCE_OPTION = true;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
new file mode 100644
index 0000000..6d5a328
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common;
+
+/**
+ * Keys shared across tests
+ */
+public interface SliderXMLConfKeysForTesting {
+
+ String KEY_TEST_HBASE_HOME = "slider.test.hbase.home";
+ String KEY_TEST_HBASE_TAR = "slider.test.hbase.tar";
+ String KEY_TEST_HBASE_APPCONF = "slider.test.hbase.appconf";
+ String KEY_TEST_ACCUMULO_HOME = "slider.test.accumulo.home";
+ String KEY_TEST_ACCUMULO_TAR = "slider.test.accumulo.tar";
+ String KEY_TEST_ACCUMULO_APPCONF = "slider.test.accumulo.appconf";
+
+ String KEY_TEST_THAW_WAIT_TIME = "slider.test.thaw.wait.seconds";
+
+ int DEFAULT_THAW_WAIT_TIME_SECONDS = 60;
+
+
+ String KEY_TEST_FREEZE_WAIT_TIME = "slider.test.freeze.wait.seconds";
+
+ int DEFAULT_TEST_FREEZE_WAIT_TIME_SECONDS = 60;
+
+ String KEY_TEST_TIMEOUT = "slider.test.timeout.seconds";
+
+ int DEFAULT_TEST_TIMEOUT_SECONDS = 10 * 60;
+
+ String KEY_TEST_HBASE_LAUNCH_TIME = "slider.test.hbase.launch.wait.seconds";
+
+ int DEFAULT_HBASE_LAUNCH_TIME_SECONDS = 60 * 3;
+
+ String KEY_TEST_HBASE_ENABLED = "slider.test.hbase.enabled";
+
+ String KEY_TEST_ACCUMULO_ENABLED = "slider.test.accumulo.enabled";
+
+ String KEY_ACCUMULO_LAUNCH_TIME =
+ "slider.test.accumulo.launch.wait.seconds";
+
+ int DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS = 60 * 3;
+ String KEY_TEST_AGENT_ENABLED = "slider.test.agent.enabled";
+
+ int DEFAULT_AGENT_LAUNCH_TIME_SECONDS = 60 * 3;
+
+ String KEY_TEST_AGENT_HOME = "slider.test.agent.home";
+ String KEY_TEST_AGENT_TAR = "slider.test.agent.tar";
+
+ String KEY_TEST_TEARDOWN_KILLALL = "slider.test.teardown.killall";
+ boolean DEFAULT_TEARDOWN_KILLALL = true;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
new file mode 100644
index 0000000..272ae6a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common;
+
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * These are the keys that can be added to <code>conf/slider-client.xml</code>.
+ */
+public interface SliderXmlConfKeys {
+ String PREFIX_PROVIDER = "slider.provider";
+ /**
+ * pattern to identify a provider
+ * {@value}
+ */
+ String KEY_PROVIDER = PREFIX_PROVIDER + ".%s";
+
+ /**
+ * HBase provider key (derived from {@link #KEY_PROVIDER}
+ * and so not found in the code itself
+ * {@value}
+ */
+ String KEY_PROVIDER_HBASE = PREFIX_PROVIDER + ".hbase";
+
+ /**
+ * Accumulo provider key (derived from {@link #KEY_PROVIDER}
+ * and so not found in the code itself
+ * {@value}
+ */
+ String KEY_PROVIDER_ACCUMULO =
+ PREFIX_PROVIDER + ".accumulo";
+
+ /**
+ * Accumulo agent key (derived from {@link #KEY_PROVIDER}
+ * and so not found in the code itself
+ * {@value}
+ */
+ String KEY_PROVIDER_AGENT = PREFIX_PROVIDER + ".agent";
+
+ /**
+ * conf option set to point to where the config came from
+ * {@value}
+ */
+ String KEY_TEMPLATE_ORIGIN = "slider.template.origin";
+
+ /**
+ * Original name for the default FS. This is still
+ * expected by applications deployed
+ */
+ String FS_DEFAULT_NAME_CLASSIC = "fs.default.name";
+
+ /**
+ * Slider principal
+ */
+ String KEY_KERBEROS_PRINCIPAL = "slider.kerberos.principal";
+
+ /**
+ * Name of the property for ACLs for Slider AM.
+ * {@value}
+ */
+ String KEY_PROTOCOL_ACL = "security.slider.protocol.acl";
+
+ /**
+ * Limit on restarts for the AM
+ * {@value}
+ */
+ String KEY_AM_RESTART_LIMIT = "slider.yarn.restart.limit";
+
+ /**
+ * Default Limit on restarts for the AM
+ * {@value}
+ */
+ int DEFAULT_AM_RESTART_LIMIT = 2;
+
+ /**
+ * Flag which is set to indicate that security should be enabled
+ * when talking to this cluster.
+ */
+ String KEY_SECURITY_ENABLED =
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION;
+
+ /**
+ * queue name
+ */
+ String KEY_YARN_QUEUE = "slider.yarn.queue";
+ String DEFAULT_YARN_QUEUE = YarnConfiguration.DEFAULT_QUEUE_NAME;
+
+ /**
+ * default priority
+ */
+ String KEY_YARN_QUEUE_PRIORITY = "slider.yarn.queue.priority";
+ int DEFAULT_YARN_QUEUE_PRIORITY = 1;
+
+
+ /**
+ * The slider base path: {@value}
+ * Defaults to HomeDir/.slider
+ */
+ String KEY_SLIDER_BASE_PATH = "slider.base.path";
+
+
+ /**
+ * Option for the permissions for the cluster directory itself: {@value}
+ */
+ String CLUSTER_DIRECTORY_PERMISSIONS =
+ "slider.cluster.directory.permissions";
+ /**
+ * Default value for the permissions :{@value}
+ */
+ String DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS = "750";
+ /**: {@value}
+ * Option for the permissions for the data directory itself
+ */
+ String DATA_DIRECTORY_PERMISSIONS = "slider.data.directory.permissions";
+ /**
+ * Default value for the data directory permissions: {@value}
+ */
+ String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750";
+
+
+ String REGISTRY_PATH = "slider.registry.path";
+
+ /**
+ * Default value for the registry: {@value}
+ */
+ String DEFAULT_REGISTRY_PATH = "/registry";
+
+
+ String REGISTRY_ZK_QUORUM = "slider.zookeeper.quorum";
+
+ /**
+ * Default value for the registry: {@value}
+ */
+ String DEFAULT_REGISTRY_ZK_QUORUM = "localhost:2181";
+
+
+ String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH =
+ "ipc.client.fallback-to-simple-auth-allowed";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
new file mode 100644
index 0000000..c8e5523
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Base args for all actions
+ */
+public abstract class AbstractActionArgs extends ArgOps implements Arguments {
+ protected static final Logger log =
+ LoggerFactory.getLogger(AbstractActionArgs.class);
+
+
+ protected AbstractActionArgs() {
+ }
+
+ /**
+ * URI of the filesystem
+ */
+ @Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
+ description = "Filesystem URI",
+ converter = URIArgumentConverter.class)
+ public URI filesystemURL;
+
+ @Parameter(names = {ARG_BASE_PATH},
+ description = "Slider base path on the filesystem",
+ converter = PathArgumentConverter.class)
+ public Path basePath;
+
+
+ /**
+ * This is the default parameter
+ */
+ @Parameter
+ public final List<String> parameters = new ArrayList<String>();
+
+
+ /**
+ * get the name: relies on arg 1 being the cluster name in all operations
+ * @return the name argument, null if there is none
+ */
+ public String getClusterName() {
+ return (parameters.isEmpty()) ? null : parameters.get(0);
+ }
+
+ /**
+ -D name=value
+
+ Define an HBase configuration option which overrides any options in
+ the configuration XML files of the image or in the image configuration
+ directory. The values will be persisted.
+ Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
+
+ */
+
+ @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
+ public final List<String> definitions = new ArrayList<String>();
+
+ /**
+ * System properties
+ */
+ @Parameter(names = {ARG_SYSPROP}, arity = 1,
+ description = "system properties in the form name value" +
+ " These are set after the JVM is started.")
+ public final List<String> sysprops = new ArrayList<String>(0);
+
+
+ @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
+ description = "hostname:port of the YARN resource manager")
+ public String manager;
+
+
+ @Parameter(names = ARG_DEBUG, description = "Debug mode")
+ public boolean debug = false;
+
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 1;
+ }
+
+ /**
+ * Get the name of the action
+ * @return
+ */
+ public abstract String getActionName() ;
+
+ /**
+ * Get the max #of params expected
+ * @return the number of params in the {@link #parameters} field;
+ */
+ public int getMaxParams() {
+ return getMinParams();
+ }
+
+ public void validate() throws BadCommandArgumentsException {
+
+ int minArgs = getMinParams();
+ int actionArgSize = parameters.size();
+ if (minArgs > actionArgSize) {
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName());
+ }
+ int maxArgs = getMaxParams();
+ if (maxArgs == -1) {
+ maxArgs = minArgs;
+ }
+ if (actionArgSize > maxArgs) {
+ String message = String.format("%s for action %s: limit is %d but saw %d",
+ ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
+ getActionName(), maxArgs,
+ actionArgSize);
+ log.error(message);
+ int index = 1;
+ for (String actionArg : parameters) {
+ log.error("[{}] \"{}\"", index++, actionArg);
+ }
+ throw new BadCommandArgumentsException(message);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + ": " + getActionName();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java
new file mode 100644
index 0000000..23ba414
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+
+/**
+ * Base class for all the delegates
+ */
+public class AbstractArgsDelegate extends ArgOps implements Arguments {
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
new file mode 100644
index 0000000..030fd59
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.providers.SliderProviderFactory;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Abstract Action to build things; shares args across build and
+ * list
+ */
+public abstract class AbstractClusterBuildingActionArgs extends AbstractActionArgs {
+
+ /**
+ * Declare the image configuration directory to use when creating or
+ * reconfiguring a slider cluster. The path must be on a filesystem visible
+ * to all nodes in the YARN cluster. Only one configuration directory can
+ * be specified.
+ */
+ @Parameter(names = ARG_CONFDIR,
+ description = "Path to cluster configuration directory in HDFS",
+ converter = PathArgumentConverter.class)
+ public Path confdir;
+
+ @Parameter(names = ARG_ZKPATH,
+ description = "Zookeeper path for the application")
+ public String appZKPath;
+
+ @Parameter(names = ARG_ZKHOSTS,
+ description = "comma separated list of the Zookeeper hosts")
+ public String zkhosts;
+
+ /**
+ * --image path
+ the full path to a .tar or .tar.gz path containing an HBase image.
+ */
+ @Parameter(names = ARG_IMAGE,
+ description = "The full path to a .tar or .tar.gz path containing the application",
+ converter = PathArgumentConverter.class)
+ public Path image;
+
+ @Parameter(names = ARG_APP_HOME,
+ description = "Home directory of a pre-installed application")
+ public String appHomeDir;
+
+ @Parameter(names = ARG_PROVIDER,
+ description = "Provider of the specific cluster application")
+ public String provider = SliderProviderFactory.DEFAULT_CLUSTER_TYPE;
+
+/*
+
+ @Parameter(names = {ARG_PACKAGE},
+ description = "URI to a slider package",
+ converter = URIArgumentConverter.class )
+ public URI packageURI;
+*/
+ @Parameter(names = {ARG_PACKAGE},
+ description = "URI to a slider package")
+ public String packageURI;
+
+ @Parameter(names = {ARG_RESOURCES},
+ description = "File defining the resources of this instance")
+ public File resources;
+
+ @Parameter(names = {ARG_TEMPLATE},
+ description = "Template application configuration")
+ public File template;
+
+ @ParametersDelegate
+ public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+
+
+ @ParametersDelegate
+ public AppAndResouceOptionArgsDelegate optionsDelegate = new AppAndResouceOptionArgsDelegate();
+
+
+ public Map<String, String> getOptionsMap() throws
+ BadCommandArgumentsException {
+ return optionsDelegate.getOptionsMap();
+ }
+
+ /**
+ * Get the role heap mapping (may be empty, but never null)
+ * @return role heap mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, Map<String, String>> getCompOptionMap() throws
+ BadCommandArgumentsException {
+ return optionsDelegate.getCompOptionMap();
+ }
+
+
+ public Map<String, String> getResourceOptionsMap() throws
+ BadCommandArgumentsException {
+ return optionsDelegate.getResourceOptionsMap();
+ }
+
+ /**
+ * Get the role heap mapping (may be empty, but never null)
+ * @return role heap mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, Map<String, String>> getResourceCompOptionMap() throws
+ BadCommandArgumentsException {
+ return optionsDelegate.getResourceCompOptionMap();
+ }
+
+ @VisibleForTesting
+ public List<String> getComponentTuples() {
+ return componentDelegate.getComponentTuples();
+ }
+
+ /**
+ * Get the role mapping (may be empty, but never null)
+ * @return role mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
+ return componentDelegate.getComponentMap();
+ }
+
+ public Path getConfdir() {
+ return confdir;
+ }
+
+ public String getAppZKPath() {
+ return appZKPath;
+ }
+
+ public String getZKhosts() {
+ return zkhosts;
+ }
+
+ public Path getImage() {
+ return image;
+ }
+
+ public String getAppHomeDir() {
+ return appHomeDir;
+ }
+
+ public String getProvider() {
+ return provider;
+ }
+
+ public ConfTree buildAppOptionsConfTree() throws BadCommandArgumentsException {
+ return buildConfTree(getOptionsMap());
+ }
+
+ public ConfTree buildResourceOptionsConfTree() throws BadCommandArgumentsException {
+ return buildConfTree(getResourceOptionsMap());
+ }
+
+ protected ConfTree buildConfTree(Map<String, String> optionsMap) throws
+ BadCommandArgumentsException {
+ ConfTree confTree = new ConfTree();
+ ConfTreeOperations ops = new ConfTreeOperations(confTree);
+ confTree.global.putAll(optionsMap);
+ Map<String, Map<String, String>> roleOptionMap = getCompOptionMap();
+ for (Map.Entry<String, Map<String, String>> entry : roleOptionMap.entrySet()) {
+ String key = entry.getKey();
+ Map<String, String> value = entry.getValue();
+ ops.getOrAddComponent(key).putAll(value);
+ }
+ return confTree;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
new file mode 100644
index 0000000..5d889e1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+
+@Parameters(commandNames = {SliderActions.ACTION_AM_SUICIDE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_AM_SUICIDE)
+public class ActionAMSuicideArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_AM_SUICIDE;
+ }
+
+ @Parameter(names = {ARG_MESSAGE},
+ description = "reason for the action")
+ public String message;
+
+ @Parameter(names = {ARG_EXITCODE},
+ description = "exit code")
+ public int exitcode = 1;
+
+ @Parameter(names = {ARG_WAIT},
+ description = "time for AM to wait before exiting")
+ public int waittime = 1000;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java
new file mode 100644
index 0000000..1a182d1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_BUILD},
+ commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
+
+public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_BUILD;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
new file mode 100644
index 0000000..cfcfb9d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+@Parameters(commandNames = {SliderActions.ACTION_CREATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class ActionCreateArgs extends AbstractClusterBuildingActionArgs
+ implements WaitTimeAccessor, LaunchArgsAccessor {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CREATE;
+ }
+
+ @ParametersDelegate
+ LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+ @Override
+ public String getRmAddress() {
+ return launchArgs.getRmAddress();
+ }
+
+ @Override
+ public int getWaittime() {
+ return launchArgs.getWaittime();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ launchArgs.setWaittime(waittime);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
new file mode 100644
index 0000000..d4acea6
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_DESTROY},
+ commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
+
+public class ActionDestroyArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_DESTROY;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
new file mode 100644
index 0000000..b0f53f8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+
+/*
+
+@Parameters(commandNames = {SliderActions.ACTION_KILL_CONTAINER},
+ commandDescription = SliderActions.DESCRIBE_ACTION_KILL_CONTAINER)
+*/
+public class ActionEchoArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_DESTROY;
+ }
+
+ @Parameter(names = {ARG_MESSAGE},
+ description = "message to echo")
+ public String message;
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
new file mode 100644
index 0000000..2d6bef2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_EXISTS},
+ commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
+
+public class ActionExistsArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_EXISTS;
+ }
+ @Parameter(names = {ARG_LIVE},
+ description = "verify that the cluster is running")
+ public boolean live;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java
new file mode 100644
index 0000000..725973e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.List;
+import java.util.Map;
+
+@Parameters(commandNames = {SliderActions.ACTION_FLEX},
+ commandDescription = SliderActions.DESCRIBE_ACTION_FLEX)
+
+public class ActionFlexArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_FLEX;
+ }
+
+ @ParametersDelegate
+ public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+
+ /**
+ * Get the component mapping (may be empty, but never null)
+ * @return mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
+ return componentDelegate.getComponentMap();
+ }
+
+ public List<String> getComponentTuples() {
+ return componentDelegate.getComponentTuples();
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
new file mode 100644
index 0000000..04f6305
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+@Parameters(commandNames = {SliderActions.ACTION_FREEZE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
+
+public class ActionFreezeArgs extends AbstractActionArgs implements
+ WaitTimeAccessor {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_FREEZE;
+ }
+
+ public static final String FREEZE_COMMAND_ISSUED = "freeze command issued";
+ @ParametersDelegate
+ public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
+
+ @Override
+ public int getWaittime() {
+ return waitDelegate.getWaittime();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ waitDelegate.setWaittime(waittime);
+ }
+
+ @Parameter(names={ARG_MESSAGE},
+ description = "reason for the operation")
+ public String message = FREEZE_COMMAND_ISSUED;
+
+ @Parameter(names = {ARG_FORCE},
+ description = "force the operation")
+ public boolean force;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionGetConfArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionGetConfArgs.java
new file mode 100644
index 0000000..b636a5e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionGetConfArgs.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_GETCONF},
+ commandDescription = SliderActions.DESCRIBE_ACTION_GETCONF)
+
+public class ActionGetConfArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_GETCONF;
+ }
+
+ //--format
+ @Parameter(names = ARG_FORMAT,
+ description = "Format for a response: [text|xml|json|properties]")
+ public String format = FORMAT_XML;
+
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "Output file for the configuration data")
+ private String output;
+
+
+ public String getFormat() {
+ return format;
+ }
+
+ public String getOutput() {
+ return output;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
new file mode 100644
index 0000000..4d4098f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_HELP, SliderActions.ACTION_USAGE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
+
+public class ActionHelpArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_HELP;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionKillContainerArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionKillContainerArgs.java
new file mode 100644
index 0000000..8c18ad8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionKillContainerArgs.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_KILL_CONTAINER},
+ commandDescription = SliderActions.DESCRIBE_ACTION_KILL_CONTAINER)
+
+public class ActionKillContainerArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_KILL_CONTAINER;
+ }
+
+ @Parameter(names = {ARG_ID},
+ description = "ID of the container")
+ public String id;
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
new file mode 100644
index 0000000..8f42b69
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_LIST},
+ commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
+
+public class ActionListArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_LIST;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
new file mode 100644
index 0000000..bab97be
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+
+import java.io.File;
+
+@Parameters(commandNames = {SliderActions.ACTION_REGISTRY},
+ commandDescription = SliderActions.DESCRIBE_ACTION_REGISTRY)
+
+public class ActionRegistryArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_REGISTRY;
+ }
+
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ //--format
+ @Parameter(names = ARG_FORMAT,
+ description = "Format for a response: [text|xml|json|properties]")
+ public String format = FORMAT_XML;
+
+
+ @Parameter(names = {ARG_DEST},
+ description = "Output destination")
+ public File dest;
+
+ @Parameter(names = {ARG_LIST},
+ description = "list services")
+ public String list;
+
+ @Parameter(names = {ARG_LISTCONF},
+ description = "list configurations")
+ public String listConf;
+
+ @Parameter(names = {ARG_GETCONF},
+ description = "get files")
+ public String getConf;
+
+
+ @Parameter(names = {ARG_LISTFILES},
+ description = "list files")
+ public String listFiles;
+
+ @Parameter(names = {ARG_GETFILES},
+ description = "get files")
+ public String getFiles;
+
+ @Override
+ public void validate() throws BadCommandArgumentsException {
+ super.validate();
+
+ //verify that at most one of the operations is set
+ int gets = s(getConf) + s(getFiles);
+ int lists = s(list) + s(listConf) + s(listFiles);
+ int set = lists + gets;
+ if (set > 1) {
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_TOO_MANY_ARGUMENTS);
+ }
+ if (dest != null && (lists > 0 || set == 0)) {
+ throw new BadCommandArgumentsException("Argument " + ARG_DEST
+ + " is only supported on 'get' operations");
+ }
+ }
+
+ @SuppressWarnings("VariableNotUsedInsideIf")
+ private int s(String arg) {
+ return arg != null ? 1 : 0;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
new file mode 100644
index 0000000..631ad86
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_STATUS},
+ commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
+
+public class ActionStatusArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_STATUS;
+ }
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "Output file for the configuration data")
+ private String output;
+
+ public String getOutput() {
+ return output;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
new file mode 100644
index 0000000..8408385
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+@Parameters(commandNames = {SliderActions.ACTION_THAW},
+ commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
+public class ActionThawArgs extends AbstractActionArgs implements
+ WaitTimeAccessor,
+ LaunchArgsAccessor {
+
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_THAW;
+ }
+
+ @Override
+ public int getWaittime() {
+ return launchArgs.getWaittime();
+ }
+
+ @ParametersDelegate
+ LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+ @Override
+ public String getRmAddress() {
+ return launchArgs.getRmAddress();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ launchArgs.setWaittime(waittime);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
new file mode 100644
index 0000000..c36ef62
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_VERSION},
+ commandDescription = SliderActions.DESCRIBE_ACTION_VERSION)
+
+public class ActionVersionArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_VERSION;
+ }
+
+ public int getMinParams() {
+ return 0;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
new file mode 100644
index 0000000..1f07de3
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Delegate for application and resource options
+ */
+public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
+
+
+ /**
+ * Options key value
+ */
+ @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
+ description = ARG_OPTION + "<name> <value>",
+ splitter = DontSplitArguments.class)
+ public List<String> optionTuples = new ArrayList<String>(0);
+
+
+ /**
+ * All the app component option triples
+ */
+ @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT, ARG_ROLEOPT}, arity = 3,
+ description = "Component option " + ARG_COMP_OPT +
+ " <component> <name> <option>",
+ splitter = DontSplitArguments.class)
+ public List<String> compOptTriples = new ArrayList<String>(0);
+
+ /**
+ * Resource Options
+ */
+ @Parameter(names = {ARG_RESOURCE_OPT, ARG_RESOURCE_OPT_SHORT}, arity = 2,
+ description = "Resource option "+ ARG_RESOURCE_OPT + "<name> <value>",
+ splitter = DontSplitArguments.class)
+ public List<String> resOptionTuples = new ArrayList<String>(0);
+
+
+ /**
+ * All the resource component option triples
+ */
+ @Parameter(names = {ARG_RES_COMP_OPT, ARG_RES_COMP_OPT_SHORT,}, arity = 3,
+ description = "Component resource option " + ARG_RES_COMP_OPT +
+ " <component> <name> <option>",
+ splitter = DontSplitArguments.class)
+ public List<String> resCompOptTriples = new ArrayList<String>(0);
+
+
+ public Map<String, String> getOptionsMap() throws
+ BadCommandArgumentsException {
+ return convertTupleListToMap(ARG_OPTION, optionTuples);
+ }
+
+ /**
+ * Get the role heap mapping (may be empty, but never null)
+ * @return role heap mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, Map<String, String>> getCompOptionMap() throws
+ BadCommandArgumentsException {
+ return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
+ }
+
+ public Map<String, String> getResourceOptionsMap() throws
+ BadCommandArgumentsException {
+ return convertTupleListToMap(ARG_RESOURCE_OPT, resOptionTuples);
+ }
+
+ /**
+ * Get the role heap mapping (may be empty, but never null)
+ * @return role heap mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, Map<String, String>> getResourceCompOptionMap() throws
+ BadCommandArgumentsException {
+ return convertTripleListToMaps(ARG_COMP_OPT, resCompOptTriples);
+ }
+
+ public void setOption(String key, String value) {
+ optionTuples.add(key);
+ optionTuples.add(value);
+ }
+
+ public void setResourceOption(String key, String value) {
+ resOptionTuples.add(key);
+ resOptionTuples.add(value);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
new file mode 100644
index 0000000..28e9702
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Static argument manipulation operations
+ */
+public class ArgOps {
+
+ private static final Logger
+ log = LoggerFactory.getLogger(ArgOps.class);
+
+ /**
+ * create a 3-tuple
+ * @param msg
+ * @param min
+ * @param max
+ * @return
+ */
+ public static List<Object> triple(String msg, int min, int max) {
+ List<Object> l = new ArrayList<Object>(3);
+ l.add(msg);
+ l.add(min);
+ l.add(max);
+ return l;
+ }
+
+ /**
+ * Create a tuple
+ * @param msg
+ * @param min
+ * @return
+ */
+ public static List<Object> tuple(String msg, int min) {
+ return triple(msg, min, min);
+ }
+
+ public static void applyFileSystemURL(URI filesystemURL, Configuration conf) {
+ if (filesystemURL != null) {
+ //filesystem argument was set -this overwrites any defaults in the
+ //configuration
+ FileSystem.setDefaultUri(conf, filesystemURL);
+ }
+ }
+
+ public static void splitPairs(Collection<String> pairs,
+ Map<String, String> dest) {
+ for (String prop : pairs) {
+ String[] keyval = prop.split("=", 2);
+ if (keyval.length == 2) {
+ dest.put(keyval[0], keyval[1]);
+ }
+ }
+ }
+
+
+ public static void applyDefinitions(Map<String, String> definitionMap,
+ Configuration conf) {
+ for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ log.debug("configuration[{}]=\"{}\"", key, val);
+ conf.set(key, val, "command line");
+ }
+ }
+
+ /**
+ * Create a map from a tuple list like ['worker','2','master','1] into a map
+ * ['worker':'2',"master":'1'];
+ * Duplicate entries also trigger errors
+ * @param description description for errors
+ * @param list list to conver to tuples
+ * @return the map of key value pairs -unordered.
+ * @throws BadCommandArgumentsException odd #of arguments received
+ */
+ public static Map<String, String> convertTupleListToMap(String description,
+ List<String> list) throws
+ BadCommandArgumentsException {
+ Map<String, String> results = new HashMap<String, String>();
+ if (list != null && !list.isEmpty()) {
+ int size = list.size();
+ if (size % 2 != 0) {
+ //odd number of elements, not permitted
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_PARSE_FAILURE + description);
+ }
+ for (int count = 0; count < size; count += 2) {
+ String key = list.get(count);
+ String val = list.get(count + 1);
+ if (results.get(key) != null) {
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_DUPLICATE_ENTRY + description
+ + ": " + key);
+ }
+ results.put(key, val);
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Create a map from a tuple list like
+ * ['worker','heapsize','5G','master','heapsize','2M'] into a map
+ * ['worker':'2',"master":'1'];
+ * Duplicate entries also trigger errors
+ * @param description
+ * @param list
+ * @return
+ * @throws BadCommandArgumentsException odd #of arguments received
+ */
+ public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
+ List<String> list) throws
+ BadCommandArgumentsException {
+ Map<String, Map<String, String>> results =
+ new HashMap<String, Map<String, String>>();
+ if (list != null && !list.isEmpty()) {
+ int size = list.size();
+ if (size % 3 != 0) {
+ //wrong number of elements, not permitted
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_PARSE_FAILURE + description);
+ }
+ for (int count = 0; count < size; count += 3) {
+ String role = list.get(count);
+ String key = list.get(count + 1);
+ String val = list.get(count + 2);
+ Map<String, String> roleMap = results.get(role);
+ if (roleMap == null) {
+ //demand create new role map
+ roleMap = new HashMap<String, String>();
+ results.put(role, roleMap);
+ }
+ if (roleMap.get(key) != null) {
+ throw new BadCommandArgumentsException(
+ ErrorStrings.ERROR_DUPLICATE_ENTRY + description
+ + ": for key " + key + " under " + role);
+ }
+ roleMap.put(key, val);
+ }
+ }
+ return results;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
new file mode 100644
index 0000000..44fecea
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+/**
+ * Here are all the arguments that may be parsed by the client or server
+ * command lines.
+ */
+public interface Arguments {
+
+ String ARG_APP_HOME = "--apphome";
+ String ARG_CONFDIR = "--appconf";
+ String ARG_COMPONENT = "--component";
+ String ARG_COMPONENT_SHORT = "--comp";
+ String ARG_COMP_OPT= "--compopt";
+ String ARG_COMP_OPT_SHORT = "--co";
+
+ String ARG_DEBUG = "--debug";
+ String ARG_DEST = "--dest";
+ String ARG_DEFINE = "-D";
+ String ARG_EXITCODE = "--exitcode";
+ /**
+ filesystem-uri: {@value}
+ */
+ String ARG_FILESYSTEM = "--fs";
+ String ARG_FILESYSTEM_LONG = "--filesystem";
+ String ARG_BASE_PATH = "--basepath";
+ String ARG_FORMAT = "--format";
+ String ARG_FORCE = "--force";
+ String ARG_GETCONF = "--getconf";
+ String ARG_GETFILES = "--getfiles";
+ String ARG_HELP = "--help";
+ String ARG_ID = "--id";
+ String ARG_IMAGE = "--image";
+ String ARG_LIST = "--list";
+ String ARG_LISTFILES = "--listfiles";
+ String ARG_LISTCONF = "--listconf";
+ String ARG_LIVE = "--live";
+ String ARG_MANAGER = "--manager";
+ String ARG_MANAGER_SHORT = "--m";
+ String ARG_MESSAGE = "--message";
+ String ARG_OPTION = "--option";
+ String ARG_OPTION_SHORT = "-O";
+ // String ARG_NAME = "--name";
+ String ARG_OUTPUT = "--out";
+ String ARG_OUTPUT_SHORT = "-o";
+ String ARG_PACKAGE = "--package";
+ String ARG_PROVIDER = "--provider";
+ String ARG_RESOURCES = "--resources";
+ String ARG_RES_COMP_OPT = "--rescompopt";
+ String ARG_RES_COMP_OPT_SHORT = "--rco";
+ String ARG_RESOURCE_MANAGER = "--rm";
+ String ARG_RESOURCE_OPT = "--resopt";
+ String ARG_RESOURCE_OPT_SHORT = "-ro";
+ String ARG_SYSPROP = "-S";
+ String ARG_TEMPLATE = "--template";
+ String ARG_WAIT = "--wait";
+ String ARG_ZKPATH = "--zkpath";
+ String ARG_ZKPORT = "--zkport";
+ String ARG_ZKHOSTS = "--zkhosts";
+
+
+ /**
+ * Deprecated
+ */
+ @Deprecated
+ String ARG_ROLE = "--role";
+ @Deprecated
+ String ARG_ROLEOPT = "--roleopt";
+
+ /**
+ * server: URI for the cluster
+ */
+ String ARG_CLUSTER_URI = "-cluster-uri";
+
+
+ /**
+ * server: Path for the resource manager instance (required)
+ */
+ String ARG_RM_ADDR = "--rm";
+
+ String FORMAT_XML = "xml";
+ String FORMAT_PROPERTIES = "properties";
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
new file mode 100644
index 0000000..44a2a7a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.util.Collection;
+
+/**
+ * Slider Client CLI Args
+ */
+
+public class ClientArgs extends CommonArgs {
+
+ /*
+
+ All the arguments for specific actions
+
+ */
+ /**
+ * This is not bonded to jcommander, it is set up
+ * after the construction to point to the relevant
+ * entry
+ */
+ private AbstractClusterBuildingActionArgs buildingActionArgs;
+ private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs();
+ private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
+ private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
+ private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
+ private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
+ private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
+ private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
+ private final ActionGetConfArgs actionGetConfArgs = new ActionGetConfArgs();
+ private final ActionKillContainerArgs actionKillContainerArgs =
+ new ActionKillContainerArgs();
+ private final ActionListArgs actionListArgs = new ActionListArgs();
+ private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
+ private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
+ private final ActionThawArgs actionThawArgs = new ActionThawArgs();
+ private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
+ private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
+
+
+ public ClientArgs(String[] args) {
+ super(args);
+ }
+
+ public ClientArgs(Collection args) {
+ super(args);
+ }
+
+ @Override
+ protected void addActionArguments() {
+
+ addActions(
+ actionAMSuicideArgs,
+ actionBuildArgs,
+ actionCreateArgs,
+ actionDestroyArgs,
+ actionExistsArgs,
+ actionFlexArgs,
+ actionFreezeArgs,
+ actionGetConfArgs,
+ actionKillContainerArgs,
+ actionListArgs,
+ actionRegistryArgs,
+ actionStatusArgs,
+ actionThawArgs,
+ actionHelpArgs,
+ actionVersionArgs
+ );
+ }
+
+ @Override
+ public void applyDefinitions(Configuration conf) throws
+ BadCommandArgumentsException {
+ super.applyDefinitions(conf);
+ //RM
+ if (getManager() != null) {
+ log.debug("Setting RM to {}", getManager());
+ conf.set(YarnConfiguration.RM_ADDRESS, getManager());
+ }
+ if ( getBasePath() != null ) {
+ log.debug("Setting basePath to {}", getBasePath());
+ conf.set(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH, getBasePath().toString());
+ }
+ }
+
+ public AbstractClusterBuildingActionArgs getBuildingActionArgs() {
+ return buildingActionArgs;
+ }
+
+ public ActionAMSuicideArgs getActionAMSuicideArgs() {
+ return actionAMSuicideArgs;
+ }
+
+ public ActionBuildArgs getActionBuildArgs() {
+ return actionBuildArgs;
+ }
+
+ public ActionCreateArgs getActionCreateArgs() {
+ return actionCreateArgs;
+ }
+
+ public ActionDestroyArgs getActionDestroyArgs() {
+ return actionDestroyArgs;
+ }
+
+ public ActionExistsArgs getActionExistsArgs() {
+ return actionExistsArgs;
+ }
+
+ public ActionFlexArgs getActionFlexArgs() {
+ return actionFlexArgs;
+ }
+
+ public ActionFreezeArgs getActionFreezeArgs() {
+ return actionFreezeArgs;
+ }
+
+ public ActionGetConfArgs getActionGetConfArgs() {
+ return actionGetConfArgs;
+ }
+
+ public ActionKillContainerArgs getActionKillContainerArgs() {
+ return actionKillContainerArgs;
+ }
+
+ public ActionListArgs getActionListArgs() {
+ return actionListArgs;
+ }
+
+ public ActionRegistryArgs getActionRegistryArgs() {
+ return actionRegistryArgs;
+ }
+
+ public ActionStatusArgs getActionStatusArgs() {
+ return actionStatusArgs;
+ }
+
+ public ActionThawArgs getActionThawArgs() {
+ return actionThawArgs;
+ }
+
+ /**
+ * Look at the chosen action and bind it as the core action for the operation.
+ * In theory this could be done by introspecting on the list of actions and
+ * choosing it without the switch statement. In practise this switch, while
+ * verbose, is easier to debug.
+ * @throws SliderException bad argument or similar
+ */
+ @Override
+ public void applyAction() throws SliderException {
+ String action = getAction();
+ if (SliderActions.ACTION_BUILD.equals(action)) {
+ bindCoreAction(actionBuildArgs);
+ //its a builder, so set those actions too
+ buildingActionArgs = actionBuildArgs;
+ } else if (SliderActions.ACTION_CREATE.equals(action)) {
+ bindCoreAction(actionCreateArgs);
+ //its a builder, so set those actions too
+ buildingActionArgs = actionCreateArgs;
+
+ } else if (SliderActions.ACTION_FREEZE.equals(action)) {
+ bindCoreAction(actionFreezeArgs);
+
+ } else if (SliderActions.ACTION_THAW.equals(action)) {
+ bindCoreAction(actionThawArgs);
+
+ } else if (SliderActions.ACTION_AM_SUICIDE.equals(action)) {
+ bindCoreAction(actionAMSuicideArgs);
+
+ } else if (SliderActions.ACTION_DESTROY.equals(action)) {
+ bindCoreAction(actionDestroyArgs);
+
+ } else if (SliderActions.ACTION_EXISTS.equals(action)) {
+ bindCoreAction(actionExistsArgs);
+
+ } else if (SliderActions.ACTION_FLEX.equals(action)) {
+ bindCoreAction(actionFlexArgs);
+
+ } else if (SliderActions.ACTION_GETCONF.equals(action)) {
+ bindCoreAction(actionGetConfArgs);
+
+ } else if (SliderActions.ACTION_HELP.equals(action) ||
+ SliderActions.ACTION_USAGE.equals(action)) {
+ bindCoreAction(actionHelpArgs);
+
+ } else if (SliderActions.ACTION_KILL_CONTAINER.equals(action)) {
+ bindCoreAction(actionKillContainerArgs);
+
+ } else if (SliderActions.ACTION_LIST.equals(action)) {
+ bindCoreAction(actionListArgs);
+
+ } else if (SliderActions.ACTION_REGISTRY.equals(action)) {
+ bindCoreAction(actionRegistryArgs);
+
+ } else if (SliderActions.ACTION_STATUS.equals(action)) {
+ bindCoreAction(actionStatusArgs);
+
+ } else if (SliderActions.ACTION_VERSION.equals(action)) {
+ bindCoreAction(actionVersionArgs);
+
+ } else if (action == null || action.isEmpty()) {
+ throw new BadCommandArgumentsException(ErrorStrings.ERROR_NO_ACTION);
+
+ } else {
+ throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
+ + " " + action);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
new file mode 100644
index 0000000..c938f8b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class contains the common argument set for all tne entry points,
+ * and the core parsing logic to verify that the action is on the list
+ * of allowed actions -and that the remaining number of arguments is
+ * in the range allowed
+ */
+
+public abstract class CommonArgs extends ArgOps implements SliderActions,
+ Arguments {
+
+ protected static final Logger log = LoggerFactory.getLogger(CommonArgs.class);
+
+
+ @Parameter(names = ARG_HELP, help = true)
+ public boolean help;
+
+
+ /**
+ -D name=value
+
+ Define an HBase configuration option which overrides any options in
+ the configuration XML files of the image or in the image configuration
+ directory. The values will be persisted.
+ Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
+
+ */
+
+ public Map<String, String> definitionMap = new HashMap<String, String>();
+ /**
+ * System properties
+ */
+ public Map<String, String> syspropsMap = new HashMap<String, String>();
+
+
+ /**
+ * fields
+ */
+ public JCommander commander;
+ private final String[] args;
+
+ private AbstractActionArgs coreAction;
+
+ /**
+ * get the name: relies on arg 1 being the cluster name in all operations
+ * @return the name argument, null if there is none
+ */
+ public String getClusterName() {
+ return coreAction.getClusterName();
+ }
+
+ public CommonArgs(String[] args) {
+ this.args = args;
+ commander = new JCommander(this);
+ }
+
+ public CommonArgs(Collection args) {
+ List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
+ this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
+ commander = new JCommander(this);
+ }
+
+
+ public String usage() {
+ StringBuilder builder = new StringBuilder("\n");
+ commander.usage(builder, " ");
+ builder.append("\nactions: ");
+ return builder.toString();
+ }
+
+
+ /**
+ * Parse routine -includes registering the action-specific argument classes
+ * and postprocess it
+ * @throws SliderException on any problem
+ */
+ public void parse() throws SliderException {
+ addActionArguments();
+ try {
+ commander.parse(getArgs());
+ } catch (ParameterException e) {
+ throw new BadCommandArgumentsException(e, "%s in %s",
+ e.toString(),
+ (getArgs() != null
+ ? (SliderUtils.join(getArgs(),
+ " ", false))
+ : "[]"));
+ }
+ //now copy back to this class some of the attributes that are common to all
+ //actions
+ postProcess();
+ }
+
+ /**
+ * Add a command
+ * @param name action
+ * @param arg
+ */
+ protected void addAction(String name, Object arg) {
+ commander.addCommand(name, arg);
+ }
+
+ protected void addActions(Object... actions) {
+ for (Object action : actions) {
+ commander.addCommand(action);
+ }
+ }
+
+ /**
+ * Override point to add a set of actions
+ */
+ protected void addActionArguments() {
+
+ }
+
+ /**
+ * validate args via {@link #validate()}
+ * then postprocess the arguments
+ */
+ public void postProcess() throws SliderException {
+ applyAction();
+ validate();
+
+ //apply entry set
+ for (Map.Entry<String, String> entry : syspropsMap.entrySet()) {
+ System.setProperty(entry.getKey(), entry.getValue());
+ }
+ }
+
+
+ /**
+ * Implementors must implement their action apply routine here
+ */
+ public abstract void applyAction() throws SliderException;
+
+
+ /**
+ * Bind the core action; this extracts any attributes that are used
+ * across routines
+ * @param action action to bind
+ */
+ protected void bindCoreAction(AbstractActionArgs action) {
+ coreAction = action;
+
+ splitPairs(coreAction.definitions, definitionMap);
+ splitPairs(coreAction.sysprops, syspropsMap);
+ }
+
+ /**
+ * Get the core action -type depends on the action
+ * @return the action class
+ */
+ public AbstractActionArgs getCoreAction() {
+ return coreAction;
+ }
+
+ /**
+ * Validate the arguments against the action requested
+ */
+ public void validate() throws BadCommandArgumentsException {
+ if (coreAction == null) {
+ throw new BadCommandArgumentsException(ErrorStrings.ERROR_NO_ACTION
+ + usage());
+ }
+ log.debug("action={}", getAction());
+ //let the action validate itself
+ coreAction.validate();
+ }
+
+ /**
+ * Apply all the definitions on the command line to the configuration
+ * @param conf config
+ */
+ public void applyDefinitions(Configuration conf) throws
+ BadCommandArgumentsException {
+ applyDefinitions(definitionMap, conf);
+ }
+
+
+ /**
+ * If the Filesystem URL was provided, it overrides anything in
+ * the configuration
+ * @param conf configuration
+ */
+ public void applyFileSystemURL(Configuration conf) {
+ ArgOps.applyFileSystemURL(getFilesystemURL(), conf);
+ }
+
+ public boolean isDebug() {
+ return coreAction.debug;
+ }
+
+
+ public URI getFilesystemURL() {
+ return coreAction.filesystemURL;
+ }
+
+ public Path getBasePath() { return coreAction.basePath; }
+
+ public String getManager() {
+ return coreAction.manager;
+ }
+
+
+// public String getRmAddress() {
+// return rmAddress;
+// }
+
+
+ public String getAction() {
+ return commander.getParsedCommand();
+ }
+
+ public List<String> getActionArgs() {
+ return coreAction.parameters;
+ }
+
+ public String[] getArgs() {
+ return args;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
new file mode 100644
index 0000000..494258e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ComponentArgsDelegate extends AbstractArgsDelegate {
+
+ /**
+ * This is a listing of the roles to create
+ */
+ @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT, ARG_ROLE},
+ arity = 2,
+ description = "--component <name> <count>",
+ splitter = DontSplitArguments.class)
+ public List<String> componentTuples = new ArrayList<String>(0);
+
+
+ /**
+ * Get the role mapping (may be empty, but never null)
+ * @return role mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
+ return convertTupleListToMap("component", componentTuples);
+ }
+
+ public List<String> getComponentTuples() {
+ return componentTuples;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
new file mode 100644
index 0000000..74ba221
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.IParameterSplitter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DontSplitArguments implements IParameterSplitter {
+
+ @Override
+ public List<String> split(String value) {
+ ArrayList<String> list = new ArrayList<String>(1);
+ list.add(value);
+ return list;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
new file mode 100644
index 0000000..f4ff4ce
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+/**
+ * Launch args for create and thaw and anything else that can start something
+ */
+public interface LaunchArgsAccessor extends WaitTimeAccessor {
+ String getRmAddress();
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
new file mode 100644
index 0000000..f0068e2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Any launch-time args
+ */
+public class LaunchArgsDelegate extends WaitArgsDelegate implements
+ LaunchArgsAccessor {
+
+
+ //TODO: do we need this?
+ @Parameter(names = ARG_RESOURCE_MANAGER,
+ description = "Resource manager hostname:port ",
+ required = false)
+ private String rmAddress;
+
+ @Override
+ public String getRmAddress() {
+ return rmAddress;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java b/slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
new file mode 100644
index 0000000..ccb526c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+import org.apache.hadoop.fs.Path;
+
+public class PathArgumentConverter extends BaseConverter<Path> {
+
+ public PathArgumentConverter(String optionName) {
+ super(optionName);
+ }
+
+ @Override
+ public Path convert(String value) {
+ return new Path(value);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
new file mode 100644
index 0000000..f9516d1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.slider.common.params;
+
+/**
+ * Parameters sent by the Client to the AM
+ */
+public class SliderAMArgs extends CommonArgs {
+
+ SliderAMCreateAction createAction = new SliderAMCreateAction();
+
+ public SliderAMArgs(String[] args) {
+ super(args);
+ }
+
+ @Override
+ protected void addActionArguments() {
+ addActions(createAction);
+ }
+
+ public String getImage() {
+ return createAction.image;
+ }
+
+ /**
+ * This is the URI in the FS to the Slider cluster; the conf file (and any
+ * other cluster-specifics) can be picked up here
+ */
+ public String getSliderClusterURI() {
+ return createAction.sliderClusterURI;
+ }
+
+ /**
+ * Am binding is simple: there is only one action
+ */
+ @Override
+ public void applyAction() {
+ bindCoreAction(createAction);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
new file mode 100644
index 0000000..0d084da
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+
+@Parameters(commandNames = {SliderActions.ACTION_CREATE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class SliderAMCreateAction extends AbstractActionArgs implements
+ LaunchArgsAccessor {
+
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CREATE;
+ }
+
+ @Parameter(names = ARG_IMAGE, description = "image", required = false)
+ public String image;
+
+ /**
+ * This is the URI in the FS to the Slider cluster; the conf file (and any
+ * other cluster-specifics) can be picked up here
+ */
+ @Parameter(names = ARG_CLUSTER_URI,
+ description = "URI to the Slider cluster", required = true)
+ public String sliderClusterURI;
+
+ @ParametersDelegate
+ LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+ @Override
+ public String getRmAddress() {
+ return launchArgs.getRmAddress();
+ }
+
+ @Override
+ public int getWaittime() {
+ return launchArgs.getWaittime();
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ launchArgs.setWaittime(waittime);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
new file mode 100644
index 0000000..2219a25
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+/**
+ * Actions.
+ * Only some of these are supported by specific Slider Services; they
+ * are listed here to ensure the names are consistent
+ */
+public interface SliderActions {
+ String ACTION_AM_SUICIDE = "am-suicide";
+ String ACTION_BUILD = "build";
+ String ACTION_CREATE = "create";
+ String ACTION_DESTROY = "destroy";
+ String ACTION_ECHO = "echo";
+ String ACTION_EXISTS = "exists";
+ String ACTION_FLEX = "flex";
+ String ACTION_FREEZE = "freeze";
+ String ACTION_GETCONF = "getconf";
+ String ACTION_HELP = "help";
+ String ACTION_KILL_CONTAINER = "kill-container";
+ String ACTION_LIST = "list";
+ String ACTION_PREFLIGHT = "preflight";
+ String ACTION_RECONFIGURE = "reconfigure";
+ String ACTION_REGISTRY = "registry";
+ String ACTION_STATUS = "status";
+ String ACTION_THAW = "thaw";
+ String ACTION_USAGE = "usage";
+ String ACTION_VERSION = "version";
+ String DESCRIBE_ACTION_AM_SUICIDE =
+ "Tell the Slider Application Master to simulate a process failure by terminating itself";
+ String DESCRIBE_ACTION_BUILD =
+ "Build a Slider cluster specification -but do not start it";
+ String DESCRIBE_ACTION_CREATE =
+ "Create a live Slider application";
+ String DESCRIBE_ACTION_DESTROY =
+ "Destroy a frozen Slider application)";
+ String DESCRIBE_ACTION_EXISTS =
+ "Probe for an application running";
+ String DESCRIBE_ACTION_FLEX = "Flex a Slider application";
+ String DESCRIBE_ACTION_FREEZE =
+ "Freeze/suspend a running application";
+ String DESCRIBE_ACTION_GETCONF =
+ "Get the configuration of an application";
+ String DESCRIBE_ACTION_KILL_CONTAINER =
+ "Kill a container in the application";
+ String DESCRIBE_ACTION_HELP = "Print help information";
+ String DESCRIBE_ACTION_LIST =
+ "List running Slider applications";
+ String DESCRIBE_ACTION_MONITOR =
+ "Monitor a running application";
+ String DESCRIBE_ACTION_REGISTRY =
+ "Query the registry of a YARN application";
+ String DESCRIBE_ACTION_STATUS =
+ "Get the status of an application";
+ String DESCRIBE_ACTION_THAW =
+ "Thaw a frozen application";
+ String DESCRIBE_ACTION_VERSION =
+ "Print the Slider version information";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java b/slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
new file mode 100644
index 0000000..b0d1ebf
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class URIArgumentConverter extends BaseConverter<URI> {
+
+ public URIArgumentConverter(String optionName) {
+ super(optionName);
+ }
+
+ @Override
+ public URI convert(String value) {
+ try {
+ return new URI(value);
+ } catch (URISyntaxException e) {
+ throw new RuntimeException("Cannot make a URI from " + value);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java b/slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
new file mode 100644
index 0000000..8894309
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public class URLArgumentConverter extends BaseConverter<URL> {
+ public URLArgumentConverter(String optionName) {
+ super(optionName);
+ }
+
+ @Override
+ public URL convert(String value) {
+ try {
+ return new URL(value);
+ } catch (MalformedURLException e) {
+ throw new RuntimeException("Cannot make a URL from " + value);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
new file mode 100644
index 0000000..1c27c01
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+
+public class WaitArgsDelegate extends AbstractArgsDelegate implements
+ WaitTimeAccessor {
+
+
+ //--wait [timeout]
+ @Parameter(names = {ARG_WAIT},
+ description = "time to wait for an action to complete")
+ public int waittime = 0;
+
+
+ @Override
+ public int getWaittime() {
+ return waittime;
+ }
+
+ @Override
+ public void setWaittime(int waittime) {
+ this.waittime = waittime;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java b/slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
new file mode 100644
index 0000000..13d4d5a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+public interface WaitTimeAccessor {
+ int getWaittime();
+ void setWaittime(int waittime);
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
new file mode 100644
index 0000000..a8c7e77
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -0,0 +1,512 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeSet;
+
+/**
+ * Methods to aid in config, both in the Configuration class and
+ * with other parts of setting up Slider-initated processes.
+ *
+ * Some of the methods take an argument of a map iterable for their sources; this allows
+ * the same method
+ */
+public class ConfigHelper {
+ private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
+
+ /**
+ * Dump the (sorted) configuration
+ * @param conf config
+ * @return the sorted keyset
+ */
+ public static TreeSet<String> dumpConf(Configuration conf) {
+ TreeSet<String> keys = sortedConfigKeys(conf);
+ for (String key : keys) {
+ log.info("{}={}", key, conf.get(key));
+ }
+ return keys;
+ }
+
+
+ /**
+ * Take a configuration and return a sorted set
+ * @param conf config
+ * @return
+ */
+ public static TreeSet<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
+ TreeSet<String> sorted = new TreeSet<String>();
+ for (Map.Entry<String, String> entry : conf) {
+ sorted.add(entry.getKey());
+ }
+ return sorted;
+ }
+
+ /**
+ * Set an entire map full of values
+ *
+ * @param config config to patch
+ * @param map map of data
+ * @param origin origin data
+ */
+ public static void addConfigMap(Configuration config,
+ Map<String, String> map,
+ String origin) throws BadConfigException {
+ addConfigMap(config, map.entrySet(), origin);
+ }
+
+ /**
+ * Set an entire map full of values
+ *
+ * @param config config to patch
+ * @param map map of data
+ * @param origin origin data
+ */
+ public static void addConfigMap(Configuration config,
+ Iterable<Map.Entry<String, String>> map,
+ String origin) throws BadConfigException {
+ for (Map.Entry<String, String> mapEntry : map) {
+ String key = mapEntry.getKey();
+ String value = mapEntry.getValue();
+ if (value == null) {
+ throw new BadConfigException("Null value for property " + key);
+ }
+ config.set(key, value, origin);
+ }
+ }
+
+
+ /**
+ * Save a config file in a destination directory on a given filesystem
+ * @param systemConf system conf used for creating filesystems
+ * @param confToSave config to save
+ * @param confdir the directory path where the file is to go
+ * @param filename the filename
+ * @return the destination path where the file was saved
+ * @throws IOException IO problems
+ */
+ public static Path saveConfig(Configuration systemConf,
+ Configuration confToSave,
+ Path confdir,
+ String filename) throws IOException {
+ FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
+ Path destPath = new Path(confdir, filename);
+ saveConfig(fs, destPath, confToSave);
+ return destPath;
+ }
+
+ /**
+ * Save a config
+ * @param fs filesystem
+ * @param destPath dest to save
+ * @param confToSave config to save
+ * @throws IOException IO problems
+ */
+ public static void saveConfig(FileSystem fs,
+ Path destPath,
+ Configuration confToSave) throws
+ IOException {
+ FSDataOutputStream fos = fs.create(destPath);
+ try {
+ confToSave.writeXml(fos);
+ } finally {
+ IOUtils.closeStream(fos);
+ }
+ }
+
+ /**
+ * Convert to an XML string
+ * @param conf configuration
+ * @return conf
+ * @throws IOException
+ */
+ public static String toXml(Configuration conf) throws IOException {
+ StringWriter writer = new StringWriter();
+ conf.writeXml(writer);
+ return writer.toString();
+ }
+
+ /**
+ * This will load and parse a configuration to an XML document
+ * @param fs filesystem
+ * @param path path
+ * @return an XML document
+ * @throws IOException IO failure
+ */
+ public Document parseConfiguration(FileSystem fs,
+ Path path) throws
+ IOException {
+ int len = (int) fs.getLength(path);
+ byte[] data = new byte[len];
+ FSDataInputStream in = fs.open(path);
+ try {
+ in.readFully(0, data);
+ } catch (IOException e) {
+ in.close();
+ }
+ ByteArrayInputStream in2;
+
+ //this is here to track down a parse issue
+ //related to configurations
+ String s = new String(data, 0, len);
+ log.debug("XML resource {} is \"{}\"", path, s);
+ in2 = new ByteArrayInputStream(data);
+ try {
+ Document document = parseConfigXML(in);
+ return document;
+ } catch (ParserConfigurationException e) {
+ throw new IOException(e);
+ } catch (SAXException e) {
+ throw new IOException(e);
+ } finally {
+ in2.close();
+ }
+
+ }
+
+ /**
+ * Load a configuration from ANY FS path. The normal Configuration
+ * loader only works with file:// URIs
+ * @param fs filesystem
+ * @param path path
+ * @return a loaded resource
+ * @throws IOException
+ */
+ public static Configuration loadConfiguration(FileSystem fs,
+ Path path) throws
+ IOException {
+ int len = (int) fs.getLength(path);
+ byte[] data = new byte[len];
+ FSDataInputStream in = fs.open(path);
+ try {
+ in.readFully(0, data);
+ } catch (IOException e) {
+ in.close();
+ }
+ ByteArrayInputStream in2;
+
+ in2 = new ByteArrayInputStream(data);
+ Configuration conf1 = new Configuration(false);
+ conf1.addResource(in2);
+ //now clone it while dropping all its sources
+ Configuration conf2 = new Configuration(false);
+ String src = path.toString();
+ for (Map.Entry<String, String> entry : conf1) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ conf2.set(key, value, src);
+ }
+ return conf2;
+ }
+
+
+ /**
+ * Generate a config file in a destination directory on the local filesystem
+ * @param confdir the directory path where the file is to go
+ * @param filename the filename
+ * @return the destination path
+ */
+ public static File saveConfig(Configuration generatingConf,
+ File confdir,
+ String filename) throws IOException {
+
+
+ File destPath = new File(confdir, filename);
+ OutputStream fos = new FileOutputStream(destPath);
+ try {
+ generatingConf.writeXml(fos);
+ } finally {
+ IOUtils.closeStream(fos);
+ }
+ return destPath;
+ }
+
+ /**
+ * Parse an XML Hadoop configuration into an XML document. x-include
+ * is supported, but as the location isn't passed in, relative
+ * URIs are out.
+ * @param in instream
+ * @return a document
+ * @throws ParserConfigurationException parser feature problems
+ * @throws IOException IO problems
+ * @throws SAXException XML is invalid
+ */
+ public static Document parseConfigXML(InputStream in) throws
+ ParserConfigurationException,
+ IOException,
+ SAXException {
+ DocumentBuilderFactory docBuilderFactory
+ = DocumentBuilderFactory.newInstance();
+ //ignore all comments inside the xml file
+ docBuilderFactory.setIgnoringComments(true);
+
+ //allow includes in the xml file
+ docBuilderFactory.setNamespaceAware(true);
+ docBuilderFactory.setXIncludeAware(true);
+ DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
+ return builder.parse(in);
+ }
+
+ /**
+ * Load a Hadoop configuration from a local file.
+ * @param file file to load
+ * @return a configuration which hasn't actually had the load triggered
+ * yet.
+ * @throws FileNotFoundException file is not there
+ * @throws IOException any other IO problem
+ */
+ public static Configuration loadConfFromFile(File file) throws
+ IOException {
+ if (!file.exists()) {
+ throw new FileNotFoundException("File not found :"
+ + file.getAbsoluteFile());
+ }
+ Configuration conf = new Configuration(false);
+ try {
+ conf.addResource(file.toURI().toURL());
+ } catch (MalformedURLException e) {
+ //should never happen...
+ throw new IOException(
+ "File " + file.toURI() + " doesn't have a valid URL");
+ }
+ return conf;
+ }
+
+ /**
+ * looks for the config under $confdir/$templateFilename; if not there
+ * loads it from /conf/templateFile.
+ * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
+ * origin to help debug what's happening
+ * @param systemConf system conf
+ * @param confdir conf dir in FS
+ * @param templateFilename filename in the confdir
+ * @param fallbackResource resource to fall back on
+ * @return loaded conf
+ * @throws IOException IO problems
+ */
+ public static Configuration loadTemplateConfiguration(Configuration systemConf,
+ Path confdir,
+ String templateFilename,
+ String fallbackResource) throws
+ IOException {
+ FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
+
+ Path templatePath = new Path(confdir, templateFilename);
+ return loadTemplateConfiguration(fs, templatePath, fallbackResource);
+ }
+
+ /**
+ * looks for the config under $confdir/$templateFilename; if not there
+ * loads it from /conf/templateFile.
+ * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
+ * origin to help debug what's happening.
+ * @param fs Filesystem
+ * @param templatePath HDFS path for template
+ * @param fallbackResource resource to fall back on, or "" for no fallback
+ * @return loaded conf
+ * @throws IOException IO problems
+ * @throws FileNotFoundException if the path doesn't have a file and there
+ * was no fallback.
+ */
+ public static Configuration loadTemplateConfiguration(FileSystem fs,
+ Path templatePath,
+ String fallbackResource) throws
+ IOException {
+ Configuration conf = null;
+ String origin;
+ if (fs.exists(templatePath)) {
+ log.debug("Loading template configuration {}", templatePath);
+ conf = loadConfiguration(fs, templatePath);
+ origin = templatePath.toString();
+ } else {
+ if (fallbackResource.isEmpty()) {
+ throw new FileNotFoundException("No config file found at " + templatePath);
+ }
+ log.debug("Template {} not found" +
+ " -reverting to classpath resource {}", templatePath, fallbackResource);
+ conf = new Configuration(false);
+ conf.addResource(fallbackResource);
+ origin = "Resource " + fallbackResource;
+ }
+ //force a get
+ conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN);
+ conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin);
+ //now set the origin
+ return conf;
+ }
+
+
+ /**
+ * For testing: dump a configuration
+ * @param conf configuration
+ * @return listing in key=value style
+ */
+ public static String dumpConfigToString(Configuration conf) {
+ TreeSet<String> sorted = sortedConfigKeys(conf);
+
+ StringBuilder builder = new StringBuilder();
+ for (String key : sorted) {
+
+ builder.append(key)
+ .append("=")
+ .append(conf.get(key))
+ .append("\n");
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Merge in one configuration above another
+ * @param base base config
+ * @param merge one to merge. This MUST be a non-default-load config to avoid
+ * merge origin confusion
+ * @param origin description of the origin for the put operation
+ * @return the base with the merged values
+ */
+ public static Configuration mergeConfigurations(Configuration base,
+ Iterable<Map.Entry<String, String>> merge,
+ String origin) {
+ for (Map.Entry<String, String> entry : merge) {
+ base.set(entry.getKey(), entry.getValue(), origin);
+ }
+ return base;
+ }
+
+ /**
+ * Register a resource as a default resource.
+ * Do not attempt to use this unless you understand that the
+ * order in which default resources are loaded affects the outcome,
+ * and that subclasses of Configuration often register new default
+ * resources
+ * @param resource the resource name
+ * @return the URL or null
+ */
+ public static URL registerDefaultResource(String resource) {
+ URL resURL = ConfigHelper.class.getClassLoader()
+ .getResource(resource);
+ if (resURL != null) {
+ Configuration.addDefaultResource(resource);
+ }
+ return resURL;
+ }
+
+ /**
+ * Load a configuration from a resource on this classpath.
+ * If the resource is not found, an empty configuration is returned
+ * @param resource the resource name
+ * @return the loaded configuration.
+ */
+ public static Configuration loadFromResource(String resource) {
+ Configuration conf = new Configuration(false);
+ URL resURL = ConfigHelper.class.getClassLoader()
+ .getResource(resource);
+ if (resURL != null) {
+ log.debug("loaded resources from {}", resURL);
+ conf.addResource(resource);
+ } else{
+ log.debug("failed to find {} on the classpath", resource);
+ }
+ return conf;
+
+ }
+
+ /**
+ * Load a resource that must be there
+ * @param resource the resource name
+ * @return the loaded configuration
+ * @throws FileNotFoundException if the resource is missing
+ */
+ public static Configuration loadMandatoryResource(String resource) throws
+ FileNotFoundException {
+ Configuration conf = new Configuration(false);
+ URL resURL = ConfigHelper.class.getClassLoader()
+ .getResource(resource);
+ if (resURL != null) {
+ log.debug("loaded resources from {}", resURL);
+ conf.addResource(resource);
+ } else {
+ throw new FileNotFoundException(resource);
+ }
+ return conf;
+ }
+
+ /**
+ * Propagate a property from a source to a dest config, with a best-effort
+ * attempt at propagating the origin.
+ * If the
+ * @param dest destination
+ * @param src source
+ * @param key key to try to copy
+ * @return true if the key was found and propagated
+ */
+ public static boolean propagate(Configuration dest,
+ Configuration src,
+ String key) {
+ String val = src.get(key);
+ if (val != null) {
+ String[] origin = src.getPropertySources(key);
+ if (origin != null && origin.length > 0) {
+ dest.set(key, val, origin[0]);
+ } else {
+ dest.set(key, val);
+ return true;
+ }
+ }
+ return false;
+ }
+
+
+ /**
+ * Take a configuration, return a hash map
+ * @param conf conf
+ * @return hash map
+ */
+ public static Map<String, String> buildMapFromConfiguration(Configuration conf) {
+ Map<String, String> map = new HashMap<String, String>();
+ return SliderUtils.mergeEntries(map, conf);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
new file mode 100644
index 0000000..a108092
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -0,0 +1,549 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
+import org.apache.slider.core.persist.Filenames;
+import org.apache.slider.core.persist.InstancePaths;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
+import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
+
+public class CoreFileSystem {
+ private static final Logger
+ log = LoggerFactory.getLogger(CoreFileSystem.class);
+
+ protected final FileSystem fileSystem;
+ protected final Configuration configuration;
+
+ public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
+ Preconditions.checkNotNull(fileSystem,
+ "Cannot create a CoreFileSystem with a null FileSystem");
+ Preconditions.checkNotNull(configuration,
+ "Cannot create a CoreFileSystem with a null Configuration");
+ this.fileSystem = fileSystem;
+ this.configuration = configuration;
+ }
+
+ public CoreFileSystem(Configuration configuration) throws IOException {
+ Preconditions.checkNotNull(configuration,
+ "Cannot create a CoreFileSystem with a null Configuration");
+ this.fileSystem = FileSystem.get(configuration);
+ this.configuration = fileSystem.getConf();
+ }
+
+ /**
+ * Get the temp path for this cluster
+ * @param clustername name of the cluster
+ * @return path for temp files (is not purged)
+ */
+ public Path getTempPathForCluster(String clustername) {
+ Path clusterDir = buildClusterDirPath(clustername);
+ return new Path(clusterDir, SliderKeys.TMP_DIR_PREFIX);
+ }
+
+ /**
+ * Returns the underlying FileSystem for this object.
+ *
+ * @return filesystem
+ */
+ public FileSystem getFileSystem() {
+ return fileSystem;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("CoreFileSystem{");
+ sb.append("fileSystem=").append(fileSystem.getUri());
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Build up the path string for a cluster instance -no attempt to
+ * create the directory is made
+ *
+ * @param clustername name of the cluster
+ * @return the path for persistent data
+ */
+ public Path buildClusterDirPath(String clustername) {
+ if (clustername == null) {
+ throw new NullPointerException();
+ }
+ Path path = getBaseApplicationPath();
+ return new Path(path, SliderKeys.CLUSTER_DIRECTORY + "/" + clustername);
+ }
+
+ /**
+ * Create the Slider cluster path for a named cluster and all its subdirs
+ * This is a directory; a mkdirs() operation is executed
+ * to ensure that it is there.
+ *
+ * @param clustername name of the cluster
+ * @return the path to the cluster directory
+ * @throws java.io.IOException trouble
+ * @throws SliderException slider-specific exceptions
+ */
+ public Path createClusterDirectories(String clustername, Configuration conf) throws
+ IOException,
+ SliderException {
+
+
+ Path clusterDirectory = buildClusterDirPath(clustername);
+ InstancePaths instancePaths = new InstancePaths(clusterDirectory);
+ createClusterDirectories(instancePaths);
+ return clusterDirectory;
+ }
+
+ /**
+ * Create the Slider cluster path for a named cluster and all its subdirs
+ * This is a directory; a mkdirs() operation is executed
+ * to ensure that it is there.
+ *
+ * @param instancePaths instance paths
+ * @return the path to the cluster directory
+ * @throws java.io.IOException trouble
+ * @throws SliderException slider-specific exceptions
+ */
+ public void createClusterDirectories(InstancePaths instancePaths) throws
+ IOException, SliderException {
+ Path clusterDirectory = instancePaths.instanceDir;
+
+ verifyDirectoryNonexistent(clusterDirectory);
+ FsPermission clusterPerms = getInstanceDirectoryPermissions();
+ createWithPermissions(clusterDirectory, clusterPerms);
+ createWithPermissions(instancePaths.snapshotConfPath, clusterPerms);
+ createWithPermissions(instancePaths.generatedConfPath, clusterPerms);
+ createWithPermissions(instancePaths.historyPath, clusterPerms);
+ createWithPermissions(instancePaths.tmpPathAM, clusterPerms);
+
+ // Data Directory
+ String dataOpts =
+ configuration.get(SliderXmlConfKeys.DATA_DIRECTORY_PERMISSIONS,
+ SliderXmlConfKeys.DEFAULT_DATA_DIRECTORY_PERMISSIONS);
+ log.debug("Setting data directory permissions to {}", dataOpts);
+ createWithPermissions(instancePaths.dataPath, new FsPermission(dataOpts));
+
+ }
+
+ /**
+ * Create a directory with the given permissions.
+ *
+ * @param dir directory
+ * @param clusterPerms cluster permissions
+ * @throws IOException IO problem
+ * @throws org.apache.slider.core.exceptions.BadClusterStateException any cluster state problem
+ */
+ public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
+ IOException,
+ BadClusterStateException {
+ if (fileSystem.isFile(dir)) {
+ // HADOOP-9361 shows some filesystems don't correctly fail here
+ throw new BadClusterStateException(
+ "Cannot create a directory over a file %s", dir);
+ }
+ log.debug("mkdir {} with perms {}", dir, clusterPerms);
+ //no mask whatoever
+ fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
+ fileSystem.mkdirs(dir, clusterPerms);
+ //and force set it anyway just to make sure
+ fileSystem.setPermission(dir, clusterPerms);
+ }
+
+ /**
+ * Get the permissions of a path
+ *
+ * @param path path to check
+ * @return the permissions
+ * @throws IOException any IO problem (including file not found)
+ */
+ public FsPermission getPathPermissions(Path path) throws IOException {
+ FileStatus status = fileSystem.getFileStatus(path);
+ return status.getPermission();
+ }
+
+ public FsPermission getInstanceDirectoryPermissions() {
+ String clusterDirPermsOct =
+ configuration.get(CLUSTER_DIRECTORY_PERMISSIONS,
+ DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
+ return new FsPermission(clusterDirPermsOct);
+ }
+
+ /**
+ * Verify that the cluster directory is not present
+ *
+ * @param clustername name of the cluster
+ * @param clusterDirectory actual directory to look for
+ * @return the path to the cluster directory
+ * @throws IOException trouble with FS
+ * @throws SliderException If the directory exists
+ */
+ public void verifyClusterDirectoryNonexistent(String clustername,
+ Path clusterDirectory) throws
+ IOException,
+ SliderException {
+ if (fileSystem.exists(clusterDirectory)) {
+ throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername,
+ clusterDirectory);
+ }
+ }
+ /**
+ * Verify that the given directory is not present
+ *
+ * @param clusterDirectory actual directory to look for
+ * @return the path to the cluster directory
+ * @throws IOException trouble with FS
+ * @throws SliderException If the directory exists
+ */
+ public void verifyDirectoryNonexistent(Path clusterDirectory) throws
+ IOException,
+ SliderException {
+ if (fileSystem.exists(clusterDirectory)) {
+ log.error("Dir {} exists: {}",
+ clusterDirectory,
+ listFSDir(clusterDirectory));
+ throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS,
+ clusterDirectory);
+ }
+ }
+
+ /**
+ * Verify that a user has write access to a directory.
+ * It does this by creating then deleting a temp file
+ *
+ * @param dirPath actual directory to look for
+ * @throws FileNotFoundException file not found
+ * @throws IOException trouble with FS
+ * @throws BadClusterStateException if the directory is not writeable
+ */
+ public void verifyDirectoryWriteAccess(Path dirPath) throws IOException,
+ SliderException {
+ verifyPathExists(dirPath);
+ Path tempFile = new Path(dirPath, "tmp-file-for-checks");
+ try {
+ FSDataOutputStream out = null;
+ out = fileSystem.create(tempFile, true);
+ IOUtils.closeStream(out);
+ fileSystem.delete(tempFile, false);
+ } catch (IOException e) {
+ log.warn("Failed to create file {}: {}", tempFile, e);
+ throw new BadClusterStateException(e,
+ "Unable to write to directory %s : %s", dirPath, e.toString());
+ }
+ }
+
+ /**
+ * Verify that a path exists
+ * @param path path to check
+ * @throws FileNotFoundException file not found
+ * @throws IOException trouble with FS
+ */
+ public void verifyPathExists(Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ throw new FileNotFoundException(path.toString());
+ }
+ }
+
+ /**
+ * Verify that a path exists
+ * @param path path to check
+ * @throws FileNotFoundException file not found or is not a file
+ * @throws IOException trouble with FS
+ */
+ public void verifyFileExists(Path path) throws IOException {
+ FileStatus status = fileSystem.getFileStatus(path);
+
+ if (!status.isFile()) {
+ throw new FileNotFoundException("Not a file: " + path.toString());
+ }
+ }
+
+ /**
+ * Create the application-instance specific temporary directory
+ * in the DFS
+ *
+ * @param clustername name of the cluster
+ * @param subdir application ID
+ * @return the path; this directory will already have been created
+ */
+ public Path createAppInstanceTempPath(String clustername, String subdir)
+ throws IOException {
+ Path tmp = getTempPathForCluster(clustername);
+ Path instancePath = new Path(tmp, subdir);
+ fileSystem.mkdirs(instancePath);
+ return instancePath;
+ }
+
+ /**
+ * Create the application-instance specific temporary directory
+ * in the DFS
+ *
+ * @param clustername name of the cluster
+ * @return the path; this directory will already have been deleted
+ */
+ public Path purgeAppInstanceTempFiles(String clustername) throws
+ IOException {
+ Path tmp = getTempPathForCluster(clustername);
+ fileSystem.delete(tmp, true);
+ return tmp;
+ }
+
+ /**
+ * Get the base path
+ *
+ * @return the base path optionally configured by {@value SliderXmlConfKeys#KEY_SLIDER_BASE_PATH}
+ */
+ public Path getBaseApplicationPath() {
+ String configuredBasePath = configuration.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH);
+ return configuredBasePath != null ? new Path(configuredBasePath) :
+ new Path(getHomeDirectory(), SliderKeys.SLIDER_BASE_DIRECTORY);
+ }
+
+ public Path getHomeDirectory() {
+ return fileSystem.getHomeDirectory();
+ }
+
+ public boolean maybeAddImagePath(Map<String, LocalResource> localResources,
+ Path imagePath) throws IOException {
+ if (imagePath != null) {
+ LocalResource resource = createAmResource(imagePath,
+ LocalResourceType.ARCHIVE);
+ localResources.put(SliderKeys.LOCAL_TARBALL_INSTALL_SUBDIR, resource);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public boolean maybeAddImagePath(Map<String, LocalResource> localResources,
+ String imagePath) throws IOException {
+
+ return imagePath != null &&
+ maybeAddImagePath(localResources, new Path(imagePath));
+ }
+
+
+
+
+ /**
+ * Create an AM resource from the
+ *
+ * @param destPath dest path in filesystem
+ * @param resourceType resource type
+ * @return the resource set up wih application-level visibility and the
+ * timestamp & size set from the file stats.
+ */
+ public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
+ FileStatus destStatus = fileSystem.getFileStatus(destPath);
+ LocalResource amResource = Records.newRecord(LocalResource.class);
+ amResource.setType(resourceType);
+ // Set visibility of the resource
+ // Setting to most private option
+ amResource.setVisibility(LocalResourceVisibility.APPLICATION);
+ // Set the resource to be copied over
+ amResource.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
+ // Set timestamp and length of file so that the framework
+ // can do basic sanity checks for the local resource
+ // after it has been copied over to ensure it is the same
+ // resource the client intended to use with the application
+ amResource.setTimestamp(destStatus.getModificationTime());
+ amResource.setSize(destStatus.getLen());
+ return amResource;
+ }
+
+ /**
+ * Register all files under a fs path as a directory to push out
+ *
+ * @param srcDir src dir
+ * @param destRelativeDir dest dir (no trailing /)
+ * @return the map of entries
+ */
+ public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException {
+ //now register each of the files in the directory to be
+ //copied to the destination
+ FileStatus[] fileset = fileSystem.listStatus(srcDir);
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>(fileset.length);
+ for (FileStatus entry : fileset) {
+
+ LocalResource resource = createAmResource(entry.getPath(),
+ LocalResourceType.FILE);
+ String relativePath = destRelativeDir + "/" + entry.getPath().getName();
+ localResources.put(relativePath, resource);
+ }
+ return localResources;
+ }
+
+ /**
+ * Submit a JAR containing a specific class, returning
+ * the resource to be mapped in
+ *
+ * @param clazz class to look for
+ * @param subdir subdirectory (expected to end in a "/")
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
+ throws IOException, SliderException {
+ File localFile = SliderUtils.findContainingJarOrFail(clazz);
+ LocalResource resource = submitFile(localFile, tempPath, subdir, jarName);
+ return resource;
+ }
+
+ /**
+ * Submit a local file to the filesystem references by the instance's cluster
+ * filesystem
+ *
+ * @param localFile filename
+ * @param subdir subdirectory (expected to end in a "/")
+ * @param destFileName destination filename
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName) throws IOException {
+ Path src = new Path(localFile.toString());
+ Path subdirPath = new Path(tempPath, subdir);
+ fileSystem.mkdirs(subdirPath);
+ Path destPath = new Path(subdirPath, destFileName);
+
+ fileSystem.copyFromLocalFile(false, true, src, destPath);
+
+ // Set the type of resource - file or archive
+ // archives are untarred at destination
+ // we don't need the jar file to be untarred for now
+ return createAmResource(destPath, LocalResourceType.FILE);
+ }
+
+ /**
+ * list entries in a filesystem directory
+ *
+ * @param path directory
+ * @return a listing, one to a line
+ * @throws IOException
+ */
+ public String listFSDir(Path path) throws IOException {
+ FileStatus[] stats = fileSystem.listStatus(path);
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.getPath().toString())
+ .append("\t")
+ .append(stat.getLen())
+ .append("\n");
+ }
+ return builder.toString();
+ }
+
+ public void touch(Path path, boolean overwrite) throws IOException {
+ FSDataOutputStream out = fileSystem.create(path, overwrite);
+ out.close();
+ }
+
+ public void cat(Path path, boolean overwrite, String data) throws IOException {
+ FSDataOutputStream out = fileSystem.create(path, overwrite);
+ byte[] bytes = data.getBytes(Charset.forName("UTF-8"));
+ out.write(bytes);
+ out.close();
+ }
+
+ /**
+ * Create a path that must exist in the cluster fs
+ * @param uri uri to create
+ * @return the path
+ * @throws SliderException if the path does not exist
+ */
+ public Path createPathThatMustExist(String uri) throws
+ SliderException,
+ IOException {
+ Path path = new Path(uri);
+ verifyPathExists(path);
+ return path;
+ }
+
+ /**
+ * Locate an application conf json in the FS. This includes a check to verify
+ * that the file is there.
+ *
+ * @param clustername name of the cluster
+ * @return the path to the spec.
+ * @throws IOException IO problems
+ * @throws SliderException if the path isn't there
+ */
+ public Path locateInstanceDefinition(String clustername) throws IOException,
+ SliderException {
+ Path clusterDirectory = buildClusterDirPath(clustername);
+ Path appConfPath =
+ new Path(clusterDirectory, Filenames.APPCONF);
+ verifyClusterSpecExists(clustername, appConfPath);
+ return appConfPath;
+ }
+
+ /**
+ * Verify that a cluster specification exists
+ * @param clustername name of the cluster (For errors only)
+ * @param clusterSpecPath cluster specification path
+ * @throws IOException IO problems
+ * @throws SliderException if the cluster specification is not present
+ */
+ public void verifyClusterSpecExists(String clustername,
+ Path clusterSpecPath) throws
+ IOException,
+ SliderException {
+ if (!fileSystem.isFile(clusterSpecPath)) {
+ log.debug("Missing specification file {}", clusterSpecPath);
+ throw UnknownApplicationInstanceException.unknownInstance(clustername
+ +
+ "\n (definition not found at "
+ +
+ clusterSpecPath);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java b/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
new file mode 100644
index 0000000..119991f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+/**
+ * A duration in milliseconds
+ */
+public class Duration {
+ public long start, finish;
+ public final long limit;
+
+ public Duration() {
+ this(0);
+ }
+
+ public Duration(long limit) {
+ this.limit = limit;
+ }
+
+ public Duration start() {
+ start = System.currentTimeMillis();
+ return this;
+ }
+
+ public void finish() {
+ finish = System.currentTimeMillis();
+ }
+
+ public long getInterval() {
+ return finish - start;
+ }
+
+ public boolean getLimitExceeded() {
+ return limit >= 0 && ((System.currentTimeMillis() - start) > limit);
+ }
+
+ @Override
+ public String toString() {
+ return "Duration " +
+ ((finish >= start)
+ ? (" of " + getInterval() + " millis")
+ : "undefined");
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java
new file mode 100644
index 0000000..294f37e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+import java.io.IOException;
+
+/**
+ * Extends Core Filesystem with operations to manipulate ClusterDescription
+ * persistent state
+ */
+public class SliderFileSystem extends CoreFileSystem {
+
+ public SliderFileSystem(FileSystem fileSystem,
+ Configuration configuration) {
+ super(fileSystem, configuration);
+ }
+
+ public SliderFileSystem(Configuration configuration) throws IOException {
+ super(configuration);
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
new file mode 100644
index 0000000..9399b2c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -0,0 +1,1353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.MissingArgException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.ClasspathConstructor;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class SliderUtils {
+
+ private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
+
+ /**
+ * Atomic bool to track whether or not process security has already been
+ * turned on (prevents re-entrancy)
+ */
+ private static final AtomicBoolean processSecurityAlreadyInitialized =
+ new AtomicBoolean(false);
+ public static final String JAVA_SECURITY_KRB5_REALM =
+ "java.security.krb5.realm";
+ public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
+
+ private SliderUtils() {
+ }
+
+ /**
+ * Implementation of set-ness, groovy definition of true/false for a string
+ * @param s string
+ * @return true iff the string is neither null nor empty
+ */
+ public static boolean isUnset(String s) {
+ return s == null || s.isEmpty();
+ }
+
+ public static boolean isSet(String s) {
+ return !isUnset(s);
+ }
+
+ /*
+ * Validates whether num is an integer
+ * @param num
+ * @param msg the message to be shown in exception
+ */
+ private static void validateNumber(String num, String msg) throws BadConfigException {
+ try {
+ Integer.parseInt(num);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(msg + num);
+ }
+ }
+
+ /*
+ * Translates the trailing JVM heapsize unit: g, G, m, M
+ * This assumes designated unit of 'm'
+ * @param heapsize
+ * @return heapsize in MB
+ */
+ public static String translateTrailingHeapUnit(String heapsize) throws BadConfigException {
+ String errMsg = "Bad heapsize: ";
+ if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
+ String num = heapsize.substring(0, heapsize.length()-1);
+ validateNumber(num, errMsg);
+ return num;
+ }
+ if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
+ String num = heapsize.substring(0, heapsize.length()-1)+"000";
+ validateNumber(num, errMsg);
+ return num;
+ }
+ // check if specified heap size is a number
+ validateNumber(heapsize, errMsg);
+ return heapsize;
+ }
+
+ /**
+ * recursive directory delete
+ * @param dir dir to delete
+ * @throws IOException on any problem
+ */
+ public static void deleteDirectoryTree(File dir) throws IOException {
+ if (dir.exists()) {
+ if (dir.isDirectory()) {
+ log.info("Cleaning up {}", dir);
+ //delete the children
+ File[] files = dir.listFiles();
+ if (files == null) {
+ throw new IOException("listfiles() failed for " + dir);
+ }
+ for (File file : files) {
+ log.info("deleting {}", file);
+ if (!file.delete()) {
+ log.warn("Unable to delete " + file);
+ }
+ }
+ if (!dir.delete()) {
+ log.warn("Unable to delete " + dir);
+ }
+ } else {
+ throw new IOException("Not a directory " + dir);
+ }
+ } else {
+ //not found, do nothing
+ log.debug("No output dir yet");
+ }
+ }
+
+ /**
+ * Find a containing JAR
+ * @param my_class class to find
+ * @return the file
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ * @throws FileNotFoundException if the class did not resolve to a file
+ */
+ public static File findContainingJarOrFail(Class clazz) throws IOException {
+ File localFile = SliderUtils.findContainingJar(clazz);
+ if (null == localFile) {
+ throw new FileNotFoundException("Could not find JAR containing " + clazz);
+ }
+ return localFile;
+ }
+
+
+ /**
+ * Find a containing JAR
+ * @param my_class class to find
+ * @return the file or null if it is not found
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ */
+ public static File findContainingJar(Class my_class) throws IOException {
+ ClassLoader loader = my_class.getClassLoader();
+ if (loader == null) {
+ throw new IOException(
+ "Class " + my_class + " does not have a classloader!");
+ }
+ String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+ Enumeration<URL> urlEnumeration = loader.getResources(class_file);
+ if (urlEnumeration == null) {
+ throw new IOException("Unable to find resources for class " + my_class);
+ }
+
+ for (Enumeration itr = urlEnumeration; itr.hasMoreElements(); ) {
+ URL url = (URL) itr.nextElement();
+ if ("jar".equals(url.getProtocol())) {
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ String jarFilePath = toReturn.replaceAll("!.*$", "");
+ return new File(jarFilePath);
+ } else {
+ log.info("could not locate JAR containing {} URL={}", my_class, url);
+ }
+ }
+ return null;
+ }
+
+ public static void checkPort(String hostname, int port, int connectTimeout)
+ throws IOException {
+ InetSocketAddress addr = new InetSocketAddress(hostname, port);
+ checkPort(hostname, addr, connectTimeout);
+ }
+
+ @SuppressWarnings("SocketOpenedButNotSafelyClosed")
+ public static void checkPort(String name,
+ InetSocketAddress address,
+ int connectTimeout)
+ throws IOException {
+ Socket socket = null;
+ try {
+ socket = new Socket();
+ socket.connect(address, connectTimeout);
+ } catch (Exception e) {
+ throw new IOException("Failed to connect to " + name
+ + " at " + address
+ + " after " + connectTimeout + "millisconds"
+ + ": " + e,
+ e);
+ } finally {
+ IOUtils.closeSocket(socket);
+ }
+ }
+
+ public static void checkURL(String name, String url, int timeout) throws
+ IOException {
+ InetSocketAddress address = NetUtils.createSocketAddr(url);
+ checkPort(name, address, timeout);
+ }
+
+ /**
+ * A required file
+ * @param role role of the file (for errors)
+ * @param filename the filename
+ * @throws ExitUtil.ExitException if the file is missing
+ * @return the file
+ */
+ public static File requiredFile(String filename, String role) throws
+ IOException {
+ if (filename.isEmpty()) {
+ throw new ExitUtil.ExitException(-1, role + " file not defined");
+ }
+ File file = new File(filename);
+ if (!file.exists()) {
+ throw new ExitUtil.ExitException(-1,
+ role + " file not found: " +
+ file.getCanonicalPath());
+ }
+ return file;
+ }
+
+ /**
+ * Normalize a cluster name then verify that it is valid
+ * @param name proposed cluster name
+ * @return true iff it is valid
+ */
+ public static boolean isClusternameValid(String name) {
+ if (name == null || name.isEmpty()) {
+ return false;
+ }
+ int first = name.charAt(0);
+ if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
+ return false;
+ }
+
+ for (int i = 0; i < name.length(); i++) {
+ int elt = (int) name.charAt(i);
+ int t = Character.getType(elt);
+ if (0 == (t & Character.LOWERCASE_LETTER)
+ && 0 == (t & Character.DECIMAL_DIGIT_NUMBER)
+ && elt != '-'
+ && elt != '_') {
+ return false;
+ }
+ if (!Character.isLetterOrDigit(elt) && elt != '-' && elt != '_') {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Copy a directory to a new FS -both paths must be qualified. If
+ * a directory needs to be created, supplied permissions can override
+ * the default values. Existing directories are not touched
+ * @param conf conf file
+ * @param srcDirPath src dir
+ * @param destDirPath dest dir
+ * @param permission permission for the dest directory; null means "default"
+ * @return # of files copies
+ */
+ public static int copyDirectory(Configuration conf,
+ Path srcDirPath,
+ Path destDirPath,
+ FsPermission permission) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
+ FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcDirPath)) {
+ throw new FileNotFoundException("Source dir not found " + srcDirPath);
+ }
+ if (!srcFS.isDirectory(srcDirPath)) {
+ throw new FileNotFoundException("Source dir not a directory " + srcDirPath);
+ }
+ FileStatus[] entries = srcFS.listStatus(srcDirPath);
+ int srcFileCount = entries.length;
+ if (srcFileCount == 0) {
+ return 0;
+ }
+ if (permission == null) {
+ permission = FsPermission.getDirDefault();
+ }
+ if (!destFS.exists(destDirPath)) {
+ new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath, permission);
+ }
+ Path[] sourcePaths = new Path[srcFileCount];
+ for (int i = 0; i < srcFileCount; i++) {
+ FileStatus e = entries[i];
+ Path srcFile = e.getPath();
+ if (srcFS.isDirectory(srcFile)) {
+ throw new IOException("Configuration dir " + srcDirPath
+ + " contains a directory " + srcFile);
+ }
+ log.debug("copying src conf file {}", srcFile);
+ sourcePaths[i] = srcFile;
+ }
+ log.debug("Copying {} files from {} to dest {}", srcFileCount,
+ srcDirPath,
+ destDirPath);
+ FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
+ return srcFileCount;
+ }
+
+
+ public static String stringify(Throwable t) {
+ StringWriter sw = new StringWriter();
+ sw.append(t.toString()).append('\n');
+ t.printStackTrace(new PrintWriter(sw));
+ return sw.toString();
+ }
+
+ /**
+ * Create a configuration with Slider-specific tuning.
+ * This is done rather than doing custom configs.
+ * @return the config
+ */
+ public static YarnConfiguration createConfiguration() {
+ YarnConfiguration conf = new YarnConfiguration();
+ patchConfiguration(conf);
+ return conf;
+ }
+
+ /**
+ * Take an existing conf and patch it for Slider's needs. Useful
+ * in Service.init & RunService methods where a shared config is being
+ * passed in
+ * @param conf configuration
+ * @return the patched configuration
+ */
+ public static Configuration patchConfiguration(Configuration conf) {
+
+ //if the fallback option is NOT set, enable it.
+ //if it is explicitly set to anything -leave alone
+ if (conf.get(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH) == null) {
+ conf.set(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH, "true");
+ }
+ return conf;
+ }
+
+ /**
+ * Take a collection, return a list containing the string value of every
+ * element in the collection.
+ * @param c collection
+ * @return a stringified list
+ */
+ public static List<String> collectionToStringList(Collection c) {
+ List<String> l = new ArrayList<String>(c.size());
+ for (Object o : c) {
+ l.add(o.toString());
+ }
+ return l;
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(Collection collection, String separator) {
+ return join(collection, separator, true);
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(Collection collection, String separator, boolean trailing) {
+ StringBuilder b = new StringBuilder();
+ for (Object o : collection) {
+ b.append(o);
+ b.append(separator);
+ }
+ return trailing?
+ b.toString()
+ : (b.substring(0, b.length() - 1));
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection strings
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator) {
+ return join(collection, separator, true);
+
+
+ }
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection strings
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator,
+ boolean trailing) {
+ return join(Arrays.asList(collection), separator, trailing);
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -except at the end
+ * @param collection strings
+ * @param separator separator string
+ * @return the list
+ */
+ public static String joinWithInnerSeparator(String separator,
+ Object... collection) {
+ StringBuilder b = new StringBuilder();
+ boolean first = true;
+
+ for (Object o : collection) {
+ if (first) {
+ first = false;
+ } else {
+ b.append(separator);
+ }
+ b.append(o.toString());
+ b.append(separator);
+ }
+ return b.toString();
+ }
+
+ public static String mandatoryEnvVariable(String key) {
+ String v = System.getenv(key);
+ if (v == null) {
+ throw new MissingArgException("Missing Environment variable " + key);
+ }
+ return v;
+ }
+
+ public static String appReportToString(ApplicationReport r, String separator) {
+ StringBuilder builder = new StringBuilder(512);
+ builder.append("application ").append(
+ r.getName()).append("/").append(r.getApplicationType());
+ builder.append(separator).append(
+ "state: ").append(r.getYarnApplicationState());
+ builder.append(separator).append("URL: ").append(r.getTrackingUrl());
+ builder.append(separator).append("Started ").append(new Date(r.getStartTime()).toGMTString());
+ long finishTime = r.getFinishTime();
+ if (finishTime>0) {
+ builder.append(separator).append("Finished ").append(new Date(finishTime).toGMTString());
+ }
+ builder.append(separator).append("RPC :").append(r.getHost()).append(':').append(r.getRpcPort());
+ String diagnostics = r.getDiagnostics();
+ if (!diagnostics.isEmpty()) {
+ builder.append(separator).append("Diagnostics :").append(diagnostics);
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Merge in one map to another -all entries in the second map are
+ * merged into the first -overwriting any duplicate keys.
+ * @param first first map -the updated one.
+ * @param second the map that is merged in
+ * @return the first map
+ */
+ public static Map<String, String> mergeMap(Map<String, String> first,
+ Map<String, String> second) {
+ first.putAll(second);
+ return first;
+ }
+
+ /**
+ * Merge a set of entries into a map. This will take the entryset of
+ * a map, or a Hadoop collection itself
+ * @param dest destination
+ * @param entries entries
+ * @return dest -with the entries merged in
+ */
+ public static Map<String, String> mergeEntries(Map<String, String> dest,
+ Iterable<Map.Entry<String, String>> entries) {
+ for (Map.Entry<String, String> entry: entries) {
+ dest.put(entry.getKey(), entry.getValue());
+ }
+ return dest;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ first.putAll(second);
+ return first;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ for (Map.Entry<T1, T2> entry : second.entrySet()) {
+ T1 key = entry.getKey();
+ if (!first.containsKey(key)) {
+ first.put(key, entry.getValue());
+ }
+ }
+ return first;
+ }
+
+ /**
+ * Convert a map to a multi-line string for printing
+ * @param map map to stringify
+ * @return a string representation of the map
+ */
+ public static String stringifyMap(Map<String, String> map) {
+ StringBuilder builder =new StringBuilder();
+ for (Map.Entry<String, String> entry: map.entrySet()) {
+ builder.append(entry.getKey())
+ .append("=\"")
+ .append(entry.getValue())
+ .append("\"\n");
+
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Get the int value of a role
+ * @param roleMap map of role key->val entries
+ * @param key key the key to look for
+ * @param defVal default value to use if the key is not in the map
+ * @param min min value or -1 for do not check
+ * @param max max value or -1 for do not check
+ * @return the int value the integer value
+ * @throws BadConfigException if the value could not be parsed
+ */
+ public static int getIntValue(Map<String, String> roleMap,
+ String key,
+ int defVal,
+ int min,
+ int max
+ ) throws BadConfigException {
+ String valS = roleMap.get(key);
+ return parseAndValidate(key, valS, defVal, min, max);
+
+ }
+
+ /**
+ * Parse an int value, replacing it with defval if undefined;
+ * @param errorKey key to use in exceptions
+ * @param defVal default value to use if the key is not in the map
+ * @param min min value or -1 for do not check
+ * @param max max value or -1 for do not check
+ * @return the int value the integer value
+ * @throws BadConfigException if the value could not be parsed
+ */
+ public static int parseAndValidate(String errorKey,
+ String valS,
+ int defVal,
+ int min, int max) throws
+ BadConfigException {
+ if (valS == null) {
+ valS = Integer.toString(defVal);
+ }
+ String trim = valS.trim();
+ int val;
+ try {
+ val = Integer.decode(trim);
+ } catch (NumberFormatException e) {
+ throw new BadConfigException("Failed to parse value of "
+ + errorKey + ": \"" + trim + "\"");
+ }
+ if (min >= 0 && val < min) {
+ throw new BadConfigException("Value of "
+ + errorKey + ": " + val + ""
+ + "is less than the minimum of " + min);
+ }
+ if (max >= 0 && val > max) {
+ throw new BadConfigException("Value of "
+ + errorKey + ": " + val + ""
+ + "is more than the maximum of " + max);
+ }
+ return val;
+ }
+
+ public static InetSocketAddress getRmAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
+ }
+
+ public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ }
+
+ /**
+ * probe to see if the RM scheduler is defined
+ * @param conf config
+ * @return true if the RM scheduler address is set to
+ * something other than 0.0.0.0
+ */
+ public static boolean isRmSchedulerAddressDefined(Configuration conf) {
+ InetSocketAddress address = getRmSchedulerAddress(conf);
+ return isAddressDefined(address);
+ }
+
+ /**
+ * probe to see if the address
+ * @param address
+ * @return true if the scheduler address is set to
+ * something other than 0.0.0.0
+ */
+ public static boolean isAddressDefined(InetSocketAddress address) {
+ return !(address.getHostName().equals("0.0.0.0"));
+ }
+
+ public static void setRmAddress(Configuration conf, String rmAddr) {
+ conf.set(YarnConfiguration.RM_ADDRESS, rmAddr);
+ }
+
+ public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
+ conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
+ }
+
+ public static boolean hasAppFinished(ApplicationReport report) {
+ return report == null ||
+ report.getYarnApplicationState().ordinal() >=
+ YarnApplicationState.FINISHED.ordinal();
+ }
+
+ public static String containerToString(Container container) {
+ if (container == null) {
+ return "null container";
+ }
+ return String.format(Locale.ENGLISH,
+ "ContainerID=%s nodeID=%s http=%s priority=%s",
+ container.getId(),
+ container.getNodeId(),
+ container.getNodeHttpAddress(),
+ container.getPriority());
+ }
+
+ /**
+ * convert an AM report to a string for diagnostics
+ * @param report the report
+ * @return the string value
+ */
+ public static String reportToString(ApplicationReport report) {
+ if (report == null) {
+ return "Null application report";
+ }
+
+ return "App " + report.getName() + "/" + report.getApplicationType() +
+ "# " +
+ report.getApplicationId() + " user " + report.getUser() +
+ " is in state " + report.getYarnApplicationState() +
+ " RPC: " + report.getHost() + ":" + report.getRpcPort() +
+ " URL" + report.getOriginalTrackingUrl();
+ }
+
+ /**
+ * Convert a YARN URL into a string value of a normal URL
+ * @param url URL
+ * @return string representatin
+ */
+ public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(url.getScheme()).append("://");
+ if (url.getHost() != null) {
+ builder.append(url.getHost()).append(":").append(url.getPort());
+ }
+ builder.append(url.getFile());
+ return builder.toString();
+ }
+
+ public static int findFreePort(int start, int limit) {
+ if (start == 0) {
+ //bail out if the default is "dont care"
+ return 0;
+ }
+ int found = 0;
+ int port = start;
+ int finish = start + limit;
+ while (found == 0 && port < finish) {
+ if (isPortAvailable(port)) {
+ found = port;
+ } else {
+ port++;
+ }
+ }
+ return found;
+ }
+
+ /**
+ * See if a port is available for listening on by trying to listen
+ * on it and seeing if that works or fails.
+ * @param port port to listen to
+ * @return true if the port was available for listening on
+ */
+ public static boolean isPortAvailable(int port) {
+ try {
+ ServerSocket socket = new ServerSocket(port);
+ socket.close();
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Build the environment map from a role option map, finding all entries
+ * beginning with "env.", adding them to a map of (prefix-removed)
+ * env vars
+ * @param roleOpts role options. This can be null, meaning the
+ * role is undefined
+ * @return a possibly empty map of environment variables.
+ */
+ public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
+ Map<String, String> env = new HashMap<String, String>();
+ if (roleOpts != null) {
+ for (Map.Entry<String, String> entry:roleOpts.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(RoleKeys.ENV_PREFIX)) {
+ String envName = key.substring(RoleKeys.ENV_PREFIX.length());
+ if (!envName.isEmpty()) {
+ env.put(envName,entry.getValue());
+ }
+ }
+ }
+ }
+ return env;
+ }
+
+ /**
+ * Apply a set of command line options to a cluster role map
+ * @param clusterRoleMap cluster role map to merge onto
+ * @param commandOptions command opts
+ */
+ public static void applyCommandLineRoleOptsToRoleMap(Map<String, Map<String, String>> clusterRoleMap,
+ Map<String, Map<String, String>> commandOptions) {
+ for (Map.Entry<String, Map<String, String>> entry: commandOptions.entrySet()) {
+ String key = entry.getKey();
+ Map<String, String> optionMap = entry.getValue();
+ Map<String, String> existingMap = clusterRoleMap.get(key);
+ if (existingMap == null) {
+ existingMap = new HashMap<String, String>();
+ }
+ log.debug("Overwriting role options with command line values {}",
+ stringifyMap(optionMap));
+ mergeMap(existingMap, optionMap);
+ //set or overwrite the role
+ clusterRoleMap.put(key, existingMap);
+ }
+ }
+
+ /**
+ * verify that the supplied cluster name is valid
+ * @param clustername cluster name
+ * @throws BadCommandArgumentsException if it is invalid
+ */
+ public static void validateClusterName(String clustername) throws
+ BadCommandArgumentsException {
+ if (!isClusternameValid(clustername)) {
+ throw new BadCommandArgumentsException(
+ "Illegal cluster name: " + clustername);
+ }
+ }
+
+ /**
+ * Verify that a Kerberos principal has been set -if not fail
+ * with an error message that actually tells you what is missing
+ * @param conf configuration to look at
+ * @param principal key of principal
+ * @throws BadConfigException if the key is not set
+ */
+ public static void verifyPrincipalSet(Configuration conf,
+ String principal) throws
+ BadConfigException {
+ String principalName = conf.get(principal);
+ if (principalName == null) {
+ throw new BadConfigException("Unset Kerberos principal : %s",
+ principal);
+ }
+ log.debug("Kerberos princial {}={}", principal, principalName);
+ }
+
+ /**
+ * Flag to indicate whether the cluster is in secure mode
+ * @param conf configuration to look at
+ * @return true if the slider client/service should be in secure mode
+ */
+ public static boolean isHadoopClusterSecure(Configuration conf) {
+ return conf.getBoolean(SliderXmlConfKeys.KEY_SECURITY_ENABLED, false);
+ }
+
+ /**
+ * Init security if the cluster configuration declares the cluster is secure
+ * @param conf configuration to look at
+ * @return true if the cluster is secure
+ * @throws IOException cluster is secure
+ * @throws BadConfigException the configuration/process is invalid
+ */
+ public static boolean maybeInitSecurity(Configuration conf) throws
+ IOException,
+ BadConfigException {
+ boolean clusterSecure = isHadoopClusterSecure(conf);
+ if (clusterSecure) {
+ log.debug("Enabling security");
+ initProcessSecurity(conf);
+ }
+ return clusterSecure;
+ }
+
+ /**
+ * Turn on security. This is setup to only run once.
+ * @param conf configuration to build up security
+ * @return true if security was initialized in this call
+ * @throws IOException IO/Net problems
+ * @throws BadConfigException the configuration and system state are inconsistent
+ */
+ public static boolean initProcessSecurity(Configuration conf) throws
+ IOException,
+ BadConfigException {
+
+ if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
+ //security is already inited
+ return false;
+ }
+
+ log.info("JVM initialized into secure mode with kerberos realm {}",
+ SliderUtils.getKerberosRealm());
+ //this gets UGI to reset its previous world view (i.e simple auth)
+ //security
+ log.debug("java.security.krb5.realm={}",
+ System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
+ log.debug("java.security.krb5.kdc={}",
+ System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
+ log.debug("hadoop.security.authentication={}",
+ conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
+ log.debug("hadoop.security.authorization={}",
+ conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION));
+ SecurityUtil.setAuthenticationMethod(
+ UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
+ log.debug("Authenticating as " + authUser.toString());
+ log.debug("Login user is {}", UserGroupInformation.getLoginUser());
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ throw new BadConfigException("Although secure mode is enabled," +
+ "the application has already set up its user as an insecure entity %s",
+ authUser);
+ }
+ if (authUser.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.SIMPLE) {
+ throw new BadConfigException("Auth User is not Kerberized %s" +
+ " -security has already been set up with the wrong authentication method",
+ authUser);
+
+ }
+
+ SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
+ SliderUtils.verifyPrincipalSet(conf,
+ DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ return true;
+ }
+
+ /**
+ * Force an early login: This catches any auth problems early rather than
+ * in RPC operations
+ * @throws IOException if the login fails
+ */
+ public static void forceLogin() throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isLoginKeytabBased()) {
+ UserGroupInformation.getLoginUser().reloginFromKeytab();
+ } else {
+ UserGroupInformation.getLoginUser().reloginFromTicketCache();
+ }
+ }
+ }
+
+ /**
+ * Submit a JAR containing a specific class and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param clazz class to look for
+ * @param libdir lib directory
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public static LocalResource putJar(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
+ LocalResource res = sliderFileSystem.submitJarWithClass(
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/"+ jarName, res);
+ return res;
+ }
+
+ public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
+ Map<String, Map<String, String>> dest =
+ new HashMap<String, Map<String, String>>();
+ for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
+ dest.put(entry.getKey(), stringMapClone(entry.getValue()));
+ }
+ return dest;
+ }
+
+ public static Map<String, String> stringMapClone(Map<String, String> src) {
+ Map<String, String> dest = new HashMap<String, String>();
+ return mergeEntries(dest, src.entrySet());
+ }
+
+ /**
+ * List a directory in the local filesystem
+ * @param dir directory
+ * @return a listing, one to a line
+ */
+ public static String listDir(File dir) {
+ if (dir == null) {
+ return "";
+ }
+ StringBuilder builder = new StringBuilder();
+ String[] confDirEntries = dir.list();
+ for (String entry : confDirEntries) {
+ builder.append(entry).append("\n");
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Create a file:// path from a local file
+ * @param file file to point the path
+ * @return a new Path
+ */
+ public static Path createLocalPath(File file) {
+ return new Path(file.toURI());
+ }
+
+ /**
+ * Get the current user -relays to
+ * {@link UserGroupInformation#getCurrentUser()}
+ * with any Slider-specific post processing and exception handling
+ * @return user info
+ * @throws IOException on a failure to get the credentials
+ */
+ public static UserGroupInformation getCurrentUser() throws IOException {
+
+ try {
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ return currentUser;
+ } catch (IOException e) {
+ log.info("Failed to grt user info", e);
+ throw e;
+ }
+ }
+
+ public static String getKerberosRealm() {
+ try {
+ return KerberosUtil.getDefaultRealm();
+ } catch (Exception e) {
+ log.debug("introspection into JVM internals failed", e);
+ return "(unknown)";
+
+ }
+ }
+
+ /**
+ * Register the client resource in
+ * {@link SliderKeys#CLIENT_RESOURCE}
+ * for Configuration instances.
+ *
+ * @return true if the resource could be loaded
+ */
+ public static URL registerClientResource() {
+ return ConfigHelper.registerDefaultResource(SliderKeys.CLIENT_RESOURCE);
+ }
+
+
+ /**
+ * Attempt to load the slider client resource. If the
+ * resource is not on the CP an empty config is returned.
+ * @return a config
+ */
+ public static Configuration loadClientConfigurationResource() {
+ return ConfigHelper.loadFromResource(SliderKeys.CLIENT_RESOURCE);
+ }
+
+ /**
+ * Convert a char sequence to a string.
+ * This ensures that comparisions work
+ * @param charSequence source
+ * @return the string equivalent
+ */
+ public static String sequenceToString(CharSequence charSequence) {
+ StringBuilder stringBuilder = new StringBuilder(charSequence);
+ return stringBuilder.toString();
+ }
+
+ /**
+ * Build up the classpath for execution
+ * -behaves very differently on a mini test cluster vs a production
+ * production one.
+ *
+ * @param sliderConfDir relative path to the dir containing slider config
+ * options to put on the classpath -or null
+ * @param libdir directory containing the JAR files
+ * @param config the configuration
+ * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
+ * (and hence the current classpath should be used, not anything built up)
+ * @return a classpath
+ */
+ public static ClasspathConstructor buildClasspath(String sliderConfDir,
+ String libdir,
+ Configuration config,
+ boolean usingMiniMRCluster) {
+
+ ClasspathConstructor classpath = new ClasspathConstructor();
+
+ // add the runtime classpath needed for tests to work
+ if (usingMiniMRCluster) {
+ // for mini cluster we pass down the java CP properties
+ // and nothing else
+ classpath.appendAll(classpath.javaVMClasspath());
+ } else {
+ classpath.addLibDir("./" + libdir);
+ if (sliderConfDir != null) {
+ classpath.addClassDirectory(sliderConfDir);
+ }
+ classpath.addRemoteClasspathEnvVar();
+ classpath.appendAll(classpath.yarnApplicationClasspath(config));
+ }
+ return classpath;
+ }
+
+ /**
+ * Verify that a path refers to a directory. If not
+ * logs the parent dir then throws an exception
+ * @param dir the directory
+ * @param errorlog log for output on an error
+ * @throws FileNotFoundException if it is not a directory
+ */
+ public static void verifyIsDir(File dir, Logger errorlog) throws FileNotFoundException {
+ if (!dir.exists()) {
+ errorlog.warn("contents of {}: {}", dir,
+ listDir(dir.getParentFile()));
+ throw new FileNotFoundException(dir.toString());
+ }
+ if (!dir.isDirectory()) {
+ errorlog.info("contents of {}: {}", dir,
+ listDir(dir.getParentFile()));
+ throw new FileNotFoundException(
+ "Not a directory: " + dir);
+ }
+ }
+
+ /**
+ * Verify that a file exists
+ * @param file file
+ * @param errorlog log for output on an error
+ * @throws FileNotFoundException
+ */
+ public static void verifyFileExists(File file, Logger errorlog) throws FileNotFoundException {
+ if (!file.exists()) {
+ errorlog.warn("contents of {}: {}", file,
+ listDir(file.getParentFile()));
+ throw new FileNotFoundException(file.toString());
+ }
+ if (!file.isFile()) {
+ throw new FileNotFoundException("Not a file: " + file.toString());
+ }
+ }
+
+ /**
+ * verify that a config option is set
+ * @param configuration config
+ * @param key key
+ * @return the value, in case it needs to be verified too
+ * @throws BadConfigException if the key is missing
+ */
+ public static String verifyOptionSet(Configuration configuration, String key,
+ boolean allowEmpty) throws BadConfigException {
+ String val = configuration.get(key);
+ if (val == null) {
+ throw new BadConfigException(
+ "Required configuration option \"%s\" not defined ", key);
+ }
+ if (!allowEmpty && val.isEmpty()) {
+ throw new BadConfigException(
+ "Configuration option \"%s\" must not be empty", key);
+ }
+ return val;
+ }
+
+ /**
+ * Verify that a keytab property is defined and refers to a non-empty file
+ *
+ * @param siteConf configuration
+ * @param prop property to look for
+ * @return the file referenced
+ * @throws BadConfigException on a failure
+ */
+ public static File verifyKeytabExists(Configuration siteConf, String prop) throws
+ BadConfigException {
+ String keytab = siteConf.get(prop);
+ if (keytab == null) {
+ throw new BadConfigException("Missing keytab property %s",
+ prop);
+
+ }
+ File keytabFile = new File(keytab);
+ if (!keytabFile.exists()) {
+ throw new BadConfigException("Missing keytab file %s defined in %s",
+ keytabFile,
+ prop);
+ }
+ if (keytabFile.length() == 0 || !keytabFile.isFile()) {
+ throw new BadConfigException("Invalid keytab file %s defined in %s",
+ keytabFile,
+ prop);
+ }
+ return keytabFile;
+ }
+
+ /**
+ * Convert an epoch time to a GMT time. This
+ * uses the deprecated Date.toString() operation,
+ * so is in one place to reduce the number of deprecation warnings.
+ * @param time timestamp
+ * @return string value as ISO-9601
+ */
+ @SuppressWarnings({"CallToDateToString", "deprecation"})
+ public static String toGMTString(long time) {
+ return new Date(time).toGMTString();
+ }
+
+ /**
+ * Add the cluster build information; this will include Hadoop details too
+ * @param cd cluster
+ * @param prefix prefix for the build info
+ */
+ public static void addBuildInfo(Map<String, String> info, String prefix) {
+
+ Properties props = SliderVersionInfo.loadVersionProperties();
+ info.put(prefix + "." + SliderVersionInfo.APP_BUILD_INFO, props.getProperty(
+ SliderVersionInfo.APP_BUILD_INFO));
+ info.put(prefix + "." + SliderVersionInfo.HADOOP_BUILD_INFO,
+ props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO));
+
+ info.put(prefix + "." + SliderVersionInfo.HADOOP_DEPLOYED_INFO,
+ VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
+ }
+
+ /**
+ * Set the time for an information (human, machine) timestamp pair of fields.
+ * The human time is the time in millis converted via the {@link Date} class.
+ * @param info info fields
+ * @param keyHumanTime name of human time key
+ * @param keyMachineTime name of machine time
+ * @param time timestamp
+ */
+ public static void setInfoTime(Map info,
+ String keyHumanTime,
+ String keyMachineTime,
+ long time) {
+ info.put(keyHumanTime, SliderUtils.toGMTString(time));
+ info.put(keyMachineTime, Long.toString(time));
+ }
+
+ public static Path extractImagePath(CoreFileSystem fs, MapOperations internalOptions) throws
+ SliderException,
+ IOException {
+ Path imagePath;
+ String imagePathOption =
+ internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ String appHomeOption = internalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME);
+ if (!isUnset(imagePathOption)) {
+ imagePath = fs.createPathThatMustExist(imagePathOption);
+ } else {
+ imagePath = null;
+ if (isUnset(appHomeOption)) {
+ throw new BadClusterStateException(ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED);
+ }
+ }
+ return imagePath;
+ }
+
+ /**
+ * trigger a JVM halt with no clean shutdown at all
+ * @param status status code for exit
+ * @param text text message
+ * @param delay delay in millis
+ * @return the timer (assuming the JVM hasn't halted yet)
+ *
+ */
+ public static Timer haltAM(int status, String text, int delay) {
+
+ Timer timer = new Timer("halt timer", false);
+ timer.schedule(new DelayedHalt(status, text), delay);
+ return timer;
+ }
+
+ public static String propertiesToString(Properties props) {
+ TreeSet<String> keys = new TreeSet<String>(props.stringPropertyNames());
+ StringBuilder builder = new StringBuilder();
+ for (String key : keys) {
+ builder.append(key)
+ .append("=")
+ .append(props.getProperty(key))
+ .append("\n");
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Callable for async/scheduled halt
+ */
+ public static class DelayedHalt extends TimerTask {
+ private final int status;
+ private final String text;
+
+ public DelayedHalt(int status, String text) {
+ this.status = status;
+ this.text = text;
+ }
+
+ @Override
+ public void run() {
+ try {
+ ExitUtil.halt(status, text);
+ //this should never be reached
+ } catch (ExitUtil.HaltException e) {
+ log.info("Halt failed");
+ }
+ }
+ }
+
+ /**
+ * This wrapps ApplicationReports and generates a string version
+ * iff the toString() operator is invoked
+ */
+ public static class OnDemandReportStringifier {
+ private final ApplicationReport report;
+
+ public OnDemandReportStringifier(ApplicationReport report) {
+ this.report = report;
+ }
+
+ @Override
+ public String toString() {
+ return appReportToString(report, "\n");
+ }
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
new file mode 100644
index 0000000..86025ee
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.util.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Locale;
+import java.util.Properties;
+
+/**
+ * Extract the version properties, which will look something like
+ * <pre>
+ * application.name=${pom.name}
+ * application.version=${pom.version}
+ * application.build=${buildNumber}
+ * application.build.java.version=${java.version}
+ * application.build.info=${pom.name}-${pom.version} Built against ${buildNumber} on ${java.version} by ${user.name}
+ * </pre>
+ *
+ * the <code>mvn process-resources</code> target will expand the properties
+ * and add the resources to target/classes, which will then look something like
+ * <pre>
+ * application.name=Slider Core
+ * application.version=0.7.1-SNAPSHOT
+ * application.build=1dd69
+ * application.build.java.version=1.7.0_45
+ * application.build.user=stevel
+ * application.build.info=Slider Core-0.7.1-SNAPSHOT Built against 1dd69 on 1.7.0_45 by stevel
+ * </pre>
+ *
+ * Note: the values will change and more properties added.
+ */
+public class SliderVersionInfo {
+ private static final Logger log = LoggerFactory.getLogger(SliderVersionInfo.class);
+
+ /**
+ * Name of the resource containing the filled-in-at-runtime props
+ */
+ public static final String VERSION_RESOURCE =
+ "org/apache/slider/providers/dynamic/application.properties";
+
+ public static final String APP_NAME = "application.name";
+ public static final String APP_VERSION = "application.version";
+ public static final String APP_BUILD = "application.build";
+ public static final String APP_BUILD_JAVA_VERSION = "application.build.java.version";
+ public static final String APP_BUILD_USER = "application.build.user";
+ public static final String APP_BUILD_INFO = "application.build.info";
+ public static final String HADOOP_BUILD_INFO = "hadoop.build.info";
+ public static final String HADOOP_DEPLOYED_INFO = "hadoop.deployed.info";
+
+
+ public static Properties loadVersionProperties() {
+ Properties props = new Properties();
+ URL resURL = SliderVersionInfo.class.getClassLoader()
+ .getResource(VERSION_RESOURCE);
+ assert resURL != null : "Null resource " + VERSION_RESOURCE;
+
+ try {
+ InputStream inStream = resURL.openStream();
+ assert inStream != null : "Null input stream from " + VERSION_RESOURCE;
+ props.load(inStream);
+ } catch (IOException e) {
+ log.warn("IOE loading " + VERSION_RESOURCE, e);
+ }
+ return props;
+ }
+
+ /**
+ * Load the version info and print it
+ * @param logger logger
+ */
+ public static void loadAndPrintVersionInfo(Logger logger) {
+ Properties props = loadVersionProperties();
+ logger.info(props.getProperty(APP_BUILD_INFO));
+ logger.info("Compiled against Hadoop {}",
+ props.getProperty(HADOOP_BUILD_INFO));
+ logger.info(getHadoopVersionString());
+ }
+
+ public static String getHadoopVersionString() {
+ return String.format(Locale.ENGLISH,
+ "Hadoop runtime version %s with source checksum %s and build date %s",
+ VersionInfo.getBranch(),
+ VersionInfo.getSrcChecksum(),
+ VersionInfo.getDate());
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/BuildHelper.java b/slider-core/src/main/java/org/apache/slider/core/build/BuildHelper.java
new file mode 100644
index 0000000..1adc15f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/build/BuildHelper.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.build;
+
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.slider.common.tools.SliderVersionInfo;
+
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * classes to help with the build
+ */
+public class BuildHelper {
+ /**
+ * Add the cluster build information; this will include Hadoop details too
+ * @param dest map to insert this too
+ * @param prefix prefix for the build info
+ */
+ public static void addBuildMetadata(Map dest, String prefix) {
+
+ Properties props = SliderVersionInfo.loadVersionProperties();
+ dest.put(prefix + "." + SliderVersionInfo.APP_BUILD_INFO,
+ props.getProperty(
+ SliderVersionInfo.APP_BUILD_INFO));
+ dest.put(prefix + "." + SliderVersionInfo.HADOOP_BUILD_INFO,
+ props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO));
+
+ dest.put(prefix + "." + SliderVersionInfo.HADOOP_DEPLOYED_INFO,
+ VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
new file mode 100644
index 0000000..8918be8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.build;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.persist.ConfPersister;
+import org.apache.slider.core.persist.InstancePaths;
+import org.apache.slider.core.persist.LockAcquireFailedException;
+import org.apache.slider.core.persist.LockHeldAction;
+import org.apache.slider.core.registry.zk.ZKPathBuilder;
+import org.apache.slider.core.registry.zk.ZookeeperUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.slider.api.OptionKeys.INTERNAL_AM_TMP_DIR;
+import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_HOME;
+import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH;
+import static org.apache.slider.api.OptionKeys.INTERNAL_DATA_DIR_PATH;
+import static org.apache.slider.api.OptionKeys.INTERNAL_GENERATED_CONF_PATH;
+import static org.apache.slider.api.OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH;
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_HOSTS;
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
+
+/**
+ * Build up the instance of a cluster.
+ */
+public class InstanceBuilder {
+
+ private final String clustername;
+ private final Configuration conf;
+ private final CoreFileSystem coreFS;
+ private final InstancePaths instancePaths;
+ private AggregateConf instanceDescription;
+
+ private static final Logger log =
+ LoggerFactory.getLogger(InstanceBuilder.class);
+
+ public InstanceBuilder(CoreFileSystem coreFileSystem,
+ Configuration conf,
+ String clustername) {
+ this.clustername = clustername;
+ this.conf = conf;
+ this.coreFS = coreFileSystem;
+ Path instanceDir = coreFileSystem.buildClusterDirPath(clustername);
+ instancePaths = new InstancePaths(instanceDir);
+
+ }
+
+ public AggregateConf getInstanceDescription() {
+ return instanceDescription;
+ }
+
+ public InstancePaths getInstancePaths() {
+ return instancePaths;
+ }
+
+
+ @Override
+ public String toString() {
+ return "Builder working with " + clustername + " at " +
+ getInstanceDir();
+ }
+
+ private Path getInstanceDir() {
+ return instancePaths.instanceDir;
+ }
+
+ /**
+ * Initial part of the build process
+ * @param instanceConf
+ * @param provider
+ */
+ public void init(
+ String provider,
+ AggregateConf instanceConf) {
+
+
+ this.instanceDescription = instanceConf;
+
+ //internal is extended
+ ConfTreeOperations internalOps = instanceConf.getInternalOperations();
+
+ Map<String, Object> md = internalOps.getConfTree().metadata;
+ long time = System.currentTimeMillis();
+ md.put(StatusKeys.INFO_CREATE_TIME_HUMAN, SliderUtils.toGMTString(time));
+ md.put(StatusKeys.INFO_CREATE_TIME_MILLIS, Long.toString(time));
+
+ MapOperations globalOptions = internalOps.getGlobalOptions();
+ BuildHelper.addBuildMetadata(md, "create");
+ SliderUtils.setInfoTime(md,
+ StatusKeys.INFO_CREATE_TIME_HUMAN,
+ StatusKeys.INFO_CREATE_TIME_MILLIS,
+ System.currentTimeMillis());
+
+ internalOps.set(INTERNAL_AM_TMP_DIR,
+ instancePaths.tmpPathAM.toUri());
+ internalOps.set(INTERNAL_SNAPSHOT_CONF_PATH,
+ instancePaths.snapshotConfPath.toUri());
+ internalOps.set(INTERNAL_GENERATED_CONF_PATH,
+ instancePaths.generatedConfPath.toUri());
+ internalOps.set(INTERNAL_DATA_DIR_PATH,
+ instancePaths.dataPath.toUri());
+
+
+ internalOps.set(OptionKeys.INTERNAL_PROVIDER_NAME, provider);
+ internalOps.set(OptionKeys.APPLICATION_NAME, clustername);
+
+ }
+
+ /**
+ * Set up the image/app home path
+ * @param appImage path in the DFS to the tar file
+ * @param appHomeDir other strategy: home dir
+ * @throws BadConfigException if both or neither are found (its an xor)
+ */
+ public void setImageDetails(
+ Path appImage,
+ String appHomeDir) throws BadConfigException {
+ boolean appHomeUnset = SliderUtils.isUnset(appHomeDir);
+ // App home or image
+ if (appImage != null) {
+ if (!appHomeUnset) {
+ // both args have been set
+ throw new BadConfigException(
+ ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED);
+ }
+ instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_IMAGE_PATH,
+ appImage.toUri());
+ } else {
+ // the alternative is app home, which now MUST be set
+ if (appHomeUnset) {
+ // both args have been set
+ throw new BadConfigException(ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED);
+
+ }
+ instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_HOME,
+ appHomeDir);
+
+ }
+ }
+
+ /**
+ * Propagate any critical principals from the current site config down to the HBase one.
+ */
+ public void propagatePrincipals() {
+ String dfsPrincipal = conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ if (dfsPrincipal != null) {
+ String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
+ DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY;
+ instanceDescription.getAppConfOperations().set(siteDfsPrincipal, dfsPrincipal);
+ }
+ }
+
+ public void propagateFilename() {
+ String fsDefaultName = conf.get(
+ CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
+ instanceDescription.getAppConfOperations().set(OptionKeys.SITE_XML_PREFIX +
+ CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ fsDefaultName
+ );
+
+ instanceDescription.getAppConfOperations().set(OptionKeys.SITE_XML_PREFIX +
+ SliderXmlConfKeys.FS_DEFAULT_NAME_CLASSIC,
+ fsDefaultName
+ );
+
+ }
+
+
+ public void takeSnapshotOfConfDir(Path appconfdir) throws
+ IOException,
+ BadConfigException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(appconfdir.toUri(), conf);
+ if (!srcFS.isDirectory(appconfdir)) {
+ throw new BadConfigException(
+ "Source Configuration directory is not valid: %s",
+ appconfdir.toString());
+ }
+ // bulk copy
+ FsPermission clusterPerms = coreFS.getInstanceDirectoryPermissions();
+ // first the original from wherever to the DFS
+ SliderUtils.copyDirectory(conf, appconfdir, instancePaths.snapshotConfPath,
+ clusterPerms);
+ }
+
+
+ /**
+ * Persist this
+ * @throws IOException
+ * @throws SliderException
+ * @throws LockAcquireFailedException
+ * @param appconfdir
+ */
+ public void persist(Path appconfdir) throws
+ IOException,
+ SliderException,
+ LockAcquireFailedException {
+ coreFS.createClusterDirectories(instancePaths);
+ ConfPersister persister =
+ new ConfPersister(coreFS, getInstanceDir());
+ ConfDirSnapshotAction action = null;
+ if (appconfdir != null) {
+ action = new ConfDirSnapshotAction(appconfdir);
+ }
+ persister.save(instanceDescription, action);
+ }
+
+ /**
+ * Add the ZK paths to the application options.
+ *
+ * @param zkBinding ZK binding
+ */
+ public void addZKBinding(ZKPathBuilder zkBinding) throws BadConfigException {
+
+ String quorum = zkBinding.getAppQuorum();
+ if (SliderUtils.isSet(quorum)) {
+ MapOperations globalAppOptions =
+ instanceDescription.getAppConfOperations().getGlobalOptions();
+ globalAppOptions.put(ZOOKEEPER_PATH, zkBinding.getAppPath());
+ globalAppOptions.put(ZOOKEEPER_QUORUM, quorum);
+ globalAppOptions.put(ZOOKEEPER_HOSTS,
+ ZookeeperUtils.convertToHostsOnlyList(quorum));
+ }
+ }
+
+ /**
+ * Class to execute the snapshotting of the configuration directory
+ * while the persistence lock is held.
+ *
+ * This guarantees that there won't be an attempt to launch a cluster
+ * until the snapshot is complete -as the write lock won't be released
+ * until afterwards.
+ */
+ private class ConfDirSnapshotAction implements LockHeldAction {
+
+ private final Path appconfdir;
+
+ private ConfDirSnapshotAction(Path appconfdir) {
+ this.appconfdir = appconfdir;
+ }
+
+ @Override
+ public void execute() throws IOException, SliderException {
+
+ takeSnapshotOfConfDir(appconfdir);
+ }
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
new file mode 100644
index 0000000..f923ef3
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.build;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.persist.ConfPersister;
+import org.apache.slider.core.persist.LockAcquireFailedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class InstanceIO {
+ protected static final Logger log =
+ LoggerFactory.getLogger(InstanceIO.class);
+
+ /**
+ * Load in an instance definition -but do not resolve it
+ * @param sliderFileSystem filesystem
+ * @param clusterDirectory CD
+ * @return
+ * @throws IOException
+ * @throws SliderException
+ */
+ public static AggregateConf loadInstanceDefinitionUnresolved(
+ CoreFileSystem sliderFileSystem,
+ Path clusterDirectory) throws
+ IOException,
+ SliderException {
+ AggregateConf instanceDefinition = new AggregateConf();
+ ConfPersister persister =
+ new ConfPersister(sliderFileSystem, clusterDirectory);
+ try {
+ persister.load(instanceDefinition);
+ } catch (LockAcquireFailedException e) {
+ log.debug("Lock acquisition failure of {}", clusterDirectory, e);
+
+ throw new BadClusterStateException(
+ "Application at %s is locked for reading",
+ clusterDirectory.toString());
+ }
+ return instanceDefinition;
+ }
+
+
+ /**
+ * Update an instance definition
+ * @param coreFS
+ * @param dir
+ * @param instanceDefinition
+ * @throws SliderException
+ * @throws IOException
+ * @throws LockAcquireFailedException
+ */
+ public static void updateInstanceDefinition(CoreFileSystem coreFS,
+ Path dir,
+ AggregateConf instanceDefinition)
+ throws SliderException, IOException, LockAcquireFailedException {
+ ConfPersister persister =
+ new ConfPersister(coreFS, dir);
+ persister.save(instanceDefinition, null);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
new file mode 100644
index 0000000..02669cd
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf;
+
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * Aggregate Configuration.
+ *
+ * It is serializable to JSON
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public final class AggregateConf {
+
+ private String name;
+ private ConfTree resources;
+ private ConfTree internal;
+ private ConfTree appConf;
+
+ private ConfTreeOperations resourceOperations;
+ private ConfTreeOperations appConfOperations;
+ private ConfTreeOperations internalOperations;
+
+ public AggregateConf() {
+ this(new ConfTree(), new ConfTree(), new ConfTree());
+ }
+
+ public AggregateConf(String name) {
+ this(new ConfTree(), new ConfTree(), new ConfTree());
+ this.name = name;
+ }
+
+ public AggregateConf(ConfTree resources,
+ ConfTree appConf,
+ ConfTree internal) {
+ setResources(resources);
+ setAppConf(appConf);
+ setInternal(internal);
+ }
+
+ public void setResources(ConfTree resources) {
+ this.resources = resources;
+ resourceOperations = new ConfTreeOperations(resources);
+ }
+
+ public void setAppConf(ConfTree appConf) {
+ this.appConf = appConf;
+ appConfOperations = new ConfTreeOperations(appConf);
+ }
+
+ public ConfTree getInternal() {
+ return internal;
+ }
+
+ public void setInternal(ConfTree internal) {
+ this.internal = internal;
+ internalOperations = new ConfTreeOperations(internal);
+ }
+
+ public ConfTree getResources() {
+ return resources;
+ }
+
+ public ConfTree getAppConf() {
+ return appConf;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @JsonIgnore
+ public ConfTreeOperations getResourceOperations() {
+ return resourceOperations;
+ }
+
+
+ @JsonIgnore
+ public ConfTreeOperations getAppConfOperations() {
+ return appConfOperations;
+ }
+
+ @JsonIgnore
+ public ConfTreeOperations getInternalOperations() {
+ return internalOperations;
+ }
+
+ /**
+ * predicate to query if all sections have data structures
+ * @return true if every section is non-null
+ */
+ @JsonIgnore
+ public boolean isComplete() {
+ return resources != null && appConf != null && internal != null;
+ }
+
+ public void validate() throws BadConfigException {
+ if (!isComplete()) {
+ throw new BadConfigException("Incomplete instance %s", this);
+ }
+ resourceOperations.validate();
+ internalOperations.validate();
+ appConfOperations.validate();
+ }
+
+ public void resolve() throws BadConfigException {
+ validate();
+ resourceOperations.resolve();
+ internalOperations.resolve();
+ appConfOperations.resolve();
+ }
+
+ /**
+ * string operation includes all the inner conftrees
+ * @return a string description
+ */
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("{");
+ sb.append(",\n\"internal\": ").append(internal);
+ sb.append(",\n\"resources\": ").append(resources);
+ sb.append(",\n\"appConf\" :").append(appConf);
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
new file mode 100644
index 0000000..f989726
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf;
+
+import org.apache.slider.core.persist.ConfTreeSerDeser;
+import org.apache.slider.core.persist.PersistKeys;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A conf tree represents one of the configuration trees
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public final class ConfTree {
+
+ /**
+ * Size of an initial map. This is kept low so the cost of having
+ * many conf trees in a process is low.
+ */
+ public static final int INITAL_MAP_CAPACITY = 3;
+
+ protected static final Logger
+ log = LoggerFactory.getLogger(ConfTree.class);
+
+ /**
+ * version counter
+ */
+ public String schema = PersistKeys.SCHEMA;
+
+ /**
+ * Metadata
+ */
+ public Map<String, Object> metadata = new HashMap<String, Object>(
+ INITAL_MAP_CAPACITY);
+
+
+ /**
+ * Global options
+ */
+ public Map<String, String> global =
+ new HashMap<String, String>(INITAL_MAP_CAPACITY);
+
+
+ /**
+ * Role options,
+ * role -> option -> value
+ */
+ public Map<String, Map<String, String>> components =
+ new HashMap<String, Map<String, String>>(INITAL_MAP_CAPACITY);
+
+
+ /**
+ * Shallow clone
+ * @return a shallow clone
+ * @throws CloneNotSupportedException
+ */
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return super.clone();
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return toJson();
+ } catch (Exception e) {
+ log.warn("Failed to convert to JSON ", e);
+ return super.toString();
+ }
+ }
+
+ /**
+ * Convert to a JSON string
+ * @return a JSON string description
+ * @throws IOException Problems mapping/writing the object
+ */
+ public String toJson() throws IOException,
+ JsonGenerationException,
+ JsonMappingException {
+ return ConfTreeSerDeser.toString(this);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
new file mode 100644
index 0000000..1beb8fd
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.persist.ConfTreeSerDeser;
+import org.apache.slider.core.persist.PersistKeys;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class ConfTreeOperations {
+
+ public final ConfTree confTree;
+ private final MapOperations globalOptions;
+
+ protected static final Logger
+ log = LoggerFactory.getLogger(ConfTreeOperations.class);
+
+
+ public ConfTreeOperations(ConfTree confTree) {
+ assert confTree != null : "null tree";
+ assert confTree.components != null : "null tree components";
+ this.confTree = confTree;
+ globalOptions = new MapOperations("global", confTree.global);
+ }
+
+ /**
+ * Get the underlying conf tree
+ * @return the tree
+ */
+ public ConfTree getConfTree() {
+ return confTree;
+ }
+
+ /**
+ * Validate the configuration
+ * @throws BadConfigException
+ */
+ public void validate() throws BadConfigException {
+ String version = confTree.schema;
+ if (version == null) {
+ throw new BadConfigException("'version' undefined");
+ }
+ if (!PersistKeys.SCHEMA.equals(version)) {
+ throw new BadConfigException(
+ "version %s incompatible with supported version %s",
+ version,
+ PersistKeys.SCHEMA);
+ }
+ }
+
+ /**
+ * Resolve a ConfTree by mapping all global options into each component
+ * -if there is none there already
+ */
+ public void resolve() {
+ for (Map.Entry<String, Map<String, String>> comp : confTree.components.entrySet()) {
+ mergeInGlobal(comp.getValue());
+ }
+ }
+
+ /**
+ * Merge any options
+ * @param component dest values
+ */
+ public void mergeInGlobal(Map<String, String> component) {
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(component, confTree.global);
+ }
+
+ /**
+ * Get operations on the global set
+ * @return a wrapped map
+ */
+ public MapOperations getGlobalOptions() {
+ return globalOptions;
+ }
+
+
+ /**
+ * look up a component and return its options
+ * @param component component name
+ * @return component mapping or null
+ */
+ public MapOperations getComponent(String component) {
+ Map<String, String> instance = confTree.components.get(component);
+ if (instance != null) {
+ return new MapOperations(component, instance);
+ }
+ return null;
+ }
+
+ /**
+ * Get at the underlying component map
+ * @return a map of components. This is the raw ConfTree data structure
+ */
+ public Map<String, Map<String, String>> getComponents() {
+ return confTree.components;
+ }
+
+ /**
+ * Get a component -adding it to the components map if
+ * none with that name exists
+ * @param name role
+ * @return role mapping
+ */
+ public MapOperations getOrAddComponent(String name) {
+ MapOperations operations = getComponent(name);
+ if (operations != null) {
+ return operations;
+ }
+ //create a new instances
+ Map<String, String> map = new HashMap<String, String>();
+ confTree.components.put(name, map);
+ return new MapOperations(name, map);
+ }
+
+
+ /*
+ * return the Set of names names
+ */
+ @JsonIgnore
+ public Set<String> getComponentNames() {
+ return new HashSet<String>(confTree.components.keySet());
+ }
+
+
+
+ /**
+ * Get a component whose presence is mandatory
+ * @param name component name
+ * @return the mapping
+ * @throws BadConfigException if the name is not there
+ */
+ public MapOperations getMandatoryComponent(String name) throws
+ BadConfigException {
+ MapOperations ops = getComponent(name);
+ if (ops == null) {
+ throw new BadConfigException("Missing component " + name);
+ }
+ return ops;
+ }
+
+ /**
+ * Set a global option, converting it to a string as needed
+ * @param key key
+ * @param value non null value
+ */
+ public void set(String key, Object value) {
+ globalOptions.put(key, value.toString());
+ }
+ /**
+ * get a global option
+ * @param key key
+ * @return value or null
+ *
+ */
+ public String get(String key) {
+ return globalOptions.get(key);
+ }
+
+ /**
+ * Propagate all global keys matching a prefix
+ * @param src source
+ * @param prefix prefix
+ */
+ public void propagateGlobalKeys(ConfTree src, String prefix) {
+ Map<String, String> global = src.global;
+ for (Map.Entry<String, String> entry : global.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(prefix)) {
+ set(key, entry.getValue());
+ }
+ }
+ }
+
+ /**
+ * Propagate all global keys matching a prefix
+ * @param src source
+ * @param prefix prefix
+ */
+ public void propagateGlobalKeys(ConfTreeOperations src, String prefix) {
+ propagateGlobalKeys(src.confTree, prefix);
+ }
+
+ /**
+ * Merge the map of a single component
+ * @param component component name
+ * @param map map to merge
+ */
+ public void mergeSingleComponentMap(String component, Map<String, String> map) {
+ MapOperations comp = getOrAddComponent(component);
+ comp.putAll(map);
+ }
+ /**
+ * Merge the map of a single component
+ * @param component component name
+ * @param map map to merge
+ */
+ public void mergeSingleComponentMapPrefix(String component,
+ Map<String, String> map,
+ String prefix,
+ boolean overwrite) {
+ MapOperations comp = getOrAddComponent(component);
+ comp.mergeMapPrefixedKeys(map,prefix, overwrite);
+ }
+
+ /**
+ * Merge in components
+ * @param commandOptions component options on the CLI
+ */
+ public void mergeComponents(Map<String, Map<String, String>> commandOptions) {
+ for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
+ mergeSingleComponentMap(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Merge in components
+ * @param commandOptions component options on the CLI
+ */
+ public void mergeComponentsPrefix(Map<String,
+ Map<String, String>> commandOptions,
+ String prefix,
+ boolean overwrite) {
+ for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
+ mergeSingleComponentMapPrefix(entry.getKey(), entry.getValue(), prefix, overwrite);
+ }
+ }
+
+ /**
+ * Merge in another tree -no overwrites of global or conf data
+ * (note that metadata does a naive putAll merge/overwrite)
+ * @param that the other tree
+ */
+ public void mergeWithoutOverwrite(ConfTree that) {
+
+ getGlobalOptions().mergeWithoutOverwrite(that.global);
+ confTree.metadata.putAll(that.metadata);
+
+ for (Map.Entry<String, Map<String, String>> entry : that.components.entrySet()) {
+ MapOperations comp = getOrAddComponent(entry.getKey());
+ comp.mergeWithoutOverwrite(entry.getValue());
+ }
+ }
+
+ /**
+ * Merge in another tree with overwrites
+ * @param that the other tree
+ */
+ public void merge(ConfTree that) {
+
+ getGlobalOptions().putAll(that.global);
+ confTree.metadata.putAll(that.metadata);
+
+ for (Map.Entry<String, Map<String, String>> entry : that.components.entrySet()) {
+ MapOperations comp = getOrAddComponent(entry.getKey());
+ comp.putAll(entry.getValue());
+ }
+ }
+
+
+ /**
+ * Load from a resource. The inner conf tree is the loaded data -unresolved
+ * @param resource resource
+ * @return loaded value
+ * @throws IOException load failure
+ */
+ public static ConfTreeOperations fromResource(String resource) throws
+ IOException {
+ ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
+ ConfTreeOperations ops = new ConfTreeOperations(
+ confTreeSerDeser.fromResource(resource) );
+ return ops;
+ }
+
+ /**
+ * Load from a resource. The inner conf tree is the loaded data -unresolved
+ * @param resource resource
+ * @return loaded value
+ * @throws IOException load failure
+ */
+ public static ConfTreeOperations fromFile(File resource) throws
+ IOException {
+ ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
+ ConfTreeOperations ops = new ConfTreeOperations(
+ confTreeSerDeser.fromFile(resource) );
+ return ops;
+ }
+
+ /**
+ * Build from an existing instance -which is cloned via JSON ser/deser
+ * @param instance the source instance
+ * @return loaded value
+ * @throws IOException load failure
+ */
+ public static ConfTreeOperations fromInstance(ConfTree instance) throws
+ IOException {
+ ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
+ ConfTreeOperations ops = new ConfTreeOperations(
+ confTreeSerDeser.fromJson(confTreeSerDeser.toJson(instance)) );
+ return ops;
+ }
+
+ /**
+ * Load from a file and merge it in
+ * @param file file
+ * @throws IOException any IO problem
+ * @throws BadConfigException if the file is invalid
+ */
+ public void mergeFile(File file) throws IOException, BadConfigException {
+ ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
+ ConfTree tree = confTreeSerDeser.fromFile(file);
+ ConfTreeOperations ops = new ConfTreeOperations(tree);
+ ops.validate();
+ merge(ops.confTree);
+ }
+
+ @Override
+ public String toString() {
+ return confTree.toString();
+ }
+
+ /**
+ * Convert to a JSON string
+ * @return a JSON string description
+ */
+ public String toJson() throws IOException,
+ JsonGenerationException,
+ JsonMappingException {
+ return confTree.toJson();
+ }
+
+ /**
+ * Get a component option
+ * @param name component name
+ * @param option option name
+ * @param defVal default value
+ * @return resolved value
+ */
+ public String getComponentOpt(String name, String option, String defVal) {
+ MapOperations roleopts = getComponent(name);
+ if (roleopts == null) {
+ return defVal;
+ }
+ return roleopts.getOption(option, defVal);
+ }
+
+ /**
+ * Get a component opt; use {@link Integer#decode(String)} so as to take hex
+ * oct and bin values too.
+ *
+ * @param name component name
+ * @param option option name
+ * @param defVal default value
+ * @return parsed value
+ * @throws NumberFormatException if the role could not be parsed.
+ */
+ public int getComponentOptInt(String name, String option, int defVal) {
+ String val = getComponentOpt(name, option, Integer.toString(defVal));
+ return Integer.decode(val);
+ }
+
+ /**
+ * Set a component option, creating the component if necessary
+ * @param component component name
+ * @param option option name
+ * @param val value
+ */
+ public void setComponentOpt(String component, String option, String val) {
+ Map<String, String> roleopts = getOrAddComponent(component);
+ roleopts.put(option, val);
+ }
+
+ /**
+ * Set an integer role option, creating the role if necessary
+ * @param role role name
+ * @param option option name
+ * @param val integer value
+ */
+ public void setRoleOpt(String role, String option, int val) {
+ setComponentOpt(role, option, Integer.toString(val));
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
new file mode 100644
index 0000000..0050d04
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Standard map operations.
+ *
+ * This delegates the standard map interface to the map passed in,
+ * so it can be used to add more actions to the map.
+ */
+public class MapOperations implements Map<String, String> {
+ private static final Logger log =
+ LoggerFactory.getLogger(MapOperations.class);
+
+ /**
+ * Global options
+ */
+ public final Map<String, String> options;
+
+ public final String name;
+
+ public MapOperations() {
+ options = new HashMap<String, String>();
+ name = "";
+ }
+
+ /**
+ * Create an instance
+ * @param name
+ * @param options
+ */
+ public MapOperations(String name, Map<String, String> options) {
+ assert options != null : "null map";
+ this.options = options;
+ this.name = name;
+ }
+
+
+ /**
+ * Get a cluster option or value
+ *
+ * @param key
+ * @param defVal
+ * @return option in map or the default
+ */
+ public String getOption(String key, String defVal) {
+ String val = options.get(key);
+ return val != null ? val : defVal;
+ }
+
+
+ /**
+ * Get a cluster option or value
+ *
+ * @param key
+ * @return the value
+ * @throws BadConfigException if the option is missing
+ */
+ public String getMandatoryOption(String key) throws BadConfigException {
+ String val = options.get(key);
+ if (val == null) {
+ if (log.isDebugEnabled()) {
+ log.debug("Missing key {} from config containing {}",
+ key, this);
+ }
+ throw new BadConfigException("Missing option " + key);
+ }
+ return val;
+ }
+
+ /**
+ * Get an integer option; use {@link Integer#decode(String)} so as to take hex
+ * oct and bin values too.
+ *
+ * @param option option name
+ * @param defVal default value
+ * @return parsed value
+ * @throws NumberFormatException if the role could not be parsed.
+ */
+ public int getOptionInt(String option, int defVal) {
+ String val = getOption(option, Integer.toString(defVal));
+ return Integer.decode(val);
+ }
+ /**
+ * Get a mandatory integer option; use {@link Integer#decode(String)} so as to take hex
+ * oct and bin values too.
+ *
+ * @param option option name
+ * @return parsed value
+ * @throws NumberFormatException if the option could not be parsed.
+ * @throws BadConfigException if the option could not be found
+ */
+ public int getMandatoryOptionInt(String option) throws BadConfigException {
+ getMandatoryOption(option);
+ return getOptionInt(option, 0);
+ }
+
+ /**
+ * Verify that an option is set: that is defined AND non-empty
+ * @param key
+ * @throws BadConfigException
+ */
+ public void verifyOptionSet(String key) throws BadConfigException {
+ if (SliderUtils.isUnset(getOption(key, null))) {
+ throw new BadConfigException("Unset option %s", key);
+ }
+ }
+
+ public void mergeWithoutOverwrite(Map<String, String> that) {
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(options, that);
+ }
+
+ /**
+ * Merge a map by prefixed keys
+ * @param that the map to merge in
+ * @param prefix prefix to match on
+ * @param overwrite flag to enable overwrite
+ */
+ public void mergeMapPrefixedKeys(Map<String, String> that,
+ String prefix,
+ boolean overwrite) {
+ for (Map.Entry<String, String> entry : that.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(prefix)) {
+ if (overwrite || get(key) == null) {
+ put(key, entry.getValue());
+ }
+ }
+ }
+ }
+
+ /**
+ * Set a property if it is not already set
+ * @param key key
+ * @param value value
+ */
+ public void putIfUnset(String key, String value) {
+ if (get(key) == null) {
+ put(key, value);
+ }
+ }
+
+ public void set(String key, Object value) {
+ assert value != null;
+ put(key, value.toString());
+ }
+
+ public int size() {
+ return options.size();
+ }
+
+ public boolean isEmpty() {
+ return options.isEmpty();
+ }
+
+ public boolean containsValue(Object value) {
+ return options.containsValue(value);
+ }
+
+ public boolean containsKey(Object key) {
+ return options.containsKey(key);
+ }
+
+ public String get(Object key) {
+ return options.get(key);
+ }
+
+ public String put(String key, String value) {
+ return options.put(key, value);
+ }
+
+ public String remove(Object key) {
+ return options.remove(key);
+ }
+
+ public void putAll(Map<? extends String, ? extends String> m) {
+ options.putAll(m);
+ }
+
+ public void clear() {
+ options.clear();
+ }
+
+ public Set<String> keySet() {
+ return options.keySet();
+ }
+
+ public Collection<String> values() {
+ return options.values();
+ }
+
+ public Set<Map.Entry<String, String>> entrySet() {
+ return options.entrySet();
+ }
+
+ @SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
+ public boolean equals(Object o) {
+ return options.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ return options.hashCode();
+ }
+
+ public boolean isSet(String key) {
+ return SliderUtils.isSet(get(key));
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append(name).append("=\n");
+
+ for (Entry<String, String> entry : options.entrySet()) {
+ builder.append(" ")
+ .append(entry.getKey())
+ .append('=')
+ .append(entry.getValue())
+ .append('\n');
+ }
+ return builder.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java
new file mode 100644
index 0000000..fdedd20
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/BadClusterStateException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+
+/**
+ * YARN cluster itself is in a bad state
+ */
+public class BadClusterStateException extends SliderException {
+ public BadClusterStateException(String message,
+ Object... args) {
+ super(EXIT_BAD_STATE, message, args);
+ }
+
+ public BadClusterStateException(Throwable throwable,
+ String message, Object... args) {
+ super(EXIT_BAD_STATE, throwable, message, args);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/BadCommandArgumentsException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/BadCommandArgumentsException.java
new file mode 100644
index 0000000..0d5d686
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/BadCommandArgumentsException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+public class BadCommandArgumentsException extends SliderException {
+ public BadCommandArgumentsException(String s, Object... args) {
+ super(EXIT_COMMAND_ARGUMENT_ERROR, s, args);
+ }
+
+ public BadCommandArgumentsException(Throwable throwable, String message,
+ Object... args) {
+ super(EXIT_COMMAND_ARGUMENT_ERROR, throwable, message, args);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/BadConfigException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/BadConfigException.java
new file mode 100644
index 0000000..65a8ea8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/BadConfigException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+/**
+ * An exception to raise on a bad configuration
+ */
+public class BadConfigException extends SliderException {
+
+ public BadConfigException(String s) {
+ super(EXIT_BAD_CONFIGURATION, s);
+ }
+
+ public BadConfigException(String message, Object... args) {
+ super(EXIT_BAD_CONFIGURATION, message, args);
+ }
+
+ public BadConfigException(
+ Throwable throwable,
+ String message, Object... args) {
+ super(EXIT_BAD_CONFIGURATION, throwable, message, args);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
new file mode 100644
index 0000000..c949c1c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+public interface ErrorStrings {
+ String E_UNSTABLE_CLUSTER = "Unstable Application Instance :";
+ String E_CLUSTER_RUNNING = "Application Instance lready running";
+ String E_ALREADY_EXISTS = "already exists";
+ String PRINTF_E_INSTANCE_ALREADY_EXISTS = "Application Instance \"%s\" already exists and is defined in %s";
+ String PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS = "Application Instance dir already exists: %s";
+ String E_MISSING_PATH = "Missing path ";
+ String E_INCOMPLETE_CLUSTER_SPEC =
+ "Cluster specification is marked as incomplete: ";
+ String E_UNKNOWN_INSTANCE = "Unknown application instance ";
+ String E_DESTROY_CREATE_RACE_CONDITION =
+ "created while it was being destroyed";
+ String E_UNKNOWN_ROLE = "Unknown role ";
+ /**
+ * ERROR Strings
+ */
+ String ERROR_NO_ACTION = "No action specified";
+ String ERROR_UNKNOWN_ACTION = "Unknown command: ";
+ String ERROR_NOT_ENOUGH_ARGUMENTS =
+ "Not enough arguments for action: ";
+ String ERROR_PARSE_FAILURE =
+ "Failed to parse ";
+ /**
+ * All the remaining values after argument processing
+ */
+ String ERROR_TOO_MANY_ARGUMENTS =
+ "Too many arguments";
+ String ERROR_DUPLICATE_ENTRY = "Duplicate entry for ";
+ String E_APPLICATION_NOT_RUNNING = "Application not running";
+ String E_FINISHED_APPLICATION = E_APPLICATION_NOT_RUNNING + ": %s state=%s ";
+ String E_NO_IMAGE_OR_HOME_DIR_SPECIFIED =
+ "Neither an image path nor binary home directory were specified";
+ String E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED =
+ "Both application image path and home dir have been provided";
+ String E_CONFIGURATION_DIRECTORY_NOT_FOUND =
+ "Configuration directory \"%s\" not found";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/MissingArgException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/MissingArgException.java
new file mode 100644
index 0000000..0faffa9
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/MissingArgException.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+
+public class MissingArgException extends RuntimeException {
+ public MissingArgException(String s) {
+ super(s);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java
new file mode 100644
index 0000000..f86b336
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+import java.io.IOException;
+
+/**
+ * Exception raised when a node cannot be found in the structure
+ * that is being examined.
+ */
+public class NoSuchNodeException extends IOException {
+
+ public NoSuchNodeException(String uuid) {
+ super("Unknown node: " + uuid);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
new file mode 100644
index 0000000..18e3157
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.core.main.ServiceLaunchException;
+
+public class SliderException extends ServiceLaunchException implements
+ SliderExitCodes {
+ public SliderException() {
+ super(EXIT_EXCEPTION_THROWN, "SliderException");
+ }
+
+ public SliderException(int code, String message) {
+ super(code, message);
+ }
+
+ public SliderException(String s) {
+ super(EXIT_EXCEPTION_THROWN, s);
+ }
+
+ public SliderException(String s, Throwable throwable) {
+ super(EXIT_EXCEPTION_THROWN, s, throwable);
+ }
+
+ /**
+ * Format the exception as you create it
+ * @param code
+ * @param message
+ * @param args
+ */
+ public SliderException(int code, String message, Object... args) {
+ super(code, String.format(message, args));
+ }
+
+ /**
+ * Format the exception, include a throwable.
+ * The throwable comes before the message so that it is out of the varargs
+ * @param code exit code
+ * @param throwable thrown
+ * @param message message
+ * @param args arguments
+ */
+ public SliderException(int code,
+ Throwable throwable,
+ String message,
+ Object... args) {
+ super(code, String.format(message, args), throwable);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java
new file mode 100644
index 0000000..deddbbc
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+public class SliderInternalStateException extends SliderException {
+ public SliderInternalStateException(String s) {
+ super(EXIT_INTERNAL_ERROR, s);
+ }
+
+ public SliderInternalStateException(String s, Throwable throwable) {
+ super(EXIT_INTERNAL_ERROR, throwable, s);
+ }
+
+ public SliderInternalStateException(String message,
+ Object... args) {
+ super(EXIT_INTERNAL_ERROR, message, args);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
new file mode 100644
index 0000000..7f59e41
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+/**
+ * An Exception to be thrown for an explicit "shut down the cluster" operation
+ * raised by the application state or other parts of the AM
+ */
+public class TriggerClusterTeardownException extends SliderException {
+
+ public TriggerClusterTeardownException(int code,
+ String message,
+ Object... args) {
+ super(code, message, args);
+ }
+
+ public TriggerClusterTeardownException(int code,
+ Throwable throwable,
+ String message, Object... args) {
+ super(code, throwable, message, args);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java
new file mode 100644
index 0000000..a1f8ae9
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+public class UnknownApplicationInstanceException extends SliderException {
+ public UnknownApplicationInstanceException(String s) {
+ super(EXIT_UNKNOWN_INSTANCE, s);
+ }
+
+ public UnknownApplicationInstanceException(String s, Throwable throwable) {
+ super(EXIT_UNKNOWN_INSTANCE, throwable, s);
+ }
+
+ public UnknownApplicationInstanceException(String message,
+ Object... args) {
+ super(EXIT_UNKNOWN_INSTANCE, message, args);
+ }
+
+ /**
+ * Create an instance with the standard exception name
+ * @param name name
+ * @return an instance to throw
+ */
+ public static UnknownApplicationInstanceException unknownInstance(String name) {
+ return new UnknownApplicationInstanceException(ErrorStrings.E_UNKNOWN_INSTANCE
+ + ": " + name);
+ }
+ public static UnknownApplicationInstanceException unknownInstance(String name,
+ Throwable throwable) {
+ UnknownApplicationInstanceException exception =
+ unknownInstance(name);
+ exception.initCause(throwable);
+ return exception;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java
new file mode 100644
index 0000000..5ad3fdc
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+import java.io.IOException;
+
+/**
+ * Called when some spinning operation timed out
+ */
+public class WaitTimeoutException extends IOException {
+ public WaitTimeoutException(String message) {
+ super(message);
+ }
+
+ public WaitTimeoutException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AMRestartSupport.java b/slider-core/src/main/java/org/apache/slider/core/launch/AMRestartSupport.java
new file mode 100644
index 0000000..eb8e25b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AMRestartSupport.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterResponsePBImpl;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.server.services.docstore.utility.SliderServiceUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.List;
+
+/**
+ * Logic needed to enable AM restart support in both submission and
+ * operation
+ */
+public class AMRestartSupport {
+ public static final String REGISTER_AM_RESPONSE =
+ "RegisterApplicationMasterResponse.getContainersFromPreviousAttempts()";
+ private static final Logger
+ log = LoggerFactory.getLogger(SliderServiceUtils.class);
+
+ /**
+ * Request that containers are kept across submissions.
+ * @param submissionContext context to query
+ * @return true if the method was applied.
+ */
+
+ public static boolean keepContainersAcrossSubmissions(
+ ApplicationSubmissionContext submissionContext) {
+ Method m = null;
+ String methName =
+ "ApplicationSubmissionContext.setKeepContainersAcrossApplicationAttempts()";
+ Class<? extends ApplicationSubmissionContext> cls =
+ submissionContext.getClass();
+ try {
+ m = cls.getDeclaredMethod("setKeepContainersAcrossApplicationAttempts",
+ boolean.class);
+ m.setAccessible(true);
+ } catch (NoSuchMethodException e) {
+ log.debug(methName + " not found");
+ } catch (SecurityException e) {
+ log.debug("No access to " + methName);
+ }
+ // AM-RESTART-SUPPORT: AM wants its old containers back on a restart
+ if (m != null) {
+ try {
+ m.invoke(submissionContext, true);
+ return true;
+ } catch (InvocationTargetException ite) {
+ log.error(methName + " got", ite);
+ } catch (IllegalAccessException iae) {
+ log.error(methName + " got", iae);
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Get the containers from a previous attempt
+ * @param response AM registration response
+ * @return a list of containers (possibly empty) if the AM provided
+ * that field in its registration, and the hadoop JAR has the relevant
+ * method to access it.
+ */
+ public static List<Container> retrieveContainersFromPreviousAttempt(
+ RegisterApplicationMasterResponse response) {
+ List<Container> liveContainers = null;
+ Method m = extractRetrieveContainersMethod(response);
+ if (m != null) {
+ try {
+ Object obj = m.invoke(response);
+ if (obj instanceof List) {
+ liveContainers = (List<Container>) obj;
+
+ }
+ } catch (InvocationTargetException ite) {
+ log.error(REGISTER_AM_RESPONSE + " got", ite);
+ } catch (IllegalAccessException iae) {
+ log.error(REGISTER_AM_RESPONSE + " got", iae);
+ }
+ }
+ return liveContainers;
+ }
+
+ /**
+ * Get the method to retrieve containers. The presence of this
+ * method indicates the Hadoop libraries are compiled with the
+ * extra fields, and that, if the client requested it, the AM
+ * will be given a list of existing containers on a restart
+ * @param response registration response
+ * @return a method or null if it is not present.
+ */
+ public static Method extractRetrieveContainersMethod(
+ RegisterApplicationMasterResponse response) {
+ Method m = null;
+ Class<? extends RegisterApplicationMasterResponse> cls =
+ response.getClass();
+ try {
+ m = cls.getDeclaredMethod("getContainersFromPreviousAttempts");
+ } catch (NoSuchMethodException e) {
+ log.debug(REGISTER_AM_RESPONSE + " not found");
+ } catch (SecurityException e) {
+ log.debug("No access to " + REGISTER_AM_RESPONSE);
+ }
+ return m;
+ }
+
+ public static boolean isAMRestartInHadoopLibrary() {
+ RegisterApplicationMasterResponse response =
+ new RegisterApplicationMasterResponsePBImpl();
+ return null != extractRetrieveContainersMethod(response);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
new file mode 100644
index 0000000..d7183e2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.MapOperations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Launcher of applications: base class
+ */
+public abstract class AbstractLauncher extends Configured {
+ private static final Logger log =
+ LoggerFactory.getLogger(AbstractLauncher.class);
+ /**
+ * Filesystem to use for the launch
+ */
+ protected final CoreFileSystem coreFileSystem;
+ /**
+ * Env vars; set up at final launch stage
+ */
+ protected final Map<String, String> envVars = new HashMap<String, String>();
+ protected final MapOperations env = new MapOperations("env", envVars);
+ protected final ContainerLaunchContext containerLaunchContext =
+ Records.newRecord(ContainerLaunchContext.class);
+ protected final List<String> commands = new ArrayList<String>(20);
+ protected final Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ private final Map<String, ByteBuffer> serviceData =
+ new HashMap<String, ByteBuffer>();
+ // security
+ Credentials credentials = new Credentials();
+
+
+ protected AbstractLauncher(Configuration conf,
+ CoreFileSystem fs) {
+ super(conf);
+ this.coreFileSystem = fs;
+ }
+
+ public AbstractLauncher(CoreFileSystem fs) {
+ this.coreFileSystem = fs;
+ }
+
+ /**
+ * Get the container. Until "completed", this isn't valid to launch.
+ * @return the container to launch
+ */
+ public ContainerLaunchContext getContainerLaunchContext() {
+ return containerLaunchContext;
+ }
+
+ /**
+ * Get the env vars to work on
+ * @return env vars
+ */
+ public MapOperations getEnv() {
+ return env;
+ }
+
+ public List<String> getCommands() {
+ return commands;
+ }
+
+ public Map<String, LocalResource> getLocalResources() {
+ return localResources;
+ }
+
+ public void addLocalResource(String subpath, LocalResource resource) {
+ localResources.put(subpath, resource);
+ }
+
+ /**
+ * Add a set of local resources
+ * @param resourceMap map of name:resource to add
+ */
+ public void addLocalResources(Map<String, LocalResource> resourceMap) {
+ localResources.putAll(resourceMap);
+ }
+
+
+ public Map<String, ByteBuffer> getServiceData() {
+ return serviceData;
+ }
+
+
+ /**
+ * Add a command line. It is converted to a single command before being
+ * added.
+ * @param cmd
+ */
+ public void addCommandLine(CommandLineBuilder cmd) {
+ commands.add(cmd.build());
+ }
+
+ public void addCommand(String cmd) {
+ commands.add(cmd);
+ }
+
+ /**
+ * Add a list of commands. Each element in the list becomes a single command
+ * @param commandList
+ */
+ public void addCommands(List<String> commandList) {
+ commands.addAll(commandList);
+ }
+
+ /**
+ * Get all commands as a string, separated by ";". This is for diagnostics
+ * @return a string descriptionof the commands
+ */
+ public String getCommandsAsString() {
+ return SliderUtils.join(getCommands(), "; ");
+ }
+
+ /**
+ * Complete the launch context (copy in env vars, etc).
+ * @return the container to launch
+ */
+ public ContainerLaunchContext completeContainerLaunch() throws IOException {
+ dumpLocalResources();
+
+ String cmdStr = SliderUtils.join(commands, " ", false);
+ log.debug("Completed setting up container command {}", cmdStr);
+ containerLaunchContext.setCommands(commands);
+
+ //fix the env variables
+ containerLaunchContext.setEnvironment(env);
+ //service data
+ containerLaunchContext.setServiceData(serviceData);
+ containerLaunchContext.setLocalResources(localResources);
+
+
+ DataOutputBuffer dob = new DataOutputBuffer();
+ credentials.writeTokenStorageToStream(dob);
+ ByteBuffer tokenBuffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ containerLaunchContext.setTokens(tokenBuffer);
+
+
+ return containerLaunchContext;
+ }
+
+ /**
+ * Dump local resources at debug level
+ */
+ private void dumpLocalResources() {
+ if (log.isDebugEnabled()) {
+ log.debug("{} resources: ", localResources.size());
+ for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
+
+ String key = entry.getKey();
+ LocalResource val = entry.getValue();
+ log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
+ }
+ }
+ }
+
+ /**
+ * This is critical for an insecure cluster -it passes
+ * down the username to YARN, and so gives the code running
+ * in containers the rights it needs to work with
+ * data.
+ * @throws IOException problems working with current user
+ */
+ protected void propagateUsernameInInsecureCluster() throws IOException {
+ //insecure cluster: propagate user name via env variable
+ String userName = UserGroupInformation.getCurrentUser().getUserName();
+ env.put("HADOOP_USER_NAME", userName);
+ }
+
+ /**
+ * Extract any resource requirements from this component's settings.
+ * All fields that are set will override the existing values -if
+ * unset that resource field will be left unchanged.
+ *
+ * Important: the configuration must already be fully resolved
+ * in order to pick up global options.
+ * @param resource resource to configure
+ * @param map map of options
+ */
+ public void extractResourceRequirements(Resource resource,
+ Map<String, String> map) {
+
+
+ if (map != null) {
+ MapOperations options = new MapOperations("", map);
+ resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY,
+ resource.getMemory()));
+ resource.setVirtualCores(options.getOptionInt(ResourceKeys.YARN_CORES,
+ resource.getVirtualCores()));
+ }
+ }
+
+
+ public void setEnv(String var, String value) {
+ env.put(var, value);
+ }
+
+ public void putEnv(Map<String, String> map) {
+ env.putAll(map);
+ }
+
+ /**
+ * Important: the configuration must already be fully resolved
+ * in order to pick up global options
+ * Copy env vars into the launch context.
+ */
+ public boolean copyEnvVars(MapOperations options) {
+ if (options == null) {
+ return false;
+ }
+ for (Map.Entry<String, String> entry : options.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(RoleKeys.ENV_PREFIX)) {
+ key = key.substring(RoleKeys.ENV_PREFIX.length());
+ env.put(key, entry.getValue());
+ }
+ }
+ return true;
+ }
+
+ public String[] dumpEnvToString() {
+
+ List<String> nodeEnv = new ArrayList<String>();
+
+ for (Map.Entry<String, String> entry : env.entrySet()) {
+ String envElt = String.format("%s=\"%s\"",
+ entry.getKey(),
+ entry.getValue());
+ log.debug(envElt);
+ nodeEnv.add(envElt);
+ }
+ String[] envDescription = nodeEnv.toArray(new String[nodeEnv.size()]);
+
+ return envDescription;
+ }
+
+ /**
+ * Suubmit an entire directory
+ * @param srcDir src path in filesystem
+ * @param destRelativeDir relative path under destination local dir
+ * @throws IOException IO problems
+ */
+ public void submitDirectory(Path srcDir, String destRelativeDir) throws
+ IOException {
+ //add the configuration resources
+ Map<String, LocalResource> confResources;
+ confResources = coreFileSystem.submitDirectory(
+ srcDir,
+ destRelativeDir);
+ addLocalResources(confResources);
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
new file mode 100644
index 0000000..85b022a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class AppMasterLauncher extends AbstractLauncher {
+
+
+ private static final Logger log =
+ LoggerFactory.getLogger(AppMasterLauncher.class);
+
+ protected final YarnClientApplication application;
+ private final String name;
+ private final String type;
+ private final ApplicationSubmissionContext submissionContext;
+ private final ApplicationId appId;
+ private final boolean secureCluster;
+ private int maxAppAttempts = 2;
+ private boolean keepContainersOverRestarts = true;
+ private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME;
+ private int priority = 1;
+ private final Resource resource = Records.newRecord(Resource.class);
+ private final SliderYarnClientImpl yarnClient;
+
+ /**
+ * Build the AM Launcher
+ * @param name app name
+ * @param type applicatin type
+ * @param conf hadoop config
+ * @param fs filesystem binding
+ * @param application precreated YARN client app instance
+ * @param secureCluster is the cluster secure?
+ * @param options map of options. All values are extracted in this constructor only
+ * -the map is not retained.
+ */
+ public AppMasterLauncher(String name,
+ String type,
+ Configuration conf,
+ CoreFileSystem fs,
+ SliderYarnClientImpl yarnClient,
+ boolean secureCluster,
+ Map<String, String> options
+ ) throws IOException, YarnException {
+ super(conf, fs);
+ this.yarnClient = yarnClient;
+ this.application = yarnClient.createApplication();
+ this.name = name;
+ this.type = type;
+ this.secureCluster = secureCluster;
+
+ submissionContext = application.getApplicationSubmissionContext();
+ appId = submissionContext.getApplicationId();
+ // set the application name;
+ submissionContext.setApplicationName(name);
+ // app type used in service enum;
+ submissionContext.setApplicationType(type);
+ extractResourceRequirements(resource, options);
+
+ }
+
+ public void setMaxAppAttempts(int maxAppAttempts) {
+ this.maxAppAttempts = maxAppAttempts;
+ }
+
+ public void setKeepContainersOverRestarts(boolean keepContainersOverRestarts) {
+ this.keepContainersOverRestarts = keepContainersOverRestarts;
+ }
+
+
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setMemory(int memory) {
+ resource.setMemory(memory);
+ }
+
+ public void setVirtualCores(int cores) {
+ resource.setVirtualCores(cores);
+ }
+
+ public ApplicationId getApplicationId() {
+ return appId;
+ }
+
+ public int getMaxAppAttempts() {
+ return maxAppAttempts;
+ }
+
+ public boolean isKeepContainersOverRestarts() {
+ return keepContainersOverRestarts;
+ }
+
+ public String getQueue() {
+ return queue;
+ }
+
+ public int getPriority() {
+ return priority;
+ }
+
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ public void setPriority(int priority) {
+ this.priority = priority;
+ }
+
+ /**
+ * Complete the launch context (copy in env vars, etc).
+ * @return the container to launch
+ */
+ public ApplicationSubmissionContext completeAppMasterLaunch() throws
+ IOException {
+
+
+
+ //queue priority
+ Priority pri = Records.newRecord(Priority.class);
+ pri.setPriority(priority);
+ submissionContext.setPriority(pri);
+
+ // Set the queue to which this application is to be submitted in the RM
+ // Queue for App master
+
+ submissionContext.setQueue(queue);
+
+
+ //container requirements
+ submissionContext.setResource(resource);
+
+ if (keepContainersOverRestarts) {
+ log.debug("Requesting cluster stays running over AM failure");
+ submissionContext.setKeepContainersAcrossApplicationAttempts(true);
+ }
+
+ submissionContext.setMaxAppAttempts(maxAppAttempts);
+
+ if (secureCluster) {
+ addSecurityTokens();
+ } else {
+ propagateUsernameInInsecureCluster();
+ }
+ completeContainerLaunch();
+ submissionContext.setAMContainerSpec(containerLaunchContext);
+ return submissionContext;
+
+ }
+
+ /**
+ * Add the security tokens if this is a secure cluster
+ * @throws IOException
+ */
+ private void addSecurityTokens() throws IOException {
+
+ String tokenRenewer = getConf().get(YarnConfiguration.RM_PRINCIPAL);
+ if (SliderUtils.isUnset(tokenRenewer)) {
+ throw new IOException(
+ "Can't get Master Kerberos principal for the RM to use as renewer: "
+ + YarnConfiguration.RM_PRINCIPAL
+ );
+ }
+
+ // For now, only getting tokens for the default file-system.
+ FileSystem fs = coreFileSystem.getFileSystem();
+ fs.addDelegationTokens(tokenRenewer, credentials);
+ }
+
+
+ public LaunchedApplication submitApplication() throws IOException, YarnException {
+ completeAppMasterLaunch();
+ log.info("Submitting application to Resource Manager");
+ ApplicationId applicationId =
+ yarnClient.submitApplication(submissionContext);
+ return new LaunchedApplication(applicationId, yarnClient);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
new file mode 100644
index 0000000..4b3297e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.tools.SliderUtils;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * build a classpath -allows for entries to be injected in front of
+ * YARN classpath as well as behind, adds appropriate separators,
+ * extraction of local classpath, etc.
+ */
+public class ClasspathConstructor {
+
+ // public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
+ public static final String CLASS_PATH_SEPARATOR = File.pathSeparator;
+ private final List<String> pathElements = new ArrayList<String>();
+
+ public ClasspathConstructor() {
+ }
+
+
+ /**
+ * Get the list of JARs from the YARN settings
+ * @param config configuration
+ */
+ public List<String> yarnApplicationClasspath(Configuration config) {
+ String[] cp = config.getTrimmedStrings(
+ YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH);
+ return cp!= null ? Arrays.asList(cp) : new ArrayList<String>(0);
+
+ }
+
+
+ @Override
+ public String toString() {
+ return buildClasspath();
+ }
+
+ public String buildClasspath() {
+ return SliderUtils.join(pathElements,
+ CLASS_PATH_SEPARATOR,
+ false);
+ }
+
+ /**
+ * Get a copy of the path list
+ * @return the JARs
+ */
+ public List<String> getPathElements() {
+ return Collections.unmodifiableList(pathElements);
+ }
+
+ /**
+ * Append an entry
+ * @param path path
+ */
+ public void append(String path) {
+ pathElements.add(path);
+ }
+
+ /**
+ * Insert a path at the front of the list. This places it ahead of
+ * the standard YARN artifacts
+ * @param path path to the JAR. Absolute or relative -on the target
+ * system
+ */
+ public void insert(String path) {
+ pathElements.add(0, path);
+ }
+
+ public void appendAll(Collection<String> paths) {
+ pathElements.addAll(paths);
+ }
+
+ public void insertAll(Collection<String> paths) {
+ pathElements.addAll(0, paths);
+ }
+
+
+ public void addLibDir(String pathToLibDir) {
+ append(buildLibDir(pathToLibDir));
+ }
+
+ public void insertLibDir(String pathToLibDir) {
+ insert(buildLibDir(pathToLibDir));
+ }
+
+ public void addClassDirectory(String pathToDir) {
+ append(buildLibDir(appendDirectoryTerminator(pathToDir)));
+ }
+
+ public void insertClassDirectory(String pathToDir) {
+ insert(buildLibDir(appendDirectoryTerminator(pathToDir)));
+ }
+
+
+ public void addRemoteClasspathEnvVar() {
+ append(ApplicationConstants.Environment.CLASSPATH.$());
+ }
+
+
+ public void insertRemoteClasspathEnvVar() {
+ append(ApplicationConstants.Environment.CLASSPATH.$());
+ }
+
+
+ /**
+ * Build a lib dir path
+ * @param pathToLibDir path to the directory; may or may not end with a
+ * trailing space
+ * @return a path to a lib dir that is compatible with the java classpath
+ */
+ public String buildLibDir(String pathToLibDir) {
+ String dir = appendDirectoryTerminator(pathToLibDir);
+ dir += "*";
+ return dir;
+ }
+
+ private String appendDirectoryTerminator(String pathToLibDir) {
+ String dir = pathToLibDir.trim();
+ if (!dir.endsWith("/")) {
+ dir += "/";
+ }
+ return dir;
+ }
+
+ public Collection<String> splitClasspath(String localpath) {
+ String separator = System.getProperty("path.separator");
+ return StringUtils.getStringCollection(localpath, separator);
+ }
+
+ public Collection<String> javaVMClasspath() {
+ return splitClasspath(System.getProperty("java.class.path"));
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
new file mode 100644
index 0000000..60d9b7c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.slider.common.tools.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Build a single command line to include in the container commands;
+ * Special support for JVM command buildup.
+ */
+public class CommandLineBuilder {
+ protected final List<String> argumentList = new ArrayList<String>(20);
+
+ public void addJavaBinary() {
+ add(
+ ApplicationConstants.Environment.JAVA_HOME.$() + "/bin/java");
+ }
+
+
+ /**
+ * Add an entry to the command list
+ * @param arg argument -this will be converted to a string
+ */
+ public void add(Object... args) {
+ for (Object arg : args) {
+ argumentList.add(arg.toString());
+ }
+ }
+
+ /**
+ * Get the value at an offset
+ * @param offset offset
+ * @return the value at that point
+ */
+ public String elt(int offset) {
+ return argumentList.get(offset);
+ }
+
+ /**
+ * Get the number of arguments
+ * @return an integer >= 0
+ */
+ public int size() {
+ return argumentList.size();
+ }
+
+ /**
+ * Append the output and error files to the tail of the command
+ * @param stdout out
+ * @param stderr error. Set this to null to append into stdout
+ */
+ public void addOutAndErrFiles(String stdout, String stderr) {
+ Preconditions.checkNotNull(stdout, "Null output file");
+ Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
+ // write out the path output
+ argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stdout);
+ if (stderr != null) {
+ argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stderr);
+ } else {
+ argumentList.add("2>&1");
+ }
+ }
+
+ /**
+ * This just returns the command line
+ * @see #build()
+ * @return the command line
+ */
+ @Override
+ public String toString() {
+ return build();
+ }
+
+ /**
+ * Build the command line
+ * @return the command line
+ */
+ public String build() {
+ return SliderUtils.join(argumentList, " ");
+ }
+
+ public List<String> getArgumentList() {
+ return argumentList;
+ }
+
+ /**
+ * Set the size of the heap if a non-empty heap is passed in.
+ * @param heap empty string or something like "128M" ,"1G" etc. The value is
+ * trimmed.
+ */
+ public void setJVMHeap(String heap) {
+ if (SliderUtils.isSet(heap)) {
+ add("-Xmx" + heap.trim());
+ }
+ }
+
+ public void enableJavaAssertions() {
+ add("-ea");
+ add("-esa");
+ }
+
+ public void sysprop(String property, String value) {
+ add("-D" + property + "=" + value);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
new file mode 100644
index 0000000..f8ea4ee
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+
+/**
+ * Code to ease launching of any container
+ */
+public class ContainerLauncher extends AbstractLauncher {
+ private static final Logger log =
+ LoggerFactory.getLogger(ContainerLauncher.class);
+ // Allocated container
+ public final Container container;
+
+ public ContainerLauncher(Configuration conf,
+ CoreFileSystem fs,
+ Container container) {
+ super(conf, fs);
+ this.container = container;
+ }
+
+ /**
+ * This code is in the dist shell examples -it's been moved here
+ * so that if it is needed, it's still here
+ * @return a remote user with a token to access the container.
+ */
+ public UserGroupInformation setupUGI() {
+ UserGroupInformation user =
+ UserGroupInformation.createRemoteUser(container.getId().toString());
+ String cmIpPortStr =
+ container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
+ final InetSocketAddress cmAddress =
+ NetUtils.createSocketAddr(cmIpPortStr);
+
+ org.apache.hadoop.yarn.api.records.Token containerToken =
+ container.getContainerToken();
+ if (containerToken != null) {
+ Token<ContainerTokenIdentifier> token =
+ ConverterUtils.convertFromYarn(containerToken,
+ cmAddress);
+ user.addToken(token);
+ }
+ return user;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java b/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java
new file mode 100644
index 0000000..e5a025c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.common.tools.Duration;
+
+import java.io.IOException;
+
+/**
+ * Launched App with logic around it.
+ */
+public class LaunchedApplication {
+
+ protected final ApplicationId applicationId;
+ protected final SliderYarnClientImpl yarnClient;
+
+ public LaunchedApplication(ApplicationId applicationId,
+ SliderYarnClientImpl yarnClient) {
+ assert applicationId != null;
+ assert yarnClient != null;
+ this.applicationId = applicationId;
+ this.yarnClient = yarnClient;
+ }
+
+ public LaunchedApplication(SliderYarnClientImpl yarnClient,
+ ApplicationReport report) {
+ this.yarnClient = yarnClient;
+ this.applicationId = report.getApplicationId();
+ }
+
+ public ApplicationId getApplicationId() {
+ return applicationId;
+ }
+
+ /**
+ * Monitor the submitted application for reaching the requested state.
+ * Will also report if the app reaches a later state (failed, killed, etc)
+ * Kill application if duration!= null & time expires.
+ * @param duration how long to wait -must be more than 0
+ * @param desiredState desired state.
+ * @return the application report -null on a timeout
+ * @throws YarnException
+ * @throws IOException
+ */
+ public ApplicationReport monitorAppToState(YarnApplicationState desiredState, Duration duration)
+ throws YarnException, IOException {
+ return yarnClient.monitorAppToState(applicationId, desiredState, duration);
+ }
+
+ /**
+ * Kill the submitted application by sending a call to the ASM
+ * @throws YarnException
+ * @throws IOException
+ */
+ public boolean forceKill(String reason)
+ throws YarnException, IOException {
+ if (applicationId != null) {
+ yarnClient.killRunningApplication(applicationId, reason);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Kill the application
+ * @return the response
+ * @throws YarnException YARN problems
+ * @throws IOException IO problems
+ */
+ public KillApplicationResponse kill(String reason) throws
+ YarnException,
+ IOException {
+ return yarnClient.killRunningApplication(applicationId, reason);
+ }
+
+ public ApplicationReport getApplicationReport()
+ throws YarnException, IOException {
+ return yarnClient.getApplicationReport(applicationId);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/RunningApplication.java b/slider-core/src/main/java/org/apache/slider/core/launch/RunningApplication.java
new file mode 100644
index 0000000..14c522c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/RunningApplication.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.server.appmaster.rpc.RpcBinder;
+
+import java.io.IOException;
+
+import static org.apache.slider.common.Constants.CONNECT_TIMEOUT;
+import static org.apache.slider.common.Constants.RPC_TIMEOUT;
+
+/**
+ * A running application built from an app report. This one
+ * can be talked to
+ */
+public class RunningApplication extends LaunchedApplication {
+
+ private final ApplicationReport applicationReport;
+ public RunningApplication(SliderYarnClientImpl yarnClient,
+ ApplicationReport applicationReport) {
+ super(yarnClient, applicationReport);
+ this.applicationReport = applicationReport;
+ }
+
+ public ApplicationReport getApplicationReport() {
+ return applicationReport;
+ }
+
+
+ /**
+ * Connect to a Slider AM
+ * @param app application report providing the details on the application
+ * @return an instance
+ * @throws YarnException
+ * @throws IOException
+ */
+ public SliderClusterProtocol connect(ApplicationReport app) throws
+ YarnException,
+ IOException {
+
+ try {
+ return RpcBinder.getProxy(yarnClient.getConfig(),
+ yarnClient.getRmClient(),
+ app,
+ CONNECT_TIMEOUT,
+ RPC_TIMEOUT);
+ } catch (InterruptedException e) {
+ throw new SliderException(SliderExitCodes.EXIT_TIMED_OUT,
+ e,
+ "Interrupted waiting for communications with the Application Master");
+ }
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/ExitCodeProvider.java b/slider-core/src/main/java/org/apache/slider/core/main/ExitCodeProvider.java
new file mode 100644
index 0000000..3059c05
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ExitCodeProvider.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.main;
+
+/**
+ * Get the exit code of an exception. Making it an interface allows
+ * us to retrofit exit codes onto existing classes
+ */
+public interface ExitCodeProvider {
+
+ /**
+ * Method to get the exit code
+ * @return the exit code
+ */
+ int getExitCode();
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/IrqHandler.java b/slider-core/src/main/java/org/apache/slider/core/main/IrqHandler.java
new file mode 100644
index 0000000..9e7a596
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/main/IrqHandler.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.main;
+
+import sun.misc.Signal;
+import sun.misc.SignalHandler;
+
+import java.io.IOException;
+
+/**
+ * This class bundles up all the compiler warnings about abuse of sun.misc
+ * interrupt handling code
+ * into one place.
+ */
+@SuppressWarnings("UseOfSunClasses")
+public final class IrqHandler implements SignalHandler {
+
+ public static final String CONTROL_C = "INT";
+ public static final String SIGTERM = "TERM";
+
+ private final String name;
+ private final Interrupted handler;
+
+ /**
+ * Create an IRQ handler bound to the specific interrupt
+ * @param name signal name
+ * @param handler handler
+ * @throws IOException
+ */
+ public IrqHandler(String name, Interrupted handler) throws IOException {
+ this.handler = handler;
+ this.name = name;
+ try {
+ Signal.handle(new Signal(name), this);
+ } catch (IllegalArgumentException e) {
+ throw new IOException(
+ "Could not set handler for signal \"" + name + "\"."
+ + "This can happen if the JVM has the -Xrs set.",
+ e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "IrqHandler for signal " + name ;
+ }
+
+ /**
+ * Handler for the JVM API for signal handling
+ * @param signal signal raised
+ */
+// @Override
+ public void handle(Signal signal) {
+ InterruptData data = new InterruptData(signal.getName(), signal.getNumber());
+ handler.interrupted(data);
+ }
+
+ /**
+ * Interrupt data to pass on.
+ */
+ public static class InterruptData {
+ public final String name;
+ public final int number;
+
+ public InterruptData(String name, int number) {
+ this.name = name;
+ this.number = number;
+ }
+
+ @Override
+ public String toString() {
+ return "signal " + name + '(' + number + ')';
+ }
+ }
+
+ /**
+ * Callback on interruption
+ */
+ public interface Interrupted {
+
+ /**
+ * Handle an interrupt
+ * @param interruptData data
+ */
+ void interrupted(InterruptData interruptData);
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java b/slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java
new file mode 100644
index 0000000..b172260
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/main/LauncherExitCodes.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.main;
+
+
+/*
+ * YARN Codes,
+ * Exit codes from 32 up are relative to a base value that
+ * we put a fair way up from the the base numbers, so that
+ * applications can have their own set of failures
+ */
+public interface LauncherExitCodes {
+ /**
+ * 0: success
+ */
+ int EXIT_SUCCESS = 0;
+
+ /**
+ * -1: generic "false" response. The operation worked but
+ * the result was not true
+ */
+ int EXIT_FALSE = -1;
+
+ /**
+ * Exit code when a client requested service termination: {@value}
+ */
+ int EXIT_CLIENT_INITIATED_SHUTDOWN = 1;
+
+ /**
+ * Exit code when targets could not be launched: {@value}
+ */
+ int EXIT_TASK_LAUNCH_FAILURE = 2;
+
+ /**
+ * Exit code when an exception was thrown from the service: {@value}
+ */
+ int EXIT_EXCEPTION_THROWN = 32;
+
+ /**
+ * Exit code when a usage message was printed: {@value}
+ */
+ int EXIT_USAGE = 33;
+
+ /**
+ * Exit code when something happened but we can't be specific: {@value}
+ */
+ int EXIT_OTHER_FAILURE = 34;
+
+ /**
+ * Exit code when a control-C, kill -3, signal was picked up: {@value}
+ */
+
+ int EXIT_INTERRUPTED = 35;
+
+ /**
+ * Exit code when the command line doesn't parse: {@value}, or
+ * when it is otherwise invalid.
+ */
+ int EXIT_COMMAND_ARGUMENT_ERROR = 36;
+
+ /**
+ * Exit code when the configurations in valid/incomplete: {@value}
+ */
+ int EXIT_BAD_CONFIGURATION = 37;
+
+ /**
+ * Exit code when the configurations in valid/incomplete: {@value}
+ */
+ int EXIT_CONNECTIVITY_PROBLEM = 38;
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/RunService.java b/slider-core/src/main/java/org/apache/slider/core/main/RunService.java
new file mode 100644
index 0000000..9a52b38
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/main/RunService.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.main;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+
+/**
+ * An interface which services can implement to have their
+ * execution managed by the ServiceLauncher.
+ * The command line options will be passed down before the
+ * {@link Service#init(Configuration)} operation is invoked via an
+ * invocation of {@link RunService#bindArgs(Configuration, String...)}
+ * After the service has been successfully started via {@link Service#start()}
+ * the {@link RunService#runService()} method is called to execute the
+ * service. When this method returns, the service launcher will exit, using
+ * the return code from the method as its exit option.
+ */
+public interface RunService extends Service {
+
+ /**
+ * Propagate the command line arguments.
+ * This method is called before {@link Service#init(Configuration)};
+ * the configuration that is returned from this operation
+ * is the one that is passed on to the init operation.
+ * This permits implemenations to change the configuration before
+ * the init operation.n
+ *
+ *
+ * @param config the initial configuration build up by the
+ * service launcher.
+ * @param args argument list list of arguments passed to the command line
+ * after any launcher-specific commands have been stripped.
+ * @return the configuration to init the service with. This MUST NOT be null.
+ * Recommended: pass down the config parameter with any changes
+ * @throws Exception any problem
+ */
+ Configuration bindArgs(Configuration config, String... args) throws Exception;
+
+ /**
+ * Run a service. This called after {@link Service#start()}
+ * @return the exit code
+ * @throws Throwable any exception to report
+ */
+ int runService() throws Throwable ;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
new file mode 100644
index 0000000..8277a51
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.main;
+
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * A service launch exception that includes an exit code;
+ * when caught by the ServiceLauncher, it will convert that
+ * into a process exit code.
+ */
+public class ServiceLaunchException extends YarnException
+ implements ExitCodeProvider, LauncherExitCodes {
+
+ private final int exitCode;
+
+ public ServiceLaunchException(int exitCode, Throwable cause) {
+ super(cause);
+ this.exitCode = exitCode;
+ }
+
+ public ServiceLaunchException(int exitCode, String message) {
+ super(message);
+ this.exitCode = exitCode;
+ }
+
+ public ServiceLaunchException(int exitCode, String message, Throwable cause) {
+ super(message, cause);
+ this.exitCode = exitCode;
+ }
+
+ @Override
+ public int getExitCode() {
+ return exitCode;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
new file mode 100644
index 0000000..2d50419
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
@@ -0,0 +1,575 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.main;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A class to launch any service by name.
+ *
+ * It's designed to be subclassed for custom entry points.
+ *
+ *
+ * Workflow
+ * <ol>
+ * <li>An instance of the class is created</li>
+ * <li>If it implements RunService, it is given the binding args off the CLI</li>
+ * <li>Its service.init() and service.start() methods are called.</li>
+ * <li>If it implements RunService, runService() is called and its return
+ * code used as the exit code.</li>
+ * <li>Otherwise: it waits for the service to stop, assuming in its start() method
+ * it begins work</li>
+ * <li>If an exception returned an exit code, that becomes the exit code of the
+ * command.</li>
+ * </ol>
+ * Error and warning messages are logged to stderr. Why? If the classpath
+ * is wrong & logger configurations not on it, then no error messages by
+ * the started app will be seen and the caller is left trying to debug
+ * using exit codes.
+ *
+ */
+@SuppressWarnings("UseOfSystemOutOrSystemErr")
+public class ServiceLauncher<S extends Service>
+ implements LauncherExitCodes, IrqHandler.Interrupted {
+ private static final Log LOG = LogFactory.getLog(ServiceLauncher.class);
+ protected static final int PRIORITY = 30;
+
+ public static final String NAME = "ServiceLauncher";
+ /**
+ * name of class for entry point strings: {@value}
+ */
+ public static final String ENTRY_POINT =
+ "org.apache.hadoop.yarn.service.launcher." + NAME;
+
+
+ public static final String USAGE_MESSAGE =
+ "Usage: " + NAME + " classname [--conf <conf file>] <service arguments> | ";
+
+ /**
+ * Name of the "--conf" argument.
+ */
+ public static final String ARG_CONF = "--conf";
+ static int SHUTDOWN_TIME_ON_INTERRUPT = 30 * 1000;
+
+ private volatile S service;
+ private int serviceExitCode;
+ private final List<IrqHandler> interruptHandlers = new ArrayList<IrqHandler>(1);
+ private Configuration configuration;
+ private String serviceClassName;
+ private static AtomicBoolean signalAlreadyReceived = new AtomicBoolean(false);
+
+
+ /**
+ * Create an instance of the launcher
+ * @param serviceClassName classname of the service
+ */
+ public ServiceLauncher(String serviceClassName) {
+ this.serviceClassName = serviceClassName;
+ }
+
+ /**
+ * Get the service. Null until and unless
+ * {@link #launchService(Configuration, String[], boolean)} has completed
+ * @return the service
+ */
+ public S getService() {
+ return service;
+ }
+
+ /**
+ * Get the configuration constructed from the command line arguments
+ * @return the configuration used to create the service
+ */
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ /**
+ * The exit code from a successful service execution
+ * @return the exit code.
+ */
+ public int getServiceExitCode() {
+ return serviceExitCode;
+ }
+
+ @Override
+ public String toString() {
+ return "ServiceLauncher for " + serviceClassName;
+ }
+
+ /**
+ * Launch the service, by creating it, initing it, starting it and then
+ * maybe running it. {@link RunService#bindArgs(Configuration, String...)} is invoked
+ * on the service between creation and init.
+ *
+ * All exceptions that occur are propagated upwards.
+ *
+ * If the method returns a status code, it means that it got as far starting
+ * the service, and if it implements {@link RunService}, that the
+ * method {@link RunService#runService()} has completed.
+ *
+ * At this point, the service is returned by {@link #getService()}.
+ *
+ * @param conf configuration
+ * @param processedArgs arguments after the configuration parameters
+ * have been stripped out.
+ * @param addShutdownHook should a shutdown hook be added to terminate
+ * this service on shutdown. Tests should set this to false.
+ * @throws ClassNotFoundException classname not on the classpath
+ * @throws IllegalAccessException not allowed at the class
+ * @throws InstantiationException not allowed to instantiate it
+ * @throws InterruptedException thread interrupted
+ * @throws IOException any IO exception
+ */
+ public int launchService(Configuration conf,
+ String[] processedArgs,
+ boolean addShutdownHook)
+ throws Throwable,
+ ClassNotFoundException,
+ InstantiationException,
+ IllegalAccessException,
+ ExitUtil.ExitException {
+
+ instantiateService(conf);
+
+ //Register the interrupt handlers
+ registerInterruptHandler();
+ //and the shutdown hook
+ if (addShutdownHook) {
+ ServiceShutdownHook shutdownHook = new ServiceShutdownHook(service);
+ ShutdownHookManager.get().addShutdownHook(shutdownHook, PRIORITY);
+ }
+ RunService runService = null;
+
+ if (service instanceof RunService) {
+ //if its a runService, pass in the conf and arguments before init)
+ runService = (RunService) service;
+ configuration = runService.bindArgs(configuration, processedArgs);
+ assert configuration != null : "null configuration returned by bindArgs()";
+ }
+
+ //some class constructors init; here this is picked up on.
+ if (!service.isInState(Service.STATE.INITED)) {
+ service.init(configuration);
+ }
+ service.start();
+ int exitCode = EXIT_SUCCESS;
+ if (runService != null) {
+ //assume that runnable services are meant to run from here
+ exitCode = runService.runService();
+ LOG.debug("Service exited with exit code " + exitCode);
+
+ } else {
+ //run the service until it stops or an interrupt happens on a different thread.
+ LOG.debug("waiting for service threads to terminate");
+ service.waitForServiceToStop(0);
+ }
+ //exit
+ serviceExitCode = exitCode;
+ return serviceExitCode;
+ }
+
+ /**
+ * Instantiate the service defined in <code>serviceClassName</code>
+ * . Sets the <code>configuration</code> field
+ * to the configuration, and <code>service</code> to the service.
+ *
+ * @param conf configuration to use
+ * @throws ClassNotFoundException no such class
+ * @throws InstantiationException no empty constructor,
+ * problems with dependencies
+ * @throws IllegalAccessException no access rights
+ */
+ public Service instantiateService(Configuration conf) throws
+ ClassNotFoundException,
+ InstantiationException,
+ IllegalAccessException,
+ ExitUtil.ExitException {
+ configuration = conf;
+
+ //Instantiate the class -this requires the service to have a public
+ // zero-argument constructor
+ Class<?> serviceClass =
+ this.getClass().getClassLoader().loadClass(serviceClassName);
+ Object instance = serviceClass.newInstance();
+ if (!(instance instanceof Service)) {
+ //not a service
+ throw new ExitUtil.ExitException(EXIT_BAD_CONFIGURATION,
+ "Not a Service class: " + serviceClassName);
+ }
+
+ service = (S) instance;
+ return service;
+ }
+
+ /**
+ * Register this class as the handler for the control-C interrupt.
+ * Can be overridden for testing.
+ * @throws IOException on a failure to add the handler
+ */
+ protected void registerInterruptHandler() throws IOException {
+ try {
+ interruptHandlers.add(new IrqHandler(IrqHandler.CONTROL_C, this));
+ interruptHandlers.add(new IrqHandler(IrqHandler.SIGTERM, this));
+ } catch (IOException e) {
+ error("Signal handler setup failed : " + e, e);
+ }
+ }
+
+ /**
+ * The service has been interrupted.
+ * Trigger something resembling an elegant shutdown;
+ * Give the service time to do this before the exit operation is called
+ * @param interruptData the interrupted data.
+ */
+ @Override
+ public void interrupted(IrqHandler.InterruptData interruptData) {
+ String message = "Service interrupted by " + interruptData.toString();
+ warn(message);
+ if (!signalAlreadyReceived.compareAndSet(false, true)) {
+ warn("Repeated interrupt: escalating to a JVM halt");
+ // signal already received. On a second request to a hard JVM
+ // halt and so bypass any blocking shutdown hooks.
+ ExitUtil.halt(EXIT_INTERRUPTED, message);
+ }
+ int shutdownTimeMillis = SHUTDOWN_TIME_ON_INTERRUPT;
+ //start an async shutdown thread with a timeout
+ ServiceForcedShutdown forcedShutdown =
+ new ServiceForcedShutdown(shutdownTimeMillis);
+ Thread thread = new Thread(forcedShutdown);
+ thread.setDaemon(true);
+ thread.start();
+ //wait for that thread to finish
+ try {
+ thread.join(shutdownTimeMillis);
+ } catch (InterruptedException ignored) {
+ //ignored
+ }
+ if (!forcedShutdown.isServiceStopped()) {
+ warn("Service did not shut down in time");
+ }
+ exit(EXIT_INTERRUPTED, message);
+ }
+
+ protected void warn(String text) {
+ System.err.println(text);
+ }
+
+
+ protected void error(String message, Throwable thrown) {
+ String text = "Exception:" + message;
+ System.err.println(text);
+ LOG.error(text, thrown);
+ }
+
+ /**
+ * Exit the code.
+ * This is method can be overridden for testing, throwing an
+ * exception instead. Any subclassed method MUST raise an
+ * {@link ExitUtil.ExitException} instance.
+ * The service launcher code assumes that after this method is invoked,
+ * no other code in the same method is called.
+ * @param exitCode code to exit
+ */
+ protected void exit(int exitCode, String message) {
+ ExitUtil.terminate(exitCode, message);
+ }
+
+ /**
+ * Exit off an exception. This can be subclassed for testing
+ * @param ee exit exception
+ */
+ protected void exit(ExitUtil.ExitException ee) {
+ ExitUtil.terminate(ee.status, ee);
+ }
+
+ /**
+ * Get the service name via {@link Service#getName()}.
+ * If the service is not instantiated, the classname is returned instead.
+ * @return the service name
+ */
+ public String getServiceName() {
+ Service s = service;
+ String name = null;
+ if (s != null) {
+ try {
+ name = s.getName();
+ } catch (Exception ignored) {
+ // ignored
+ }
+ }
+ if (name != null) {
+ return "service " + name;
+ } else {
+ return "service classname " + serviceClassName;
+ }
+ }
+
+ /**
+ * Parse the command line, building a configuration from it, then
+ * launch the service and wait for it to finish. finally, exit
+ * passing the status code to the {@link #exit(int)} method.
+ * @param args arguments to the service. arg[0] is
+ * assumed to be the service classname and is automatically
+ */
+ public void launchServiceAndExit(List<String> args) {
+
+ //Currently the config just the default
+ Configuration conf = new Configuration();
+ String[] processedArgs = extractConfigurationArgs(conf, args);
+ ExitUtil.ExitException ee = launchServiceRobustly(conf, processedArgs);
+ exit(ee);
+ }
+
+ /**
+ * Extract the configuration arguments and apply them to the configuration,
+ * building an array of processed arguments to hand down to the service.
+ *
+ * @param conf configuration to update
+ * @param args main arguments. args[0] is assumed to be the service
+ * classname and is skipped
+ * @return the processed list.
+ */
+ public static String[] extractConfigurationArgs(Configuration conf,
+ List<String> args) {
+
+ //convert args to a list
+ int argCount = args.size();
+ if (argCount <= 1 ) {
+ return new String[0];
+ }
+ List<String> argsList = new ArrayList<String>(argCount);
+ ListIterator<String> arguments = args.listIterator();
+ //skip that first entry
+ arguments.next();
+ while (arguments.hasNext()) {
+ String arg = arguments.next();
+ if (arg.equals(ARG_CONF)) {
+ //the argument is a --conf file tuple: extract the path and load
+ //it in as a configuration resource.
+
+ //increment the loop iterator
+ if (!arguments.hasNext()) {
+ //overshot the end of the file
+ exitWithMessage(EXIT_COMMAND_ARGUMENT_ERROR,
+ ARG_CONF + ": missing configuration file after ");
+ }
+ File file = new File(arguments.next());
+ if (!file.exists()) {
+ exitWithMessage(EXIT_COMMAND_ARGUMENT_ERROR,
+ ARG_CONF + ": configuration file not found: " + file);
+ }
+ try {
+ conf.addResource(file.toURI().toURL());
+ } catch (MalformedURLException e) {
+ exitWithMessage(EXIT_COMMAND_ARGUMENT_ERROR,
+ ARG_CONF + ": configuration file path invalid: " + file);
+ }
+ } else {
+ argsList.add(arg);
+ }
+ }
+ String[] processedArgs = new String[argsList.size()];
+ argsList.toArray(processedArgs);
+ return processedArgs;
+ }
+
+ /**
+ * Launch a service catching all exceptions and downgrading them to exit codes
+ * after logging.
+ * @param conf configuration to use
+ * @param processedArgs command line after the launcher-specific arguments have
+ * been stripped out
+ * @return an exit exception, which will have a status code of 0 if it worked
+ */
+ public ExitUtil.ExitException launchServiceRobustly(Configuration conf,
+ String[] processedArgs) {
+ ExitUtil.ExitException exitException;
+ try {
+ int exitCode = launchService(conf, processedArgs, true);
+ if (service != null) {
+ Throwable failure = service.getFailureCause();
+ if (failure != null) {
+ //the service exited with a failure.
+ //check what state it is in
+ Service.STATE failureState = service.getFailureState();
+ if (failureState == Service.STATE.STOPPED) {
+ //the failure occurred during shutdown, not important enough to bother
+ //the user as it may just scare them
+ LOG.debug("Failure during shutdown: " + failure, failure);
+ } else {
+ //throw it for the catch handlers to deal with
+ throw failure;
+ }
+ }
+ }
+ exitException = new ExitUtil.ExitException(exitCode,
+ "In " + serviceClassName);
+ //either the service succeeded, or an error raised during shutdown,
+ //which we don't worry that much about
+ } catch (ExitUtil.ExitException ee) {
+ exitException = ee;
+ } catch (Throwable thrown) {
+ int exitCode;
+ String message = thrown.getMessage();
+ if (message == null) {
+ message = thrown.toString();
+ }
+ LOG.error(message) ;
+ if (thrown instanceof ExitCodeProvider) {
+ exitCode = ((ExitCodeProvider) thrown).getExitCode();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("While running " + getServiceName() + ": " + message, thrown);
+ }
+ } else {
+ //not any of the service launcher exceptions -assume something worse
+ error(message, thrown);
+ exitCode = EXIT_EXCEPTION_THROWN;
+ }
+ exitException = new ExitUtil.ExitException(exitCode, message);
+ exitException.initCause(thrown);
+ }
+ return exitException;
+ }
+
+
+ /**
+ * Build a log message for starting up and shutting down.
+ * This was grabbed from the ToolRunner code.
+ * @param classname the class of the server
+ * @param args arguments
+ */
+ public static String startupShutdownMessage(String classname,
+ List<String> args) {
+ final String hostname = NetUtils.getHostname();
+
+ return toStartupShutdownString("STARTUP_MSG: ", new String[]{
+ "Starting " + classname,
+ " host = " + hostname,
+ " args = " + args,
+ " version = " + VersionInfo.getVersion(),
+ " classpath = " + System.getProperty("java.class.path"),
+ " build = " + VersionInfo.getUrl() + " -r "
+ + VersionInfo.getRevision()
+ + "; compiled by '" + VersionInfo.getUser()
+ + "' on " + VersionInfo.getDate(),
+ " java = " + System.getProperty("java.version")
+ });
+ }
+
+ /**
+ * Exit with a printed message
+ * @param status status code
+ * @param message message
+ */
+ private static void exitWithMessage(int status, String message) {
+ System.err.println(message);
+ ExitUtil.terminate(status);
+ }
+
+ private static String toStartupShutdownString(String prefix, String[] msg) {
+ StringBuilder b = new StringBuilder(prefix);
+ b.append("\n/************************************************************");
+ for (String s : msg) {
+ b.append("\n").append(prefix).append(s);
+ }
+ b.append("\n************************************************************/");
+ return b.toString();
+ }
+
+ /**
+ * forced shutdown runnable.
+ */
+ protected class ServiceForcedShutdown implements Runnable {
+
+ private final int shutdownTimeMillis;
+ private boolean serviceStopped;
+
+ public ServiceForcedShutdown(int shutdownTimeoutMillis) {
+ this.shutdownTimeMillis = shutdownTimeoutMillis;
+ }
+
+ @Override
+ public void run() {
+ if (service != null) {
+ service.stop();
+ serviceStopped = service.waitForServiceToStop(shutdownTimeMillis);
+ } else {
+ serviceStopped = true;
+ }
+ }
+
+ private boolean isServiceStopped() {
+ return serviceStopped;
+ }
+ }
+
+ /**
+ * The real main function, which takes the arguments as a list
+ * arg 0 must be the service classname
+ * @param argsList the list of arguments
+ */
+ public static void serviceMain(List<String> argsList) {
+ if (argsList.isEmpty()) {
+ exitWithMessage(EXIT_USAGE, USAGE_MESSAGE);
+ } else {
+ String serviceClassName = argsList.get(0);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(startupShutdownMessage(serviceClassName, argsList));
+ StringBuilder builder = new StringBuilder();
+ for (String arg : argsList) {
+ builder.append('"').append(arg).append("\" ");
+ }
+ LOG.debug(builder.toString());
+ }
+ Thread.setDefaultUncaughtExceptionHandler(
+ new YarnUncaughtExceptionHandler());
+
+ ServiceLauncher serviceLauncher = new ServiceLauncher<Service>(serviceClassName);
+ serviceLauncher.launchServiceAndExit(argsList);
+ }
+ }
+
+ /**
+ * This is the main entry point for the service launcher.
+ * @param args command line arguments.
+ */
+ public static void main(String[] args) {
+ List<String> argsList = Arrays.asList(args);
+ serviceMain(argsList);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java b/slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java
new file mode 100644
index 0000000..96cfdb3
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ServiceShutdownHook.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.main;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.util.ShutdownHookManager;
+
+import java.lang.ref.WeakReference;
+
+/**
+ * JVM Shutdown hook for Service which will stop the
+ * Service gracefully in case of JVM shutdown.
+ * This hook uses a weak reference to the service, so
+ * does not cause services to be retained after they have
+ * been stopped and deferenced elsewhere.
+ */
+public class ServiceShutdownHook implements Runnable {
+ private static final Log LOG = LogFactory.getLog(ServiceShutdownHook.class);
+
+ private WeakReference<Service> serviceRef;
+ private Runnable hook;
+
+ public ServiceShutdownHook(Service service) {
+ serviceRef = new WeakReference<Service>(service);
+ }
+
+ public void register(int priority) {
+ unregister();
+ hook = this;
+ ShutdownHookManager.get().addShutdownHook(hook, priority);
+ }
+
+ public void unregister() {
+ if (hook != null) {
+ try {
+ ShutdownHookManager.get().removeShutdownHook(hook);
+ } catch (IllegalStateException e) {
+ LOG.info("Failed to unregister shutdown hook",e);
+ }
+ hook = null;
+ }
+ }
+
+// @Override
+ public void run() {
+ Service service = serviceRef.get();
+ if (service == null) {
+ return;
+ }
+ try {
+ // Stop the Service
+ service.stop();
+ } catch (Throwable t) {
+ LOG.info("Error stopping " + service.getName(), t);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
new file mode 100644
index 0000000..ba47393
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * Class to implement persistence of a configuration.
+ *
+ * This code contains the logic to acquire and release locks.
+ * # writelock MUST be acquired exclusively for writes. This is done
+ * by creating the file with no overwrite
+ * # shared readlock MUST be acquired for reads. This is done by creating the readlock
+ * file with overwrite forbidden -but treating a failure as a sign that
+ * the lock exists, and therefore the operation can continue.
+ * # releaselock is only released if the client created it.
+ * # after acquiring either lock, client must check for the alternate lock
+ * existing. If it is, release lock and fail.
+ *
+ * There's one small race here: multiple readers; first reader releases lock
+ * while second is in use.
+ *
+ * Strict Fix: client checks for readlock after read completed.
+ * If it is not there, problem: fail. But this massively increases the risk of
+ * false negatives.
+ *
+ * This isn't 100% perfect, because of the condition where the owner releases
+ * a lock, a writer grabs its lock & writes to it, the reader gets slightly
+ * contaminated data:
+ * own-share-delete-write-own-release(shared)-delete
+ *
+ * We are assuming that the rate of change is low enough that this is rare, and
+ * of limited damage.
+ *
+ * ONCE A CLUSTER IS RUNNING, ONLY THE AM MAY PERSIST UPDATES VIA ITS APIs
+ *
+ * That is: outside the AM, a writelock MUST only be acquired after verifying there is no
+ * running application.
+ */
+public class ConfPersister {
+ private static final Logger log =
+ LoggerFactory.getLogger(ConfPersister.class);
+
+
+ private final ConfTreeSerDeser confTreeSerDeser =new ConfTreeSerDeser();
+
+ private final CoreFileSystem coreFS;
+ private final FileSystem fileSystem;
+ private final Path persistDir;
+ private final Path internal, resources, app_conf;
+ private final Path writelock, readlock;
+
+ public ConfPersister(CoreFileSystem coreFS, Path persistDir) {
+ this.coreFS = coreFS;
+ this.persistDir = persistDir;
+ internal = new Path(persistDir, Filenames.INTERNAL);
+ resources = new Path(persistDir, Filenames.RESOURCES);
+ app_conf = new Path(persistDir, Filenames.APPCONF);
+ writelock = new Path(persistDir, Filenames.WRITELOCK);
+ readlock = new Path(persistDir, Filenames.READLOCK);
+ fileSystem = coreFS.getFileSystem();
+ }
+
+ /**
+ * Get the target directory
+ * @return the directory for persistence
+ */
+ public Path getPersistDir() {
+ return persistDir;
+ }
+
+ /**
+ * Make the persistent directory
+ * @throws IOException IO failure
+ */
+ private void mkPersistDir() throws IOException {
+ coreFS.getFileSystem().mkdirs(persistDir);
+ }
+
+ @Override
+ public String toString() {
+ return "Persister to " + persistDir;
+ }
+
+ /**
+ * Acquire the writelock
+ * @throws IOException IO
+ * @throws LockAcquireFailedException
+ */
+ @VisibleForTesting
+ void acquireWritelock() throws IOException,
+ LockAcquireFailedException {
+ mkPersistDir();
+ long now = System.currentTimeMillis();
+ try {
+ coreFS.cat(writelock, false, new Date(now).toGMTString());
+ } catch (FileAlreadyExistsException e) {
+ // filesystems should raise this (HDFS does)
+ throw new LockAcquireFailedException(writelock);
+ } catch (IOException e) {
+ // some filesystems throw a generic IOE
+ throw new LockAcquireFailedException(writelock, e);
+ }
+ //here the lock is acquired, but verify there is no readlock
+ boolean lockFailure;
+ try {
+ lockFailure = readLockExists();
+ } catch (IOException e) {
+ lockFailure = true;
+ }
+ if (lockFailure) {
+ releaseWritelock();
+ throw new LockAcquireFailedException(readlock);
+ }
+ }
+
+ @VisibleForTesting
+ boolean readLockExists() throws IOException {
+ return fileSystem.exists(readlock);
+ }
+
+ /**
+ * Release the writelock if it is present.
+ * IOExceptions are logged
+ */
+ @VisibleForTesting
+ boolean releaseWritelock() {
+ try {
+ return fileSystem.delete(writelock, false);
+ } catch (IOException e) {
+ log.warn("IOException releasing writelock {} ", writelock, e);
+ }
+ return false;
+ }
+
+ /**
+ * Acquire the writelock
+ * @throws IOException IO
+ * @throws LockAcquireFailedException
+ * @throws FileNotFoundException if the target dir does not exist
+ */
+ @VisibleForTesting
+ boolean acquireReadLock() throws FileNotFoundException,
+ IOException,
+ LockAcquireFailedException {
+ if (!coreFS.getFileSystem().exists(persistDir)) {
+ throw new FileNotFoundException(persistDir.toString());
+ }
+ long now = System.currentTimeMillis();
+ boolean owner;
+ try {
+ coreFS.cat(readlock, false, new Date(now).toGMTString());
+ owner = true;
+ } catch (IOException e) {
+ owner = false;
+ }
+ //here the lock is acquired, but verify there is no readlock
+ boolean lockFailure;
+ try {
+ lockFailure = writelockExists();
+ } catch (IOException e) {
+ lockFailure = true;
+ }
+ if (lockFailure) {
+ releaseReadlock(owner);
+ throw new LockAcquireFailedException(writelock);
+ }
+ return owner;
+ }
+
+ @VisibleForTesting
+ boolean writelockExists() throws IOException {
+ return fileSystem.exists(writelock);
+ }
+
+ /**
+ * Release the writelock if it is present.
+ * IOExceptions are downgraded to failures
+ * @return true if the lock was present and then released
+ */
+ @VisibleForTesting
+ boolean releaseReadlock(boolean owner) {
+ if (owner) {
+ try {
+ return fileSystem.delete(readlock, false);
+ } catch (IOException e) {
+ log.warn("IOException releasing writelock {} ", readlock, e);
+ }
+ }
+ return false;
+ }
+
+ private void saveConf(AggregateConf conf) throws IOException {
+ confTreeSerDeser.save(fileSystem, internal, conf.getInternal(), true);
+ confTreeSerDeser.save(fileSystem, resources, conf.getResources(), true);
+ confTreeSerDeser.save(fileSystem, app_conf, conf.getAppConf(), true);
+ }
+
+ private void loadConf(AggregateConf conf) throws IOException {
+ conf.setInternal(confTreeSerDeser.load(fileSystem, internal));
+ conf.setResources(confTreeSerDeser.load(fileSystem, resources));
+ conf.setAppConf(confTreeSerDeser.load(fileSystem, app_conf));
+ }
+
+
+ private void maybeExecLockHeldAction(LockHeldAction action) throws
+ IOException,
+ SliderException {
+ if (action != null) {
+ action.execute();
+ }
+ }
+
+ /**
+ * Save the configuration
+ * @param conf configuration to fill in
+ * @param action
+ * @throws IOException IO problems
+ * @throws LockAcquireFailedException the lock could not be acquired
+ */
+ public void save(AggregateConf conf, LockHeldAction action) throws
+ IOException,
+ SliderException,
+ LockAcquireFailedException {
+ acquireWritelock();
+ try {
+ saveConf(conf);
+ maybeExecLockHeldAction(action);
+ } finally {
+ releaseWritelock();
+ }
+ }
+
+ /**
+ * Load the configuration. If a lock failure is raised, the
+ * contents of the configuration MAY have changed -lock race conditions
+ * are looked for on exit
+ * @param conf configuration to fill in
+ * @throws IOException IO problems
+ * @throws LockAcquireFailedException the lock could not be acquired
+ */
+ public void load(AggregateConf conf) throws
+ FileNotFoundException,
+ IOException,
+ SliderException,
+ LockAcquireFailedException {
+ boolean owner = acquireReadLock();
+ try {
+ loadConf(conf);
+ } finally {
+ releaseReadlock(owner);
+ }
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/ConfTreeSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/ConfTreeSerDeser.java
new file mode 100644
index 0000000..8271ef1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/ConfTreeSerDeser.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+import org.apache.slider.core.conf.ConfTree;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.JsonMappingException;
+
+import java.io.IOException;
+
+/**
+ * Conf tree to JSON binding
+ */
+public class ConfTreeSerDeser extends JsonSerDeser<ConfTree> {
+ public ConfTreeSerDeser() {
+ super(ConfTree.class);
+ }
+
+
+ private static final ConfTreeSerDeser staticinstance = new ConfTreeSerDeser();
+
+ /**
+ * Convert a tree instance to a JSON string -sync access to a shared ser/deser
+ * object instance
+ * @param instance object to convert
+ * @return a JSON string description
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public static String toString(ConfTree instance) throws IOException,
+ JsonGenerationException,
+ JsonMappingException {
+ synchronized (staticinstance) {
+ return staticinstance.toJson(instance);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/Filenames.java b/slider-core/src/main/java/org/apache/slider/core/persist/Filenames.java
new file mode 100644
index 0000000..06ecc51
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/Filenames.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+public interface Filenames {
+
+ String RESOURCES = "resources.json";
+ String APPCONF = "app_config.json";
+ String INTERNAL = "internal.json";
+ String WRITELOCK = "writelock";
+ String READLOCK = "readlock";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java b/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
new file mode 100644
index 0000000..c0af279
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.SliderKeys;
+
+/**
+ * Build up all the paths of an instance relative to the supplied instance
+ * directory.
+ */
+public class InstancePaths {
+
+ public final Path instanceDir;
+ public final Path snapshotConfPath;
+ public final Path generatedConfPath;
+ public final Path historyPath;
+ public final Path dataPath;
+ public final Path tmpPath;
+ public final Path tmpPathAM;
+
+ public InstancePaths(Path instanceDir) {
+ this.instanceDir = instanceDir;
+ snapshotConfPath =
+ new Path(instanceDir, SliderKeys.SNAPSHOT_CONF_DIR_NAME);
+ generatedConfPath =
+ new Path(instanceDir, SliderKeys.GENERATED_CONF_DIR_NAME);
+ historyPath = new Path(instanceDir, SliderKeys.HISTORY_DIR_NAME);
+ dataPath = new Path(instanceDir, SliderKeys.DATA_DIR_NAME);
+ tmpPath = new Path(instanceDir, SliderKeys.TMP_DIR_PREFIX);
+ tmpPathAM = new Path(tmpPath, "appmaster");
+ }
+
+ @Override
+ public String toString() {
+ return "instance at " + instanceDir;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
new file mode 100644
index 0000000..57663ad
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Support for marshalling objects to and from JSON.
+ * This class is NOT thread safe; it constructs an object mapper
+ * as an instance field.
+ * @param <T>
+ */
+public class JsonSerDeser<T> {
+
+ private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class);
+ private static final String UTF_8 = "UTF-8";
+
+ private final Class classType;
+ private final ObjectMapper mapper;
+
+ /**
+ * Create an instance bound to a specific type
+ * @param classType
+ */
+ public JsonSerDeser(Class classType) {
+ this.classType = classType;
+ this.mapper = new ObjectMapper();
+ mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ /**
+ * Convert from JSON
+ * @param json input
+ * @return the parsed JSON
+ * @throws IOException IO
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromJson(String json)
+ throws IOException, JsonParseException, JsonMappingException {
+ try {
+ return (T) (mapper.readValue(json, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json : " + e + "\n" + json, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from a JSON file
+ * @param jsonFile input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromFile(File jsonFile)
+ throws IOException, JsonParseException, JsonMappingException {
+ try {
+ return (T) (mapper.readValue(jsonFile, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json file {}: {}", jsonFile, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Convert from a JSON file
+ * @param resource input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public T fromResource(String resource)
+ throws IOException, JsonParseException, JsonMappingException {
+ InputStream resStream = null;
+ try {
+ resStream = this.getClass().getResourceAsStream(resource);
+ if (resStream == null) {
+ throw new FileNotFoundException(resource);
+ }
+
+ return (T) (mapper.readValue(resStream, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json resource {}: {}", resource, e);
+ throw e;
+ } finally {
+ IOUtils.closeStream(resStream);
+ }
+ }
+
+ /**
+ * clone by converting to JSON and back again.
+ * This is much less efficient than any Java clone process.
+ * @param instance instance to duplicate
+ * @return a new instance
+ * @throws IOException problems.
+ */
+ public T fromInstance(T instance) throws IOException {
+ return fromJson(toJson(instance));
+ }
+
+ /**
+ * Deserialize from a byte array
+ * @param b
+ * @return
+ * @throws IOException
+ */
+ public T fromBytes(byte[] b) throws IOException {
+ String json = new String(b, 0, b.length, UTF_8);
+ return fromJson(json);
+ }
+
+ /**
+ * Load from a Hadoop filesystem
+ * @param fs filesystem
+ * @param path path
+ * @return a loaded CD
+ * @throws IOException IO problems
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public T load(FileSystem fs, Path path)
+ throws IOException, JsonParseException, JsonMappingException {
+ FileStatus status = fs.getFileStatus(path);
+ long len = status.getLen();
+ byte[] b = new byte[(int) len];
+ FSDataInputStream dataInputStream = fs.open(path);
+ int count = dataInputStream.read(b);
+ if (count != len) {
+ throw new EOFException("Read finished prematurely");
+ }
+ return fromBytes(b);
+ }
+
+
+ /**
+ * Save a cluster description to a hadoop filesystem
+ * @param fs filesystem
+ * @param path path
+ * @param overwrite should any existing file be overwritten
+ * @throws IOException IO exception
+ */
+ public void save(FileSystem fs, Path path, T instance,
+ boolean overwrite) throws
+ IOException {
+ FSDataOutputStream dataOutputStream = fs.create(path, overwrite);
+ writeJsonAsBytes(instance, dataOutputStream);
+ }
+
+ /**
+ * Write the json as bytes -then close the file
+ * @param dataOutputStream an outout stream that will always be closed
+ * @throws IOException on any failure
+ */
+ private void writeJsonAsBytes(T instance,
+ DataOutputStream dataOutputStream) throws
+ IOException {
+ try {
+ String json = toJson(instance);
+ byte[] b = json.getBytes(UTF_8);
+ dataOutputStream.write(b);
+ } finally {
+ dataOutputStream.close();
+ }
+ }
+
+
+ /**
+ * Convert an object to a JSON string
+ * @param o object to convert
+ * @return a JSON string description
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public String toJson(T instance) throws IOException,
+ JsonGenerationException,
+ JsonMappingException {
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ return mapper.writeValueAsString(instance);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/LockAcquireFailedException.java b/slider-core/src/main/java/org/apache/slider/core/persist/LockAcquireFailedException.java
new file mode 100644
index 0000000..da58520
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/LockAcquireFailedException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+import org.apache.hadoop.fs.Path;
+
+public class LockAcquireFailedException extends Exception {
+
+ private final Path path;
+
+ public LockAcquireFailedException(Path path) {
+ super("Failed to acquire lock " +path);
+ this.path = path;
+ }
+
+ public LockAcquireFailedException(Path path, Throwable cause) {
+ super("Failed to acquire lock " + path, cause);
+ this.path = path;
+ }
+
+ public Path getPath() {
+ return path;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/LockHeldAction.java b/slider-core/src/main/java/org/apache/slider/core/persist/LockHeldAction.java
new file mode 100644
index 0000000..6659687
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/LockHeldAction.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.io.IOException;
+
+/**
+ * Optional action to add while the lock is held; this is needed to execute
+ * some other persistent operations within the scope at the same lock
+ * without inserting too much code into the persister
+ */
+public interface LockHeldAction {
+
+ /**
+ * Execute the action
+ * @throws IOException on any failure
+ */
+ public void execute() throws IOException, SliderException;
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java b/slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
new file mode 100644
index 0000000..1964459
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+public class PersistKeys {
+
+ public static final String SCHEMA =
+ "http://example.org/specification/v2.0.0";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/YARNRegistryClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/YARNRegistryClient.java
new file mode 100644
index 0000000..31147ef
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/YARNRegistryClient.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.client.SliderYarnClientImpl;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Client code for interacting with a registry of service instances.
+ * The initial logic just enumerates service instances in the YARN RM
+ */
+public class YARNRegistryClient {
+
+ final SliderYarnClientImpl yarnClient;
+ final String username;
+ final Configuration conf;
+
+
+ public YARNRegistryClient(SliderYarnClientImpl yarnClient,
+ String username,
+ Configuration conf) {
+ this.yarnClient = yarnClient;
+ this.username = username;
+ this.conf = conf;
+ }
+
+ /**
+ * find all live instances of a specific app -if there is >1 in the cluster,
+ * this returns them all. State should be running or less
+ * @param appname application name
+ * @return the list of all matching application instances
+ */
+ public List<ApplicationReport> findAllLiveInstances(String appname)
+ throws YarnException, IOException {
+
+ return yarnClient.findAllLiveInstances(username, appname);
+ }
+
+
+ /**
+ * Find an instance of a application belong to the current user
+ * @param appname application name
+ * @return the app report or null if none is found
+ * @throws YarnException YARN issues
+ * @throws IOException IO problems
+ */
+ public ApplicationReport findInstance(String appname) throws
+ YarnException,
+ IOException {
+ List<ApplicationReport> instances = listInstances();
+ return yarnClient.findClusterInInstanceList(instances, appname);
+ }
+
+ /**
+ * List instances belonging to a specific user
+ * @return a possibly empty list of AMs
+ */
+ public List<ApplicationReport> listInstances()
+ throws YarnException, IOException {
+ return yarnClient.listInstances(username);
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
new file mode 100644
index 0000000..fb77363
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Represents a set of configurations for an application, component, etc.
+ * Json serialisable; accessors are synchronized
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedConfigSet {
+
+ public Map<String, PublishedConfiguration> configurations =
+ new HashMap<String, PublishedConfiguration>();
+
+ public void put(String name, PublishedConfiguration conf) {
+ configurations.put(name, conf);
+ }
+
+ public PublishedConfiguration get(String name) {
+ return configurations.get(name);
+ }
+
+ public boolean contains(String name) {
+ return configurations.containsKey(name);
+ }
+
+ public int size() {
+ return configurations.size();
+ }
+
+ public Set<String> keys() {
+ TreeSet<String> keys = new TreeSet<String>();
+ keys.addAll(configurations.keySet());
+ return keys;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
new file mode 100644
index 0000000..59ebe83
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have
+ * to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedConfiguration {
+
+ public String description;
+
+ public int elements;
+
+ public long updated;
+
+ public String updatedTime;
+
+ public void setUpdated(long updated) {
+ this.updated = updated;
+ this.updatedTime = new Date(updated).toString();
+ }
+
+ public long getUpdated() {
+ return updated;
+ }
+
+ @JsonIgnore
+ private Map<String, String> values = new HashMap<String, String>();
+
+ /**
+ * Set the values from an iterable (this includes a Hadoop Configuration
+ * and Java properties object).
+ * Any existing value set is discarded
+ * @param entries entries to put
+ */
+ @JsonIgnore
+ public void putValues(Iterable<Map.Entry<String, String>> entries) {
+ values = new HashMap<String, String>();
+ for (Map.Entry<String, String> entry : entries) {
+ values.put(entry.getKey(), entry.getValue());
+ }
+
+ }
+
+ /**
+ * Convert to Hadoop XML
+ * @return
+ */
+ public Configuration asConfiguration() {
+ Configuration conf = new Configuration(false);
+ try {
+ ConfigHelper.addConfigMap(conf, values, "");
+ } catch (BadConfigException e) {
+ // triggered on a null value; switch to a runtime (and discard the stack)
+ throw new RuntimeException(e.toString());
+ }
+ return conf;
+ }
+
+ public String asConfigurationXML() throws IOException {
+ return ConfigHelper.toXml(asConfiguration());
+ }
+
+ /**
+ * Convert values to properties
+ * @return a property file
+ */
+ public Properties asProperties() {
+ Properties props = new Properties();
+ props.putAll(values);
+ return props;
+ }
+
+ /**
+ * Return the values as json string
+ * @return
+ * @throws IOException
+ */
+ public String asJson() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ String json = mapper.writeValueAsString(values);
+ return json;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedContent.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedContent.java
new file mode 100644
index 0000000..67a9b35
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedContent.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+public class PublishedContent {
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java
new file mode 100644
index 0000000..a286ba4
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/CommonRegistryConstants.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.info;
+
+public class CommonRegistryConstants {
+
+ public static final String WEB_UI = "org.apache.http.UI";
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
new file mode 100644
index 0000000..1eb87c6
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.info;
+
+/**
+ * These are constants unique to the Slider AM
+ */
+public class CustomRegistryConstants {
+
+ public static final String MANAGEMENT_REST_API =
+ "org.apache.slider.management";
+ public static final String REGISTRY_REST_API =
+ "org.apache.slider.registry";
+
+ public static final String PUBLISHER_REST_API =
+ "org.apache.slider.publisher";
+
+ public static final String AGENT_REST_API =
+ "org.apache.slider.publisher";
+
+ public static final String AM_IPC_PROTOCOL =
+ "org.apache.slider.appmaster";
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java
new file mode 100644
index 0000000..cde282b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.info;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * A registry document
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class RegisteredDocument {
+ public String contentType;
+ public String url;
+ public String description;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
new file mode 100644
index 0000000..1a34655
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.info;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class RegisteredEndpoint {
+
+ // standard types
+
+ /**
+ * URL: {@value}
+ */
+ public static final String TYPE_URL = "url";
+
+ /**
+ * hostname: {@value}
+ */
+ public static final String TYPE_HOSTNAME = "hostname";
+
+ /**
+ * "hostname:port" pair: {@value}
+ */
+ public static final String TYPE_ADDRESS = "address";
+
+ // standard protocols
+
+ /**
+ * Generic TCP protocol: {@value}
+ */
+
+ public static final String PROTOCOL_TCP = "tcp";
+
+ /**
+ * Generic TCP protocol: {@value}
+ */
+
+ public static final String PROTOCOL_UDP = "udp";
+
+ /**
+ * HTTP: {@value}
+ */
+
+ public static final String PROTOCOL_HTTP = "http";
+
+ /**
+ * HTTPS: {@value}
+ */
+
+ public static final String PROTOCOL_HTTPS = "http";
+
+ /**
+ * Classic "Writable" Hadoop IPC: {@value}
+ */
+ public static final String PROTOCOL_HADOOP_RPC = "org.apache.hadoop.ipc.RPC";
+
+ /**
+ * Protocol buffer based Hadoop IPC: {@value}
+ */
+ public static final String PROTOCOL_HADOOP_PROTOBUF = "org.apache.hadoop.ipc.Protobuf";
+
+ public String value;
+ public String protocol = "";
+ public String type = "";
+ public String description = "";
+
+ public RegisteredEndpoint() {
+ }
+
+ public RegisteredEndpoint(String value,
+ String protocol,
+ String type,
+ String description) {
+ this.value = value;
+ this.protocol = protocol;
+ this.type = type;
+ this.description = description;
+ }
+
+ /**
+ * Build an endpoint instance from a URI, extracting
+ * the protocol from it
+ * @param uri URI to set the value to
+ * @param description description
+ */
+ public RegisteredEndpoint(URI uri,
+ String description) {
+
+ this.value = uri.toString();
+ this.protocol = uri.getScheme();
+ this.type = TYPE_URL;
+ this.description = description;
+ }
+ /**
+ * Build an endpoint instance from a URI, extracting
+ * the protocol from it
+ * @param uri URI to set the value to
+ * @param description description
+ */
+ public RegisteredEndpoint(InetSocketAddress address,
+ String protocol,
+ String description) {
+
+ this.value = address.toString();
+ this.protocol = protocol;
+ this.type = TYPE_ADDRESS;
+ this.description = description;
+ }
+
+ /**
+ * Build an endpoint instance from a URL, extracting
+ * the protocol from it
+ * @param url URL to set the value to
+ * @param description description
+ */
+ public RegisteredEndpoint(URL url,
+ String description) throws URISyntaxException {
+ this(url.toURI(), description);
+ }
+
+ /**
+ * Get the value as a URL
+ * @return URL of the value -if the value type is URL
+ * @throws SliderException if the value is of the wrong type, or unparsable
+ */
+ public URL asURL() throws SliderException {
+ verifyEndpointType(TYPE_URL);
+ try {
+ return new URL(value);
+ } catch (MalformedURLException e) {
+ throw new SliderException(-1, e,
+ "could not create a URL from %s : %s", value, e.toString());
+ }
+ }
+
+
+
+ public void verifyEndpointType(String desiredType) throws SliderException {
+ if (!type.equals(desiredType)) {
+ throw new SliderException(-1, "Body of endpoint is of type %s and not %s",
+ type, desiredType);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java
new file mode 100644
index 0000000..441a7de
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.info;
+
+/**
+ * Fieldnames in registry entries
+ */
+public class RegistryFields {
+
+ public static final String ENDPOINTS = "endpoints";
+ public static final String INTERNAL_VIEW = "internalView";
+ public static final String EXTERNAL_VIEW = "externalView";
+
+ public static final String PROTOCOL = "protocol";
+ public static final String VALUE = "value";
+ public static final String DESCRIPTION = "description";
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
new file mode 100644
index 0000000..61167e0
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.info;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class RegistryView {
+
+ /**
+ * Endpoints
+ */
+ public Map<String, RegisteredEndpoint> endpoints = new HashMap<String, RegisteredEndpoint>(2);
+
+ public String configurationsURL;
+
+ public String documentsURL;
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
new file mode 100644
index 0000000..74b70f8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.info;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.Serializable;
+
+/**
+ * Service instance data to serialize with JSON
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ServiceInstanceData implements Serializable {
+
+ public String name;
+ public String id;
+ public long registrationTimeUTC;
+
+ /**
+ * Anything can go into the internal view, it's specific
+ * to the provider
+ */
+ public RegistryView internalView = new RegistryView();
+
+ /**
+ * External view of the service: public information
+ */
+ public RegistryView externalView = new RegistryView();
+
+
+}
+
+
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/zk/BlockingZKWatcher.java b/slider-core/src/main/java/org/apache/slider/core/registry/zk/BlockingZKWatcher.java
new file mode 100644
index 0000000..68c617c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/zk/BlockingZKWatcher.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.zk;
+
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class BlockingZKWatcher implements Watcher {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(BlockingZKWatcher.class);
+ private final AtomicBoolean connectedFlag = new AtomicBoolean(false);
+
+ @Override
+ public void process(WatchedEvent event) {
+ log.info("ZK binding callback received");
+ connectedFlag.set(true);
+ synchronized (connectedFlag) {
+ try {
+ connectedFlag.notify();
+ } catch (Exception e) {
+ log.warn("failed while waiting for notification", e);
+ }
+ }
+ }
+
+ /**
+ * Wait for a flag to go true
+ * @param timeout timeout in millis
+ */
+
+ public void waitForZKConnection(int timeout) throws InterruptedException {
+ synchronized (connectedFlag) {
+ if (!connectedFlag.get()) {
+ log.info("waiting for ZK event");
+ //wait a bit
+ connectedFlag.wait(timeout);
+ }
+ }
+ assert connectedFlag.get();
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZKCallback.java b/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZKCallback.java
new file mode 100644
index 0000000..0fd137e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZKCallback.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.zk;
+
+import org.apache.zookeeper.Watcher;
+
+/**
+ * Relays ZK watcher events to a closure
+ */
+public abstract class ZKCallback implements Watcher {
+
+ public ZKCallback() {
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZKIntegration.java b/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZKIntegration.java
new file mode 100644
index 0000000..d039b2e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZKIntegration.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.zk;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+
+public class ZKIntegration implements Watcher {
+
+/**
+ * Base path for services
+ */
+ public static String ZK_SERVICES = "services";
+ /**
+ * Base path for all Slider references
+ */
+ public static String ZK_SLIDER = "slider";
+ public static String ZK_USERS = "users";
+ public static String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
+ public static String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
+
+ public static List<String> ZK_USERS_PATH_LIST = new ArrayList<String>();
+ static {
+ ZK_USERS_PATH_LIST.add(ZK_SERVICES);
+ ZK_USERS_PATH_LIST.add(ZK_SLIDER);
+ ZK_USERS_PATH_LIST.add(ZK_USERS);
+ }
+
+ public static int SESSION_TIMEOUT = 5000;
+ protected static final Logger log =
+ LoggerFactory.getLogger(ZKIntegration.class);
+ private ZooKeeper zookeeper;
+ private final String username;
+ private final String clustername;
+ private final String userPath;
+ private int sessionTimeout = SESSION_TIMEOUT;
+/**
+ flag to set to indicate that the user path should be created if
+ it is not already there
+ */
+ private final AtomicBoolean toInit = new AtomicBoolean(false);
+ private final boolean createClusterPath;
+ private final Watcher watchEventHandler;
+ private final String zkConnection;
+ private final boolean canBeReadOnly;
+
+ protected ZKIntegration(String zkConnection,
+ String username,
+ String clustername,
+ boolean canBeReadOnly,
+ boolean createClusterPath,
+ Watcher watchEventHandler
+ ) throws IOException {
+ this.username = username;
+ this.clustername = clustername;
+ this.watchEventHandler = watchEventHandler;
+ this.zkConnection = zkConnection;
+ this.canBeReadOnly = canBeReadOnly;
+ this.createClusterPath = createClusterPath;
+ this.userPath = mkSliderUserPath(username);
+ }
+
+ public void init() throws IOException {
+ assert zookeeper == null;
+ log.debug("Binding ZK client to {}", zkConnection);
+ zookeeper = new ZooKeeper(zkConnection, sessionTimeout, this, canBeReadOnly);
+ }
+
+ /**
+ * Create an instance bonded to the specific closure
+ * @param zkConnection
+ * @param username
+ * @param clustername
+ * @param canBeReadOnly
+ * @param watchEventHandler
+ * @return
+ * @throws IOException
+ */
+ public static ZKIntegration newInstance(String zkConnection, String username, String clustername, boolean createClusterPath, boolean canBeReadOnly, Watcher watchEventHandler) throws IOException {
+
+ return new ZKIntegration(zkConnection,
+ username,
+ clustername,
+ canBeReadOnly,
+ createClusterPath,
+ watchEventHandler);
+ }
+
+ public String getConnectionString() {
+ return zkConnection;
+ }
+
+ public String getClusterPath() {
+ return mkClusterPath(username, clustername);
+ }
+
+ public boolean getConnected() {
+ return zookeeper.getState().isConnected();
+ }
+
+ public boolean getAlive() {
+ return zookeeper.getState().isAlive();
+ }
+
+ public ZooKeeper.States getState() {
+ return zookeeper.getState();
+ }
+
+ public Stat getClusterStat() throws KeeperException, InterruptedException {
+ return stat(getClusterPath());
+ }
+
+ public boolean exists(String path) throws
+ KeeperException,
+ InterruptedException {
+ return stat(path) != null;
+ }
+
+ public Stat stat(String path) throws KeeperException, InterruptedException {
+ return zookeeper.exists(path, false);
+ }
+
+ @Override
+ public String toString() {
+ return "ZK integration bound @ " + zkConnection + ": " + zookeeper;
+ }
+
+/**
+ * Event handler to notify of state events
+ * @param event
+ */
+ @Override
+ public void process(WatchedEvent event) {
+ log.debug("{}", event);
+ try {
+ maybeInit();
+ } catch (Exception e) {
+ log.error("Failed to init", e);
+ }
+ if (watchEventHandler != null) {
+ watchEventHandler.process(event);
+ }
+ }
+
+ private void maybeInit() throws KeeperException, InterruptedException {
+ if (!toInit.getAndSet(true) && createClusterPath) {
+ log.debug("initing");
+ //create the user path
+ mkPath(ZK_USERS_PATH_LIST, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ //create the specific user
+ createPath(userPath, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ }
+ }
+
+ /**
+ * Create a path under a parent, don't care if it already exists
+ * As the path isn't returned, this isn't the way to create sequentially
+ * numbered nodes.
+ * @param parent parent dir. Must have a trailing / if entry!=null||empty
+ * @param entry entry -can be null or "", in which case it is not appended
+ * @param acl
+ * @param createMode
+ * @return the path if created; null if not
+ */
+ public String createPath(String parent,
+ String entry,
+ List<ACL> acl,
+ CreateMode createMode) throws KeeperException, InterruptedException {
+ //initial create of full path
+ assert acl != null;
+ assert acl.size() > 0;
+ assert parent != null;
+ String path = parent;
+ if (entry != null) {
+ path = path + entry;
+ }
+ try {
+ log.debug("Creating ZK path {}", path);
+ return zookeeper.create(path, null, acl, createMode);
+ } catch (KeeperException.NodeExistsException ignored) {
+ //node already there
+ log.debug("node already present:{}",path);
+ return null;
+ }
+ }
+
+ /**
+ * Recursive path create
+ * @param path
+ * @param data
+ * @param acl
+ * @param createMode
+ */
+ public void mkPath(List<String> paths,
+ List<ACL> acl,
+ CreateMode createMode) throws KeeperException, InterruptedException {
+ String history = "/";
+ for (String entry : paths) {
+ createPath(history, ((String) entry), acl, createMode);
+ history = history + entry + "/";
+ }
+ }
+
+/**
+ * Blocking enum of users
+ * @return an unordered list of clusters under a user
+ */
+ public List<String> getClusters() throws KeeperException, InterruptedException {
+ return zookeeper.getChildren(userPath, (Watcher) null);
+ }
+
+ /**
+ * Delete a node, does not throw an exception if the path is not fond
+ * @param path path to delete
+ * @return true if the path could be deleted, false if there was no node to delete
+ *
+ */
+ public boolean delete(String path) throws
+ InterruptedException,
+ KeeperException {
+ try {
+ zookeeper.delete(path, -1);
+ return true;
+ } catch (KeeperException.NoNodeException ignored) {
+ return false;
+ }
+ }
+
+/**
+ * Build the path to a cluster; exists once the cluster has come up.
+ * Even before that, a ZK watcher could wait for it.
+ * @param username user
+ * @param clustername name of the cluster
+ * @return a strin
+ */
+ public static String mkClusterPath(String username, String clustername) {
+ return mkSliderUserPath(username) + "/" + clustername;
+ }
+/**
+ * Build the path to a cluster; exists once the cluster has come up.
+ * Even before that, a ZK watcher could wait for it.
+ * @param username user
+ * @return a string
+ */
+ public static String mkSliderUserPath(String username) {
+ return SVC_SLIDER_USERS + "/" + username;
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZKPathBuilder.java b/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZKPathBuilder.java
new file mode 100644
index 0000000..c822749
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZKPathBuilder.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.zk;
+
+import java.util.Locale;
+
+public final class ZKPathBuilder {
+
+ private final String username, appname, clustername;
+ private final String quorum;
+
+ private String appPath;
+ private String registryPath;
+ private final String appQuorum;
+
+ public ZKPathBuilder(String username,
+ String appname,
+ String clustername,
+ String quorum,
+ String appQuorum) {
+ this.username = username;
+ this.appname = appname;
+ this.clustername = clustername;
+ this.quorum = quorum;
+ appPath = buildAppPath();
+ registryPath = buildRegistryPath();
+ this.appQuorum = appQuorum;
+ }
+
+ public String buildAppPath() {
+ return String.format(Locale.ENGLISH, "/yarnapps_%s_%s_%s", appname,
+ username, clustername);
+
+ }
+
+ public String buildRegistryPath() {
+ return String.format(Locale.ENGLISH, "/services_%s_%s_%s", appname,
+ username, clustername);
+
+ }
+
+ public String getQuorum() {
+ return quorum;
+ }
+
+ public String getAppQuorum() {
+ return appQuorum;
+ }
+
+ public String getAppPath() {
+ return appPath;
+ }
+
+ public void setAppPath(String appPath) {
+ this.appPath = appPath;
+ }
+
+ public String getRegistryPath() {
+ return registryPath;
+ }
+
+ public void setRegistryPath(String registryPath) {
+ this.registryPath = registryPath;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZookeeperUtils.java b/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZookeeperUtils.java
new file mode 100644
index 0000000..a998cf1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/zk/ZookeeperUtils.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.zk;
+
+import com.google.common.net.HostAndPort;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ZookeeperUtils {
+
+ public static String buildConnectionString(String zkHosts, int port) {
+ String zkPort = Integer.toString(port);
+ //parse the hosts
+ String[] hostlist = zkHosts.split(",", 0);
+ String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",");
+ //this quorum has a trailing comma
+ quorum = quorum.substring(0, quorum.length() - 1);
+ return quorum;
+ }
+
+ /**
+ * Take a quorum list and split it to (trimmed) pairs
+ * @param hostPortQuorumList list of form h1:port, h2:port2,...
+ * @return a possibly empty list of values between commas. They may not be
+ * valid hostname:port pairs
+ */
+ public static List<String> splitToPairs(String hostPortQuorumList) {
+ // split an address hot
+ String[] strings = StringUtils.getStrings(hostPortQuorumList);
+ List<String> tuples = new ArrayList<String>(strings.length);
+ for (String s : strings) {
+ tuples.add(s.trim());
+ }
+ return tuples;
+ }
+
+ /**
+ * Split a quorum list into a list of hostnames and ports
+ * @param hostPortQuorumList split to a list of hosts and ports
+ * @return a list of values
+ */
+ public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) {
+ // split an address hot
+ String[] strings = StringUtils.getStrings(hostPortQuorumList);
+ List<HostAndPort> list = new ArrayList<HostAndPort>(strings.length);
+ for (String s : strings) {
+ list.add(HostAndPort.fromString(s.trim()));
+ }
+ return list;
+ }
+
+ /**
+ * Build up to a hosts only list
+ * @param hostAndPorts
+ * @return a list of the hosts only
+ */
+ public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) {
+ StringBuilder sb = new StringBuilder();
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ sb.append(hostAndPort.getHostText()).append(",");
+ }
+ if (sb.length() > 0) {
+ sb.delete(sb.length() - 1, sb.length());
+ }
+ return sb.toString();
+ }
+
+ public static String buildQuorumEntry(HostAndPort hostAndPort,
+ int defaultPort) {
+ String s = hostAndPort.toString();
+ if (hostAndPort.hasPort()) {
+ return s;
+ } else {
+ return s + ":" + defaultPort;
+ }
+ }
+
+ /**
+ * Build a quorum list, injecting a ":defaultPort" ref if needed on
+ * any entry without one
+ * @param hostAndPorts
+ * @param defaultPort
+ * @return
+ */
+ public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
+ List<String> entries = new ArrayList<String>(hostAndPorts.size());
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ entries.add(buildQuorumEntry(hostAndPort, defaultPort));
+ }
+ return SliderUtils.join(entries, ",", false);
+ }
+
+ public static String convertToHostsOnlyList(String quorum) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
+ return ZookeeperUtils.buildHostsOnlyList(hostAndPorts);
+ }
+
+ public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts =
+ ZookeeperUtils.splitToHostsAndPorts(quorum);
+ if (hostAndPorts.isEmpty()) {
+ throw new BadConfigException("empty zookeeper quorum");
+ }
+ return hostAndPorts;
+ }
+
+ public static int getFirstPort(String quorum, int defVal) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
+ int port = hostAndPorts.get(0).getPortOrDefault(defVal);
+ return port;
+
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
new file mode 100644
index 0000000..1736636
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
+import static org.apache.slider.api.ResourceKeys.DEF_YARN_CORES;
+import static org.apache.slider.api.ResourceKeys.DEF_YARN_MEMORY;
+import static org.apache.slider.api.ResourceKeys.YARN_CORES;
+import static org.apache.slider.api.ResourceKeys.YARN_MEMORY;
+
+public abstract class AbstractClientProvider extends Configured {
+ private static final Logger log =
+ LoggerFactory.getLogger(AbstractClientProvider.class);
+ protected static final ProviderUtils providerUtils =
+ new ProviderUtils(log);
+
+ public static final String PROVIDER_RESOURCE_BASE =
+ "org/apache/slider/providers/";
+ public static final String PROVIDER_RESOURCE_BASE_ROOT =
+ "/" + PROVIDER_RESOURCE_BASE;
+
+ public AbstractClientProvider(Configuration conf) {
+ super(conf);
+ }
+
+ public abstract String getName();
+
+ public abstract List<ProviderRole> getRoles();
+
+ /**
+ * Validate the instance definition.
+ * @param clusterSpec
+ */
+ public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ SliderException {
+
+ List<ProviderRole> roles = getRoles();
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+ for (ProviderRole role : roles) {
+ String name = role.name;
+ MapOperations component = resources.getComponent(name);
+ if (component != null) {
+ String instances = component.get(COMPONENT_INSTANCES);
+ if (instances == null) {
+ String message = "No instance count provided for " + name;
+ log.error("{} with \n{}", message, resources.toString());
+ throw new BadClusterStateException(message);
+ }
+ String ram = component.get(YARN_MEMORY);
+ String cores = component.get(YARN_CORES);
+
+
+ providerUtils.getRoleResourceRequirement(ram,
+ DEF_YARN_MEMORY,
+ Integer.MAX_VALUE);
+ providerUtils.getRoleResourceRequirement(cores,
+ DEF_YARN_CORES,
+ Integer.MAX_VALUE);
+ }
+ }
+ }
+
+
+ /**
+ * Any provider-side alteration of a configuration can take place here.
+ * @param aggregateConf config to patch
+ * @throws IOException IO problems
+ * @throws SliderException Slider-specific issues
+ */
+ public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws
+ SliderException,
+ IOException {
+ //default: do nothing
+ }
+
+
+ /**
+ * Prepare the AM settings for launch
+ * @param fileSystem filesystem
+ * @param serviceConf configuration of the client
+ * @param launcher launcher to set up
+ * @param instanceDescription instance description being launched
+ * @param snapshotConfDirPath
+ * @param generatedConfDirPath
+ * @param clientConfExtras
+ * @param libdir
+ * @param tempPath
+ * @param miniClusterTestRun flag set to true on a mini cluster run
+ * @throws IOException
+ * @throws SliderException
+ */
+ public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
+ Configuration serviceConf,
+ AbstractLauncher launcher,
+ AggregateConf instanceDescription,
+ Path snapshotConfDirPath,
+ Path generatedConfDirPath,
+ Configuration clientConfExtras,
+ String libdir,
+ Path tempPath,
+ boolean miniClusterTestRun)
+ throws IOException, SliderException {
+
+ }
+
+ /**
+ * Load in and merge in templates. Null arguments means "no such template"
+ * @param instanceConf instance to patch
+ * @param internalTemplate patch to internal.json
+ * @param resourceTemplate path to resources.json
+ * @param appConfTemplate path to app_conf.json
+ * @throws IOException any IO problems
+ */
+ protected void mergeTemplates(AggregateConf instanceConf,
+ String internalTemplate,
+ String resourceTemplate,
+ String appConfTemplate) throws IOException {
+ if (internalTemplate != null) {
+ ConfTreeOperations template =
+ ConfTreeOperations.fromResource(internalTemplate);
+ instanceConf.getInternalOperations()
+ .mergeWithoutOverwrite(template.confTree);
+ }
+
+ if (resourceTemplate != null) {
+ ConfTreeOperations resTemplate =
+ ConfTreeOperations.fromResource(resourceTemplate);
+ instanceConf.getResourceOperations()
+ .mergeWithoutOverwrite(resTemplate.confTree);
+ }
+
+ if (appConfTemplate != null) {
+ ConfTreeOperations template =
+ ConfTreeOperations.fromResource(appConfTemplate);
+ instanceConf.getAppConfOperations()
+ .mergeWithoutOverwrite(template.confTree);
+ }
+
+ }
+
+ /**
+ * This is called pre-launch to validate that the cluster specification
+ * is valid. This can include checking that the security options
+ * are in the site files prior to launch, that there are no conflicting operations
+ * etc.
+ *
+ * This check is made prior to every launch of the cluster -so can
+ * pick up problems which manually edited cluster files have added,
+ * or from specification files from previous versions.
+ *
+ * The provider MUST NOT change the remote specification. This is
+ * purely a pre-launch validation of options.
+ *
+ *
+ * @param sliderFileSystem filesystem
+ * @param clustername name of the cluster
+ * @param configuration cluster configuration
+ * @param instanceDefinition cluster specification
+ * @param clusterDirPath directory of the cluster
+ * @param generatedConfDirPath path to place generated artifacts
+ * @param secure flag to indicate that the cluster is secure
+ * @throws SliderException on any validation issue
+ * @throws IOException on any IO problem
+ */
+ public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
+ String clustername,
+ Configuration configuration,
+ AggregateConf instanceDefinition,
+ Path clusterDirPath,
+ Path generatedConfDirPath,
+ boolean secure) throws
+ SliderException,
+ IOException {
+ validateInstanceDefinition(instanceDefinition);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
new file mode 100644
index 0000000..ea875ad
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.main.ExitCodeProvider;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
+import org.apache.slider.server.services.curator.RegistryBinderService;
+import org.apache.slider.server.services.docstore.utility.ForkedProcessService;
+import org.apache.slider.server.services.docstore.utility.Parent;
+import org.apache.slider.server.services.docstore.utility.SequenceService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The base class for provider services. It lets the implementations
+ * add sequences of operations, and propagates service failures
+ * upstream
+ */
+public abstract class AbstractProviderService
+ extends SequenceService
+ implements
+ ProviderCore,
+ SliderKeys,
+ ProviderService {
+ private static final Logger log =
+ LoggerFactory.getLogger(AbstractProviderService.class);
+ protected AggregateConf instanceDefinition;
+ protected StateAccessForProviders stateAccessor;
+ protected AgentRestOperations restOps;
+ protected RegistryBinderService<ServiceInstanceData> registry;
+
+ public AbstractProviderService(String name) {
+ super(name);
+ }
+
+ @Override
+ public Configuration getConf() {
+ return getConfig();
+ }
+
+ public StateAccessForProviders getStateAccessor() {
+ return stateAccessor;
+ }
+
+ public void setStateAccessor(StateAccessForProviders stateAccessor) {
+ this.stateAccessor = stateAccessor;
+ }
+
+ @Override
+ public void bind(StateAccessForProviders stateAccessor,
+ RegistryBinderService<ServiceInstanceData> registry) {
+ this.stateAccessor = stateAccessor;
+ this.registry = registry;
+ }
+
+ @Override
+ public AgentRestOperations getAgentRestOperations() {
+ return restOps;
+ }
+
+ public void setAgentRestOperations(AgentRestOperations agentRestOperations) {
+ this.restOps = agentRestOperations;
+ }
+
+ /**
+ * Load a specific XML configuration file for the provider config
+ * @param confDir configuration directory
+ * @param siteXMLFilename provider-specific filename
+ * @return a configuration to be included in status
+ * @throws BadCommandArgumentsException argument problems
+ * @throws IOException IO problems
+ */
+ protected Configuration loadProviderConfigurationInformation(File confDir,
+ String siteXMLFilename)
+ throws BadCommandArgumentsException, IOException {
+ Configuration siteConf;
+ File siteXML = new File(confDir, siteXMLFilename);
+ if (!siteXML.exists()) {
+ throw new BadCommandArgumentsException(
+ "Configuration directory %s doesn't contain %s - listing is %s",
+ confDir, siteXMLFilename, SliderUtils.listDir(confDir));
+ }
+
+ //now read it in
+ siteConf = ConfigHelper.loadConfFromFile(siteXML);
+ log.info("{} file is at {}", siteXMLFilename, siteXML);
+ log.info(ConfigHelper.dumpConfigToString(siteConf));
+ return siteConf;
+ }
+
+ /**
+ * No-op implementation of this method.
+ *
+ * {@inheritDoc}
+ */
+ @Override
+ public void validateApplicationConfiguration(AggregateConf instance,
+ File confDir,
+ boolean secure) throws
+ IOException,
+ SliderException {
+
+ }
+
+ /**
+ * Scan through the roles and see if it is supported.
+ * @param role role to look for
+ * @return true if the role is known about -and therefore
+ * that a launcher thread can be deployed to launch it
+ */
+ @Override
+ public boolean isSupportedRole(String role) {
+ Collection<ProviderRole> roles = getRoles();
+ for (ProviderRole providedRole : roles) {
+ if (providedRole.name.equals(role)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ @Override // ExitCodeProvider
+ public int getExitCode() {
+ Throwable cause = getFailureCause();
+ if (cause != null) {
+ //failed for some reason
+ if (cause instanceof ExitCodeProvider) {
+ return ((ExitCodeProvider) cause).getExitCode();
+ }
+ }
+ ForkedProcessService lastProc = latestProcess();
+ if (lastProc == null) {
+ return 0;
+ } else {
+ return lastProc.getExitCode();
+ }
+ }
+
+ /**
+ * Return the latest forked process service that ran
+ * @return the forkes service
+ */
+ protected ForkedProcessService latestProcess() {
+ Service current = getCurrentService();
+ Service prev = getPreviousService();
+
+ Service latest = current != null ? current : prev;
+ if (latest instanceof ForkedProcessService) {
+ return (ForkedProcessService) latest;
+ } else {
+ //its a composite object, so look inside it for a process
+ if (latest instanceof Parent) {
+ return getFPSFromParentService((Parent) latest);
+ } else {
+ //no match
+ return null;
+ }
+ }
+ }
+
+
+ /**
+ * Given a parent service, find the one that is a forked process
+ * @param parent parent
+ * @return the forked process service or null if there is none
+ */
+ protected ForkedProcessService getFPSFromParentService(Parent parent) {
+ List<Service> services = parent.getServices();
+ for (Service s : services) {
+ if (s instanceof ForkedProcessService) {
+ return (ForkedProcessService) s;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * if we are already running, start this service
+ */
+ protected void maybeStartCommandSequence() {
+ if (isInState(STATE.STARTED)) {
+ startNextService();
+ }
+ }
+
+ /**
+ * Create a new forked process service with the given
+ * name, environment and command list -then add it as a child
+ * for execution in the sequence.
+ *
+ * @param name command name
+ * @param env environment
+ * @param commands command line
+ * @throws IOException
+ * @throws SliderException
+ */
+ protected ForkedProcessService queueCommand(String name,
+ Map<String, String> env,
+ List<String> commands) throws
+ IOException,
+ SliderException {
+ ForkedProcessService process = buildProcess(name, env, commands);
+ //register the service for lifecycle management; when this service
+ //is terminated, so is the master process
+ addService(process);
+ return process;
+ }
+
+ public ForkedProcessService buildProcess(String name,
+ Map<String, String> env,
+ List<String> commands) throws
+ IOException,
+ SliderException {
+ ForkedProcessService process;
+ process = new ForkedProcessService(name);
+ process.init(getConfig());
+ process.build(env, commands);
+ return process;
+ }
+
+
+ /*
+ * Build the provider status, can be empty
+ * @return the provider status - map of entries to add to the info section
+ */
+ @Override
+ public Map<String, String> buildProviderStatus() {
+ return new HashMap<String, String>();
+ }
+
+ /* non-javadoc
+ * @see org.apache.slider.providers.ProviderService#buildMonitorDetails(org.apache.slider.api.ClusterDescription)
+ */
+ @Override
+ public Map<String,URL> buildMonitorDetails(ClusterDescription clusterDesc) {
+ return Collections.emptyMap();
+ }
+
+ protected String getInfoAvoidingNull(ClusterDescription clusterDesc, String key) {
+ String value = clusterDesc.getInfo(key);
+
+ return null == value ? "N/A" : value;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
new file mode 100644
index 0000000..444c041
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+/**
+ * Placement values
+ */
+public class PlacementPolicy {
+
+ public static final int DEFAULT = 0;
+ public static final int EXCLUDE_FROM_FLEXING = 1;
+ public static final int NO_DATA_LOCALITY = 2;
+ public static final int ANTI_AFFINITY_REQUIRED = 4;
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicyOptions.java b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicyOptions.java
new file mode 100644
index 0000000..e61f944
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicyOptions.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+public enum PlacementPolicyOptions {
+
+ EXCLUDE_FROM_FLEXING,
+ NO_DATA_LOCALITY,
+ ANTI_AFFINITY_REQUIRED,
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
new file mode 100644
index 0000000..f1883fc
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.util.List;
+public interface ProviderCore {
+
+ String getName();
+
+ List<ProviderRole> getRoles();
+
+ Configuration getConf();
+
+ void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ SliderException;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
new file mode 100644
index 0000000..7502267
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+/**
+ * Provider role and key for use in app requests.
+ *
+ * This class uses the role name as the key for hashes and in equality tests,
+ * and ignores the other values.
+ */
+public final class ProviderRole {
+ public final String name;
+ public final int id;
+ public final int placementPolicy;
+
+ public ProviderRole(String name, int id) {
+ this(name, id, PlacementPolicy.DEFAULT);
+ }
+
+ public ProviderRole(String name, int id, int policy) {
+ this.name = name;
+ this.id = id;
+ this.placementPolicy = policy;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ ProviderRole that = (ProviderRole) o;
+ if (!name.equals(that.name)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "ProviderRole{" +
+ "name='" + name + '\'' +
+ ", id=" + id +
+ ", policy=" + placementPolicy +
+ '}';
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
new file mode 100644
index 0000000..bf771f3
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.main.ExitCodeProvider;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
+import org.apache.slider.server.services.curator.RegistryBinderService;
+import org.apache.slider.server.services.docstore.utility.EventCallback;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Map;
+
+public interface ProviderService extends ProviderCore, Service,
+ ExitCodeProvider {
+
+ /**
+ * Set up the entire container launch context
+ * @param containerLauncher
+ * @param instanceDefinition
+ * @param container
+ * @param role
+ * @param sliderFileSystem
+ * @param generatedConfPath
+ * @param appComponent
+ * @param containerTmpDirPath
+ */
+ void buildContainerLaunchContext(ContainerLauncher containerLauncher,
+ AggregateConf instanceDefinition,
+ Container container,
+ String role,
+ SliderFileSystem sliderFileSystem,
+ Path generatedConfPath,
+ MapOperations resourceComponent,
+ MapOperations appComponent,
+ Path containerTmpDirPath) throws
+ IOException,
+ SliderException;
+
+ /**
+ * Execute a process in the AM
+ * @param instanceDefinition cluster description
+ * @param confDir configuration directory
+ * @param env environment
+ * @param execInProgress the callback for the exec events
+ * @return true if a process was actually started
+ * @throws IOException
+ * @throws SliderException
+ */
+ boolean exec(AggregateConf instanceDefinition,
+ File confDir,
+ Map<String, String> env,
+ EventCallback execInProgress) throws IOException,
+ SliderException;
+
+ /**
+ * Scan through the roles and see if it is supported.
+ * @param role role to look for
+ * @return true if the role is known about -and therefore
+ * that a launcher thread can be deployed to launch it
+ */
+ boolean isSupportedRole(String role);
+
+ /**
+ * Load a specific XML configuration file for the provider config
+ * @param confDir configuration directory
+ * @param siteXMLFilename provider-specific filename
+ * @return a configuration to be included in status
+ * @throws BadCommandArgumentsException
+ * @throws IOException
+ */
+ Configuration loadProviderConfigurationInformation(File confDir)
+ throws BadCommandArgumentsException, IOException;
+
+ /**
+ * This is a validation of the application configuration on the AM.
+ * Here is where things like the existence of keytabs and other
+ * not-seen-client-side properties can be tested, before
+ * the actual process is spawned.
+ * @param instanceDefinition clusterSpecification
+ * @param confDir configuration directory
+ * @param secure flag to indicate that secure mode checks must exist
+ * @throws IOException IO problemsn
+ * @throws SliderException any failure
+ */
+ void validateApplicationConfiguration(AggregateConf instanceDefinition,
+ File confDir,
+ boolean secure
+ ) throws IOException, SliderException;
+
+ /*
+ * Build the provider status, can be empty
+ * @return the provider status - map of entries to add to the info section
+ */
+ Map<String, String> buildProviderStatus();
+
+ /**
+ * Build a map of data intended for the AM webapp that is specific
+ * about this provider. The key is some text to be displayed, and the
+ * value can be a URL that will create an anchor over the key text.
+ *
+ * If no anchor is needed/desired, insert the key with a null value.
+ * @return
+ */
+ Map<String,URL> buildMonitorDetails(ClusterDescription clusterSpec);
+
+ /**
+ * bind operation -invoked before the service is started
+ * @param stateAccessor interface offering read access to the state
+ * @param registry
+ */
+ void bind(StateAccessForProviders stateAccessor,
+ RegistryBinderService<ServiceInstanceData> registry);
+
+ /**
+ * Returns the agent rest operations interface.
+ * @return the interface if available, null otherwise.
+ */
+ AgentRestOperations getAgentRestOperations();
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
new file mode 100644
index 0000000..4a9cb6d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -0,0 +1,478 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.SliderInternalStateException;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/**
+ * this is a factoring out of methods handy for providers. It's bonded to a log at
+ * construction time
+ */
+public class ProviderUtils implements RoleKeys {
+
+ protected final Logger log;
+
+ /**
+ * Create an instace
+ * @param log log directory to use -usually the provider
+ */
+
+ public ProviderUtils(Logger log) {
+ this.log = log;
+ }
+
+ /**
+ * Add oneself to the classpath. This does not work
+ * on minicluster test runs where the JAR is not built up
+ * @param providerResources map of provider resources to add these entries to
+ * @param provider provider to add
+ * @param jarName name of the jar to use
+ * @param sliderFileSystem target filesystem
+ * @param tempPath path in the cluster FS for temp files
+ * @param libdir relative directory to place resources
+ * @param miniClusterTestRun
+ * @return true if the class was found in a JAR
+ *
+ * @throws FileNotFoundException if the JAR was not found and this is NOT
+ * a mini cluster test run
+ * @throws IOException IO problems
+ * @throws SliderException any Slider problem
+ */
+ public static boolean addProviderJar(Map<String, LocalResource> providerResources,
+ Object provider,
+ String jarName,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libdir,
+ boolean miniClusterTestRun) throws
+ IOException,
+ SliderException {
+ try {
+ SliderUtils.putJar(providerResources,
+ sliderFileSystem,
+ provider.getClass(),
+ tempPath,
+ libdir,
+ jarName);
+ return true;
+ } catch (FileNotFoundException e) {
+ if (miniClusterTestRun) {
+ return false;
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Add a set of dependencies to the provider resources being built up,
+ * by copying them from the local classpath to the remote one, then
+ * registering them
+ * @param providerResources map of provider resources to add these entries to
+ * @param sliderFileSystem target filesystem
+ * @param tempPath path in the cluster FS for temp files
+ * @param libdir relative directory to place resources
+ * @param resources list of resource names (e.g. "hbase.jar"
+ * @param classes list of classes where classes[i] refers to a class in
+ * resources[i]
+ * @throws IOException IO problems
+ * @throws SliderException any Slider problem
+ */
+ public static void addDependencyJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libdir,
+ String[] resources,
+ Class[] classes
+ ) throws
+ IOException,
+ SliderException {
+ if (resources.length != classes.length) {
+ throw new SliderInternalStateException(
+ "mismatch in Jar names [%d] and classes [%d]",
+ resources.length,
+ classes.length);
+ }
+ int size = resources.length;
+ for (int i = 0; i < size; i++) {
+ String jarName = resources[i];
+ Class clazz = classes[i];
+ SliderUtils.putJar(providerResources,
+ sliderFileSystem,
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ }
+
+ }
+
+ /**
+ * build the log directory
+ * @return the log dir
+ */
+ public String getLogdir() throws IOException {
+ String logdir = System.getenv("LOGDIR");
+ if (logdir == null) {
+ logdir =
+ SliderKeys.TMP_LOGDIR_PREFIX + UserGroupInformation.getCurrentUser().getShortUserName();
+ }
+ return logdir;
+ }
+
+
+ public void validateNodeCount(AggregateConf instanceDescription,
+ String name, int min, int max) throws
+ BadCommandArgumentsException {
+ MapOperations component =
+ instanceDescription.getResourceOperations().getComponent(name);
+ int count;
+ if (component == null) {
+ count = 0;
+ } else {
+ count = component.getOptionInt(ResourceKeys.COMPONENT_INSTANCES, 0);
+ }
+ validateNodeCount(name, count, min, max);
+ }
+
+ /**
+ * Validate the node count and heap size values of a node class
+ *
+ * @param name node class name
+ * @param count requested node count
+ * @param min requested heap size
+ * @param max
+ * @throws BadCommandArgumentsException if the values are out of range
+ */
+ public void validateNodeCount(String name,
+ int count,
+ int min,
+ int max) throws BadCommandArgumentsException {
+ if (count < min) {
+ throw new BadCommandArgumentsException(
+ "requested no of %s nodes: %d is below the minimum of %d", name, count,
+ min);
+ }
+ if (max > 0 && count > max) {
+ throw new BadCommandArgumentsException(
+ "requested no of %s nodes: %d is above the maximum of %d", name, count,
+ max);
+ }
+ }
+
+ /**
+ * copy all options beginning site. into the site.xml
+ * @param clusterSpec cluster specification
+ * @param sitexml map for XML file to build up
+ */
+ public void propagateSiteOptions(ClusterDescription clusterSpec,
+ Map<String, String> sitexml) {
+ Map<String, String> options = clusterSpec.options;
+ propagateSiteOptions(options, sitexml);
+ }
+
+ public void propagateSiteOptions(Map<String, String> options,
+ Map<String, String> sitexml) {
+ propagateSiteOptions(options, sitexml, "");
+ }
+
+ public void propagateSiteOptions(Map<String, String> options,
+ Map<String, String> sitexml,
+ String configName) {
+ propagateSiteOptions(options, sitexml, configName, null);
+ }
+
+ public void propagateSiteOptions(Map<String, String> options,
+ Map<String, String> sitexml,
+ String configName,
+ Map<String,String> tokenMap) {
+ String prefix = OptionKeys.SITE_XML_PREFIX +
+ (configName.length() > 0 ? configName + "." : "");
+ for (Map.Entry<String, String> entry : options.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(prefix)) {
+ String envName = key.substring(prefix.length());
+ if (!envName.isEmpty()) {
+ String value = entry.getValue();
+ if (tokenMap != null) {
+ for (Map.Entry<String,String> token : tokenMap.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ token.getValue());
+ }
+ }
+ sitexml.put(envName, value);
+ }
+ }
+ }
+ }
+
+ /**
+ * Propagate an option from the cluster specification option map
+ * to the site XML map, using the site key for the name
+ * @param clusterSpec cluster specification
+ * @param optionKey key in the option map
+ * @param sitexml map for XML file to build up
+ * @param siteKey key to assign the value to in the site XML
+ * @throws BadConfigException if the option is missing from the cluster spec
+ */
+ public void propagateOption(MapOperations global,
+ String optionKey,
+ Map<String, String> sitexml,
+ String siteKey) throws BadConfigException {
+ sitexml.put(siteKey, global.getMandatoryOption(optionKey));
+ }
+
+
+ /**
+ * Build the image dir. This path is relative and only valid at the far end
+ * @param clusterSpec cluster spec
+ * @param bindir bin subdir
+ * @param script script in bin subdir
+ * @return the path to the script
+ * @throws FileNotFoundException if a file is not found, or it is not a directory*
+ */
+ public String buildPathToHomeDir(AggregateConf instanceDefinition
+ ,
+ String bindir,
+ String script) throws
+ FileNotFoundException,
+ BadConfigException {
+ MapOperations globalOptions =
+ instanceDefinition.getInternalOperations().getGlobalOptions();
+ String applicationHome =
+ globalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME);
+ String imagePath =
+ globalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ return buildPathToHomeDir(imagePath, applicationHome, bindir, script);
+ }
+
+ public String buildPathToHomeDir(String imagePath,
+ String applicationHome,
+ String bindir, String script) throws
+ FileNotFoundException {
+ String path;
+ File scriptFile;
+ if (imagePath!=null) {
+ File tarball = new File(SliderKeys.LOCAL_TARBALL_INSTALL_SUBDIR);
+ scriptFile = findBinScriptInExpandedArchive(tarball, bindir, script);
+ // now work back from the script to build the relative path
+ // to the binary which will be valid remote or local
+ StringBuilder builder = new StringBuilder();
+ builder.append(SliderKeys.LOCAL_TARBALL_INSTALL_SUBDIR);
+ builder.append("/");
+ //for the script, we want the name of ../..
+ File archive = scriptFile.getParentFile().getParentFile();
+ builder.append(archive.getName());
+ path = builder.toString();
+
+ } else {
+ // using a home directory which is required to be present on
+ // the local system -so will be absolute and resolvable
+ File homedir = new File(applicationHome);
+ path = homedir.getAbsolutePath();
+
+ //this is absolute, resolve its entire path
+ SliderUtils.verifyIsDir(homedir, log);
+ File bin = new File(homedir, bindir);
+ SliderUtils.verifyIsDir(bin, log);
+ scriptFile = new File(bin, script);
+ SliderUtils.verifyFileExists(scriptFile, log);
+ }
+ return path;
+ }
+
+
+ /**
+ * Build the image dir. This path is relative and only valid at the far end
+ * @param internal internal options
+ * @param bindir bin subdir
+ * @param script script in bin subdir
+ * @return the path to the script
+ * @throws FileNotFoundException if a file is not found, or it is not a directory*
+ */
+ public String buildPathToScript(AggregateConf instance,
+ String bindir,
+ String script) throws FileNotFoundException {
+ return buildPathToScript(instance.getInternalOperations(), bindir, script);
+ }
+ /**
+ * Build the image dir. This path is relative and only valid at the far end
+ * @param internal internal options
+ * @param bindir bin subdir
+ * @param script script in bin subdir
+ * @return the path to the script
+ * @throws FileNotFoundException if a file is not found, or it is not a directory*
+ */
+ public String buildPathToScript(ConfTreeOperations internal,
+ String bindir,
+ String script) throws FileNotFoundException {
+
+ String homedir = buildPathToHomeDir(
+ internal.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH),
+ internal.get(OptionKeys.INTERNAL_APPLICATION_HOME),
+ bindir,
+ script);
+ return buildScriptPath(bindir, script, homedir);
+ }
+
+
+
+ public String buildScriptPath(String bindir, String script, String homedir) {
+ StringBuilder builder = new StringBuilder(homedir);
+ builder.append("/");
+ builder.append(bindir);
+ builder.append("/");
+ builder.append(script);
+ return builder.toString();
+ }
+
+
+ public static String convertToAppRelativePath(File file) {
+ return convertToAppRelativePath(file.getPath());
+ }
+
+ public static String convertToAppRelativePath(String path) {
+ return ApplicationConstants.Environment.PWD.$() + "/" + path;
+ }
+
+
+ public static void validatePathReferencesLocalDir(String meaning, String path)
+ throws BadConfigException {
+ File file = new File(path);
+ if (!file.exists()) {
+ throw new BadConfigException("%s directory %s not found", meaning, file);
+ }
+ if (!file.isDirectory()) {
+ throw new BadConfigException("%s is not a directory: %s", meaning, file);
+ }
+ }
+
+ /**
+ * get the user name
+ * @return the user name
+ */
+ public String getUserName() throws IOException {
+ return UserGroupInformation.getCurrentUser().getShortUserName();
+ }
+
+ /**
+ * Find a script in an expanded archive
+ * @param base base directory
+ * @param bindir bin subdir
+ * @param script script in bin subdir
+ * @return the path to the script
+ * @throws FileNotFoundException if a file is not found, or it is not a directory
+ */
+ public File findBinScriptInExpandedArchive(File base,
+ String bindir,
+ String script)
+ throws FileNotFoundException {
+
+ SliderUtils.verifyIsDir(base, log);
+ File[] ls = base.listFiles();
+ if (ls == null) {
+ //here for the IDE to be happy, as the previous check will pick this case
+ throw new FileNotFoundException("Failed to list directory " + base);
+ }
+
+ log.debug("Found {} entries in {}", ls.length, base);
+ List<File> directories = new LinkedList<File>();
+ StringBuilder dirs = new StringBuilder();
+ for (File file : ls) {
+ log.debug("{}", false);
+ if (file.isDirectory()) {
+ directories.add(file);
+ dirs.append(file.getPath()).append(" ");
+ }
+ }
+ if (directories.size() > 1) {
+ throw new FileNotFoundException(
+ "Too many directories in archive to identify binary: " + dirs);
+ }
+ if (directories.isEmpty()) {
+ throw new FileNotFoundException(
+ "No directory found in archive " + base);
+ }
+ File archive = directories.get(0);
+ File bin = new File(archive, bindir);
+ SliderUtils.verifyIsDir(bin, log);
+ File scriptFile = new File(bin, script);
+ SliderUtils.verifyFileExists(scriptFile, log);
+ return scriptFile;
+ }
+
+ /**
+ * Return any additional arguments (argv) to provide when starting this role
+ *
+ * @param roleOptions
+ * The options for this role
+ * @return A non-null String which contains command line arguments for this role, or the empty string.
+ */
+ public static String getAdditionalArgs(Map<String,String> roleOptions) {
+ if (roleOptions.containsKey(RoleKeys.ROLE_ADDITIONAL_ARGS)) {
+ String additionalArgs = roleOptions.get(RoleKeys.ROLE_ADDITIONAL_ARGS);
+ if (null != additionalArgs) {
+ return additionalArgs;
+ }
+ }
+
+ return "";
+ }
+
+ public int getRoleResourceRequirement(String val,
+ int defVal,
+ int maxVal) {
+ if (val==null) {
+ val = Integer.toString(defVal);
+ }
+ Integer intVal;
+ if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
+ intVal = maxVal;
+ } else {
+ intVal = Integer.decode(val);
+ }
+ return intVal;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java b/slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
new file mode 100644
index 0000000..4fc617f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for factories
+ */
+public abstract class SliderProviderFactory extends Configured {
+
+ public static final String DEFAULT_CLUSTER_TYPE = AgentKeys.PROVIDER_AGENT;
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(SliderProviderFactory.class);
+ public static final String PROVIDER_NOT_FOUND =
+ "Unable to find provider of application type %s";
+
+ public SliderProviderFactory(Configuration conf) {
+ super(conf);
+ }
+
+ protected SliderProviderFactory() {
+ }
+
+ public abstract AbstractClientProvider createClientProvider();
+
+ public abstract ProviderService createServerProvider();
+
+ /**
+ * Create a provider for a specific application
+ * @param application app
+ * @return app instance
+ * @throws SliderException on any instantiation problem
+ */
+ public static SliderProviderFactory createSliderProviderFactory(String application) throws
+ SliderException {
+ Configuration conf = loadSliderConfiguration();
+ if (application == null) {
+ application = DEFAULT_CLUSTER_TYPE;
+ }
+ String providerKey =
+ String.format(SliderXmlConfKeys.KEY_PROVIDER, application);
+ if (application.contains(".")) {
+ log.debug("Treating {} as a classname", application);
+ String name = "classname.key";
+ conf.set(name, application);
+ providerKey = name;
+ }
+
+ Class<? extends SliderProviderFactory> providerClass;
+ try {
+ providerClass = conf.getClass(providerKey, null, SliderProviderFactory.class);
+ } catch (RuntimeException e) {
+ throw new BadClusterStateException(e, "Failed to load provider %s: %s", application, e);
+ }
+ if (providerClass == null) {
+ throw new BadClusterStateException(PROVIDER_NOT_FOUND, application);
+ }
+
+ Exception ex;
+ try {
+ SliderProviderFactory providerFactory = providerClass.newInstance();
+ providerFactory.setConf(conf);
+ return providerFactory;
+ } catch (InstantiationException e) {
+ ex = e;
+ } catch (IllegalAccessException e) {
+ ex = e;
+ } catch (Exception e) {
+ ex = e;
+ }
+ //by here the operation failed and ex is set to the value
+ throw new BadClusterStateException(ex,
+ "Failed to create an instance of %s : %s",
+ providerClass,
+ ex);
+ }
+
+ public static Configuration loadSliderConfiguration() {
+ Configuration conf = new Configuration();
+ conf.addResource(SliderKeys.SLIDER_XML);
+ return conf;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
new file mode 100644
index 0000000..d8dc4d1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/** This class implements the client-side aspects of the agent deployer */
+public class AgentClientProvider extends AbstractClientProvider
+ implements AgentKeys, SliderKeys {
+
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(AgentClientProvider.class);
+ protected static final String NAME = "agent";
+ private static final ProviderUtils providerUtils = new ProviderUtils(log);
+
+
+ protected AgentClientProvider(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public List<ProviderRole> getRoles() {
+ return AgentRoles.getRoles();
+ }
+
+ @Override //Client
+ public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
+ String clustername,
+ Configuration configuration,
+ AggregateConf instanceDefinition,
+ Path clusterDirPath,
+ Path generatedConfDirPath,
+ boolean secure) throws
+ SliderException,
+ IOException {
+ super.preflightValidateClusterConfiguration(sliderFileSystem, clustername,
+ configuration,
+ instanceDefinition,
+ clusterDirPath,
+ generatedConfDirPath, secure);
+
+ String appDef = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ sliderFileSystem.verifyFileExists(new Path(appDef));
+
+ String agentConf = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getMandatoryOption(AgentKeys.AGENT_CONF);
+ sliderFileSystem.verifyFileExists(new Path(agentConf));
+
+ String appHome = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
+ if (SliderUtils.isUnset(appHome)) {
+ String agentImage = instanceDefinition.getInternalOperations().
+ get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ sliderFileSystem.verifyFileExists(new Path(agentImage));
+ }
+ }
+
+ @Override
+ public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ SliderException {
+ super.validateInstanceDefinition(instanceDefinition);
+ log.debug("Validating conf {}", instanceDefinition);
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+ ConfTreeOperations appConf =
+ instanceDefinition.getAppConfOperations();
+
+ providerUtils.validateNodeCount(instanceDefinition, ROLE_NODE,
+ 0, -1);
+
+ Set<String> names = resources.getComponentNames();
+ names.remove(SliderKeys.COMPONENT_AM);
+ Map<Integer, String> priorityMap = new HashMap<Integer, String>();
+ for (String name : names) {
+ MapOperations component = resources.getMandatoryComponent(name);
+
+ // Validate count against the metainfo.xml
+
+ int priority =
+ component.getMandatoryOptionInt(ResourceKeys.COMPONENT_PRIORITY);
+ if (priority <= 0) {
+ throw new BadConfigException("Component %s %s value out of range %d",
+ name,
+ ResourceKeys.COMPONENT_PRIORITY,
+ priority);
+ }
+
+ String existing = priorityMap.get(priority);
+ if (existing != null) {
+ throw new BadConfigException(
+ "Component %s has a %s value %d which duplicates that of %s",
+ name,
+ ResourceKeys.COMPONENT_PRIORITY,
+ priority,
+ existing);
+ }
+ priorityMap.put(priority, name);
+ }
+
+ try {
+ // Validate the app definition
+ instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ } catch (BadConfigException bce) {
+ throw new BadConfigException("Application definition must be provided." + bce.getMessage());
+ }
+
+ String appHome = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
+ String agentImage = instanceDefinition.getInternalOperations().
+ get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+
+ if (SliderUtils.isUnset(appHome) && SliderUtils.isUnset(agentImage)) {
+ throw new BadConfigException("Either agent package path " +
+ AgentKeys.PACKAGE_PATH + " or image root " +
+ OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH
+ + " must be provided");
+ }
+
+ try {
+ // Validate the agent config
+ instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getMandatoryOption(AgentKeys.AGENT_CONF);
+ } catch (BadConfigException bce) {
+ throw new BadConfigException("Agent config "+ AgentKeys.AGENT_CONF
+ + " property must be provided.");
+ }
+ }
+
+ @Override
+ public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
+ Configuration serviceConf,
+ AbstractLauncher launcher,
+ AggregateConf instanceDescription,
+ Path snapshotConfDirPath,
+ Path generatedConfDirPath,
+ Configuration clientConfExtras,
+ String libdir,
+ Path tempPath, boolean miniClusterTestRun) throws
+ IOException,
+ SliderException {
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
new file mode 100644
index 0000000..7136fd9
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+/*
+
+ */
+public interface AgentKeys {
+
+ String PROVIDER_AGENT = "agent";
+ String ROLE_NODE = "node";
+ /**
+ * {@value}
+ */
+ String CONF_FILE = "agent.conf";
+ /**
+ * {@value}
+ */
+ String REGION_SERVER = "regionserver";
+ /**
+ * What is the command for hbase to print a version: {@value}
+ */
+ String COMMAND_VERSION = "version";
+ String ACTION_START = "start";
+ String ACTION_STOP = "stop";
+ /**
+ * Config directory : {@value}
+ */
+ String ARG_CONFIG = "--config";
+ /**
+ * Template stored in the slider classpath -to use if there is
+ * no site-specific template
+ * {@value}
+ */
+ String CONF_RESOURCE = "org/apache/slider/providers/agent/conf/";
+ /* URL to talk back to Agent Controller*/
+ String CONTROLLER_URL = "agent.controller.url";
+ /**
+ * The location of pre-installed agent path.
+ * This can be also be dynamically computed based on Yarn installation of agent.
+ */
+ String PACKAGE_PATH = "agent.package.root";
+ /**
+ * The location of the script implementing the command.
+ */
+ String SCRIPT_PATH = "agent.script";
+ /**
+ * Execution home for the agent.
+ */
+ String APP_HOME = "app.home";
+ /**
+ * Name of the service.
+ */
+ String SERVICE_NAME = "app.name";
+ String ARG_LABEL = "--label";
+ String ARG_HOST = "--host";
+ String ARG_PORT = "--port";
+ String AGENT_MAIN_SCRIPT_ROOT = "./infra/agent/slider-agent/";
+ String AGENT_MAIN_SCRIPT = "agent/main.py";
+
+ String APP_DEF = "application.def";
+ String AGENT_VERSION = "agent.version";
+ String AGENT_CONF = "agent.conf";
+
+ String AGENT_INSTALL_DIR = "infra/agent";
+ String APP_DEFINITION_DIR = "app/definition";
+ String AGENT_CONFIG_FILE = "infra/conf/agent.ini";
+ String AGENT_VERSION_FILE = "infra/version";
+
+ String JAVA_HOME = "java_home";
+ String PACKAGE_LIST = "package_list";
+ String COMPONENT_SCRIPT = "role.script";
+ String WAIT_HEARTBEAT = "wait.heartbeat";
+ String PYTHON_EXE = "python";
+}
+
+
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java
new file mode 100644
index 0000000..d5ca749
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.providers.SliderProviderFactory;
+
+public class AgentProviderFactory extends SliderProviderFactory {
+
+ public static final String CLASSNAME =
+ "org.apache.slider.providers.agent.AgentProviderFactory";
+
+ public AgentProviderFactory() {
+ }
+
+ public AgentProviderFactory(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return new AgentClientProvider(getConf());
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return new AgentProviderService();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
new file mode 100644
index 0000000..d70825a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -0,0 +1,613 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
+import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
+import org.apache.slider.server.appmaster.web.rest.agent.CommandReport;
+import org.apache.slider.server.appmaster.web.rest.agent.ExecutionCommand;
+import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat;
+import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse;
+import org.apache.slider.server.appmaster.web.rest.agent.Register;
+import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
+import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
+import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand;
+import org.apache.slider.server.services.curator.CuratorServiceInstance;
+import org.apache.slider.server.services.docstore.utility.EventCallback;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.slider.core.registry.info.RegistryFields.DESCRIPTION;
+import static org.apache.slider.core.registry.info.RegistryFields.ENDPOINTS;
+import static org.apache.slider.core.registry.info.RegistryFields.EXTERNAL_VIEW;
+import static org.apache.slider.core.registry.info.RegistryFields.PROTOCOL;
+import static org.apache.slider.core.registry.info.RegistryFields.VALUE;
+
+/** This class implements the server-side aspects of an agent deployment */
+public class AgentProviderService extends AbstractProviderService implements
+ ProviderCore,
+ AgentKeys,
+ SliderKeys, AgentRestOperations {
+
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(AgentProviderService.class);
+ private static final ProviderUtils providerUtils = new ProviderUtils(log);
+ private static final String LABEL_MAKER = "___";
+ private AgentClientProvider clientProvider;
+ private Map<String, ComponentInstanceState> componentStatuses = new HashMap<String, ComponentInstanceState>();
+ private Map<String, List<String>> roleHostMapping = new HashMap<String, List<String>>();
+ private AtomicInteger taskId = new AtomicInteger(0);
+
+ public AgentProviderService() {
+ super("AgentProviderService");
+ setAgentRestOperations(this);
+ }
+
+ @Override
+ public List<ProviderRole> getRoles() {
+ return AgentRoles.getRoles();
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ clientProvider = new AgentClientProvider(conf);
+ }
+
+ @Override
+ public Configuration loadProviderConfigurationInformation(File confDir) throws
+ BadCommandArgumentsException,
+ IOException {
+ return new Configuration(false);
+ }
+
+ @Override
+ public void validateInstanceDefinition(AggregateConf instanceDefinition)
+ throws
+ SliderException {
+ clientProvider.validateInstanceDefinition(instanceDefinition);
+ }
+
+ @Override
+ public void buildContainerLaunchContext(ContainerLauncher launcher,
+ AggregateConf instanceDefinition,
+ Container container,
+ String role,
+ SliderFileSystem fileSystem,
+ Path generatedConfPath,
+ MapOperations resourceComponent,
+ MapOperations appComponent,
+ Path containerTmpDirPath) throws
+ IOException,
+ SliderException {
+
+ this.instanceDefinition = instanceDefinition;
+ log.info("Build launch context for Agent");
+ log.debug(instanceDefinition.toString());
+
+ // Set the environment
+ launcher.putEnv(SliderUtils.buildEnvMap(appComponent));
+
+ String workDir = ApplicationConstants.Environment.PWD.$();
+ launcher.setEnv("AGENT_WORK_ROOT", workDir);
+ log.info("AGENT_WORK_ROOT set to {}", workDir);
+ String logDir = ApplicationConstants.Environment.LOG_DIRS.$();
+ launcher.setEnv("AGENT_LOG_ROOT", logDir);
+ log.info("AGENT_LOG_ROOT set to {}", logDir);
+
+ //local resources
+
+ // TODO: Should agent need to support App Home
+ String scriptPath = new File(AgentKeys.AGENT_MAIN_SCRIPT_ROOT, AgentKeys.AGENT_MAIN_SCRIPT).getPath();
+ String appHome = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
+ if (SliderUtils.isSet(appHome)) {
+ scriptPath = new File(appHome, AgentKeys.AGENT_MAIN_SCRIPT).getPath();
+ }
+
+ String agentImage = instanceDefinition.getInternalOperations().
+ get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ if (agentImage != null) {
+ LocalResource agentImageRes = fileSystem.createAmResource(new Path(agentImage), LocalResourceType.ARCHIVE);
+ launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes);
+ }
+
+ log.info("Using {} for agent.", scriptPath);
+ String appDef = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ LocalResource appDefRes = fileSystem.createAmResource(
+ fileSystem.getFileSystem().resolvePath(new Path(appDef)),
+ LocalResourceType.ARCHIVE);
+ launcher.addLocalResource(AgentKeys.APP_DEFINITION_DIR, appDefRes);
+
+ String agentConf = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getMandatoryOption(AgentKeys.AGENT_CONF);
+ LocalResource agentConfRes = fileSystem.createAmResource(
+ fileSystem.getFileSystem().resolvePath(new Path(agentConf)),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(AgentKeys.AGENT_CONFIG_FILE, agentConfRes);
+
+ String agentVer = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getOption(AgentKeys.AGENT_VERSION, null);
+ if (agentVer != null) {
+ LocalResource agentVerRes = fileSystem.createAmResource(
+ fileSystem.getFileSystem().resolvePath(new Path(agentVer)),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(AgentKeys.AGENT_VERSION_FILE, agentVerRes);
+ }
+
+ String label = getContainerLabel(container, role);
+ setRoleHostMapping(role, container.getNodeId().getHost());
+ CommandLineBuilder operation = new CommandLineBuilder();
+
+ operation.add(AgentKeys.PYTHON_EXE);
+
+ operation.add(scriptPath);
+ operation.add(ARG_LABEL, label);
+ operation.add(ARG_HOST);
+ operation.add(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
+ operation.add(ARG_PORT);
+ operation.add(getClusterInfoPropertyValue(StatusKeys.INFO_AM_WEB_PORT));
+
+ launcher.addCommand(operation.build());
+
+ // initialize the component instance state
+ componentStatuses.put(label,
+ new ComponentInstanceState(
+ role,
+ container.getId().toString(),
+ getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
+ }
+
+ protected void setRoleHostMapping(String role, String host) {
+ List<String> hosts = roleHostMapping.get(role);
+ if (hosts == null) {
+ hosts = new ArrayList<String>();
+ }
+ hosts.add(host);
+ roleHostMapping.put(role, hosts);
+ }
+
+ private List<String> getHostsForRole(String role) {
+ return roleHostMapping.get(role);
+ }
+
+ private String getContainerLabel(Container container, String role) {
+ return container.getId().toString() + LABEL_MAKER + role;
+ }
+
+ protected String getClusterInfoPropertyValue(String name) {
+ StateAccessForProviders accessor = getStateAccessor();
+ assert accessor.isApplicationLive();
+ ClusterDescription description = accessor.getClusterStatus();
+ return description.getInfo(name);
+ }
+
+ /**
+ * Run this service
+ *
+ * @param instanceDefinition component description
+ * @param confDir local dir with the config
+ * @param env environment variables above those generated by
+ * @param execInProgress callback for the event notification
+ *
+ * @throws IOException IO problems
+ * @throws SliderException anything internal
+ */
+ @Override
+ public boolean exec(AggregateConf instanceDefinition,
+ File confDir,
+ Map<String, String> env,
+ EventCallback execInProgress) throws
+ IOException,
+ SliderException {
+
+ return false;
+ }
+
+ /**
+ * Build the provider status, can be empty
+ *
+ * @return the provider status - map of entries to add to the info section
+ */
+ public Map<String, String> buildProviderStatus() {
+ Map<String, String> stats = new HashMap<String, String>();
+ return stats;
+ }
+
+ @Override
+ public boolean isSupportedRole(String role) {
+ return true;
+ }
+
+ @Override
+ public RegistrationResponse handleRegistration(Register registration) {
+ // dummy impl
+ RegistrationResponse response = new RegistrationResponse();
+ String label = registration.getHostname();
+ if (componentStatuses.containsKey(label)) {
+ response.setResponseStatus(RegistrationStatus.OK);
+ } else {
+ response.setResponseStatus(RegistrationStatus.FAILED);
+ response.setLog("Label not recognized.");
+ }
+ return response;
+ }
+
+ private Command getCommand(String commandVal) {
+ if (commandVal.equals(Command.START.toString())) {
+ return Command.START;
+ }
+ if (commandVal.equals(Command.INSTALL.toString())) {
+ return Command.INSTALL;
+ }
+
+ return Command.NOP;
+ }
+
+ private CommandResult getCommandResult(String commandResVal) {
+ if (commandResVal.equals(CommandResult.COMPLETED.toString())) {
+ return CommandResult.COMPLETED;
+ }
+ if (commandResVal.equals(CommandResult.FAILED.toString())) {
+ return CommandResult.FAILED;
+ }
+ if (commandResVal.equals(CommandResult.IN_PROGRESS.toString())) {
+ return CommandResult.IN_PROGRESS;
+ }
+
+ throw new IllegalArgumentException("Unrecognized value " + commandResVal);
+ }
+
+ @Override
+ public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
+ // dummy impl
+ HeartBeatResponse response = new HeartBeatResponse();
+ long id = heartBeat.getResponseId();
+ response.setResponseId(id + 1L);
+
+ String label = heartBeat.getHostname();
+ String roleName = getRoleName(label);
+ StateAccessForProviders accessor = getStateAccessor();
+ String scriptPath;
+ scriptPath = accessor.getInstanceDefinitionSnapshot().
+ getAppConfOperations().getComponentOpt(roleName, AgentKeys.COMPONENT_SCRIPT, null);
+ if (scriptPath == null) {
+ log.error("role.script is unavailable for " + roleName + ". Commands will not be sent.");
+ return response;
+ }
+
+ if (!componentStatuses.containsKey(label)) {
+ return response;
+ }
+ ComponentInstanceState componentStatus = componentStatuses.get(label);
+
+ List<CommandReport> reports = heartBeat.getReports();
+ if (reports != null && !reports.isEmpty()) {
+ CommandReport report = reports.get(0);
+ CommandResult result = getCommandResult(report.getStatus());
+ Command command = getCommand(report.getRoleCommand());
+ componentStatus.applyCommandResult(result, command);
+ log.info("Component operation. Status: {}", result);
+ }
+
+ int waitForCount = accessor.getInstanceDefinitionSnapshot().
+ getAppConfOperations().getComponentOptInt(roleName, AgentKeys.WAIT_HEARTBEAT, 0);
+
+ if (id < waitForCount) {
+ log.info("Waiting until heartbeat count {}. Current val: {}", waitForCount, id);
+ componentStatuses.put(roleName, componentStatus);
+ return response;
+ }
+
+ Command command = componentStatus.getNextCommand();
+ if (Command.NOP != command) {
+ try {
+ componentStatus.commandIssued(command);
+ if (command == Command.INSTALL) {
+ log.info("Installing component ...");
+ addInstallCommand(roleName, response, scriptPath);
+ } else if (command == Command.START) {
+ log.info("Starting component ...");
+ addStartCommand(roleName, response, scriptPath);
+ }
+ } catch (SliderException e) {
+ componentStatus.applyCommandResult(CommandResult.FAILED, command);
+ log.warn("Component instance failed operation.", e);
+ }
+ }
+
+ return response;
+ }
+
+ private String getRoleName(String label) {
+ return label.substring(label.indexOf(LABEL_MAKER) + LABEL_MAKER.length());
+ }
+
+ protected void addInstallCommand(String roleName, HeartBeatResponse response, String scriptPath)
+ throws SliderException {
+ assert getStateAccessor().isApplicationLive();
+ ConfTreeOperations appConf = getStateAccessor().getAppConfSnapshot();
+ ConfTreeOperations resourcesConf = getStateAccessor().getResourcesSnapshot();
+ ConfTreeOperations internalsConf = getStateAccessor().getInternalsSnapshot();
+
+ ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
+ prepareExecutionCommand(cmd);
+ String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
+ cmd.setClusterName(clusterName);
+ cmd.setRoleCommand(Command.INSTALL.toString());
+ cmd.setServiceName(clusterName);
+ cmd.setComponentName(roleName);
+ cmd.setRole(roleName);
+ Map<String, String> hostLevelParams = new TreeMap<String, String>();
+ hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
+ hostLevelParams.put(PACKAGE_LIST, "[{\"type\":\"tarball\",\"name\":\"" +
+ appConf.getGlobalOptions().getMandatoryOption(
+ PACKAGE_LIST) + "\"}]");
+ cmd.setHostLevelParams(hostLevelParams);
+
+ setInstallCommandConfigurations(cmd);
+
+ cmd.setCommandParams(setCommandParameters(scriptPath, false));
+
+ cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
+ response.addExecutionCommand(cmd);
+ }
+
+ private void prepareExecutionCommand(ExecutionCommand cmd) {
+ cmd.setTaskId(taskId.incrementAndGet());
+ cmd.setCommandId(cmd.getTaskId() + "-1");
+ }
+
+ private Map<String, String> setCommandParameters(String scriptPath, boolean recordConfig) {
+ Map<String, String> cmdParams = new TreeMap<String, String>();
+ cmdParams.put("service_package_folder",
+ "${AGENT_WORK_ROOT}/work/app/definition/package");
+ cmdParams.put("script", scriptPath);
+ cmdParams.put("schema_version", "2.0");
+ cmdParams.put("command_timeout", "300");
+ cmdParams.put("script_type", "PYTHON");
+ cmdParams.put("record_config", Boolean.toString(recordConfig));
+ return cmdParams;
+ }
+
+ private void setInstallCommandConfigurations(ExecutionCommand cmd) {
+ ConfTreeOperations appConf = getStateAccessor().getAppConfSnapshot();
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
+ cmd.setConfigurations(configurations);
+ }
+
+ protected void addStatusCommand(String roleName, HeartBeatResponse response, String scriptPath)
+ throws SliderException {
+ assert getStateAccessor().isApplicationLive();
+ ConfTreeOperations appConf = getStateAccessor().getAppConfSnapshot();
+ ConfTreeOperations internalsConf = getStateAccessor().getInternalsSnapshot();
+
+ StatusCommand cmd = new StatusCommand();
+ String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
+
+ cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
+ cmd.setComponentName(roleName);
+ cmd.setServiceName(clusterName);
+ cmd.setClusterName(clusterName);
+ cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
+
+ Map<String, String> hostLevelParams = new TreeMap<String, String>();
+ hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
+ cmd.setHostLevelParams(hostLevelParams);
+
+ cmd.setCommandParams(setCommandParameters(scriptPath, false));
+
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
+
+ cmd.setConfigurations(configurations);
+
+ response.addStatusCommand(cmd);
+ }
+
+ protected void addGetConfigCommand(String roleName, HeartBeatResponse response)
+ throws SliderException {
+ assert getStateAccessor().isApplicationLive();
+ ConfTreeOperations internalsConf = getStateAccessor().getInternalsSnapshot();
+
+ StatusCommand cmd = new StatusCommand();
+ String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
+
+ cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
+ cmd.setComponentName(roleName);
+ cmd.setServiceName(clusterName);
+ cmd.setClusterName(clusterName);
+ cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND);
+
+ response.addStatusCommand(cmd);
+ }
+
+ protected void addStartCommand(String roleName, HeartBeatResponse response, String scriptPath) throws
+ SliderException {
+ assert getStateAccessor().isApplicationLive();
+ ConfTreeOperations appConf = getStateAccessor().getAppConfSnapshot();
+ ConfTreeOperations internalsConf = getStateAccessor().getInternalsSnapshot();
+
+ ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
+ prepareExecutionCommand(cmd);
+ String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
+ String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME);
+ cmd.setHostname(hostName);
+ cmd.setClusterName(clusterName);
+ cmd.setRoleCommand(Command.START.toString());
+ cmd.setServiceName(clusterName);
+ cmd.setComponentName(roleName);
+ cmd.setRole(roleName);
+ Map<String, String> hostLevelParams = new TreeMap<String, String>();
+ hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
+ cmd.setHostLevelParams(hostLevelParams);
+
+ cmd.setCommandParams(setCommandParameters(scriptPath, true));
+
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
+
+ cmd.setConfigurations(configurations);
+ response.addExecutionCommand(cmd);
+ }
+
+ private Map<String, Map<String, String>> buildCommandConfigurations(ConfTreeOperations appConf) {
+
+ Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
+ Map<String, String> tokens = getStandardTokenMap(appConf);
+
+ List<String> configs = getApplicationConfigurationTypes(appConf);
+
+ //Add global
+ for (String configType : configs) {
+ addNamedConfiguration(configType, appConf.getGlobalOptions().options,
+ configurations, tokens);
+ }
+
+ return configurations;
+ }
+
+ private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) {
+ Map<String, String> tokens = new HashMap<String, String>();
+ String nnuri = appConf.get("site.fs.defaultFS");
+ tokens.put("${NN_URI}", nnuri);
+ tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
+ tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
+ return tokens;
+ }
+
+ private List<String> getApplicationConfigurationTypes(ConfTreeOperations appConf) {
+ // for now, reading this from appConf. In the future, modify this method to
+ // process metainfo.xml
+ List<String> configList = new ArrayList<String>();
+ configList.add("global");
+
+ String configTypes = appConf.get("config_types");
+ String[] configs = configTypes.split(",");
+
+ configList.addAll(Arrays.asList(configs));
+
+ // remove duplicates. mostly worried about 'global' being listed
+ return new ArrayList<String>(new HashSet<String>(configList));
+ }
+
+ private void addNamedConfiguration(String configName, Map<String, String> sourceConfig,
+ Map<String, Map<String, String>> configurations,
+ Map<String, String> tokens) {
+ Map<String, String> config = new HashMap<String, String>();
+ if (configName.equals("global")) {
+ addDefaultGlobalConfig(config);
+ }
+ // add role hosts to tokens
+ addRoleRelatedTokens(tokens);
+ providerUtils.propagateSiteOptions(sourceConfig, config, configName, tokens);
+ configurations.put(configName, config);
+ }
+
+ protected void addRoleRelatedTokens(Map<String, String> tokens) {
+ for (Map.Entry<String, List<String>> entry : roleHostMapping.entrySet()) {
+ String tokenName = entry.getKey().toUpperCase(Locale.ENGLISH) + "_HOST";
+ String hosts = StringUtils.join(",", entry.getValue());
+ tokens.put("${" + tokenName + "}", hosts);
+ }
+ }
+
+ private void addDefaultGlobalConfig(Map<String, String> config) {
+ config.put("app_log_dir", "${AGENT_LOG_ROOT}/app/log");
+ config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run");
+ config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
+ }
+
+ @Override
+ public Map<String, URL> buildMonitorDetails(ClusterDescription clusterDesc) {
+ Map<String, URL> details = new LinkedHashMap<String, URL>();
+ buildEndpointDetails(details);
+ buildRoleHostDetails(details);
+ return details;
+ }
+
+ private void buildRoleHostDetails(Map<String, URL> details) {
+ for (Map.Entry<String, List<String>> entry : roleHostMapping.entrySet()) {
+ details.put(entry.getKey() + " Host(s): " + entry.getValue(),
+ null);
+ }
+ }
+
+ private void buildEndpointDetails(Map<String, URL> details) {
+ try {
+ List<CuratorServiceInstance<ServiceInstanceData>> services =
+ registry.listInstances(SliderKeys.APP_TYPE);
+ assert services.size() == 1;
+ CuratorServiceInstance<ServiceInstanceData> service = services.get(0);
+ Map payload = (Map) service.getPayload();
+ Map<String, Map<String, String>> endpoints =
+ (Map) ((Map) payload.get(EXTERNAL_VIEW)).get(ENDPOINTS);
+ for (Map.Entry<String, Map<String, String>> endpoint : endpoints.entrySet()) {
+ if ("http".equals(endpoint.getValue().get(PROTOCOL))) {
+ URL url = new URL(endpoint.getValue().get(VALUE));
+ details.put(endpoint.getValue().get(DESCRIPTION),
+ url);
+ }
+ }
+ } catch (IOException e) {
+ log.error("Error creating list of slider URIs", e);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
new file mode 100644
index 0000000..d8aefc6
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.providers.ProviderRole;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AgentRoles {
+
+ /**
+ * List of roles
+ */
+ protected static final List<ProviderRole> ROLES =
+ new ArrayList<ProviderRole>();
+
+ public static final int KEY_NODE =
+ SliderKeys.ROLE_AM_PRIORITY_INDEX + 1;
+ /**
+ * Initialize role list
+ */
+/*
+ static {
+ ROLES.add(new ProviderRole(AgentKeys.ROLE_NODE, KEY_NODE));
+ }
+*/
+
+
+ public static List<ProviderRole> getRoles() {
+ return ROLES;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java b/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
new file mode 100644
index 0000000..541dcc2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+/** The states a component instance can be. */
+public enum Command {
+ NOP, // do nothing
+ INSTALL, // Install the component
+ START // Start the component
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java b/slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
new file mode 100644
index 0000000..f318096
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+/** Command results. */
+public enum CommandResult {
+ IN_PROGRESS, // Command is in progress
+ COMPLETED, // Command has successfully completed
+ FAILED // Command has failed
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
new file mode 100644
index 0000000..eea77d7
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** This class implements a simple state machine for component instances. */
+public class ComponentInstanceState {
+ public static final Logger log =
+ LoggerFactory.getLogger(ComponentInstanceState.class);
+ private static int MAX_FAILURE_TOLERATED = 3;
+ private static String INVALID_TRANSITION_ERROR =
+ "Result {0} for command {1} is not expected for component {2} in state {3}.";
+ private State state = State.INIT;
+ private State targetState = State.STARTED;
+ private int failuresSeen = 0;
+ private final String compName;
+ private final String containerId;
+ private final String applicationId;
+
+
+ public ComponentInstanceState(String compName,
+ String containerId,
+ String applicationId) {
+ this.compName = compName;
+ this.containerId = containerId;
+ this.applicationId = applicationId;
+ }
+
+ public void commandIssued(Command command) {
+ Command expected = getNextCommand();
+ if (expected != command) {
+ throw new IllegalArgumentException("Command " + command + " is not allowed is state " + state);
+ }
+ this.state = this.state.getNextState(command);
+ }
+
+ public void applyCommandResult(CommandResult result, Command command) {
+ if (!this.state.couldHaveIssued(command)) {
+ throw new IllegalStateException("Invalid command " + command + " for state " + this.state);
+ }
+
+ try {
+ if (result == CommandResult.FAILED) {
+ failuresSeen++;
+ } else if (result == CommandResult.COMPLETED) {
+ failuresSeen = 0;
+ }
+ this.state = this.state.getNextState(result);
+ } catch (IllegalArgumentException e) {
+ String message = String.format(INVALID_TRANSITION_ERROR,
+ result.toString(),
+ command.toString(),
+ compName,
+ state.toString());
+ log.warn(message);
+ throw new IllegalStateException(message);
+ }
+ }
+
+ public boolean hasPendingCommand() {
+ if (this.state.canIssueCommands() &&
+ this.state != this.targetState &&
+ failuresSeen < MAX_FAILURE_TOLERATED) {
+ return true;
+ }
+
+ return false;
+ }
+
+ public Command getNextCommand() {
+ if (!hasPendingCommand()) {
+ return Command.NOP;
+ }
+
+ return this.state.getSupportedCommand();
+ }
+
+ public State getState() {
+ return state;
+ }
+
+ @VisibleForTesting
+ protected void setState(State state) {
+ this.state = state;
+ }
+
+ @Override
+ public int hashCode() {
+ int hashCode = 1;
+
+ hashCode = hashCode ^ (compName != null ? compName.hashCode() : 0);
+ hashCode = hashCode ^ (containerId != null ? containerId.hashCode() : 0);
+ hashCode = hashCode ^ (applicationId != null ? applicationId.hashCode() : 0);
+ return hashCode;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ComponentInstanceState that = (ComponentInstanceState) o;
+
+ if (this.compName != null ?
+ !this.compName.equals(that.compName) : this.compName != null) {
+ return false;
+ }
+
+ if (this.containerId != null ?
+ !this.containerId.equals(that.containerId) : this.containerId != null) {
+ return false;
+ }
+
+ if (this.applicationId != null ?
+ !this.applicationId.equals(that.applicationId) : this.applicationId != null) {
+ return false;
+ }
+
+ return true;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/State.java b/slider-core/src/main/java/org/apache/slider/providers/agent/State.java
new file mode 100644
index 0000000..09732a5
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/State.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+/** The states a component instance can be. */
+public enum State {
+ INIT, // Not installed
+ INSTALLING, // Being installed
+ INSTALLED, // Installed (or stopped)
+ STARTING, // Starting
+ STARTED, // Started
+ INSTALL_FAILED; // Install failed, start failure in INSTALLED
+
+ /**
+ * Indicates whether or not it is a valid state to produce a command.
+ *
+ * @return true if command can be issued for this state.
+ */
+ public boolean canIssueCommands() {
+ switch (this) {
+ case INSTALLING:
+ case STARTING:
+ case STARTED:
+ return false;
+ default:
+ return true;
+ }
+ }
+
+ /**
+ * Returns valid command in this state.
+ *
+ * @return command allowed in this state.
+ */
+ public Command getSupportedCommand() {
+ switch (this) {
+ case INIT:
+ case INSTALL_FAILED:
+ return Command.INSTALL;
+ case INSTALLED:
+ return Command.START;
+ default:
+ return Command.NOP;
+ }
+ }
+
+ /**
+ * Returns next state based on the command result.
+ *
+ * @return next state.
+ */
+ public State getNextState(CommandResult result) throws IllegalArgumentException {
+ switch (result) {
+ case IN_PROGRESS:
+ if (this == State.INSTALLING || this == State.STARTING) {
+ return this;
+ } else {
+ throw new IllegalArgumentException(result + " is not valid for " + this);
+ }
+ case COMPLETED:
+ if (this == State.INSTALLING) {
+ return State.INSTALLED;
+ } else if (this == State.STARTING) {
+ return State.STARTED;
+ } else {
+ throw new IllegalArgumentException(result + " is not valid for " + this);
+ }
+ case FAILED:
+ if (this == State.INSTALLING) {
+ return State.INSTALL_FAILED;
+ } else if (this == State.STARTING) {
+ return State.INSTALLED;
+ } else {
+ throw new IllegalArgumentException(result + " is not valid for " + this);
+ }
+ default:
+ throw new IllegalArgumentException("Bad command result " + result);
+ }
+ }
+
+ /**
+ * Returns next state based on the command.
+ *
+ * @return next state.
+ */
+ public State getNextState(Command command) throws IllegalArgumentException {
+ switch (command) {
+ case INSTALL:
+ if (this == State.INIT || this == State.INSTALL_FAILED) {
+ return State.INSTALLING;
+ } else {
+ throw new IllegalArgumentException(command + " is not valid for " + this);
+ }
+ case START:
+ if (this == State.INSTALLED) {
+ return State.STARTING;
+ } else {
+ throw new IllegalArgumentException(command + " is not valid for " + this);
+ }
+ case NOP:
+ return this;
+ default:
+ throw new IllegalArgumentException("Bad command " + command);
+ }
+ }
+
+ public boolean couldHaveIssued(Command command) {
+ if ((this == State.INSTALLING && command == Command.INSTALL) ||
+ (this == State.STARTING && command == Command.START)) {
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
new file mode 100644
index 0000000..74f0257
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+/**
+ *
+ */
+public class CommandScript {
+ String script;
+ String scriptType;
+ long timeout;
+
+ public String getScript() {
+ return script;
+ }
+
+ public void setScript(String script) {
+ this.script = script;
+ }
+
+ public String getScriptType() {
+ return scriptType;
+ }
+
+ public void setScriptType(String scriptType) {
+ this.scriptType = scriptType;
+ }
+
+ public long getTimeout() {
+ return timeout;
+ }
+
+ public void setTimeout(long timeout) {
+ this.timeout = timeout;
+ }
+
+ public CommandScript() {
+
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
new file mode 100644
index 0000000..391a4ee
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+/**
+ *
+ */
+public class Component {
+ String name;
+ String category;
+ CommandScript commandScript;
+
+ public Component() {
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getCategory() {
+ return category;
+ }
+
+ public void setCategory(String category) {
+ this.category = category;
+ }
+
+ public CommandScript getCommandScript() {
+ return commandScript;
+ }
+
+ public void addCommandScript(CommandScript commandScript) {
+ this.commandScript = commandScript;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java
new file mode 100644
index 0000000..57effac
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigurationDependencies.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class ConfigurationDependencies {
+ List<String> configTypes;
+
+ public ConfigurationDependencies() {
+ configTypes = new ArrayList<String>();
+ }
+
+ public void setConfigType(String type) {
+ configTypes.add(type);
+ }
+
+ public List<String> getConfigTypes() {
+ return configTypes;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
new file mode 100644
index 0000000..21e8b24
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ *
+ */
+public class Metainfo {
+ String schemaVersion;
+ List<Service> services;
+
+ public Metainfo() {
+ services = new ArrayList<Service>();
+ }
+
+ public String getSchemaVersion() {
+ return schemaVersion;
+ }
+
+ public void setSchemaVersion(String schemaVersion) {
+ this.schemaVersion = schemaVersion;
+ }
+
+ public void addService(Service service) {
+ services.add(service);
+ }
+
+ public List<Service> getServices() {
+ return Collections.unmodifiableList(services);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
new file mode 100644
index 0000000..370366b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.commons.digester.Digester;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class MetainfoParser {
+
+ public Metainfo parse(InputStream metainfoStream) throws IOException {
+ Digester digester = new Digester();
+ digester.setValidating(false);
+
+ digester.addObjectCreate("metainfo", Metainfo.class);
+ digester.addBeanPropertySetter("metainfo/schemaVersion");
+
+ digester.addObjectCreate("*/service", Service.class);
+ digester.addBeanPropertySetter("*/service/name");
+ digester.addBeanPropertySetter("*/service/comment");
+ digester.addBeanPropertySetter("*/service/version");
+
+ digester.addObjectCreate("*/component", Component.class);
+ digester.addBeanPropertySetter("*/component/name");
+ digester.addBeanPropertySetter("*/component/category");
+ digester.addSetNext("*/component", "addComponent");
+
+ digester.addObjectCreate("*/commandScript", CommandScript.class);
+ digester.addBeanPropertySetter("*/commandScript/script");
+ digester.addBeanPropertySetter("*/commandScript/scriptType");
+ digester.addBeanPropertySetter("*/commandScript/timeout");
+ digester.addSetNext("*/commandScript", "addCommandScript");
+
+ digester.addObjectCreate("*/osSpecific", OSSpecific.class);
+ digester.addBeanPropertySetter("*/osSpecific/osType");
+ digester.addObjectCreate("*/package", OSPackage.class);
+ digester.addBeanPropertySetter("*/package/type");
+ digester.addBeanPropertySetter("*/package/name");
+ digester.addSetNext("*/package", "addOSPackage");
+ digester.addSetNext("*/osSpecific", "addOSSpecific");
+
+ digester.addObjectCreate("*/configuration-dependencies",
+ ConfigurationDependencies.class);
+ digester.addBeanPropertySetter("*/config-type", "configType");
+ digester.addSetNext("*/configuration-dependencies", "setConfigDependencies");
+
+ digester.addSetNext("*/service", "addService");
+
+ try {
+ return (Metainfo) digester.parse(metainfoStream);
+ } catch (IOException e) {
+
+ } catch (SAXException e) {
+
+ } finally {
+ metainfoStream.close();
+ }
+
+ return null;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
new file mode 100644
index 0000000..334f96b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+/**
+ *
+ */
+public class OSPackage {
+ String type;
+ String name;
+
+ public OSPackage() {
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
new file mode 100644
index 0000000..7c36e8e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class OSSpecific {
+ String osType;
+ List<OSPackage> packages;
+
+ public OSSpecific() {
+ packages = new ArrayList<OSPackage>();
+ }
+
+ public String getOsType() {
+ return osType;
+ }
+
+ public void setOsType(String osType) {
+ this.osType = osType;
+ }
+
+ public void addOSPackage(OSPackage osPackage) {
+ packages.add(osPackage);
+ }
+
+ public List<OSPackage> getPackages() {
+ return packages;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Service.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Service.java
new file mode 100644
index 0000000..de738dd
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Service.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class Service {
+ String name;
+ String comment;
+ String version;
+ List<Component> components;
+ List<OSSpecific> osSpecifics;
+ ConfigurationDependencies configDependencies;
+
+ public Service() {
+ components = new ArrayList<Component>();
+ osSpecifics = new ArrayList<OSSpecific>();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public ConfigurationDependencies getConfigDependencies() {
+ return configDependencies;
+ }
+
+ public void setConfigDependencies(ConfigurationDependencies configDependencies) {
+ this.configDependencies = configDependencies;
+ }
+
+ public void addComponent(Component component) {
+ components.add(component);
+ }
+
+ public List<Component> getComponents() {
+ return components;
+ }
+
+ public void addOSSpecific(OSSpecific osSpecific) {
+ osSpecifics.add(osSpecific);
+ }
+
+ public List<OSSpecific> getOSSpecifics() {
+ return osSpecifics;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/todo.md b/slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
new file mode 100644
index 0000000..dfd1373
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
@@ -0,0 +1,22 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# todo list
+
+* Retry on failure
+ * Agent can toleate a configurable number of failures (e.g. 3) before giving up
+* Agent should separate out hostname and label that is received for registration
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
new file mode 100644
index 0000000..9b3b9ef
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.slideram;
+
+import com.beust.jcommander.JCommander;
+import com.google.gson.GsonBuilder;
+import org.apache.curator.CuratorZookeeperClient;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.apache.curator.x.discovery.server.entity.ServiceNames;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider;
+import org.codehaus.jackson.node.JsonNodeFactory;
+import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * handles the setup of the Slider AM.
+ * This keeps aspects of role, cluster validation and Clusterspec setup
+ * out of the core slider client
+ */
+public class SliderAMClientProvider extends AbstractClientProvider implements
+ SliderKeys {
+
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(SliderAMClientProvider.class);
+ protected static final String NAME = "SliderAM";
+ public static final String INSTANCE_RESOURCE_BASE = PROVIDER_RESOURCE_BASE_ROOT +
+ "slideram/instance/";
+ public static final String INTERNAL_JSON =
+ INSTANCE_RESOURCE_BASE + "internal.json";
+ public static final String APPCONF_JSON =
+ INSTANCE_RESOURCE_BASE + "appconf.json";
+ public static final String RESOURCES_JSON =
+ INSTANCE_RESOURCE_BASE + "resources.json";
+
+ public SliderAMClientProvider(Configuration conf) {
+ super(conf);
+ }
+
+ /**
+ * List of roles
+ */
+ public static final List<ProviderRole> ROLES =
+ new ArrayList<ProviderRole>();
+
+ public static final int KEY_AM = ROLE_AM_PRIORITY_INDEX;
+
+ /**
+ * Initialize role list
+ */
+ static {
+ ROLES.add(new ProviderRole(COMPONENT_AM, KEY_AM,
+ PlacementPolicy.EXCLUDE_FROM_FLEXING));
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public List<ProviderRole> getRoles() {
+ return ROLES;
+ }
+
+
+ @Override //Client
+ public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
+ String clustername,
+ Configuration configuration,
+ AggregateConf instanceDefinition,
+ Path clusterDirPath,
+ Path generatedConfDirPath,
+ boolean secure) throws
+ SliderException,
+ IOException {
+
+ super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure);
+ //add a check for the directory being writeable by the current user
+ String
+ dataPath = instanceDefinition.getInternalOperations()
+ .getGlobalOptions()
+ .getMandatoryOption(
+ OptionKeys.INTERNAL_DATA_DIR_PATH);
+
+ Path path = new Path(dataPath);
+ sliderFileSystem.verifyDirectoryWriteAccess(path);
+ Path historyPath = new Path(clusterDirPath, SliderKeys.HISTORY_DIR_NAME);
+ sliderFileSystem.verifyDirectoryWriteAccess(historyPath);
+ }
+
+ /**
+ * The Slider AM sets up all the dependency JARs above slider.jar itself
+ * {@inheritDoc}
+ */
+ public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
+ Configuration serviceConf,
+ AbstractLauncher launcher,
+ AggregateConf instanceDescription,
+ Path snapshotConfDirPath,
+ Path generatedConfDirPath,
+ Configuration clientConfExtras,
+ String libdir,
+ Path tempPath, boolean miniClusterTestRun)
+ throws IOException, SliderException {
+
+ Map<String, LocalResource> providerResources =
+ new HashMap<String, LocalResource>();
+
+
+ ProviderUtils.addProviderJar(providerResources,
+ this,
+ SLIDER_JAR,
+ fileSystem,
+ tempPath,
+ libdir,
+ miniClusterTestRun);
+
+ Class<?>[] classes = {
+ JCommander.class,
+ GsonBuilder.class,
+
+ CuratorFramework.class,
+ CuratorZookeeperClient.class,
+ ServiceInstance.class,
+ ServiceNames.class,
+
+ JacksonJaxbJsonProvider.class,
+ JsonFactory.class,
+ JsonNodeFactory.class,
+ JaxbAnnotationIntrospector.class,
+
+ };
+ String[] jars =
+ {
+ JCOMMANDER_JAR,
+ GSON_JAR,
+
+ "curator-framework.jar",
+ "curator-client.jar",
+ "curator-x-discovery.jar",
+ "curator-x-discovery-service.jar",
+
+ "jackson-jaxrs",
+ "jackson-core-asl",
+ "jackson-mapper-asl",
+ "jackson-xc",
+
+ };
+ ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath,
+ libdir, jars,
+ classes);
+
+ launcher.addLocalResources(providerResources);
+ //also pick up all env variables from a map
+ launcher.copyEnvVars(
+ instanceDescription.getInternalOperations().getOrAddComponent(
+ SliderKeys.COMPONENT_AM));
+ }
+
+ /**
+ * Update the AM resource with any local needs
+ * @param capability capability to update
+ */
+ public void prepareAMResourceRequirements(MapOperations sliderAM,
+ Resource capability) {
+ capability.setMemory(sliderAM.getOptionInt(
+ ResourceKeys.YARN_MEMORY,
+ capability.getMemory()));
+ capability.setVirtualCores(
+ sliderAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores()));
+ }
+
+ /**
+ * Extract any JVM options from the cluster specification and
+ * add them to the command line
+ */
+ public void addJVMOptions(AggregateConf aggregateConf,
+ CommandLineBuilder cmdLine) throws
+ BadConfigException {
+ MapOperations sliderAM =
+ aggregateConf.getAppConfOperations().getMandatoryComponent(
+ SliderKeys.COMPONENT_AM);
+ cmdLine.sysprop("java.net.preferIPv4Stack", "true");
+ cmdLine.sysprop("java.awt.headless", "true");
+ String heap = sliderAM.getOption(RoleKeys.JVM_HEAP,
+ DEFAULT_JVM_HEAP);
+ cmdLine.setJVMHeap(heap);
+ String jvmopts = sliderAM.getOption(RoleKeys.JVM_OPTS, "");
+ if (SliderUtils.isSet(jvmopts)) {
+ cmdLine.add(jvmopts);
+ }
+ }
+
+
+ @Override
+ public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws
+ SliderException,
+ IOException {
+ mergeTemplates(aggregateConf,
+ INTERNAL_JSON, RESOURCES_JSON, APPCONF_JSON
+ );
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java
new file mode 100644
index 0000000..533ee54
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.core.main.LauncherExitCodes;
+
+public class AMUtils {
+ /**
+ * Map an exit code from a process
+ * @param exitCode
+ * @return an exit code
+ */
+ public static int mapProcessExitCodeToYarnExitCode(int exitCode) {
+ switch (exitCode) {
+ case LauncherExitCodes.EXIT_SUCCESS:
+ return LauncherExitCodes.EXIT_SUCCESS;
+ //remap from a planned shutdown to a failure
+ case LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN:
+ return SliderExitCodes.EXIT_PROCESS_FAILED;
+ default:
+ return exitCode;
+ }
+ }
+
+ public static boolean isMappedExitAFailure(int mappedExitCode) {
+ return mappedExitCode!=LauncherExitCodes.EXIT_SUCCESS
+ && mappedExitCode!= LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AsyncRMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AsyncRMOperationHandler.java
new file mode 100644
index 0000000..171c021
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/AsyncRMOperationHandler.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.slider.server.appmaster.state.RMOperationHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hands off RM operations to the Resource Manager
+ */
+public class AsyncRMOperationHandler extends RMOperationHandler {
+ protected static final Logger log =
+ LoggerFactory.getLogger(AsyncRMOperationHandler.class);
+ private final AMRMClientAsync client;
+
+ public AsyncRMOperationHandler(AMRMClientAsync client) {
+ this.client = client;
+ }
+
+ @Override
+ public void releaseAssignedContainer(ContainerId containerId) {
+ log.debug("Releasing container {}", containerId);
+
+ client.releaseAssignedContainer(containerId);
+ }
+
+ @Override
+ public void addContainerRequest(AMRMClient.ContainerRequest req) {
+ client.addContainerRequest(req);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java
new file mode 100644
index 0000000..50c99f3
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+
+/**
+ * Callback for container start requests
+ */
+public interface ContainerStartOperation {
+ /**
+ * Add a node to the list of starting
+ * nodes then trigger the NM start operation with the given
+ * launch context
+ * @param container container
+ * @param ctx context
+ * @param instance node details
+ */
+ void startContainer(Container container,
+ ContainerLaunchContext ctx,
+ RoleInstance instance) ;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
new file mode 100644
index 0000000..65b88cf
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/PrivilegedConnectToCM.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+
+/**
+ * Implement privileged connection to the CM
+ *
+ */
+public class PrivilegedConnectToCM implements PrivilegedAction<ContainerManagementProtocol> {
+ final SliderAppMaster appMaster;
+ final InetSocketAddress cmAddress;
+
+ public PrivilegedConnectToCM(SliderAppMaster appMaster,
+ InetSocketAddress cmAddress) {
+ this.appMaster = appMaster;
+ this.cmAddress = cmAddress;
+ }
+
+
+ @Override //PrivilegedAction
+ public ContainerManagementProtocol run() {
+ return ((ContainerManagementProtocol) appMaster.getProxy(
+ ContainerManagementProtocol.class,
+ cmAddress));
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java
new file mode 100644
index 0000000..d7f79f1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.server.appmaster.state.AbstractRecordFactory;
+
+public class ProtobufRecordFactory extends AbstractRecordFactory {
+ public Resource newResource() {
+ return Records.newRecord(Resource.class);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
new file mode 100644
index 0000000..b9856cb
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A service for launching containers
+ */
+public class RoleLaunchService extends AbstractService {
+ protected static final Logger log =
+ LoggerFactory.getLogger(RoleLaunchService.class);
+ /**
+ * How long to expect launcher threads to shut down on AM termination:
+ * {@value}
+ */
+ public static final int LAUNCHER_THREAD_SHUTDOWN_TIME = 10000;
+ /**
+ * Map of launched threads.
+ * These are retained so that at shutdown time the AM can signal
+ * all threads to stop.
+ *
+ * However, we don't want to run out of memory even if many containers
+ * get launched over time, so the AM tries to purge this
+ * of the latest launched thread when the RoleLauncher signals
+ * the AM that it has finished
+ */
+ private final Map<RoleLauncher, Thread> launchThreads =
+ new HashMap<RoleLauncher, Thread>();
+
+ /**
+ * Callback to whatever has the task of actually running the container
+ * start operation
+ */
+ private final ContainerStartOperation containerStarter;
+
+
+ private final ProviderService provider;
+ /**
+ * Filesystem to use for the launch
+ */
+ private final SliderFileSystem fs;
+
+ /**
+ * Path in the launch filesystem that refers to a configuration directory
+ * -the interpretation of it is left to the Provider
+ */
+ private final Path generatedConfDirPath;
+ /**
+ * Path in the launch filesystem that refers to a temp directory
+ * which will be cleaned up at (some) time in the future
+ */
+ private final Path launcherTmpDirPath;
+
+ /**
+ * Thread group for the launchers; gives them all a useful name
+ * in stack dumps
+ */
+ private final ThreadGroup launcherThreadGroup = new ThreadGroup("launcher");
+
+ private Map<String, String> envVars;
+
+ /**
+ * Construct an instance of the launcher
+ * @param startOperation the callback to start the opreation
+ * @param provider the provider
+ * @param fs filesystem
+ * @param generatedConfDirPath path in the FS for the generated dir
+ * @param envVars
+ * @param launcherTmpDirPath
+ */
+ public RoleLaunchService(ContainerStartOperation startOperation,
+ ProviderService provider,
+ SliderFileSystem fs,
+ Path generatedConfDirPath,
+ Map<String, String> envVars, Path launcherTmpDirPath) {
+ super("RoleLaunchService");
+ containerStarter = startOperation;
+ this.fs = fs;
+ this.generatedConfDirPath = generatedConfDirPath;
+ this.launcherTmpDirPath = launcherTmpDirPath;
+ this.provider = provider;
+ this.envVars = envVars;
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ joinAllLaunchedThreads();
+ super.serviceStop();
+ }
+
+ /**
+ * Start an asychronous launch operation
+ * @param container container target
+ * @param role role
+ * @param clusterSpec cluster spec to use for template
+ */
+ public void launchRole(Container container,
+ RoleStatus role,
+ AggregateConf clusterSpec) {
+ String roleName = role.getName();
+ //emergency step: verify that this role is handled by the provider
+ assert provider.isSupportedRole(roleName) : "unsupported role";
+ RoleLaunchService.RoleLauncher launcher =
+ new RoleLaunchService.RoleLauncher(container,
+ role.getProviderRole(),
+ clusterSpec,
+ clusterSpec.getResourceOperations()
+ .getOrAddComponent(roleName),
+ clusterSpec.getAppConfOperations()
+ .getOrAddComponent(roleName) );
+ launchThread(launcher,
+ String.format("%s-%s", roleName,
+ container.getId().toString())
+ );
+ }
+
+
+ public void launchThread(RoleLauncher launcher, String name) {
+ Thread launchThread = new Thread(launcherThreadGroup,
+ launcher,
+ name);
+
+ // launch and start the container on a separate thread to keep
+ // the main thread unblocked
+ // as all containers may not be allocated at one go.
+ synchronized (launchThreads) {
+ launchThreads.put(launcher, launchThread);
+ }
+ launchThread.start();
+ }
+
+ /**
+ * Method called by a launcher thread when it has completed;
+ * this removes the launcher of the map of active
+ * launching threads.
+ * @param launcher launcher that completed
+ * @param ex any exception raised
+ */
+ public void launchedThreadCompleted(RoleLauncher launcher, Exception ex) {
+ log.debug("Launched thread {} completed", launcher, ex);
+ synchronized (launchThreads) {
+ launchThreads.remove(launcher);
+ }
+ }
+
+ /**
+ Join all launched threads
+ needed for when we time out
+ and we need to release containers
+ */
+ private void joinAllLaunchedThreads() {
+
+
+ //first: take a snapshot of the thread list
+ List<Thread> liveThreads;
+ synchronized (launchThreads) {
+ liveThreads = new ArrayList<Thread>(launchThreads.values());
+ }
+ int size = liveThreads.size();
+ if (size > 0) {
+ log.info("Waiting for the completion of {} threads", size);
+ for (Thread launchThread : liveThreads) {
+ try {
+ launchThread.join(LAUNCHER_THREAD_SHUTDOWN_TIME);
+ } catch (InterruptedException e) {
+ log.info("Exception thrown in thread join: " + e, e);
+ }
+ }
+ }
+ }
+
+
+ /**
+ * Thread that runs on the AM to launch a region server.
+ */
+ private class RoleLauncher implements Runnable {
+
+ // Allocated container
+ public final Container container;
+ public final String containerRole;
+ private final MapOperations resourceComponent;
+ private final MapOperations appComponent;
+ private final AggregateConf instanceDefinition;
+ public final ProviderRole role;
+
+ public RoleLauncher(Container container,
+ ProviderRole role,
+ AggregateConf instanceDefinition,
+ MapOperations resourceComponent,
+ MapOperations appComponent) {
+ assert container != null;
+ assert role != null;
+ assert resourceComponent != null;
+ assert appComponent != null;
+ this.container = container;
+ this.containerRole = role.name;
+ this.role = role;
+ this.resourceComponent = resourceComponent;
+ this.appComponent = appComponent;
+ this.instanceDefinition = instanceDefinition;
+ }
+
+ @Override
+ public String toString() {
+ return "RoleLauncher{" +
+ "container=" + container.getId() +
+ ", containerRole='" + containerRole + '\'' +
+ '}';
+ }
+
+ @Override
+ public void run() {
+ Exception ex = null;
+ try {
+ ContainerLauncher containerLauncher = new ContainerLauncher(getConfig(),
+ fs,
+ container);
+
+
+ containerLauncher.setupUGI();
+ containerLauncher.putEnv(envVars);
+
+ log.debug("Launching container {} into role {}",
+ container.getId(),
+ containerRole);
+
+
+ //now build up the configuration data
+ Path containerTmpDirPath =
+ new Path(launcherTmpDirPath, container.getId().toString());
+ provider.buildContainerLaunchContext(containerLauncher,
+ instanceDefinition,
+ container,
+ containerRole,
+ fs,
+ generatedConfDirPath,
+ resourceComponent,
+ appComponent,
+ containerTmpDirPath
+ );
+
+ RoleInstance instance = new RoleInstance(container);
+ String[] envDescription = containerLauncher.dumpEnvToString();
+
+ String commandsAsString = containerLauncher.getCommandsAsString();
+ log.info("Starting container with command: {}",
+ commandsAsString);
+
+ instance.command = commandsAsString;
+ instance.role = containerRole;
+ instance.roleId = role.id;
+ instance.environment = envDescription;
+ containerStarter.startContainer(container,
+ containerLauncher.completeContainerLaunch(),
+ instance);
+ } catch (Exception e) {
+ log.error("Exception thrown while trying to start {}: {}",
+ containerRole, e);
+ ex = e;
+ } finally {
+ launchedThreadCompleted(this, ex);
+ }
+ }
+
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
new file mode 100644
index 0000000..7b4ed06
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -0,0 +1,1546 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import com.google.protobuf.BlockingService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceStateChangeListener;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.WebApps;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.proto.SliderClusterAPI;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.params.AbstractActionArgs;
+import org.apache.slider.common.params.SliderAMArgs;
+import org.apache.slider.common.params.SliderAMCreateAction;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.common.tools.SliderVersionInfo;
+import org.apache.slider.core.build.InstanceIO;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.SliderInternalStateException;
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.main.RunService;
+import org.apache.slider.core.main.ServiceLauncher;
+import org.apache.slider.core.persist.ConfTreeSerDeser;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.info.CommonRegistryConstants;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
+import org.apache.slider.core.registry.info.RegisteredEndpoint;
+import org.apache.slider.core.registry.info.RegistryView;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.providers.SliderProviderFactory;
+import org.apache.slider.providers.slideram.SliderAMClientProvider;
+import org.apache.slider.server.appmaster.rpc.RpcBinder;
+import org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider;
+import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
+import org.apache.slider.server.appmaster.state.AbstractRMOperation;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.ContainerAssignment;
+import org.apache.slider.server.appmaster.state.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.state.RMOperationHandler;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.web.SliderAMWebApp;
+import org.apache.slider.server.appmaster.web.SliderAmFilterInitializer;
+import org.apache.slider.server.appmaster.web.SliderAmIpFilter;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.WebAppApiImpl;
+import org.apache.slider.server.services.curator.RegistryBinderService;
+import org.apache.slider.server.services.curator.RegistryConsts;
+import org.apache.slider.server.services.curator.RegistryNaming;
+import org.apache.slider.server.services.docstore.utility.AbstractSliderLaunchedService;
+import org.apache.slider.server.services.docstore.utility.EventCallback;
+import org.apache.slider.server.services.docstore.utility.RpcService;
+import org.apache.slider.server.services.docstore.utility.WebAppService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_MANAGEMENT;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_PUBLISHER;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.WS_CONTEXT_ROOT;
+
+/**
+ * This is the AM, which directly implements the callbacks from the AM and NM
+ */
+public class SliderAppMaster extends AbstractSliderLaunchedService
+ implements AMRMClientAsync.CallbackHandler,
+ NMClientAsync.CallbackHandler,
+ RunService,
+ SliderExitCodes,
+ SliderKeys,
+ SliderClusterProtocol,
+ ServiceStateChangeListener,
+ RoleKeys,
+ EventCallback,
+ ContainerStartOperation {
+ protected static final Logger log =
+ LoggerFactory.getLogger(SliderAppMaster.class);
+
+ /**
+ * log for YARN events
+ */
+ protected static final Logger LOG_YARN = log;
+
+ public static final String SERVICE_CLASSNAME =
+ "org.apache.slider.server.appmaster.SliderAppMaster";
+ public static final String SERVICE_CLASSNAME_SHORT =
+ "SliderAppMaster";
+
+
+ /**
+ * time to wait from shutdown signal being rx'd to telling
+ * the AM: {@value}
+ */
+ public static final int TERMINATION_SIGNAL_PROPAGATION_DELAY = 1000;
+
+ public static final int HEARTBEAT_INTERVAL = 1000;
+ public static final int NUM_RPC_HANDLERS = 5;
+
+ /** YARN RPC to communicate with the Resource Manager or Node Manager */
+ private YarnRPC yarnRPC;
+
+ /** Handle to communicate with the Resource Manager*/
+ private AMRMClientAsync asyncRMClient;
+
+ private RMOperationHandler rmOperationHandler;
+
+ /** Handle to communicate with the Node Manager*/
+ public NMClientAsync nmClientAsync;
+
+ YarnConfiguration conf;
+ /**
+ * token blob
+ */
+ private ByteBuffer allTokens;
+
+ private RpcService rpcService;
+
+ /**
+ * Secret manager
+ */
+ ClientToAMTokenSecretManager secretManager;
+
+ /** Hostname of the container*/
+ private String appMasterHostname = "";
+ /* Port on which the app master listens for status updates from clients*/
+ private int appMasterRpcPort = 0;
+ /** Tracking url to which app master publishes info for clients to monitor*/
+ private String appMasterTrackingUrl = "";
+
+ /** Application Attempt Id ( combination of attemptId and fail count )*/
+ private ApplicationAttemptId appAttemptID;
+
+ /**
+ * Security info client to AM key returned after registration
+ */
+ private ByteBuffer clientToAMKey;
+
+ /**
+ * App ACLs
+ */
+ protected Map<ApplicationAccessType, String> applicationACLs;
+
+ /**
+ * Ongoing state of the cluster: containers, nodes they
+ * live on, etc.
+ */
+ private final AppState appState = new AppState(new ProtobufRecordFactory());
+
+
+ /**
+ * model the state using locks and conditions
+ */
+ private final ReentrantLock AMExecutionStateLock = new ReentrantLock();
+ private final Condition isAMCompleted = AMExecutionStateLock.newCondition();
+
+ private int amExitCode = 0;
+
+ /**
+ * Flag set if the AM is to be shutdown
+ */
+ private final AtomicBoolean amCompletionFlag = new AtomicBoolean(false);
+
+ private volatile boolean success = true;
+
+ /**
+ * Flag to set if the process exit code was set before shutdown started
+ */
+ private boolean spawnedProcessExitedBeforeShutdownTriggered;
+
+
+ /** Arguments passed in : raw*/
+ private SliderAMArgs serviceArgs;
+
+ /**
+ * ID of the AM container
+ */
+ private ContainerId appMasterContainerID;
+
+ /**
+ * ProviderService of this cluster
+ */
+ private ProviderService providerService;
+
+ private RegistryBinderService<ServiceInstanceData> registry;
+
+ /**
+ * Record of the max no. of cores allowed in this cluster
+ */
+ private int containerMaxCores;
+
+
+ /**
+ * limit container memory
+ */
+ private int containerMaxMemory;
+ private String amCompletionReason;
+
+ private RoleLaunchService launchService;
+
+ //username -null if it is not known/not to be set
+ private String hadoop_user_name;
+ private String service_user_name;
+
+ private SliderAMWebApp webApp;
+ private InetSocketAddress rpcServiceAddress;
+
+ /**
+ * Service Constructor
+ */
+ public SliderAppMaster() {
+ super("AppMasterService");
+ }
+
+
+
+ /* =================================================================== */
+/* service lifecycle methods */
+/* =================================================================== */
+
+ @Override //AbstractService
+ public synchronized void serviceInit(Configuration conf) throws Exception {
+
+ // Load in the server configuration - if it is actually on the Classpath
+ Configuration serverConf =
+ ConfigHelper.loadFromResource(SERVER_RESOURCE);
+ ConfigHelper.mergeConfigurations(conf, serverConf, SERVER_RESOURCE);
+
+ AbstractActionArgs action = serviceArgs.getCoreAction();
+ SliderAMCreateAction createAction = (SliderAMCreateAction) action;
+ //sort out the location of the AM
+ serviceArgs.applyDefinitions(conf);
+ serviceArgs.applyFileSystemURL(conf);
+
+ String rmAddress = createAction.getRmAddress();
+ if (rmAddress != null) {
+ log.debug("Setting rm address from the command line: {}", rmAddress);
+ SliderUtils.setRmSchedulerAddress(conf, rmAddress);
+ }
+ serviceArgs.applyDefinitions(conf);
+ serviceArgs.applyFileSystemURL(conf);
+ //init security with our conf
+ if (SliderUtils.isHadoopClusterSecure(conf)) {
+ log.info("Secure mode with kerberos realm {}",
+ SliderUtils.getKerberosRealm());
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ log.debug("Authenticating as " + ugi.toString());
+ SliderUtils.verifyPrincipalSet(conf,
+ DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ // always enforce protocol to be token-based.
+ conf.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ SaslRpcServer.AuthMethod.TOKEN.toString());
+ }
+ log.info("Login user is {}", UserGroupInformation.getLoginUser());
+
+ //look at settings of Hadoop Auth, to pick up a problem seen once
+ checkAndWarnForAuthTokenProblems();
+
+ //init all child services
+ super.serviceInit(conf);
+ }
+
+/* =================================================================== */
+/* RunService methods called from ServiceLauncher */
+/* =================================================================== */
+
+ /**
+ * pick up the args from the service launcher
+ * @param config
+ * @param args argument list
+ */
+ @Override // RunService
+ public Configuration bindArgs(Configuration config, String... args) throws
+ Exception {
+ config = super.bindArgs(config, args);
+ serviceArgs = new SliderAMArgs(args);
+ serviceArgs.parse();
+ //yarn-ify
+ YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
+ return SliderUtils.patchConfiguration(yarnConfiguration);
+ }
+
+
+ /**
+ * this is called by service launcher; when it returns the application finishes
+ * @return the exit code to return by the app
+ * @throws Throwable
+ */
+ @Override
+ public int runService() throws Throwable {
+ SliderVersionInfo.loadAndPrintVersionInfo(log);
+
+ //dump the system properties if in debug mode
+ if (log.isDebugEnabled()) {
+ log.debug("System properties:\n" +
+ SliderUtils.propertiesToString(System.getProperties()));
+ }
+
+ //choose the action
+ String action = serviceArgs.getAction();
+ List<String> actionArgs = serviceArgs.getActionArgs();
+ int exitCode = EXIT_SUCCESS;
+ if (action.equals(SliderActions.ACTION_HELP)) {
+ log.info(getName() + serviceArgs.usage());
+ exitCode = SliderExitCodes.EXIT_USAGE;
+ } else if (action.equals(SliderActions.ACTION_CREATE)) {
+ exitCode = createAndRunCluster(actionArgs.get(0));
+ } else {
+ throw new SliderException("Unimplemented: " + action);
+ }
+ log.info("Exiting AM; final exit code = {}", exitCode);
+ return exitCode;
+ }
+
+
+ /**
+ * Initialize a newly created service then add it.
+ * Because the service is not started, this MUST be done before
+ * the AM itself starts, or it is explicitly added after
+ * @param service the service to init
+ */
+ public Service initAndAddService(Service service) {
+ service.init(getConfig());
+ addService(service);
+ return service;
+ }
+
+ /* =================================================================== */
+
+ /**
+ * Create and run the cluster.
+ * @return exit code
+ * @throws Throwable on a failure
+ */
+ private int createAndRunCluster(String clustername) throws Throwable {
+
+ //load the cluster description from the cd argument
+ String sliderClusterDir = serviceArgs.getSliderClusterURI();
+ URI sliderClusterURI = new URI(sliderClusterDir);
+ Path clusterDirPath = new Path(sliderClusterURI);
+ SliderFileSystem fs = getClusterFS();
+
+ // build up information about the running application -this
+ // will be passed down to the cluster status
+ MapOperations appInformation = new MapOperations();
+
+ AggregateConf instanceDefinition =
+ InstanceIO.loadInstanceDefinitionUnresolved(fs, clusterDirPath);
+
+ log.info("Deploying cluster {}:", instanceDefinition);
+
+ //REVISIT: why is this done?
+ appState.updateInstanceDefinition(instanceDefinition);
+ File confDir = getLocalConfDir();
+ if (!confDir.exists() || !confDir.isDirectory()) {
+ log.info("Conf dir {} does not exist.", confDir);
+ File parentFile = confDir.getParentFile();
+ log.info("Parent dir {}:\n{}", parentFile, SliderUtils.listDir(parentFile));
+ }
+
+ Configuration serviceConf = getConfig();
+ // Try to get the proper filtering of static resources through the yarn proxy working
+ serviceConf.set("hadoop.http.filter.initializers",
+ SliderAmFilterInitializer.NAME);
+ serviceConf.set(SliderAmIpFilter.WS_CONTEXT_ROOT, WS_CONTEXT_ROOT);
+
+ conf = new YarnConfiguration(serviceConf);
+ //get our provider
+ MapOperations globalInternalOptions =
+ instanceDefinition.getInternalOperations().getGlobalOptions();
+ String providerType = globalInternalOptions.getMandatoryOption(
+ OptionKeys.INTERNAL_PROVIDER_NAME);
+ log.info("Cluster provider type is {}", providerType);
+ SliderProviderFactory factory =
+ SliderProviderFactory.createSliderProviderFactory(
+ providerType);
+ providerService = factory.createServerProvider();
+ // init the provider BUT DO NOT START IT YET
+ initAndAddService(providerService);
+
+ InetSocketAddress address = SliderUtils.getRmSchedulerAddress(conf);
+ log.info("RM is at {}", address);
+ yarnRPC = YarnRPC.create(conf);
+
+ /*
+ * Extract the container ID. This is then
+ * turned into an (incompete) container
+ */
+ appMasterContainerID = ConverterUtils.toContainerId(
+ SliderUtils.mandatoryEnvVariable(
+ ApplicationConstants.Environment.CONTAINER_ID.name())
+ );
+ appAttemptID = appMasterContainerID.getApplicationAttemptId();
+
+ ApplicationId appid = appAttemptID.getApplicationId();
+ log.info("AM for ID {}", appid.getId());
+
+ appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID,
+ appMasterContainerID.toString());
+ appInformation.put(StatusKeys.INFO_AM_APP_ID,
+ appid.toString());
+ appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID,
+ appAttemptID.toString());
+
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ Credentials credentials =
+ currentUser.getCredentials();
+ DataOutputBuffer dob = new DataOutputBuffer();
+ credentials.writeTokenStorageToStream(dob);
+ dob.close();
+ // Now remove the AM->RM token so that containers cannot access it.
+ Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<?> token = iter.next();
+ log.info("Token {}", token.getKind());
+ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
+ iter.remove();
+ }
+ }
+ allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+
+ // set up secret manager
+ secretManager = new ClientToAMTokenSecretManager(appAttemptID, null);
+
+ // if not a secure cluster, extract the username -it will be
+ // propagated to workers
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ hadoop_user_name = System.getenv(HADOOP_USER_NAME);
+ service_user_name = hadoop_user_name;
+ log.info(HADOOP_USER_NAME + "='{}'", hadoop_user_name);
+ } else {
+ service_user_name = UserGroupInformation.getCurrentUser().getUserName();
+ }
+
+ Map<String, String> envVars;
+
+ /**
+ * It is critical this section is synchronized, to stop async AM events
+ * arriving while registering a restarting AM.
+ */
+ synchronized (appState) {
+ int heartbeatInterval = HEARTBEAT_INTERVAL;
+
+ //add the RM client -this brings the callbacks in
+ asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval,
+ this);
+ addService(asyncRMClient);
+ //wrap it for the app state model
+ rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient);
+ //now bring it up
+ deployChildService(asyncRMClient);
+
+
+ //nmclient relays callbacks back to this class
+ nmClientAsync = new NMClientAsyncImpl("nmclient", this);
+ deployChildService(nmClientAsync);
+
+ //bring up the Slider RPC service
+ startSliderRPCServer();
+
+ rpcServiceAddress = rpcService.getConnectAddress();
+ appMasterHostname = rpcServiceAddress.getHostName();
+ appMasterRpcPort = rpcServiceAddress.getPort();
+ appMasterTrackingUrl = null;
+ log.info("AM Server is listening at {}:{}", appMasterHostname,
+ appMasterRpcPort);
+ appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname);
+ appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort);
+
+
+ //registry
+
+
+ registry = startRegistrationService();
+
+ //build the role map
+ List<ProviderRole> providerRoles =
+ new ArrayList<ProviderRole>(providerService.getRoles());
+ providerRoles.addAll(SliderAMClientProvider.ROLES);
+
+ // Start up the WebApp and track the URL for it
+ webApp = new SliderAMWebApp(registry);
+ WebApps.$for(SliderAMWebApp.BASE_PATH, WebAppApi.class,
+ new WebAppApiImpl(this, appState, providerService), "ws")
+ .with(serviceConf)
+ .start(webApp);
+ appMasterTrackingUrl = "http://" + appMasterHostname + ":" + webApp.port();
+ WebAppService<SliderAMWebApp> webAppService =
+ new WebAppService<SliderAMWebApp>("slider", webApp);
+
+ webAppService.init(conf);
+ webAppService.start();
+ addService(webAppService);
+
+ appInformation.put(StatusKeys.INFO_AM_WEB_URL, appMasterTrackingUrl + "/");
+ appInformation.set(StatusKeys.INFO_AM_WEB_PORT, webApp.port());
+
+ // Register self with ResourceManager
+ // This will start heartbeating to the RM
+ // address = SliderUtils.getRmSchedulerAddress(asyncRMClient.getConfig());
+ log.info("Connecting to RM at {},address tracking URL={}",
+ appMasterRpcPort, appMasterTrackingUrl);
+ RegisterApplicationMasterResponse response = asyncRMClient
+ .registerApplicationMaster(appMasterHostname,
+ appMasterRpcPort,
+ appMasterTrackingUrl);
+ Resource maxResources =
+ response.getMaximumResourceCapability();
+ containerMaxMemory = maxResources.getMemory();
+ containerMaxCores = maxResources.getVirtualCores();
+ appState.setContainerLimits(maxResources.getMemory(),
+ maxResources.getVirtualCores());
+ // set the RM-defined maximum cluster values
+ appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores));
+ appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory));
+
+ boolean securityEnabled = UserGroupInformation.isSecurityEnabled();
+ if (securityEnabled) {
+ secretManager.setMasterKey(
+ response.getClientToAMTokenMasterKey().array());
+ applicationACLs = response.getApplicationACLs();
+
+ //tell the server what the ACLs are
+ rpcService.getServer().refreshServiceAcl(conf, new SliderAMPolicyProvider());
+ }
+
+ // extract container list
+ List<Container> liveContainers =
+ response.getContainersFromPreviousAttempts();
+
+ //now validate the installation
+ Configuration providerConf =
+ providerService.loadProviderConfigurationInformation(confDir);
+
+ providerService.validateApplicationConfiguration(instanceDefinition,
+ confDir,
+ securityEnabled);
+
+ //determine the location for the role history data
+ Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME);
+
+ //build the instance
+ appState.buildInstance(instanceDefinition,
+ providerConf,
+ providerRoles,
+ fs.getFileSystem(),
+ historyDir,
+ liveContainers,
+ appInformation);
+
+ // add the AM to the list of nodes in the cluster
+
+ appState.buildAppMasterNode(appMasterContainerID,
+ appMasterHostname,
+ webApp.port(),
+ appMasterHostname + ":" + webApp.port());
+
+ // build up environment variables that the AM wants set in every container
+ // irrespective of provider and role.
+ envVars = new HashMap<String, String>();
+ if (hadoop_user_name != null) {
+ envVars.put(HADOOP_USER_NAME, hadoop_user_name);
+ }
+ }
+ String rolesTmpSubdir = appMasterContainerID.toString() + "/roles";
+
+ String amTmpDir = globalInternalOptions.getMandatoryOption(OptionKeys.INTERNAL_AM_TMP_DIR);
+
+ Path tmpDirPath = new Path(amTmpDir);
+ Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir);
+ fs.getFileSystem().mkdirs(launcherTmpDirPath);
+
+ //launcher service
+ launchService = new RoleLaunchService(this,
+ providerService,
+ fs,
+ new Path(getGeneratedConfDir()),
+ envVars,
+ launcherTmpDirPath);
+
+ deployChildService(launchService);
+
+ appState.noteAMLaunched();
+
+
+ //Give the provider restricted access to the state, registry
+ providerService.bind(appState, registry);
+ registerServiceInstance(clustername, appid);
+
+
+ // launch the provider; this is expected to trigger a callback that
+ // starts the node review process
+ launchProviderService(instanceDefinition, confDir);
+
+
+ try {
+ //now block waiting to be told to exit the process
+ waitForAMCompletionSignal();
+ //shutdown time
+ } finally {
+ finish();
+ }
+
+ return amExitCode;
+ }
+
+
+ /**
+ * This registers the service instance and its external values
+ * @param instanceName name of this instance
+ * @param appid application ID
+ * @throws Exception
+ */
+ private void registerServiceInstance(String instanceName,
+ ApplicationId appid) throws Exception {
+ // the registry is running, so register services
+ URL amWeb = new URL(appMasterTrackingUrl);
+ String serviceName = SliderKeys.APP_TYPE;
+ int id = appid.getId();
+ String appRegistryName = RegistryNaming.createRegistryName(instanceName,
+ service_user_name,
+ serviceName);
+ String registryId =
+ RegistryNaming.createUniqueInstanceId(instanceName, service_user_name, serviceName, id);
+
+ List<String> serviceInstancesRunning = registry.instanceIDs(serviceName);
+ log.info("service instances already running: {}", serviceInstancesRunning);
+
+
+ // now publish yarn-site.xml
+ PublishedConfiguration pubconf = new PublishedConfiguration();
+ pubconf.description = "YARN site settings";
+ pubconf.putValues(new YarnConfiguration());
+ appState.getPublishedConfigurations().put("yarn-site.xml", pubconf);
+
+ ServiceInstanceData instanceData = new ServiceInstanceData();
+
+ RegisteredEndpoint webUI =
+ new RegisteredEndpoint(amWeb, "Application Master Web UI");
+
+
+ // public REST services
+
+ RegistryView externalView = instanceData.externalView;
+ externalView.endpoints.put(CommonRegistryConstants.WEB_UI, webUI);
+
+ externalView.endpoints.put(
+ CustomRegistryConstants.MANAGEMENT_REST_API,
+ new RegisteredEndpoint(
+ new URL(amWeb, SLIDER_PATH_MANAGEMENT),
+ "Management REST API" )
+ );
+
+ externalView.endpoints.put(
+ CustomRegistryConstants.REGISTRY_REST_API,
+ new RegisteredEndpoint(
+ new URL(amWeb, RegistryConsts.REGISTRY_RESOURCE_PATH),
+ "Registry Web Service" )
+ );
+
+ URL publisherURL = new URL(amWeb, SLIDER_PATH_PUBLISHER);
+ externalView.endpoints.put(
+ CustomRegistryConstants.PUBLISHER_REST_API,
+ new RegisteredEndpoint(
+ publisherURL,
+ "Publisher Service" )
+ );
+
+ // IPC services
+ externalView.endpoints.put(
+ CustomRegistryConstants.AM_IPC_PROTOCOL,
+ new RegisteredEndpoint(rpcServiceAddress,
+ RegisteredEndpoint.PROTOCOL_HADOOP_PROTOBUF,
+ "Slider AM RPC") );
+
+ /**
+ * Set the configurations URL.
+ */
+ externalView.configurationsURL = publisherURL.toExternalForm();
+
+
+ // internal services
+
+ instanceData.internalView.endpoints.put(
+ CustomRegistryConstants.AGENT_REST_API,
+ new RegisteredEndpoint(
+ new URL(amWeb, SLIDER_PATH_AGENTS),
+ "Agent REST API" )
+ );
+
+
+ registry.register(
+ appRegistryName,
+ registryId,
+ amWeb,
+ instanceData);
+ }
+
+ /**
+ * looks for a specific case where a token file is provided as an environment
+ * variable, yet the file is not there.
+ *
+ * This surfaced (once) in HBase, where its HDFS library was looking for this,
+ * and somehow the token was missing. This is a check in the AM so that
+ * if the problem re-occurs, the AM can fail with a more meaningful message.
+ *
+ */
+ private void checkAndWarnForAuthTokenProblems() {
+ String fileLocation =
+ System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
+ if (fileLocation != null) {
+ File tokenFile = new File(fileLocation);
+ if (!tokenFile.exists()) {
+ log.warn("Token file {} specified in {} not found", tokenFile,
+ UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION);
+ }
+ }
+ }
+
+ /**
+ * Build the configuration directory passed in or of the target FS
+ * @return the file
+ */
+ public File getLocalConfDir() {
+ File confdir =
+ new File(SliderKeys.PROPAGATED_CONF_DIR_NAME).getAbsoluteFile();
+ return confdir;
+ }
+
+ /**
+ * Get the path to the DFS configuration that is defined in the cluster specification
+ * @return the generated configuration dir
+ */
+ public String getGeneratedConfDir() {
+ return getInstanceDefinition()
+ .getInternalOperations().
+ getGlobalOptions().get(OptionKeys.INTERNAL_GENERATED_CONF_PATH);
+ }
+
+ /**
+ * Get the filesystem of this cluster
+ * @return the FS of the config
+ */
+ public SliderFileSystem getClusterFS() throws IOException {
+ return new SliderFileSystem(getConfig());
+ }
+
+ /**
+ * Block until it is signalled that the AM is done
+ */
+ private void waitForAMCompletionSignal() {
+ AMExecutionStateLock.lock();
+ try {
+ if (!amCompletionFlag.get()) {
+ log.debug("blocking until signalled to terminate");
+ isAMCompleted.awaitUninterruptibly();
+ }
+ } finally {
+ AMExecutionStateLock.unlock();
+ }
+ //add a sleep here for about a second. Why? it
+ //stops RPC calls breaking so dramatically when the cluster
+ //is torn down mid-RPC
+ try {
+ Thread.sleep(TERMINATION_SIGNAL_PROPAGATION_DELAY);
+ } catch (InterruptedException ignored) {
+ //ignored
+ }
+ }
+
+ /**
+ * Declare that the AM is complete
+ * @param exitCode exit code for the aM
+ * @param reason reason for termination
+ */
+ public synchronized void signalAMComplete(int exitCode, String reason) {
+ amCompletionReason = reason;
+ AMExecutionStateLock.lock();
+ try {
+ amCompletionFlag.set(true);
+ amExitCode = exitCode;
+ isAMCompleted.signal();
+ } finally {
+ AMExecutionStateLock.unlock();
+ }
+ }
+
+ /**
+ * shut down the cluster
+ */
+ private synchronized void finish() {
+ FinalApplicationStatus appStatus;
+ log.info("Triggering shutdown of the AM: {}", amCompletionReason);
+
+ String appMessage = amCompletionReason;
+ //stop the daemon & grab its exit code
+ int exitCode = amExitCode;
+ success = exitCode == 0 || exitCode == 3;
+
+ appStatus = success ? FinalApplicationStatus.SUCCEEDED:
+ FinalApplicationStatus.FAILED;
+ if (!spawnedProcessExitedBeforeShutdownTriggered) {
+ //stopped the forked process but don't worry about its exit code
+ exitCode = stopForkedProcess();
+ log.debug("Stopped forked process: exit code={}", exitCode);
+ }
+
+ //stop any launches in progress
+ launchService.stop();
+
+
+ //now release all containers
+ releaseAllContainers();
+
+ // When the application completes, it should send a finish application
+ // signal to the RM
+ log.info("Application completed. Signalling finish to RM");
+
+
+ //if there were failed containers and the app isn't already down as failing, it is now
+ int failedContainerCount = appState.getFailedCountainerCount();
+ if (failedContainerCount != 0 &&
+ appStatus == FinalApplicationStatus.SUCCEEDED) {
+ appStatus = FinalApplicationStatus.FAILED;
+ appMessage =
+ "Completed with exit code = " + exitCode + " - " + getContainerDiagnosticInfo();
+ success = false;
+ }
+ try {
+ log.info("Unregistering AM status={} message={}", appStatus, appMessage);
+ asyncRMClient.unregisterApplicationMaster(appStatus, appMessage, null);
+ } catch (YarnException e) {
+ log.info("Failed to unregister application: " + e, e);
+ } catch (IOException e) {
+ log.info("Failed to unregister application: " + e, e);
+ }
+ }
+
+ /**
+ * Get diagnostics info about containers
+ */
+ private String getContainerDiagnosticInfo() {
+ return appState.getContainerDiagnosticInfo();
+ }
+
+ public Object getProxy(Class protocol, InetSocketAddress addr) {
+ return yarnRPC.getProxy(protocol, addr, getConfig());
+ }
+
+ /**
+ * Start the slider RPC server
+ */
+ private void startSliderRPCServer() throws IOException {
+ SliderClusterProtocolPBImpl protobufRelay = new SliderClusterProtocolPBImpl(this);
+ BlockingService blockingService = SliderClusterAPI.SliderClusterProtocolPB
+ .newReflectiveBlockingService(
+ protobufRelay);
+
+ rpcService = new RpcService(RpcBinder.createProtobufServer(
+ new InetSocketAddress("0.0.0.0", 0),
+ getConfig(),
+ secretManager,
+ NUM_RPC_HANDLERS,
+ blockingService,
+ null));
+ deployChildService(rpcService);
+ }
+
+
+/* =================================================================== */
+/* AMRMClientAsync callbacks */
+/* =================================================================== */
+
+ /**
+ * Callback event when a container is allocated.
+ *
+ * The app state is updated with the allocation, and builds up a list
+ * of assignments and RM opreations. The assignments are
+ * handed off into the pool of service launchers to asynchronously schedule
+ * container launch operations.
+ *
+ * The operations are run in sequence; they are expected to be 0 or more
+ * release operations (to handle over-allocations)
+ *
+ * @param allocatedContainers list of containers that are now ready to be
+ * given work.
+ */
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+ @Override //AMRMClientAsync
+ public void onContainersAllocated(List<Container> allocatedContainers) {
+ LOG_YARN.info("onContainersAllocated({})", allocatedContainers.size());
+ List<ContainerAssignment> assignments = new ArrayList<ContainerAssignment>();
+ List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
+
+ //app state makes all the decisions
+ appState.onContainersAllocated(allocatedContainers, assignments, operations);
+
+ //for each assignment: instantiate that role
+ for (ContainerAssignment assignment : assignments) {
+ RoleStatus role = assignment.role;
+ Container container = assignment.container;
+ launchService.launchRole(container, role, getInstanceDefinition());
+ }
+
+ //for all the operations, exec them
+ rmOperationHandler.execute(operations);
+ log.info("Diagnostics: " + getContainerDiagnosticInfo());
+ }
+
+ @Override //AMRMClientAsync
+ public synchronized void onContainersCompleted(List<ContainerStatus> completedContainers) {
+ LOG_YARN.info("onContainersCompleted([{}]", completedContainers.size());
+ for (ContainerStatus status : completedContainers) {
+ ContainerId containerId = status.getContainerId();
+ LOG_YARN.info("Container Completion for" +
+ " containerID={}," +
+ " state={}," +
+ " exitStatus={}," +
+ " diagnostics={}",
+ containerId, status.getState(),
+ status.getExitStatus(),
+ status.getDiagnostics());
+
+ // non complete containers should not be here
+ assert (status.getState() == ContainerState.COMPLETE);
+ AppState.NodeCompletionResult result = appState.onCompletedNode(conf, status);
+ if (result.containerFailed) {
+ RoleInstance ri = result.roleInstance;
+ log.error("Role instance {} failed ", ri);
+ }
+ }
+
+ // ask for more containers if any failed
+ // In the case of Slider, we don't expect containers to complete since
+ // Slider is a long running application. Keep track of how many containers
+ // are completing. If too many complete, abort the application
+ // TODO: this needs to be better thought about (and maybe something to
+ // better handle in Yarn for long running apps)
+
+ try {
+ reviewRequestAndReleaseNodes();
+ } catch (SliderInternalStateException e) {
+ log.warn("Exception while flexing nodes", e);
+ }
+ }
+
+ /**
+ * Implementation of cluster flexing.
+ * It should be the only way that anything -even the AM itself on startup-
+ * asks for nodes.
+ * @return true if the any requests were made
+ * @throws IOException
+ */
+ private boolean flexCluster(ConfTree updated)
+ throws IOException, SliderInternalStateException, BadConfigException {
+
+ appState.updateResourceDefinitions(updated);
+
+ // ask for more containers if needed
+ return reviewRequestAndReleaseNodes();
+ }
+
+ /**
+ * Look at where the current node state is -and whether it should be changed
+ */
+ private synchronized boolean reviewRequestAndReleaseNodes()
+ throws SliderInternalStateException {
+ log.debug("in reviewRequestAndReleaseNodes()");
+ if (amCompletionFlag.get()) {
+ log.info("Ignoring node review operation: shutdown in progress");
+ return false;
+ }
+ try {
+ List<AbstractRMOperation> allOperations = appState.reviewRequestAndReleaseNodes();
+ //now apply the operations
+ rmOperationHandler.execute(allOperations);
+ return !allOperations.isEmpty();
+ } catch (TriggerClusterTeardownException e) {
+
+ //App state has decided that it is time to exit
+ log.error("Cluster teardown triggered %s", e);
+ signalAMComplete(e.getExitCode(), e.toString());
+ return false;
+ }
+ }
+
+ /**
+ * Shutdown operation: release all containers
+ */
+ private void releaseAllContainers() {
+ //now apply the operations
+ rmOperationHandler.execute(appState.releaseAllContainers());
+ }
+
+ /**
+ * RM wants to shut down the AM
+ */
+ @Override //AMRMClientAsync
+ public void onShutdownRequest() {
+ LOG_YARN.info("Shutdown Request received");
+ signalAMComplete(EXIT_CLIENT_INITIATED_SHUTDOWN, "Shutdown requested from RM");
+ }
+
+ /**
+ * Monitored nodes have been changed
+ * @param updatedNodes list of updated nodes
+ */
+ @Override //AMRMClientAsync
+ public void onNodesUpdated(List<NodeReport> updatedNodes) {
+ LOG_YARN.info("Nodes updated");
+ }
+
+ /**
+ * heartbeat operation; return the ratio of requested
+ * to actual
+ * @return progress
+ */
+ @Override //AMRMClientAsync
+ public float getProgress() {
+ return appState.getApplicationProgressPercentage();
+ }
+
+ @Override //AMRMClientAsync
+ public void onError(Throwable e) {
+ //callback says it's time to finish
+ LOG_YARN.error("AMRMClientAsync.onError() received " + e, e);
+ signalAMComplete(EXIT_EXCEPTION_THROWN, "AMRMClientAsync.onError() received " + e);
+ }
+
+/* =================================================================== */
+/* SliderClusterProtocol */
+/* =================================================================== */
+
+ @Override //SliderClusterProtocol
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion,
+ int clientMethodsHash) throws
+ IOException {
+ return ProtocolSignature.getProtocolSignature(
+ this, protocol, clientVersion, clientMethodsHash);
+ }
+
+
+
+ @Override //SliderClusterProtocol
+ public long getProtocolVersion(String protocol, long clientVersion) throws
+ IOException {
+ return SliderClusterProtocol.versionID;
+ }
+
+
+/* =================================================================== */
+/* SliderClusterProtocol */
+/* =================================================================== */
+
+ @Override //SliderClusterProtocol
+ public Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request) throws
+ IOException,
+ YarnException {
+ SliderUtils.getCurrentUser();
+ String message = request.getMessage();
+ log.info("SliderAppMasterApi.stopCluster: {}",message);
+ signalAMComplete(EXIT_CLIENT_INITIATED_SHUTDOWN, message);
+ return Messages.StopClusterResponseProto.getDefaultInstance();
+ }
+
+ @Override //SliderClusterProtocol
+ public Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) throws
+ IOException,
+ YarnException {
+ SliderUtils.getCurrentUser();
+
+ String payload = request.getClusterSpec();
+ ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
+ ConfTree updated = confTreeSerDeser.fromJson(payload);
+ boolean flexed = flexCluster(updated);
+ return Messages.FlexClusterResponseProto.newBuilder().setResponse(flexed).build();
+ }
+
+ @Override //SliderClusterProtocol
+ public Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(
+ Messages.GetJSONClusterStatusRequestProto request) throws
+ IOException,
+ YarnException {
+ SliderUtils.getCurrentUser();
+ String result;
+ //quick update
+ //query and json-ify
+ ClusterDescription cd;
+ cd = getCurrentClusterStatus();
+ result = cd.toJsonString();
+ String stat = result;
+ return Messages.GetJSONClusterStatusResponseProto.newBuilder()
+ .setClusterSpec(stat)
+ .build();
+ }
+
+ /**
+ * Get the current cluster status, including any provider-specific info
+ * @return a status document
+ */
+ public ClusterDescription getCurrentClusterStatus() {
+ ClusterDescription cd;
+ synchronized (this) {
+ updateClusterStatus();
+ cd = getClusterDescription();
+ }
+ return cd;
+ }
+
+
+ @Override
+ public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition(
+ Messages.GetInstanceDefinitionRequestProto request) throws
+ IOException,
+ YarnException {
+
+ log.info("Received call to getInstanceDefinition()");
+ String internal;
+ String resources;
+ String app;
+ synchronized (appState) {
+ AggregateConf instanceDefinition = appState.getInstanceDefinition();
+ internal = instanceDefinition.getInternal().toJson();
+ resources = instanceDefinition.getResources().toJson();
+ app = instanceDefinition.getAppConf().toJson();
+ }
+ assert internal != null;
+ assert resources != null;
+ assert app != null;
+ log.info("Generating getInstanceDefinition Response");
+ Messages.GetInstanceDefinitionResponseProto.Builder builder =
+ Messages.GetInstanceDefinitionResponseProto.newBuilder();
+ builder.setInternal(internal);
+ builder.setResources(resources);
+ builder.setApplication(app);
+ return builder.build();
+ }
+
+
+ @Override //SliderClusterProtocol
+ public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request) throws
+ IOException,
+ YarnException {
+ SliderUtils.getCurrentUser();
+ String role = request.getRole();
+ Messages.ListNodeUUIDsByRoleResponseProto.Builder builder =
+ Messages.ListNodeUUIDsByRoleResponseProto.newBuilder();
+ List<RoleInstance> nodes = appState.enumLiveNodesInRole(role);
+ for (RoleInstance node : nodes) {
+ builder.addUuid(node.id);
+ }
+ return builder.build();
+ }
+
+ @Override //SliderClusterProtocol
+ public Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request) throws
+ IOException,
+ YarnException {
+ SliderUtils.getCurrentUser();
+ RoleInstance instance = appState.getLiveInstanceByContainerID(
+ request.getUuid());
+ return Messages.GetNodeResponseProto.newBuilder()
+ .setClusterNode(instance.toProtobuf())
+ .build();
+ }
+
+ @Override //SliderClusterProtocol
+ public Messages.GetClusterNodesResponseProto getClusterNodes(Messages.GetClusterNodesRequestProto request) throws
+ IOException,
+ YarnException {
+ SliderUtils.getCurrentUser();
+ List<RoleInstance>
+ clusterNodes = appState.getLiveInstancesByContainerIDs(
+ request.getUuidList());
+
+ Messages.GetClusterNodesResponseProto.Builder builder =
+ Messages.GetClusterNodesResponseProto.newBuilder();
+ for (RoleInstance node : clusterNodes) {
+ builder.addClusterNode(node.toProtobuf());
+ }
+ //at this point: a possibly empty list of nodes
+ return builder.build();
+ }
+
+ @Override
+ public Messages.EchoResponseProto echo(Messages.EchoRequestProto request) throws
+ IOException,
+ YarnException {
+ Messages.EchoResponseProto.Builder builder =
+ Messages.EchoResponseProto.newBuilder();
+ String text = request.getText();
+ log.info("Echo request size ={}", text.length());
+ log.info(text);
+ //now return it
+ builder.setText(text);
+ return builder.build();
+ }
+
+ @Override
+ public Messages.KillContainerResponseProto killContainer(Messages.KillContainerRequestProto request) throws
+ IOException,
+ YarnException {
+ String containerID = request.getId();
+ log.info("Kill Container {}", containerID);
+ //throws NoSuchNodeException if it is missing
+ RoleInstance instance =
+ appState.getLiveInstanceByContainerID(containerID);
+ List<AbstractRMOperation> opsList =
+ new LinkedList<AbstractRMOperation>();
+ ContainerReleaseOperation release =
+ new ContainerReleaseOperation(instance.getId());
+ opsList.add(release);
+ //now apply the operations
+ rmOperationHandler.execute(opsList);
+ Messages.KillContainerResponseProto.Builder builder =
+ Messages.KillContainerResponseProto.newBuilder();
+ builder.setSuccess(true);
+ return builder.build();
+ }
+
+ @Override
+ public Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request) throws
+ IOException,
+ YarnException {
+ int signal = request.getSignal();
+ String text = request.getText();
+ int delay = request.getDelay();
+ log.info("AM Suicide with signal {}, message {} delay = {}", signal, text, delay);
+ SliderUtils.haltAM(signal, text, delay);
+ Messages.AMSuicideResponseProto.Builder builder =
+ Messages.AMSuicideResponseProto.newBuilder();
+ return builder.build();
+ }
+
+ /* =================================================================== */
+/* END */
+/* =================================================================== */
+
+ /**
+ * Update the cluster description with anything interesting
+ */
+ public synchronized void updateClusterStatus() {
+ Map<String, String> providerStatus = providerService.buildProviderStatus();
+ assert providerStatus != null : "null provider status";
+ appState.refreshClusterStatus(providerStatus);
+ }
+
+ /**
+ * Launch the provider service
+ *
+ * @param cd
+ * @param confDir
+ * @throws IOException
+ * @throws SliderException
+ */
+ protected synchronized void launchProviderService(AggregateConf instanceDefinition,
+ File confDir)
+ throws IOException, SliderException {
+ Map<String, String> env = new HashMap<String, String>();
+ boolean execStarted = providerService.exec(instanceDefinition, confDir, env, this);
+ if (execStarted) {
+ providerService.registerServiceListener(this);
+ providerService.start();
+ } else {
+ // didn't start, so don't register
+ providerService.start();
+ // and send the started event ourselves
+ eventCallbackEvent();
+ }
+ }
+
+
+ /* =================================================================== */
+ /* EventCallback from the child or ourselves directly */
+ /* =================================================================== */
+
+ @Override // EventCallback
+ public void eventCallbackEvent() {
+ // signalled that the child process is up.
+ appState.noteAMLive();
+ // now ask for the cluster nodes
+ try {
+ flexCluster(getInstanceDefinition().getResources());
+ } catch (Exception e) {
+ //this may happen in a separate thread, so the ability to act is limited
+ log.error("Failed to flex cluster nodes", e);
+ //declare a failure
+ finish();
+ }
+ }
+
+ /* =================================================================== */
+ /* ServiceStateChangeListener */
+ /* =================================================================== */
+
+ /**
+ * Received on listening service termination.
+ * @param service the service that has changed.
+ */
+ @Override //ServiceStateChangeListener
+ public void stateChanged(Service service) {
+ if (service == providerService) {
+ //its the current master process in play
+ int exitCode = providerService.getExitCode();
+ int mappedProcessExitCode =
+ AMUtils.mapProcessExitCodeToYarnExitCode(exitCode);
+ boolean shouldTriggerFailure = !amCompletionFlag.get()
+ && (AMUtils.isMappedExitAFailure(mappedProcessExitCode));
+
+
+
+ if (shouldTriggerFailure) {
+ //this wasn't expected: the process finished early
+ spawnedProcessExitedBeforeShutdownTriggered = true;
+ log.info(
+ "Process has exited with exit code {} mapped to {} -triggering termination",
+ exitCode,
+ mappedProcessExitCode);
+
+ //tell the AM the cluster is complete
+ signalAMComplete(mappedProcessExitCode,
+ "Spawned master exited with raw " + exitCode + " mapped to " +
+ mappedProcessExitCode);
+ } else {
+ //we don't care
+ log.info(
+ "Process has exited with exit code {} mapped to {} -ignoring",
+ exitCode,
+ mappedProcessExitCode);
+ }
+ }
+ }
+
+ /**
+ * stop forked process if it the running process var is not null
+ * @return the process exit code
+ */
+ protected synchronized Integer stopForkedProcess() {
+ providerService.stop();
+ return providerService.getExitCode();
+ }
+
+ /**
+ * Async start container request
+ * @param container container
+ * @param ctx context
+ * @param instance node details
+ */
+ @Override // ContainerStartOperation
+ public void startContainer(Container container,
+ ContainerLaunchContext ctx,
+ RoleInstance instance) {
+ // Set up tokens for the container too. Today, for normal shell commands,
+ // the container in distribute-shell doesn't need any tokens. We are
+ // populating them mainly for NodeManagers to be able to download any
+ // files in the distributed file-system. The tokens are otherwise also
+ // useful in cases, for e.g., when one is running a "hadoop dfs" command
+ // inside the distributed shell.
+ ctx.setTokens(allTokens.duplicate());
+ appState.containerStartSubmitted(container, instance);
+ nmClientAsync.startContainerAsync(container, ctx);
+ }
+
+ @Override // NMClientAsync.CallbackHandler
+ public void onContainerStopped(ContainerId containerId) {
+ // do nothing but log: container events from the AM
+ // are the source of container halt details to react to
+ log.info("onContainerStopped {} ", containerId);
+ }
+
+ @Override // NMClientAsync.CallbackHandler
+ public void onContainerStarted(ContainerId containerId,
+ Map<String, ByteBuffer> allServiceResponse) {
+ LOG_YARN.info("Started Container {} ", containerId);
+ RoleInstance cinfo = appState.onNodeManagerContainerStarted(containerId);
+ if (cinfo != null) {
+ LOG_YARN.info("Deployed instance of role {}", cinfo.role);
+ //trigger an async container status
+ nmClientAsync.getContainerStatusAsync(containerId,
+ cinfo.container.getNodeId());
+ } else {
+ //this is a hypothetical path not seen. We react by warning
+ log.error("Notified of started container that isn't pending {} - releasing",
+ containerId);
+ //then release it
+ asyncRMClient.releaseAssignedContainer(containerId);
+ }
+ }
+
+ @Override // NMClientAsync.CallbackHandler
+ public void onStartContainerError(ContainerId containerId, Throwable t) {
+ LOG_YARN.error("Failed to start Container " + containerId, t);
+ appState.onNodeManagerContainerStartFailed(containerId, t);
+ }
+
+ @Override // NMClientAsync.CallbackHandler
+ public void onContainerStatusReceived(ContainerId containerId,
+ ContainerStatus containerStatus) {
+ LOG_YARN.debug("Container Status: id={}, status={}", containerId,
+ containerStatus);
+ }
+
+ @Override // NMClientAsync.CallbackHandler
+ public void onGetContainerStatusError(
+ ContainerId containerId, Throwable t) {
+ LOG_YARN.error("Failed to query the status of Container {}", containerId);
+ }
+
+ @Override // NMClientAsync.CallbackHandler
+ public void onStopContainerError(ContainerId containerId, Throwable t) {
+ LOG_YARN.warn("Failed to stop Container {}", containerId);
+ }
+
+ /**
+ The cluster description published to callers
+ This is used as a synchronization point on activities that update
+ the CD, and also to update some of the structures that
+ feed in to the CD
+ */
+ public ClusterDescription getClusterSpec() {
+ return appState.getClusterSpec();
+ }
+
+ public AggregateConf getInstanceDefinition() {
+ return appState.getInstanceDefinition();
+ }
+
+ /**
+ * This is the status, the live model
+ */
+ public ClusterDescription getClusterDescription() {
+ return appState.getClusterStatus();
+ }
+
+ public ProviderService getProviderService() {
+ return providerService;
+ }
+
+ /**
+ * Get the username for the slider cluster as set in the environment
+ * @return the username or null if none was set/it is a secure cluster
+ */
+ public String getHadoop_user_name() {
+ return hadoop_user_name;
+ }
+
+ /**
+ * This is the main entry point for the service launcher.
+ * @param args command line arguments.
+ */
+ public static void main(String[] args) {
+
+ //turn the args to a list
+ List<String> argsList = Arrays.asList(args);
+ //create a new list, as the ArrayList type doesn't push() on an insert
+ List<String> extendedArgs = new ArrayList<String>(argsList);
+ //insert the service name
+ extendedArgs.add(0, SERVICE_CLASSNAME);
+ //now have the service launcher do its work
+ ServiceLauncher.serviceMain(extendedArgs);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
new file mode 100644
index 0000000..0cef1d8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.rpc;
+
+import com.google.protobuf.BlockingService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtocolProxy;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RpcEngine;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.tools.Duration;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+
+public class RpcBinder {
+ protected static final Logger log =
+ LoggerFactory.getLogger(RpcBinder.class);
+
+ public static Server createProtobufServer(InetSocketAddress addr,
+ Configuration conf,
+ SecretManager<? extends TokenIdentifier> secretManager,
+ int numHandlers,
+ BlockingService blockingService,
+ String portRangeConfig) throws
+ IOException {
+ Class<SliderClusterProtocolPB> sliderClusterAPIClass = registerSliderAPI(
+ conf);
+ RPC.Server server = new RPC.Builder(conf).setProtocol(sliderClusterAPIClass)
+ .setInstance(blockingService)
+ .setBindAddress(addr.getHostName())
+ .setPort(addr.getPort())
+ .setNumHandlers(numHandlers)
+ .setVerbose(false)
+ .setSecretManager(secretManager)
+ .setPortRangeConfig(
+ portRangeConfig)
+ .build();
+ log.debug(
+ "Adding protocol " + sliderClusterAPIClass.getCanonicalName() + " to the server");
+ server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, sliderClusterAPIClass,
+ blockingService);
+ return server;
+ }
+
+ /**
+ * Add the protobuf engine to the configuration. Harmless and inexpensive
+ * if repeated
+ * @param conf configuration to patch
+ * @return the protocol class
+ */
+ public static Class<SliderClusterProtocolPB> registerSliderAPI(
+ Configuration conf) {
+ Class<SliderClusterProtocolPB> sliderClusterAPIClass =
+ SliderClusterProtocolPB.class;
+ RPC.setProtocolEngine(conf, sliderClusterAPIClass, ProtobufRpcEngine.class);
+
+ //quick sanity check here
+ assert verifyBondedToProtobuf(conf, sliderClusterAPIClass);
+
+ return sliderClusterAPIClass;
+ }
+
+ /**
+ * Verify that the conf is set up for protobuf transport of Slider RPC
+ * @param conf configuration
+ * @param sliderClusterAPIClass class for the API
+ * @return
+ */
+ public static boolean verifyBondedToProtobuf(Configuration conf,
+ Class<SliderClusterProtocolPB> sliderClusterAPIClass) {
+ return conf.getClass("rpc.engine." + sliderClusterAPIClass.getName(),
+ RpcEngine.class) .equals(ProtobufRpcEngine.class);
+ }
+
+
+ public static SliderClusterProtocol connectToServer(InetSocketAddress addr,
+ UserGroupInformation currentUser,
+ Configuration conf,
+ int rpcTimeout) throws IOException {
+ Class<SliderClusterProtocolPB> sliderClusterAPIClass = registerSliderAPI(
+ conf);
+
+ log.debug("Connecting to Slider AM at {}", addr);
+ ProtocolProxy<SliderClusterProtocolPB> protoProxy =
+ RPC.getProtocolProxy(sliderClusterAPIClass,
+ 1,
+ addr,
+ currentUser,
+ conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ rpcTimeout,
+ null);
+ SliderClusterProtocolPB endpoint = protoProxy.getProxy();
+ return new SliderClusterProtocolProxy(endpoint);
+ }
+
+
+ /**
+ * This loops for a limited period trying to get the Proxy -
+ * by doing so it handles AM failover
+ * @param conf configuration to patch and use
+ * @param rmClient client of the resource manager
+ * @param application application to work with
+ * @param connectTimeout timeout for the whole proxy operation to timeout
+ * (milliseconds). Use 0 to indicate "do not attempt to wait" -fail fast.
+ * @param rpcTimeout timeout for RPCs to block during communications
+ * @return the proxy
+ * @throws IOException IO problems
+ * @throws YarnException Slider-generated exceptions related to the binding
+ * failing. This can include the application finishing or timeouts
+ * @throws InterruptedException if a sleep operation waiting for
+ * the cluster to respond is interrupted.
+ */
+ @SuppressWarnings("NestedAssignment")
+ public static SliderClusterProtocol getProxy(final Configuration conf,
+ final ApplicationClientProtocol rmClient,
+ ApplicationReport application,
+ final int connectTimeout,
+
+ final int rpcTimeout) throws
+ IOException,
+ YarnException,
+ InterruptedException {
+ ApplicationId appId;
+ appId = application.getApplicationId();
+ Duration timeout = new Duration(connectTimeout);
+ timeout.start();
+ Exception exception = null;
+ YarnApplicationState state = null;
+ while (application != null &&
+ (state= application.getYarnApplicationState()).equals(
+ YarnApplicationState.RUNNING)) {
+
+ try {
+ return getProxy(conf, application, rpcTimeout);
+ } catch (IOException e) {
+ if (connectTimeout <= 0 || timeout.getLimitExceeded()) {
+ throw e;
+ }
+ exception = e;
+ } catch (YarnException e) {
+ if (connectTimeout <= 0 || timeout.getLimitExceeded()) {
+ throw e;
+ }
+ exception = e;
+
+ }
+ //at this point: app failed to work
+ log.debug("Could not connect to {}. Waiting for getting the latest AM address...",
+ appId);
+ Thread.sleep(1000);
+ //or get the app report
+ application =
+ rmClient.getApplicationReport(GetApplicationReportRequest.newInstance(
+ appId)).getApplicationReport();
+ }
+ //get here if the app is no longer running. Raise a specific
+ //exception but init it with the previous failure
+ throw new BadClusterStateException(
+ exception,
+ ErrorStrings.E_FINISHED_APPLICATION, appId, state );
+ }
+
+ public static SliderClusterProtocol getProxy(final Configuration conf,
+ ApplicationReport application,
+ final int rpcTimeout) throws
+ IOException,
+ SliderException,
+ InterruptedException {
+
+ String host = application.getHost();
+ int port = application.getRpcPort();
+ String address = host + ":" + port;
+ if (host == null || 0 == port) {
+ throw new SliderException(SliderExitCodes.EXIT_CONNECTIVITY_PROBLEM,
+ "Slider instance " + application.getName()
+ + " isn't providing a valid address for the" +
+ " Slider RPC protocol: " + address);
+ }
+
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ final UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(
+ currentUser.getUserName());
+ final InetSocketAddress serviceAddr = NetUtils.createSocketAddrForHost(
+ application.getHost(), application.getRpcPort());
+ SliderClusterProtocol realProxy;
+
+ log.debug("Connecting to {}", serviceAddr);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ org.apache.hadoop.yarn.api.records.Token clientToAMToken =
+ application.getClientToAMToken();
+ Token<ClientToAMTokenIdentifier> token =
+ ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr);
+ newUgi.addToken(token);
+ realProxy =
+ newUgi.doAs(new PrivilegedExceptionAction<SliderClusterProtocol>() {
+ @Override
+ public SliderClusterProtocol run() throws IOException {
+ return connectToServer(serviceAddr, newUgi, conf, rpcTimeout);
+ }
+ });
+ } else {
+ return connectToServer(serviceAddr, newUgi, conf, rpcTimeout);
+ }
+ return realProxy;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java
new file mode 100644
index 0000000..a40078a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderAMPolicyProvider.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.rpc;
+
+import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.security.authorize.Service;
+import org.apache.slider.common.SliderXmlConfKeys;
+
+/**
+ * {@link PolicyProvider} for Slider protocols.
+ */
+
+public class SliderAMPolicyProvider extends PolicyProvider {
+
+ private static final Service[] services =
+ new Service[] {
+ new Service(SliderXmlConfKeys.KEY_PROTOCOL_ACL, SliderClusterProtocolPB.class)
+ };
+
+ @SuppressWarnings("ReturnOfCollectionOrArrayField")
+ @Override
+ public Service[] getServices() {
+ return services;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java
new file mode 100644
index 0000000..9d4ea99
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.rpc;
+
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.api.proto.SliderClusterAPI;
+
+public interface SliderClusterProtocolPB extends SliderClusterAPI.SliderClusterProtocolPB.BlockingInterface{
+
+ public static final long versionID = SliderClusterProtocol.versionID;
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
new file mode 100644
index 0000000..d3bc98a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.rpc;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.api.proto.Messages;
+
+import java.io.IOException;
+
+/**
+ * Relay from Protobuf to internal RPC.
+ *
+ */
+public class SliderClusterProtocolPBImpl implements SliderClusterProtocolPB {
+
+ private SliderClusterProtocol real;
+
+ public SliderClusterProtocolPBImpl(SliderClusterProtocol real) {
+ this.real = real;
+ }
+
+ private ServiceException wrap(Exception e) {
+ if (e instanceof ServiceException) {
+ return (ServiceException) e;
+ }
+ return new ServiceException(e);
+ }
+
+ public long getProtocolVersion(String protocol, long clientVersion) throws
+ IOException {
+ return SliderClusterProtocol.versionID;
+ }
+
+ @Override
+ public Messages.StopClusterResponseProto stopCluster(RpcController controller,
+ Messages.StopClusterRequestProto request) throws
+ ServiceException {
+ try {
+ return real.stopCluster(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+ @Override
+ public Messages.FlexClusterResponseProto flexCluster(RpcController controller,
+ Messages.FlexClusterRequestProto request) throws
+ ServiceException {
+ try {
+ return real.flexCluster(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+ @Override
+ public Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(
+ RpcController controller,
+ Messages.GetJSONClusterStatusRequestProto request) throws ServiceException {
+ try {
+ return real.getJSONClusterStatus(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+
+ @Override
+ public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition(
+ RpcController controller,
+ Messages.GetInstanceDefinitionRequestProto request) throws
+ ServiceException {
+ try {
+ return real.getInstanceDefinition(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+ @Override
+ public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(
+ RpcController controller,
+ Messages.ListNodeUUIDsByRoleRequestProto request) throws ServiceException {
+ try {
+ return real.listNodeUUIDsByRole(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+ @Override
+ public Messages.GetNodeResponseProto getNode(RpcController controller,
+ Messages.GetNodeRequestProto request) throws
+ ServiceException {
+ try {
+ return real.getNode(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+ @Override
+ public Messages.GetClusterNodesResponseProto getClusterNodes(RpcController controller,
+ Messages.GetClusterNodesRequestProto request) throws
+ ServiceException {
+ try {
+ return real.getClusterNodes(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+ @Override
+ public Messages.EchoResponseProto echo(RpcController controller,
+ Messages.EchoRequestProto request) throws
+ ServiceException {
+ try {
+ return real.echo(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+ @Override
+ public Messages.KillContainerResponseProto killContainer(RpcController controller,
+ Messages.KillContainerRequestProto request) throws
+ ServiceException {
+ try {
+ return real.killContainer(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+
+ @Override
+ public Messages.AMSuicideResponseProto amSuicide(RpcController controller,
+ Messages.AMSuicideRequestProto request) throws
+ ServiceException {
+ try {
+ return real.amSuicide(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
new file mode 100644
index 0000000..d16b9c8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.rpc;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.ipc.ProtobufHelper;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.api.proto.Messages;
+
+import java.io.IOException;
+
+public class SliderClusterProtocolProxy implements SliderClusterProtocol {
+
+ final SliderClusterProtocolPB endpoint;
+ private static final RpcController NULL_CONTROLLER = null;
+
+ public SliderClusterProtocolProxy(SliderClusterProtocolPB endpoint) {
+ this.endpoint = endpoint;
+ }
+
+ private IOException convert(ServiceException se) {
+ return ProtobufHelper.getRemoteException(se);
+ }
+
+ @Override
+ public Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request) throws
+ IOException,
+ YarnException {
+ try {
+ return endpoint.stopCluster(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
+ public Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) throws
+ IOException,
+ YarnException {
+ try {
+ return endpoint.flexCluster(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
+ public Messages.GetJSONClusterStatusResponseProto getJSONClusterStatus(
+ Messages.GetJSONClusterStatusRequestProto request) throws
+ IOException,
+ YarnException {
+ try {
+ return endpoint.getJSONClusterStatus(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+
+ @Override
+ public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition(
+ Messages.GetInstanceDefinitionRequestProto request) throws
+ IOException,
+ YarnException {
+ try {
+ return endpoint.getInstanceDefinition(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
+ public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request) throws
+ IOException,
+ YarnException {
+ try {
+ return endpoint.listNodeUUIDsByRole(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
+ public Messages.GetNodeResponseProto getNode(Messages.GetNodeRequestProto request) throws
+ IOException,
+ YarnException {
+ try {
+ return endpoint.getNode(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
+ public Messages.GetClusterNodesResponseProto getClusterNodes(Messages.GetClusterNodesRequestProto request) throws
+ IOException,
+ YarnException {
+ try {
+ return endpoint.getClusterNodes(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+
+ @Override
+ public Messages.EchoResponseProto echo(Messages.EchoRequestProto request) throws
+ IOException,
+ YarnException {
+ try {
+ return endpoint.echo(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+
+ @Override
+ public Messages.KillContainerResponseProto killContainer(Messages.KillContainerRequestProto request) throws
+ IOException,
+ YarnException {
+ try {
+ return endpoint.killContainer(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
+ public Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request) throws
+ IOException,
+ YarnException {
+ try {
+ return endpoint.amSuicide(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion,
+ int clientMethodsHash) throws
+ IOException {
+ if (!protocol.equals(RPC.getProtocolName(SliderClusterProtocolPB.class))) {
+ throw new IOException("Serverside implements " +
+ RPC.getProtocolName(SliderClusterProtocolPB.class) +
+ ". The following requested protocol is unknown: " +
+ protocol);
+ }
+
+ return ProtocolSignature.getProtocolSignature(clientMethodsHash,
+ RPC.getProtocolVersion(
+ SliderClusterProtocol.class),
+ SliderClusterProtocol.class);
+ }
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion) throws
+ IOException {
+ return SliderClusterProtocol.versionID;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java
new file mode 100644
index 0000000..215db1a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderRPCSecurityInfo.java
@@ -0,0 +1,85 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.slider.server.appmaster.rpc;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector;
+import org.apache.slider.common.SliderXmlConfKeys;
+
+import java.lang.annotation.Annotation;
+
+/**
+ * This is where security may go
+ */
+public class SliderRPCSecurityInfo extends SecurityInfo {
+
+ @Override
+ public KerberosInfo getKerberosInfo(Class<?> protocol, Configuration conf) {
+ if (!protocol.equals(SliderClusterProtocolPB.class)) {
+ return null;
+ }
+ return new KerberosInfo() {
+
+ @Override
+ public Class<? extends Annotation> annotationType() {
+ return null;
+ }
+
+ @Override
+ public String serverPrincipal() {
+ return SliderXmlConfKeys.KEY_KERBEROS_PRINCIPAL;
+ }
+
+ @Override
+ public String clientPrincipal() {
+ return null;
+ }
+ };
+ }
+
+ @Override
+ public TokenInfo getTokenInfo(Class<?> protocol, Configuration conf) {
+ if (!protocol.equals(SliderClusterProtocolPB.class)) {
+ return null;
+ }
+ return new TokenInfo() {
+
+ @Override
+ public Class<? extends Annotation> annotationType() {
+ return null;
+ }
+
+ @Override
+ public Class<? extends TokenSelector<? extends TokenIdentifier>>
+ value() {
+ return ClientToAMTokenSelector.class;
+ }
+
+ @Override
+ public String toString() {
+ return "SliderClusterProtocolPB token info";
+ }
+ };
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRMOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRMOperation.java
new file mode 100644
index 0000000..e3e595f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRMOperation.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+public class AbstractRMOperation {
+
+ /**
+ * Execute the operation
+ * @param asyncRMClient client
+ */
+ public void execute(RMOperationHandler handler) {
+
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java
new file mode 100644
index 0000000..e9655f0
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+
+/**
+ * Factory supplying records created by the App state; entry point
+ * for mock code.
+ */
+public abstract class AbstractRecordFactory {
+
+ public abstract Resource newResource();
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
new file mode 100644
index 0000000..37f2bd7
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -0,0 +1,1680 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterDescriptionKeys;
+import org.apache.slider.api.ClusterDescriptionOperations;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.exceptions.SliderInternalStateException;
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.providers.ProviderRole;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.slider.api.ResourceKeys.DEF_YARN_CORES;
+import static org.apache.slider.api.ResourceKeys.DEF_YARN_MEMORY;
+import static org.apache.slider.api.ResourceKeys.YARN_CORES;
+import static org.apache.slider.api.ResourceKeys.YARN_MEMORY;
+import static org.apache.slider.api.RoleKeys.ROLE_FAILED_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_FAILED_STARTING_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_RELEASING_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_REQUESTED_INSTANCES;
+
+
+/**
+ * The model of all the ongoing state of a Slider AM.
+ *
+ * concurrency rules: any method which begins with <i>build</i>
+ * is not synchronized and intended to be used during
+ * initialization.
+ */
+public class AppState implements StateAccessForProviders {
+ protected static final Logger log =
+ LoggerFactory.getLogger(AppState.class);
+
+ private final AbstractRecordFactory recordFactory;
+
+ /**
+ * Flag set to indicate the application is live -this only happens
+ * after the buildInstance operation
+ */
+ boolean applicationLive = false;
+
+ /**
+ * The definition of the instance. Flexing updates the resources section
+ This is used as a synchronization point on activities that update
+ the CD, and also to update some of the structures that
+ feed in to the CD
+ */
+ private AggregateConf instanceDefinition;
+
+ private long snapshotTime;
+ private AggregateConf instanceDefinitionSnapshot;
+
+ /**
+ * snapshot of resources as of last update time
+ */
+ private ConfTreeOperations resourcesSnapshot;
+ private ConfTreeOperations appConfSnapshot;
+ private ConfTreeOperations internalsSnapshot;
+
+ /**
+ * This is the status, the live model
+ */
+ private ClusterDescription clusterStatus = new ClusterDescription();
+
+ /**
+ * Metadata provided by the AM for use in filling in status requests
+ */
+ private Map<String, String> applicationInfo;
+
+ /**
+ * Client properties created via the provider -static for the life
+ * of the application
+ */
+ private Map<String, String> clientProperties = new HashMap<String, String>();
+
+ /**
+ The cluster description published to callers
+ This is used as a synchronization point on activities that update
+ the CD, and also to update some of the structures that
+ feed in to the CD
+ */
+ private ClusterDescription clusterSpec = new ClusterDescription();
+
+ private final PublishedConfigSet
+ publishedConfigurations = new PublishedConfigSet();
+
+
+ private final Map<Integer, RoleStatus> roleStatusMap =
+ new ConcurrentHashMap<Integer, RoleStatus>();
+
+ private final Map<String, ProviderRole> roles =
+ new ConcurrentHashMap<String, ProviderRole>();
+
+ /**
+ * The master node.
+ */
+ private RoleInstance appMasterNode;
+
+ /**
+ * Hash map of the containers we have. This includes things that have
+ * been allocated but are not live; it is a superset of the live list
+ */
+ private final ConcurrentMap<ContainerId, RoleInstance> activeContainers =
+ new ConcurrentHashMap<ContainerId, RoleInstance>();
+
+ /**
+ * Hash map of the containers we have released, but we
+ * are still awaiting acknowledgements on. Any failure of these
+ * containers is treated as a successful outcome
+ */
+ private final ConcurrentMap<ContainerId, Container> containersBeingReleased =
+ new ConcurrentHashMap<ContainerId, Container>();
+
+ /**
+ * Counter for completed containers ( complete denotes successful or failed )
+ */
+ private final AtomicInteger completedContainerCount = new AtomicInteger();
+
+ /**
+ * Count of failed containers
+
+ */
+ private final AtomicInteger failedContainerCount = new AtomicInteger();
+
+ /**
+ * # of started containers
+ */
+ private final AtomicInteger startedContainers = new AtomicInteger();
+
+ /**
+ * # of containers that failed to start
+ */
+ private final AtomicInteger startFailedContainers = new AtomicInteger();
+
+ /**
+ * Track the number of surplus containers received and discarded
+ */
+ private final AtomicInteger surplusContainers = new AtomicInteger();
+
+
+ /**
+ * Map of requested nodes. This records the command used to start it,
+ * resources, etc. When container started callback is received,
+ * the node is promoted from here to the containerMap
+ */
+ private final Map<ContainerId, RoleInstance> startingNodes =
+ new ConcurrentHashMap<ContainerId, RoleInstance>();
+
+ /**
+ * List of completed nodes. This isn't kept in the CD as it gets too
+ * big for the RPC responses. Indeed, we should think about how deep to get this
+ */
+ private final Map<ContainerId, RoleInstance> completedNodes
+ = new ConcurrentHashMap<ContainerId, RoleInstance>();
+
+ /**
+ * Nodes that failed to start.
+ * Again, kept out of the CD
+ */
+ private final Map<ContainerId, RoleInstance> failedNodes =
+ new ConcurrentHashMap<ContainerId, RoleInstance>();
+
+ /**
+ * Nodes that came assigned to a role above that
+ * which were asked for -this appears to happen
+ */
+ private final Set<ContainerId> surplusNodes = new HashSet<ContainerId>();
+
+ /**
+ * Map of containerID -> cluster nodes, for status reports.
+ * Access to this should be synchronized on the clusterDescription
+ */
+ private final Map<ContainerId, RoleInstance> liveNodes =
+ new ConcurrentHashMap<ContainerId, RoleInstance>();
+ private final AtomicInteger completionOfNodeNotInLiveListEvent =
+ new AtomicInteger();
+ private final AtomicInteger completionOfUnknownContainerEvent =
+ new AtomicInteger();
+
+
+ /**
+ * Record of the max no. of cores allowed in this cluster
+ */
+ private int containerMaxCores;
+
+ /**
+ * limit container memory
+ */
+ private int containerMaxMemory;
+
+ private RoleHistory roleHistory;
+ private Configuration publishedProviderConf;
+ private long startTimeThreshold;
+
+ private int failureThreshold = 10;
+
+ public AppState(AbstractRecordFactory recordFactory) {
+ this.recordFactory = recordFactory;
+ }
+
+ public int getFailedCountainerCount() {
+ return failedContainerCount.get();
+ }
+
+ /**
+ * Increment the count and return the new value
+ * @return the latest failed container count
+ */
+ public int incFailedCountainerCount() {
+ return failedContainerCount.incrementAndGet();
+ }
+
+ public int getStartFailedCountainerCount() {
+ return startFailedContainers.get();
+ }
+
+ /**
+ * Increment the count and return the new value
+ * @return the latest failed container count
+ */
+ public int incStartedCountainerCount() {
+ return startedContainers.incrementAndGet();
+ }
+
+ public int getStartedCountainerCount() {
+ return startedContainers.get();
+ }
+
+ /**
+ * Increment the count and return the new value
+ * @return the latest failed container count
+ */
+ public int incStartFailedCountainerCount() {
+ return startFailedContainers.incrementAndGet();
+ }
+
+
+ public AtomicInteger getStartFailedContainers() {
+ return startFailedContainers;
+ }
+
+ public AtomicInteger getCompletionOfNodeNotInLiveListEvent() {
+ return completionOfNodeNotInLiveListEvent;
+ }
+
+ public AtomicInteger getCompletionOfUnknownContainerEvent() {
+ return completionOfUnknownContainerEvent;
+ }
+
+ @Override
+ public Map<Integer, RoleStatus> getRoleStatusMap() {
+ return roleStatusMap;
+ }
+
+ protected Map<String, ProviderRole> getRoleMap() {
+ return roles;
+ }
+
+ private Map<ContainerId, RoleInstance> getStartingNodes() {
+ return startingNodes;
+ }
+
+ private Map<ContainerId, RoleInstance> getCompletedNodes() {
+ return completedNodes;
+ }
+
+
+ @Override
+ public PublishedConfigSet getPublishedConfigurations() {
+ return publishedConfigurations;
+ }
+
+
+ @Override
+ public Map<ContainerId, RoleInstance> getFailedNodes() {
+ return failedNodes;
+ }
+
+ @Override
+ public Map<ContainerId, RoleInstance> getLiveNodes() {
+ return liveNodes;
+ }
+
+ /**
+ * Get the desired cluster state
+ * @return the specification of the cluter
+ */
+ public ClusterDescription getClusterSpec() {
+ return clusterSpec;
+ }
+
+ @Override
+ public ClusterDescription getClusterStatus() {
+ return clusterStatus;
+ }
+
+ @VisibleForTesting
+ protected void setClusterStatus(ClusterDescription clusterDesc) {
+ this.clusterStatus = clusterDesc;
+ }
+
+ /**
+ * Set the instance definition -this also builds the (now obsolete)
+ * cluster specification from it.
+ *
+ * Important: this is for early binding and must not be used after the build
+ * operation is complete.
+ * @param definition
+ * @throws BadConfigException
+ */
+ public synchronized void updateInstanceDefinition(AggregateConf definition) throws
+ BadConfigException,
+ IOException {
+ this.instanceDefinition = definition;
+ onInstanceDefinitionUpdated();
+ }
+
+ public synchronized AggregateConf getInstanceDefinition() {
+ return instanceDefinition;
+ }
+
+ /**
+ * Get the role history of the application
+ * @return the role history
+ */
+ @VisibleForTesting
+ public RoleHistory getRoleHistory() {
+ return roleHistory;
+ }
+
+ /**
+ * Get the path used for history files
+ * @return the directory used for history files
+ */
+ @VisibleForTesting
+ public Path getHistoryPath() {
+ return roleHistory.getHistoryPath();
+ }
+
+ /**
+ * Set the container limits -the max that can be asked for,
+ * which are used when the "max" values are requested
+ * @param maxMemory maximum memory
+ * @param maxCores maximum cores
+ */
+ public void setContainerLimits(int maxMemory, int maxCores) {
+ containerMaxCores = maxCores;
+ containerMaxMemory = maxMemory;
+ }
+
+ @Override
+ public ConfTreeOperations getResourcesSnapshot() {
+ return resourcesSnapshot;
+ }
+
+ @Override
+ public ConfTreeOperations getAppConfSnapshot() {
+ return appConfSnapshot;
+ }
+
+ @Override
+ public ConfTreeOperations getInternalsSnapshot() {
+ return internalsSnapshot;
+ }
+
+ @Override
+ public boolean isApplicationLive() {
+ return applicationLive;
+ }
+
+
+ @Override
+ public long getSnapshotTime() {
+ return snapshotTime;
+ }
+
+ @Override
+ public AggregateConf getInstanceDefinitionSnapshot() {
+ return instanceDefinitionSnapshot;
+ }
+
+ /**
+ * Build up the application state
+ * @param instanceDefinition definition of the applicatin instance
+ * @param publishedProviderConf any configuration info to be published by a provider
+ * @param providerRoles roles offered by a provider
+ * @param fs filesystem
+ * @param historyDir directory containing history files
+ * @param liveContainers list of live containers supplied on an AM restart
+ * @param applicationInfo
+ */
+ public synchronized void buildInstance(AggregateConf instanceDefinition,
+ Configuration publishedProviderConf,
+ List<ProviderRole> providerRoles,
+ FileSystem fs,
+ Path historyDir,
+ List<Container> liveContainers,
+ Map<String, String> applicationInfo) throws
+ BadClusterStateException,
+ BadConfigException,
+ IOException {
+ this.publishedProviderConf = publishedProviderConf;
+ this.applicationInfo = applicationInfo != null ? applicationInfo
+ : new HashMap<String, String>();
+
+ clientProperties = new HashMap<String, String>();
+
+
+ Set<String> confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf);
+
+// Add the -site configuration properties
+ for (String key : confKeys) {
+ String val = publishedProviderConf.get(key);
+ clientProperties.put(key, val);
+ }
+
+
+ // set the cluster specification (once its dependency the client properties
+ // is out the way
+
+ updateInstanceDefinition(instanceDefinition);
+
+
+ //build the initial role list
+ for (ProviderRole providerRole : providerRoles) {
+ buildRole(providerRole);
+ }
+
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+
+ Set<String> roleNames = resources.getComponentNames();
+ for (String name : roleNames) {
+ if (!roles.containsKey(name)) {
+ // this is a new value
+ log.info("Adding new role {}", name);
+ MapOperations resComponent = resources.getComponent(name);
+ ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent);
+ buildRole(dynamicRole);
+ providerRoles.add(dynamicRole);
+ }
+ }
+ //then pick up the requirements
+ buildRoleRequirementsFromResources();
+
+
+ //set the livespan
+ MapOperations globalInternalOpts =
+ instanceDefinition.getInternalOperations().getGlobalOptions();
+ startTimeThreshold = globalInternalOpts.getOptionInt(
+ OptionKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
+ OptionKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE);
+
+ failureThreshold = globalInternalOpts.getOptionInt(
+ OptionKeys.INTERNAL_CONTAINER_FAILURE_THRESHOLD,
+ OptionKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD);
+ initClusterStatus();
+
+
+ // add the roles
+ roleHistory = new RoleHistory(providerRoles);
+ roleHistory.onStart(fs, historyDir);
+
+ //rebuild any live containers
+ rebuildModelFromRestart(liveContainers);
+
+ //mark as live
+ applicationLive = true;
+ }
+
+ public void initClusterStatus() {
+ //copy into cluster status.
+ ClusterDescription status = ClusterDescription.copy(clusterSpec);
+ status.state = ClusterDescription.STATE_CREATED;
+ MapOperations infoOps = new MapOperations("info", status.info);
+ infoOps.mergeWithoutOverwrite(applicationInfo);
+ SliderUtils.addBuildInfo(infoOps, "status");
+
+ long now = now();
+ status.setInfoTime(StatusKeys.INFO_LIVE_TIME_HUMAN,
+ StatusKeys.INFO_LIVE_TIME_MILLIS,
+ now);
+ SliderUtils.setInfoTime(infoOps,
+ StatusKeys.INFO_LIVE_TIME_HUMAN,
+ StatusKeys.INFO_LIVE_TIME_MILLIS,
+ now);
+ if (0 == status.createTime) {
+ status.createTime = now;
+ SliderUtils.setInfoTime(infoOps,
+ StatusKeys.INFO_CREATE_TIME_HUMAN,
+ StatusKeys.INFO_CREATE_TIME_MILLIS,
+ now);
+ }
+ status.state = ClusterDescription.STATE_LIVE;
+
+ //set the app state to this status
+ setClusterStatus(status);
+ }
+
+ /**
+ * Build a dynamic provider role
+ * @param name name of role
+ * @return a new provider role
+ * @throws BadConfigException bad configuration
+ */
+ public ProviderRole createDynamicProviderRole(String name,
+ MapOperations component) throws
+ BadConfigException {
+ String priOpt = component.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY);
+ int pri = SliderUtils.parseAndValidate("value of " + name + " " +
+ ResourceKeys.COMPONENT_PRIORITY,
+ priOpt, 0, 1, -1
+ );
+ log.info("Role {} assigned priority {}", name, pri);
+ String placementOpt = component.getOption(
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY, "0");
+ int placement = SliderUtils.parseAndValidate("value of " + name + " " +
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ placementOpt, 0, 0, -1
+ );
+ return new ProviderRole(name, pri, placement);
+ }
+
+
+ /**
+ * Actions to perform when an instance definition is updated
+ * Currently: resolve the configuration
+ * updated the cluster spec derivative
+ * @throws BadConfigException
+ */
+ private synchronized void onInstanceDefinitionUpdated() throws
+ BadConfigException,
+ IOException {
+ instanceDefinition.resolve();
+
+ //note the time
+ snapshotTime = now();
+ //snapshot all three sectons
+ resourcesSnapshot =
+ ConfTreeOperations.fromInstance(instanceDefinition.getResources());
+ appConfSnapshot =
+ ConfTreeOperations.fromInstance(instanceDefinition.getAppConf());
+ internalsSnapshot =
+ ConfTreeOperations.fromInstance(instanceDefinition.getInternal());
+ //build a new aggregate from the snapshots
+ instanceDefinitionSnapshot = new AggregateConf(resourcesSnapshot.confTree,
+ appConfSnapshot.confTree,
+ internalsSnapshot.confTree);
+
+ clusterSpec =
+ ClusterDescriptionOperations.buildFromInstanceDefinition(
+ instanceDefinition);
+
+
+// Add the -site configuration properties
+ for (Map.Entry<String, String> prop : clientProperties.entrySet()) {
+ clusterSpec.clientProperties.put(prop.getKey(), prop.getValue());
+ }
+
+ }
+
+ /**
+ * The resource configuration is updated -review and update state
+ * @param resources updated resources specification
+ */
+ public synchronized void updateResourceDefinitions(ConfTree resources) throws
+ BadConfigException,
+ IOException {
+ log.debug("Updating resources to {}", resources);
+
+ instanceDefinition.setResources(resources);
+ onInstanceDefinitionUpdated();
+
+
+ //propagate the role table
+
+ Map<String, Map<String, String>> updated = resources.components;
+ getClusterStatus().roles = SliderUtils.deepClone(updated);
+ getClusterStatus().updateTime = now();
+ buildRoleRequirementsFromResources();
+ }
+
+ /**
+ * build the role requirements from the cluster specification
+ */
+ private void buildRoleRequirementsFromResources() throws
+ BadConfigException {
+ //now update every role's desired count.
+ //if there are no instance values, that role count goes to zero
+
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+
+ // Add all the existing roles
+ for (RoleStatus roleStatus : getRoleStatusMap().values()) {
+ int currentDesired = roleStatus.getDesired();
+ String role = roleStatus.getName();
+ MapOperations comp =
+ resources.getComponent(role);
+ int desiredInstanceCount =
+ resources.getComponentOptInt(role, ResourceKeys.COMPONENT_INSTANCES, 0);
+ if (desiredInstanceCount == 0) {
+ log.warn("Role {} has 0 instances specified", role);
+ }
+ if (currentDesired != desiredInstanceCount) {
+ log.info("Role {} flexed from {} to {}", role, currentDesired,
+ desiredInstanceCount);
+ roleStatus.setDesired(desiredInstanceCount);
+ }
+ }
+ //now the dynamic ones. Iterate through the the cluster spec and
+ //add any role status entries not in the role status
+ Set<String> roleNames = resources.getComponentNames();
+ for (String name : roleNames) {
+ if (!roles.containsKey(name)) {
+ // this is a new value
+ log.info("Adding new role {}", name);
+ ProviderRole dynamicRole = createDynamicProviderRole(name,
+ resources.getComponent(name));
+ buildRole(dynamicRole);
+ roleHistory.addNewProviderRole(dynamicRole);
+ }
+ }
+ }
+
+ /**
+ * Add knowledge of a role.
+ * This is a build-time operation that is not synchronized, and
+ * should be used while setting up the system state -before servicing
+ * requests.
+ * @param providerRole role to add
+ */
+ public void buildRole(ProviderRole providerRole) throws BadConfigException {
+ //build role status map
+ int priority = providerRole.id;
+ if (roleStatusMap.containsKey(priority)) {
+ throw new BadConfigException("Duplicate Provider Key: %s and %s",
+ providerRole,
+ roleStatusMap.get(priority));
+ }
+ roleStatusMap.put(priority,
+ new RoleStatus(providerRole));
+ roles.put(providerRole.name, providerRole);
+ }
+
+ /**
+ * build up the special master node, which lives
+ * in the live node set but has a lifecycle bonded to the AM
+ * @param containerId the AM master
+ * @param host
+ * @param nodeHttpAddress
+ */
+ public void buildAppMasterNode(ContainerId containerId,
+ String host,
+ int amPort,
+ String nodeHttpAddress) {
+ Container container = new ContainerPBImpl();
+ container.setId(containerId);
+ NodeId nodeId = NodeId.newInstance(host, amPort);
+ container.setNodeId(nodeId);
+ container.setNodeHttpAddress(nodeHttpAddress);
+ RoleInstance am = new RoleInstance(container);
+ am.role = SliderKeys.COMPONENT_AM;
+ appMasterNode = am;
+ //it is also added to the set of live nodes
+ getLiveNodes().put(containerId, am);
+ }
+
+ /**
+ * Note that the master node has been launched,
+ * though it isn't considered live until any forked
+ * processes are running. It is NOT registered with
+ * the role history -the container is incomplete
+ * and it will just cause confusion
+ */
+ public void noteAMLaunched() {
+ getLiveNodes().put(appMasterNode.getContainerId(), appMasterNode);
+ }
+
+ /**
+ * AM declares ourselves live in the cluster description.
+ * This is meant to be triggered from the callback
+ * indicating the spawned process is up and running.
+ */
+ public void noteAMLive() {
+ appMasterNode.state = ClusterDescription.STATE_LIVE;
+ }
+
+ public RoleInstance getAppMasterNode() {
+ return appMasterNode;
+ }
+
+ @Override
+ public RoleStatus lookupRoleStatus(int key) {
+ RoleStatus rs = getRoleStatusMap().get(key);
+ if (rs == null) {
+ throw new RuntimeException("Cannot find role for role ID " + key);
+ }
+ return rs;
+ }
+
+ @Override
+ public RoleStatus lookupRoleStatus(Container c) throws YarnRuntimeException {
+ return lookupRoleStatus(ContainerPriority.extractRole(c));
+ }
+
+
+ @Override
+ public RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException {
+ ProviderRole providerRole = roles.get(name);
+ if (providerRole == null) {
+ throw new YarnRuntimeException("Unknown role " + name);
+ }
+ return lookupRoleStatus(providerRole.id);
+ }
+
+ @Override
+ public synchronized List<RoleInstance> cloneActiveContainerList() {
+ Collection<RoleInstance> values = activeContainers.values();
+ return new ArrayList<RoleInstance>(values);
+ }
+
+ @Override
+ public int getNumActiveContainers() {
+ return activeContainers.size();
+ }
+
+ @Override
+ public RoleInstance getActiveContainer(ContainerId id) {
+ return activeContainers.get(id);
+ }
+
+ @Override
+ public synchronized List<RoleInstance> cloneLiveContainerInfoList() {
+ List<RoleInstance> allRoleInstances;
+ Collection<RoleInstance> values = getLiveNodes().values();
+ allRoleInstances = new ArrayList<RoleInstance>(values);
+ return allRoleInstances;
+ }
+
+
+ @Override
+ public synchronized RoleInstance getLiveInstanceByContainerID(String containerId)
+ throws NoSuchNodeException {
+ Collection<RoleInstance> nodes = getLiveNodes().values();
+ for (RoleInstance node : nodes) {
+ if (containerId.equals(node.id)) {
+ return node;
+ }
+ }
+ //at this point: no node
+ throw new NoSuchNodeException(containerId);
+ }
+
+ @Override
+ public synchronized List<RoleInstance> getLiveInstancesByContainerIDs(
+ Collection<String> containerIDs) {
+ //first, a hashmap of those containerIDs is built up
+ Set<String> uuidSet = new HashSet<String>(containerIDs);
+ List<RoleInstance> nodes = new ArrayList<RoleInstance>(uuidSet.size());
+ Collection<RoleInstance> clusterNodes = getLiveNodes().values();
+
+ for (RoleInstance node : clusterNodes) {
+ if (uuidSet.contains(node.id)) {
+ nodes.add(node);
+ }
+ }
+ //at this point: a possibly empty list of nodes
+ return nodes;
+ }
+
+ /**
+ * Enum all nodes by role.
+ * @param role role, or "" for all roles
+ * @return a list of nodes, may be empty
+ */
+ public synchronized List<RoleInstance> enumLiveNodesInRole(String role) {
+ List<RoleInstance> nodes = new ArrayList<RoleInstance>();
+ Collection<RoleInstance> allRoleInstances = getLiveNodes().values();
+ for (RoleInstance node : allRoleInstances) {
+ if (role.isEmpty() || role.equals(node.role)) {
+ nodes.add(node);
+ }
+ }
+ return nodes;
+ }
+
+
+ /**
+ * Build an instance map.
+ * @return the map of Role name to list of role instances
+ */
+ private synchronized Map<String, List<String>> createRoleToInstanceMap() {
+ Map<String, List<String>> map = new HashMap<String, List<String>>();
+ for (RoleInstance node : getLiveNodes().values()) {
+ List<String> containers = map.get(node.role);
+ if (containers == null) {
+ containers = new ArrayList<String>();
+ map.put(node.role, containers);
+ }
+ containers.add(node.id);
+ }
+ return map;
+ }
+ /**
+ * Build an instance map to send over the wire
+ * @return the map of Role name to list of Cluster Nodes, ready
+ */
+ private synchronized Map<String, Map<String, ClusterNode>> createRoleToClusterNodeMap() {
+ Map<String, Map<String, ClusterNode>> map =
+ new HashMap<String, Map<String, ClusterNode>>();
+ for (RoleInstance node : getLiveNodes().values()) {
+
+ Map<String, ClusterNode> containers = map.get(node.role);
+ if (containers == null) {
+ containers = new HashMap<String, ClusterNode>();
+ map.put(node.role, containers);
+ }
+ Messages.RoleInstanceState pbuf = node.toProtobuf();
+ ClusterNode clusterNode = ClusterNode.fromProtobuf(pbuf);
+ containers.put(clusterNode.name, clusterNode);
+ }
+ return map;
+ }
+
+ /**
+ * Notification called just before the NM is asked to
+ * start a container
+ * @param container container to start
+ * @param instance clusterNode structure
+ */
+ public void containerStartSubmitted(Container container,
+ RoleInstance instance) {
+ instance.state = ClusterDescription.STATE_SUBMITTED;
+ instance.container = container;
+ instance.createTime = now();
+ getStartingNodes().put(container.getId(), instance);
+ activeContainers.put(container.getId(), instance);
+ roleHistory.onContainerStartSubmitted(container, instance);
+ }
+
+ /**
+ * Note that a container has been submitted for release; update internal state
+ * and mark the associated ContainerInfo released field to indicate that
+ * while it is still in the active list, it has been queued for release.
+ *
+ * @param container container
+ * @throws SliderInternalStateException if there is no container of that ID
+ * on the active list
+ */
+ public synchronized void containerReleaseSubmitted(Container container)
+ throws SliderInternalStateException {
+ ContainerId id = container.getId();
+ //look up the container
+ RoleInstance info = getActiveContainer(id);
+ if (info == null) {
+ throw new SliderInternalStateException(
+ "No active container with ID " + id.toString());
+ }
+ //verify that it isn't already released
+ if (containersBeingReleased.containsKey(id)) {
+ throw new SliderInternalStateException(
+ "Container %s already queued for release", id);
+ }
+ info.released = true;
+ containersBeingReleased.put(id, info.container);
+ RoleStatus role = lookupRoleStatus(info.roleId);
+ role.incReleasing();
+ roleHistory.onContainerReleaseSubmitted(container);
+ }
+
+
+ /**
+ * Set up the resource requirements with all that this role needs,
+ * then create the container request itself.
+ * @param role role to ask an instance of
+ * @param capability a resource to set up
+ * @return the request for a new container
+ */
+ public AMRMClient.ContainerRequest buildContainerResourceAndRequest(
+ RoleStatus role,
+ Resource capability) {
+ buildResourceRequirements(role, capability);
+ //get the role history to select a suitable node, if available
+ AMRMClient.ContainerRequest containerRequest =
+ createContainerRequest(role, capability);
+ return containerRequest;
+ }
+
+ /**
+ * Create a container request.
+ * Update internal state, such as the role request count
+ * This is where role history information will be used for placement decisions -
+ * @param role role
+ * @param resource requirements
+ * @return the container request to submit
+ */
+ public AMRMClient.ContainerRequest createContainerRequest(RoleStatus role,
+ Resource resource) {
+
+
+ AMRMClient.ContainerRequest request;
+ int key = role.getKey();
+ request = roleHistory.requestNode(role, resource);
+ role.incRequested();
+
+ return request;
+ }
+
+
+ /**
+ * Get the value of a YARN requirement (cores, RAM, etc).
+ * These are returned as integers, but there is special handling of the
+ * string {@link ResourceKeys#YARN_RESOURCE_MAX}, which triggers
+ * the return of the maximum value.
+ * @param name component to get from
+ * @param option option name
+ * @param defVal default value
+ * @param maxVal value to return if the max val is requested
+ * @return parsed value
+ * @throws NumberFormatException if the role could not be parsed.
+ */
+ private int getResourceRequirement(ConfTreeOperations resources,
+ String name,
+ String option,
+ int defVal,
+ int maxVal) {
+
+ String val = resources.getComponentOpt(name, option,
+ Integer.toString(defVal));
+ Integer intVal;
+ if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
+ intVal = maxVal;
+ } else {
+ intVal = Integer.decode(val);
+ }
+ return intVal;
+ }
+
+ /**
+ * Build up the resource requirements for this role from the
+ * cluster specification, including substituing max allowed values
+ * if the specification asked for it.
+ * @param role role
+ * @param capability capability to set up
+ */
+ public void buildResourceRequirements(RoleStatus role, Resource capability) {
+ // Set up resource requirements from role values
+ String name = role.getName();
+ ConfTreeOperations resources = getResourcesSnapshot();
+ int cores = getResourceRequirement(resources,
+ name,
+ YARN_CORES,
+ DEF_YARN_CORES,
+ containerMaxCores);
+ capability.setVirtualCores(cores);
+ int ram = getResourceRequirement(resources, name,
+ YARN_MEMORY,
+ DEF_YARN_MEMORY,
+ containerMaxMemory);
+ capability.setMemory(ram);
+ }
+
+ /**
+ * add a launched container to the node map for status responses
+ * @param container id
+ * @param node node details
+ */
+ private void addLaunchedContainer(Container container, RoleInstance node) {
+ node.container = container;
+ if (node.role == null) {
+ throw new RuntimeException(
+ "Unknown role for node " + node);
+ }
+ getLiveNodes().put(node.getContainerId(), node);
+ //tell role history
+ roleHistory.onContainerStarted(container);
+ }
+
+ /**
+ * container start event
+ * @param containerId container that is to be started
+ * @return the role instance, or null if there was a problem
+ */
+ public synchronized RoleInstance onNodeManagerContainerStarted(ContainerId containerId) {
+ try {
+ return innerOnNodeManagerContainerStarted(containerId);
+ } catch (YarnRuntimeException e) {
+ log.error("NodeManager callback on started container {} failed",
+ containerId,
+ e);
+ return null;
+ }
+ }
+
+ /**
+ * container start event handler -throwing an exception on problems
+ * @param containerId container that is to be started
+ * @return the role instance
+ * @throws RuntimeException on problems
+ */
+ @VisibleForTesting
+ public RoleInstance innerOnNodeManagerContainerStarted(ContainerId containerId) {
+ incStartedCountainerCount();
+ RoleInstance instance = activeContainers.get(containerId);
+ if (instance == null) {
+ //serious problem
+ throw new YarnRuntimeException("Container not in active containers start "+
+ containerId);
+ }
+ if (instance.role == null) {
+ throw new YarnRuntimeException("Component instance has no instance name " +
+ instance);
+ }
+ instance.startTime = now();
+ RoleInstance starting = getStartingNodes().remove(containerId);
+ if (null == starting) {
+ throw new YarnRuntimeException(
+ "Container "+ containerId +"%s is already started");
+ }
+ instance.state = ClusterDescription.STATE_LIVE;
+ RoleStatus roleStatus = lookupRoleStatus(instance.roleId);
+ roleStatus.incStarted();
+ Container container = instance.container;
+ addLaunchedContainer(container, instance);
+ return instance;
+ }
+
+ /**
+ * update the application state after a failure to start a container.
+ * This is perhaps where blacklisting could be most useful: failure
+ * to start a container is a sign of a more serious problem
+ * than a later exit.
+ *
+ * -relayed from NMClientAsync.CallbackHandler
+ * @param containerId failing container
+ * @param thrown what was thrown
+ */
+ public synchronized void onNodeManagerContainerStartFailed(ContainerId containerId,
+ Throwable thrown) {
+ activeContainers.remove(containerId);
+ incFailedCountainerCount();
+ incStartFailedCountainerCount();
+ RoleInstance instance = getStartingNodes().remove(containerId);
+ if (null != instance) {
+ RoleStatus roleStatus = lookupRoleStatus(instance.roleId);
+ if (null != thrown) {
+ instance.diagnostics = SliderUtils.stringify(thrown);
+ }
+ roleStatus.noteFailed(null);
+ roleStatus.incStartFailed();
+ getFailedNodes().put(containerId, instance);
+ roleHistory.onNodeManagerContainerStartFailed(instance.container);
+ }
+ }
+
+ /**
+ * Is a role short lived by the threshold set for this application
+ * @param instance instance
+ * @return true if the instance is considered short live
+ */
+ @VisibleForTesting
+ public boolean isShortLived(RoleInstance instance) {
+ long time = now();
+ long started = instance.startTime;
+ boolean shortlived;
+ if (started > 0) {
+ long duration = time - started;
+ shortlived = duration < startTimeThreshold;
+ } else {
+ // never even saw a start event
+ shortlived = true;
+ }
+ return shortlived;
+ }
+
+ /**
+ * Current time in milliseconds. Made protected for
+ * the option to override it in tests.
+ * @return the current time.
+ */
+ protected long now() {
+ return System.currentTimeMillis();
+ }
+
+ /**
+ * This is a very small class to send a triple result back from
+ * the completion operation
+ */
+ public static class NodeCompletionResult {
+ public boolean surplusNode = false;
+ public RoleInstance roleInstance;
+ public boolean containerFailed;
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("NodeCompletionResult{");
+ sb.append("surplusNode=").append(surplusNode);
+ sb.append(", roleInstance=").append(roleInstance);
+ sb.append(", containerFailed=").append(containerFailed);
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+
+ /**
+ * handle completed node in the CD -move something from the live
+ * server list to the completed server list
+ * @param status the node that has just completed
+ * @return NodeCompletionResult
+ */
+ public synchronized NodeCompletionResult onCompletedNode(ContainerStatus status) {
+ return onCompletedNode(null, status);
+ }
+
+ /**
+ * handle completed node in the CD -move something from the live
+ * server list to the completed server list
+ * @param amConf YarnConfiguration
+ * @param status the node that has just completed
+ * @return NodeCompletionResult
+ */
+ public synchronized NodeCompletionResult onCompletedNode(YarnConfiguration amConf,
+ ContainerStatus status) {
+ ContainerId containerId = status.getContainerId();
+ NodeCompletionResult result = new NodeCompletionResult();
+ RoleInstance roleInstance;
+
+ if (containersBeingReleased.containsKey(containerId)) {
+ log.info("Container was queued for release");
+ Container container = containersBeingReleased.remove(containerId);
+ RoleStatus roleStatus = lookupRoleStatus(container);
+ log.info("decrementing role count for role {}", roleStatus.getName());
+ roleStatus.decReleasing();
+ roleStatus.decActual();
+ roleStatus.incCompleted();
+ roleHistory.onReleaseCompleted(container);
+
+ } else if (surplusNodes.remove(containerId)) {
+ //its a surplus one being purged
+ result.surplusNode = true;
+ } else {
+ //a container has failed
+ result.containerFailed = true;
+ roleInstance = activeContainers.remove(containerId);
+ if (roleInstance != null) {
+ //it was active, move it to failed
+ incFailedCountainerCount();
+ failedNodes.put(containerId, roleInstance);
+ } else {
+ // the container may have been noted as failed already, so look
+ // it up
+ roleInstance = failedNodes.get(containerId);
+ }
+ if (roleInstance != null) {
+ int roleId = roleInstance.roleId;
+ log.info("Failed container in role {}", roleId);
+ try {
+ RoleStatus roleStatus = lookupRoleStatus(roleId);
+ roleStatus.decActual();
+ boolean shortLived = isShortLived(roleInstance);
+ String message;
+ if (roleInstance.container != null) {
+ String user = null;
+ try {
+ user = SliderUtils.getCurrentUser().getShortUserName();
+ } catch (IOException ioe) {
+ }
+ String completedLogsUrl = null;
+ Container c = roleInstance.container;
+ String url = null;
+ if (amConf != null) {
+ url = amConf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
+ }
+ if (user != null && url != null) {
+ completedLogsUrl = url
+ + "/" + c.getNodeId() + "/" + roleInstance.getContainerId() + "/ctx/" + user;
+ }
+ message = String.format("Failure %s on host %s" +
+ (completedLogsUrl != null ? ", see %s" : ""), roleInstance.getContainerId(),
+ c.getNodeId().getHost(), completedLogsUrl);
+ } else {
+ message = String.format("Failure %s",
+ containerId.toString());
+ }
+ roleStatus.noteFailed(message);
+ //have a look to see if it short lived
+ if (shortLived) {
+ roleStatus.incStartFailed();
+ }
+
+ if (roleInstance.container != null) {
+ roleHistory.onFailedContainer(roleInstance.container, shortLived);
+ }
+
+ } catch (YarnRuntimeException e1) {
+ log.error("Failed container of unknown role {}", roleId);
+ }
+ } else {
+ //this isn't a known container.
+
+ log.error("Notified of completed container {} that is not in the list" +
+ " of active or failed containers", containerId);
+ completionOfUnknownContainerEvent.incrementAndGet();
+ }
+ }
+
+ if (result.surplusNode) {
+ //a surplus node
+ return result;
+ }
+
+ //record the complete node's details; this pulls it from the livenode set
+ //remove the node
+ ContainerId id = status.getContainerId();
+ RoleInstance node = getLiveNodes().remove(id);
+ if (node == null) {
+ log.warn("Received notification of completion of unknown node {}", id);
+ completionOfNodeNotInLiveListEvent.incrementAndGet();
+
+ } else {
+ node.state = ClusterDescription.STATE_DESTROYED;
+ node.exitCode = status.getExitStatus();
+ node.diagnostics = status.getDiagnostics();
+ getCompletedNodes().put(id, node);
+ result.roleInstance = node;
+ }
+ return result;
+ }
+
+
+ /**
+ * Return the percentage done that Slider is to have YARN display in its
+ * Web UI
+ * @return an number from 0 to 100
+ */
+ public synchronized float getApplicationProgressPercentage() {
+ float percentage;
+ int desired = 0;
+ float actual = 0;
+ for (RoleStatus role : getRoleStatusMap().values()) {
+ desired += role.getDesired();
+ actual += role.getActual();
+ }
+ if (desired == 0) {
+ percentage = 100;
+ } else {
+ percentage = actual / desired;
+ }
+ return percentage;
+ }
+
+ @Override
+ public void refreshClusterStatus() {
+ refreshClusterStatus(null);
+ }
+
+ /**
+ * Update the cluster description with anything interesting
+ * @param providerStatus status from the provider for the cluster info section
+ */
+ public void refreshClusterStatus(Map<String, String> providerStatus) {
+ ClusterDescription cd = getClusterStatus();
+ long now = now();
+ cd.setInfoTime(StatusKeys.INFO_STATUS_TIME_HUMAN,
+ StatusKeys.INFO_STATUS_TIME_MILLIS,
+ now);
+ if (providerStatus != null) {
+ for (Map.Entry<String, String> entry : providerStatus.entrySet()) {
+ cd.setInfo(entry.getKey(),entry.getValue());
+ }
+ }
+ MapOperations infoOps = new MapOperations("info",cd.info);
+ infoOps.mergeWithoutOverwrite(applicationInfo);
+ SliderUtils.addBuildInfo(infoOps, "status");
+ cd.statistics = new HashMap<String, Map<String, Integer>>();
+
+ // build the map of node -> container IDs
+ Map<String, List<String>> instanceMap = createRoleToInstanceMap();
+ cd.instances = instanceMap;
+
+ //build the map of node -> containers
+ Map<String, Map<String, ClusterNode>> clusterNodes =
+ createRoleToClusterNodeMap();
+ cd.status = new HashMap<String, Object>();
+ cd.status.put(ClusterDescriptionKeys.KEY_CLUSTER_LIVE, clusterNodes);
+
+
+ for (RoleStatus role : getRoleStatusMap().values()) {
+ String rolename = role.getName();
+ List<String> instances = instanceMap.get(rolename);
+ int nodeCount = instances != null ? instances.size(): 0;
+ cd.setRoleOpt(rolename, ResourceKeys.COMPONENT_INSTANCES,
+ role.getDesired());
+ cd.setRoleOpt(rolename, RoleKeys.ROLE_ACTUAL_INSTANCES, nodeCount);
+ cd.setRoleOpt(rolename, ROLE_REQUESTED_INSTANCES, role.getRequested());
+ cd.setRoleOpt(rolename, ROLE_RELEASING_INSTANCES, role.getReleasing());
+ cd.setRoleOpt(rolename, ROLE_FAILED_INSTANCES, role.getFailed());
+ cd.setRoleOpt(rolename, ROLE_FAILED_STARTING_INSTANCES, role.getStartFailed());
+ Map<String, Integer> stats = role.buildStatistics();
+ cd.statistics.put(rolename, stats);
+ }
+
+ Map<String, Integer> sliderstats = new HashMap<String, Integer>();
+ sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED,
+ completedContainerCount.get());
+ sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED,
+ failedContainerCount.get());
+ sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, liveNodes.size());
+ sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED,
+ startedContainers.get());
+ sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED,
+ startFailedContainers.get());
+ sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_SURPLUS,
+ surplusContainers.get());
+ sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_UNKNOWN_COMPLETED,
+ completionOfUnknownContainerEvent.get());
+ cd.statistics.put(SliderKeys.COMPONENT_AM, sliderstats);
+
+ }
+
+ /**
+ * Look at where the current node state is -and whether it should be changed
+ */
+ public synchronized List<AbstractRMOperation> reviewRequestAndReleaseNodes()
+ throws SliderInternalStateException, TriggerClusterTeardownException {
+ log.debug("in reviewRequestAndReleaseNodes()");
+ List<AbstractRMOperation> allOperations =
+ new ArrayList<AbstractRMOperation>();
+ for (RoleStatus roleStatus : getRoleStatusMap().values()) {
+ if (!roleStatus.getExcludeFromFlexing()) {
+ List<AbstractRMOperation> operations = reviewOneRole(roleStatus);
+ allOperations.addAll(operations);
+ }
+ }
+ return allOperations;
+ }
+
+ public void checkFailureThreshold(RoleStatus role) throws
+ TriggerClusterTeardownException {
+ int failures = role.getFailed();
+
+ if (failures > failureThreshold) {
+ throw new TriggerClusterTeardownException(
+ SliderExitCodes.EXIT_DEPLOYMENT_FAILED,
+ ErrorStrings.E_UNSTABLE_CLUSTER +
+ " - failed with role %s failing %d times (%d in startup); threshold is %d - last failure: %s",
+ role.getName(),
+ role.getFailed(),
+ role.getStartFailed(),
+ failureThreshold,
+ role.getFailureMessage());
+ }
+ }
+
+ /**
+ * Look at the allocation status of one role, and trigger add/release
+ * actions if the number of desired role instances doesnt equal
+ * (actual+pending)
+ * @param role role
+ * @return a list of operations
+ * @throws SliderInternalStateException if the operation reveals that
+ * the internal state of the application is inconsistent.
+ */
+ public List<AbstractRMOperation> reviewOneRole(RoleStatus role)
+ throws SliderInternalStateException, TriggerClusterTeardownException {
+ List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
+ int delta;
+ String details;
+ int expected;
+ String name = role.getName();
+ synchronized (role) {
+ delta = role.getDelta();
+ details = role.toString();
+ expected = role.getDesired();
+ }
+
+ log.info(details);
+ checkFailureThreshold(role);
+
+ if (delta > 0) {
+ log.info("{}: Asking for {} more nodes(s) for a total of {} ", name,
+ delta, expected);
+ //more workers needed than we have -ask for more
+ for (int i = 0; i < delta; i++) {
+ Resource capability = recordFactory.newResource();
+ AMRMClient.ContainerRequest containerAsk =
+ buildContainerResourceAndRequest(role, capability);
+ log.info("Container ask is {}", containerAsk);
+ if (containerAsk.getCapability().getMemory() >
+ this.containerMaxMemory) {
+ log.warn(
+ "Memory requested: " + containerAsk.getCapability().getMemory() +
+ " > " +
+ this.containerMaxMemory);
+ }
+ operations.add(new ContainerRequestOperation(containerAsk));
+ }
+ } else if (delta < 0) {
+ log.info("{}: Asking for {} fewer node(s) for a total of {}", name,
+ -delta,
+ expected);
+ //reduce the number expected (i.e. subtract the delta)
+
+ //then pick some containers to kill
+ int excess = -delta;
+
+ // get the nodes to release
+ int roleId = role.getKey();
+ List<NodeInstance> nodesForRelease =
+ roleHistory.findNodesForRelease(roleId, excess);
+
+ for (NodeInstance node : nodesForRelease) {
+ RoleInstance possible = findRoleInstanceOnHost(node, roleId);
+ if (possible == null) {
+ throw new SliderInternalStateException(
+ "Failed to find a container to release on node %s", node.hostname);
+ }
+ containerReleaseSubmitted(possible.container);
+ operations.add(new ContainerReleaseOperation(possible.getId()));
+
+ }
+
+ }
+
+ return operations;
+ }
+
+
+ /**
+ * Find a container running on a specific host -looking
+ * into the node ID to determine this.
+ *
+ * @param node node
+ * @param roleId role the container must be in
+ * @return a container or null if there are no containers on this host
+ * that can be released.
+ */
+ private RoleInstance findRoleInstanceOnHost(NodeInstance node, int roleId) {
+ Collection<RoleInstance> targets = cloneActiveContainerList();
+ String hostname = node.hostname;
+ for (RoleInstance ri : targets) {
+ if (hostname.equals(RoleHistoryUtils.hostnameOf(ri.container))
+ && ri.roleId == roleId
+ && containersBeingReleased.get(ri.getContainerId()) == null) {
+ return ri;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Release all containers.
+ * @return a list of operations to execute
+ */
+ public synchronized List<AbstractRMOperation> releaseAllContainers() {
+
+ Collection<RoleInstance> targets = cloneActiveContainerList();
+ log.info("Releasing {} containers", targets.size());
+ List<AbstractRMOperation> operations =
+ new ArrayList<AbstractRMOperation>(targets.size());
+ for (RoleInstance instance : targets) {
+ Container possible = instance.container;
+ ContainerId id = possible.getId();
+ if (!instance.released) {
+ try {
+ containerReleaseSubmitted(possible);
+ } catch (SliderInternalStateException e) {
+ log.warn("when releasing container {} :", possible, e);
+ }
+ operations.add(new ContainerReleaseOperation(id));
+ }
+ }
+ return operations;
+ }
+
+ /**
+ * Event handler for allocated containers: builds up the lists
+ * of assignment actions (what to run where), and possibly
+ * a list of release operations
+ * @param allocatedContainers the containers allocated
+ * @param assignments the assignments of roles to containers
+ * @param releaseOperations any release operations
+ */
+ public synchronized void onContainersAllocated(List<Container> allocatedContainers,
+ List<ContainerAssignment> assignments,
+ List<AbstractRMOperation> releaseOperations) {
+ assignments.clear();
+ releaseOperations.clear();
+ List<Container> ordered = roleHistory.prepareAllocationList(allocatedContainers);
+ for (Container container : ordered) {
+ String containerHostInfo = container.getNodeId().getHost()
+ + ":" +
+ container.getNodeId().getPort();
+ int allocated;
+ int desired;
+ //get the role
+ ContainerId cid = container.getId();
+ RoleStatus role = lookupRoleStatus(container);
+
+
+ //dec requested count
+ role.decRequested();
+ //inc allocated count -this may need to be dropped in a moment,
+ // but us needed to update the logic below
+ allocated = role.incActual();
+
+ //look for (race condition) where we get more back than we asked
+ desired = role.getDesired();
+
+ roleHistory.onContainerAllocated( container, desired, allocated );
+
+ if (allocated > desired) {
+ log.info("Discarding surplus container {} on {}", cid,
+ containerHostInfo);
+ releaseOperations.add(new ContainerReleaseOperation(cid));
+ //register as a surplus node
+ surplusNodes.add(cid);
+ surplusContainers.incrementAndGet();
+ //and, as we aren't binding it to role, dec that role's actual count
+ role.decActual();
+ } else {
+
+ String roleName = role.getName();
+ log.info("Assigning role {} to container" +
+ " {}," +
+ " on {}:{},",
+ roleName,
+ cid,
+ container.getNodeId().getHost(),
+ container.getNodeId().getPort()
+ );
+
+ assignments.add(new ContainerAssignment(container, role));
+ //add to the history
+ roleHistory.onContainerAssigned(container);
+ }
+ }
+ }
+
+ /**
+ * Get diagnostics info about containers
+ */
+ public String getContainerDiagnosticInfo() {
+ StringBuilder builder = new StringBuilder();
+ for (RoleStatus roleStatus : getRoleStatusMap().values()) {
+ builder.append(roleStatus).append('\n');
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Event handler for the list of active containers on restart.
+ * Sets the info key {@link StatusKeys#INFO_CONTAINERS_AM_RESTART}
+ * to the size of the list passed down (and does not set it if none were)
+ * @param liveContainers the containers allocated
+ * @return true if a rebuild took place (even if size 0)
+ * @throws RuntimeException on problems
+ */
+ private boolean rebuildModelFromRestart(List<Container> liveContainers) throws
+ BadClusterStateException {
+ if (liveContainers == null) {
+ return false;
+ }
+ for (Container container : liveContainers) {
+ addRestartedContainer(container);
+ }
+ clusterStatus.setInfo(StatusKeys.INFO_CONTAINERS_AM_RESTART,
+ Integer.toString(liveContainers.size()));
+ return true;
+ }
+
+ /**
+ * Add a restarted container by walking it through the create/submit/start
+ * lifecycle, so building up the internal structures
+ * @param container container that was running before the AM restarted
+ * @throws RuntimeException on problems
+ */
+ private void addRestartedContainer(Container container) throws
+ BadClusterStateException {
+ String containerHostInfo = container.getNodeId().getHost()
+ + ":" +
+ container.getNodeId().getPort();
+ // get the container ID
+ ContainerId cid = container.getId();
+
+ // get the role
+ int roleId = ContainerPriority.extractRole(container);
+ RoleStatus role =
+ lookupRoleStatus(roleId);
+ // increment its count
+ role.incActual();
+ String roleName = role.getName();
+
+ log.info("Rebuilding container {} in role {} on {},",
+ cid,
+ roleName,
+ containerHostInfo);
+
+ //update app state internal structures and maps
+
+ RoleInstance instance = new RoleInstance(container);
+ instance.command = roleName;
+ instance.role = roleName;
+ instance.roleId = roleId;
+ instance.environment = new String[0];
+ instance.container = container;
+ instance.createTime = now();
+ instance.state = ClusterDescription.STATE_LIVE;
+ activeContainers.put(cid, instance);
+ //role history gets told
+ roleHistory.onContainerAssigned(container);
+ // pretend the container has just had its start actions submitted
+ containerStartSubmitted(container, instance);
+ // now pretend it has just started
+ innerOnNodeManagerContainerStarted(cid);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
new file mode 100644
index 0000000..3be3777
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Container;
+
+public class ContainerAssignment {
+
+ public final Container container;
+ public final RoleStatus role;
+
+ public ContainerAssignment(Container container,
+ RoleStatus role) {
+ this.container = container;
+ this.role = role;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
new file mode 100644
index 0000000..ccd9a64
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Class containing the logic to build/split container priorities into the
+ * different fields used by Slider
+ *
+ * The original design here had a requestID merged with the role, to
+ * track outstanding requests. However, this isn't possible, so
+ * the request ID has been dropped. A "location specified" flag was
+ * added to indicate whether or not the request was for a specific location
+ * -though this is currently unused.
+ *
+ * The methods are effectively surplus -but retained to preserve the
+ * option of changing behavior in future
+ */
+public final class ContainerPriority {
+
+ public static int buildPriority(int role,
+ boolean locationSpecified) {
+ return (role) ;
+ }
+
+
+ public static Priority createPriority(int role,
+ boolean locationSpecified) {
+ Priority pri = Records.newRecord(Priority.class);
+ pri.setPriority(ContainerPriority.buildPriority(role,
+ locationSpecified));
+ return pri;
+ }
+
+
+ public static int extractRole(int priority) {
+ return priority ;
+ }
+
+ /**
+ * Map from a container to a role key by way of its priority
+ * @param container container
+ * @return role key
+ */
+ public static int extractRole(Container container) {
+ Priority priority = container.getPriority();
+ assert priority != null;
+ return extractRole(priority.getPriority());
+ }
+
+ /**
+ * Map from a container to a role key by way of its priority
+ * @param container container
+ * @return role key
+ */
+ public static int extractRole(Priority priorityRecord) {
+ return extractRole(priorityRecord.getPriority());
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseOperation.java
new file mode 100644
index 0000000..8e73f19
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseOperation.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+public class ContainerReleaseOperation extends AbstractRMOperation {
+
+ private final ContainerId containerId;
+
+ public ContainerReleaseOperation(ContainerId containerId) {
+ this.containerId = containerId;
+ }
+
+ public ContainerId getContainerId() {
+ return containerId;
+ }
+
+ @Override
+ public void execute(RMOperationHandler handler) {
+ handler.releaseAssignedContainer(containerId);
+ }
+
+ @Override
+ public String toString() {
+ return "release container " + containerId;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerRequestOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerRequestOperation.java
new file mode 100644
index 0000000..25c3d60
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerRequestOperation.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+
+public class ContainerRequestOperation extends AbstractRMOperation {
+
+ private final AMRMClient.ContainerRequest request;
+
+ public ContainerRequestOperation(AMRMClient.ContainerRequest request) {
+ this.request = request;
+ }
+
+ public AMRMClient.ContainerRequest getRequest() {
+ return request;
+ }
+
+ @Override
+ public void execute(RMOperationHandler handler) {
+ handler.addContainerRequest(request);
+ }
+
+ @Override
+ public String toString() {
+ return "request container ";
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
new file mode 100644
index 0000000..a9e5a8c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+/**
+ * Information about the state of a role on a specific node instance.
+ * No fields are synchronized; sync on the instance to work with it
+ *
+ The two fields `releasing` and `requested` are used to track the ongoing
+ state of YARN requests; they do not need to be persisted across freeze/thaw
+ cycles. They may be relevant across AM restart, but without other data
+ structures in the AM, not enough to track what the AM was up to before
+ it was restarted. The strategy will be to ignore unexpected allocation
+ responses (which may come from pre-restart) requests, while treating
+ unexpected container release responses as failures.
+
+ The `active` counter is only decremented after a container release response
+ has been received.
+
+ Accesses are synchronized.
+ */
+public class NodeEntry {
+
+ public final int index;
+
+ public NodeEntry(int index) {
+ this.index = index;
+ }
+
+ /**
+ * instance explicitly requested on this node: it's OK if an allocation
+ * comes in that has not been (and when that happens, this count should
+ * not drop)
+ */
+ private int requested;
+ private int starting;
+ private int startFailed;
+ private int failed;
+ /**
+ * Number of live nodes.
+ */
+ private int live;
+ private int releasing;
+ private long lastUsed;
+
+ /**
+ * Is the node available for assignments. This does not track
+ * whether or not there are any outstanding requests for this node
+ * @return true if there are no role instances here
+ * other than some being released.
+ */
+ public synchronized boolean isAvailable() {
+ return getActive() == 0 && (requested == 0) && starting == 0;
+ }
+
+ /**
+ * return no of active instances -those that could be released as they
+ * are live and not already being released
+ * @return a number, possibly 0
+ */
+ public synchronized int getActive() {
+ return (live - releasing);
+ }
+
+ /**
+ * Return true if the node is not busy, and it
+ * has not been used since the absolute time
+ * @param absoluteTime time
+ * @return true if the node could be cleaned up
+ */
+ public synchronized boolean notUsedSince(long absoluteTime) {
+ return isAvailable() && lastUsed < absoluteTime;
+ }
+
+ public synchronized int getLive() {
+ return live;
+ }
+
+ public int getStarting() {
+ return starting;
+ }
+
+ /**
+ * Set the live value directly -used on AM restart
+ * @param v value
+ */
+ public synchronized void setLive(int v) {
+ live = v;
+ }
+
+ private void incLive() {
+ ++live;
+ }
+
+ private synchronized void decLive() {
+ live = RoleHistoryUtils.decToFloor(live);
+ }
+
+ public synchronized void onStarting() {
+ ++starting;
+ }
+
+ private void decStarting() {
+ starting = RoleHistoryUtils.decToFloor(starting);
+ }
+
+ public synchronized void onStartCompleted() {
+ decStarting();
+ incLive();
+ }
+
+ /**
+ * start failed -decrement the starting flag.
+ * @return true if the node is now available
+ */
+ public synchronized boolean onStartFailed() {
+ decStarting();
+ ++startFailed;
+ ++failed;
+ return isAvailable();
+ }
+
+ /**
+ * no of requests made of this role of this node. If it goes above
+ * 1 there's a problem
+ */
+ public synchronized int getRequested() {
+ return requested;
+ }
+
+ /**
+ * request a node:
+ */
+ public synchronized void request() {
+ ++requested;
+ }
+
+ /**
+ * A request made explicitly to this node has completed
+ */
+ public synchronized void requestCompleted() {
+ requested = RoleHistoryUtils.decToFloor(requested);
+ }
+
+ /**
+ * No of instances in release state
+ */
+ public synchronized int getReleasing() {
+ return releasing;
+ }
+
+ /**
+ * Release an instance -which is no longer marked as active
+ */
+ public synchronized void release() {
+ assert live > 0 : "no live nodes to release";
+ releasing++;
+ }
+
+ /**
+ * completion event, which can be a planned or unplanned
+ * planned: dec our release count
+ * unplanned: dec our live count
+ * @param wasReleased true if this was planned
+ * @return true if this node is now available
+ */
+ public synchronized boolean containerCompleted(boolean wasReleased) {
+ if (wasReleased) {
+ releasing = RoleHistoryUtils.decToFloor(releasing);
+ } else {
+ ++failed;
+ }
+ decLive();
+ return isAvailable();
+ }
+
+ /**
+ * Time last used.
+ */
+ public synchronized long getLastUsed() {
+ return lastUsed;
+ }
+
+ public synchronized void setLastUsed(long lastUsed) {
+ this.lastUsed = lastUsed;
+ }
+
+ public int getStartFailed() {
+ return startFailed;
+ }
+
+ public synchronized int getFailed() {
+ return failed;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("NodeEntry{");
+ sb.append("requested=").append(requested);
+ sb.append(", starting=").append(starting);
+ sb.append(", live=").append(live);
+ sb.append(", failed=").append(failed);
+ sb.append(", startFailed=").append(startFailed);
+ sb.append(", releasing=").append(releasing);
+ sb.append(", lastUsed=").append(lastUsed);
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
new file mode 100644
index 0000000..66513d9
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * A node instance -stores information about a node in the cluster.
+ *
+ * Operations on the array/set of roles are synchronized.
+ */
+public class NodeInstance {
+
+ public final String hostname;
+
+ private final List<NodeEntry> nodeEntries;
+
+ /**
+ * Create an instance and the (empty) array of nodes
+ * @param roles role count -the no. of roles
+ */
+ public NodeInstance(String hostname, int roles) {
+ this.hostname = hostname;
+ nodeEntries = new ArrayList<NodeEntry>(roles);
+ }
+
+ /**
+ * Get the entry for a role -if present
+ * @param role role index
+ * @return the entry
+ * null if the role is out of range
+ */
+ public synchronized NodeEntry get(int role) {
+ for (NodeEntry nodeEntry : nodeEntries) {
+ if (nodeEntry.index == role) {
+ return nodeEntry;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get the entry for a role -if present
+ * @param role role index
+ * @return the entry
+ * @throws ArrayIndexOutOfBoundsException if the role is out of range
+ */
+ public synchronized NodeEntry getOrCreate(int role) {
+ NodeEntry entry = get(role);
+ if (entry == null) {
+ entry = new NodeEntry(role);
+ nodeEntries.add(entry);
+ }
+ return entry;
+ }
+
+ /**
+ * Cout the number of active role instances on this node
+ * @param role role index
+ * @return 0 if there are none, otherwise the #of nodes that are running and
+ * not being released already.
+ */
+ public int getActiveRoleInstances(int role) {
+ NodeEntry nodeEntry = get(role);
+ return (nodeEntry != null ) ? nodeEntry.getActive() : 0;
+ }
+
+ /**
+ * Get the entry for a role -and remove it if present
+ * @param role the role index
+ * @return the entry that WAS there
+ */
+ public synchronized NodeEntry remove(int role) {
+ NodeEntry nodeEntry = get(role);
+ if (nodeEntry != null) {
+ nodeEntries.remove(nodeEntry);
+ }
+ return nodeEntry;
+ }
+
+ public synchronized void set(int role, NodeEntry nodeEntry) {
+ remove(role);
+ nodeEntries.add(nodeEntry);
+ }
+
+ /**
+ * run through each entry; gc'ing & removing old ones
+ * @param absoluteTime age in millis
+ * @return true if there are still entries left
+ */
+ public synchronized boolean purgeUnusedEntries(long absoluteTime) {
+ boolean active = false;
+ ListIterator<NodeEntry> entries = nodeEntries.listIterator();
+ while (entries.hasNext()) {
+ NodeEntry entry = entries.next();
+ if (entry.notUsedSince(absoluteTime)) {
+ entries.remove();
+ } else {
+ active = true;
+ }
+ }
+ return active;
+ }
+
+ @Override
+ public String toString() {
+ return hostname;
+ }
+
+ /**
+ * Full dump of entry including children
+ * @return a multi-line description fo the node
+ */
+ public String toFullString() {
+ final StringBuilder sb =
+ new StringBuilder(toString());
+ int i = 0;
+ for (NodeEntry entry : nodeEntries) {
+ sb.append(String.format("\n [%02d] ", i++));
+ sb.append(entry.toString());
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Equality test is purely on the hostname of the node address
+ * @param o other
+ * @return true if the hostnames are equal
+ */
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ NodeInstance that = (NodeInstance) o;
+ return hostname.equals(that.hostname);
+ }
+
+ @Override
+ public int hashCode() {
+ return hostname.hashCode();
+ }
+
+ /**
+ * A comparator for sorting entries where the role is newer than
+ * the other.
+ * This sort only compares the lastUsed field, not whether the
+ * node is in use or not
+ */
+ public static class newerThan implements Comparator<NodeInstance>,
+ Serializable {
+
+ final int role;
+
+ public newerThan(int role) {
+ this.role = role;
+ }
+
+ @Override
+ public int compare(NodeInstance o1, NodeInstance o2) {
+ long age = o1.getOrCreate(role).getLastUsed();
+ long age2 = o2.getOrCreate(role).getLastUsed();
+
+ if (age > age2) {
+ return -1;
+ } else if (age < age2) {
+ return 1;
+ }
+ // equal
+ return 0;
+ }
+ }
+
+ /**
+ * A comparator for sorting entries where the role is newer than
+ * the other.
+ * This sort only compares the lastUsed field, not whether the
+ * node is in use or not
+ */
+ public static class moreActiveThan implements Comparator<NodeInstance>,
+ Serializable {
+
+ final int role;
+
+ public moreActiveThan(int role) {
+ this.role = role;
+ }
+
+ @Override
+ public int compare(NodeInstance left, NodeInstance right) {
+ int activeLeft = left.getActiveRoleInstances(role);
+ int activeRight = right.getActiveRoleInstances(role);
+ return activeRight - activeLeft;
+ }
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
new file mode 100644
index 0000000..c638843
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+
+/**
+ * Node map map -and methods to work with it.
+ * Not Synchronized: caller is expected to lock access.
+ */
+public class NodeMap extends HashMap<String, NodeInstance> {
+ protected static final Logger log =
+ LoggerFactory.getLogger(NodeMap.class);
+
+ /**
+ * number of roles
+ */
+ private final int roleSize;
+
+ /**
+ * Construct
+ * @param roleSize number of roles
+ */
+ public NodeMap(int roleSize) {
+ this.roleSize = roleSize;
+ }
+
+ /**
+ * Get the node instance for the specific node -creating it if needed
+ * @param hostname node
+ * @return the instance
+ */
+ public NodeInstance getOrCreate(String hostname) {
+ NodeInstance node = get(hostname);
+ if (node == null) {
+ node = new NodeInstance(hostname, roleSize);
+ put(hostname, node);
+ }
+ return node;
+ }
+
+ /**
+ * List the active nodes
+ * @param role role
+ * @return a possibly empty sorted list of all nodes that are active
+ * in that role
+ */
+ public List<NodeInstance> listActiveNodes(int role) {
+ List<NodeInstance> nodes = new ArrayList<NodeInstance>();
+ for (NodeInstance instance : values()) {
+ if (instance.getActiveRoleInstances(role) > 0) {
+ nodes.add(instance);
+ }
+ }
+ Collections.sort(nodes, new NodeInstance.moreActiveThan(role));
+ return nodes;
+ }
+
+ /**
+ * purge the history of all nodes that have been inactive since the absolute time
+ * @param absoluteTime time
+ * @return the number purged
+ */
+ public int purgeUnusedEntries(long absoluteTime) {
+ int purged = 0;
+ Iterator<Map.Entry<String, NodeInstance>> iterator =
+ entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<String, NodeInstance> entry = iterator.next();
+ NodeInstance ni = entry.getValue();
+ if (!ni.purgeUnusedEntries(absoluteTime)) {
+ iterator.remove();
+ purged ++;
+ }
+ }
+ return purged;
+ }
+
+
+
+ /**
+ * Find a list of node for release; algorithm may make its own
+ * decisions on which to release.
+ * @param role role index
+ * @param count number of nodes to release
+ * @return a possibly empty list of nodes.
+ */
+ public List<NodeInstance> findNodesForRelease(int role, int count) {
+ List<NodeInstance> targets = new ArrayList<NodeInstance>(count);
+ List<NodeInstance> active = listActiveNodes(role);
+ List<NodeInstance> multiple = new ArrayList<NodeInstance>();
+ int nodesRemaining = count;
+ log.debug("searching for {} nodes with candidate set size {}",
+ nodesRemaining, active.size());
+ ListIterator<NodeInstance> it = active.listIterator();
+ while (it.hasNext() && nodesRemaining > 0) {
+ NodeInstance ni = it.next();
+ int load = ni.getActiveRoleInstances(role);
+ log.debug("Node {} load={}", ni, load);
+ assert load != 0;
+ if (load == 1) {
+ // at the tail of the list, from here active[*] is a load=1 entry
+ break;
+ }
+ // load is >1. Add an entry to the target list FOR EACH INSTANCE ABOVE 1
+ for (int i = 0; i < (load - 1) && nodesRemaining > 0; i++) {
+ nodesRemaining--;
+ log.debug("Push {} #{}", ni, i);
+ targets.add(ni);
+ }
+ // and add to the multiple list
+ multiple.add(ni);
+ // then pop it from the active list
+ it.remove();
+ }
+ //here either the number is found or there is still some left.
+
+ if (nodesRemaining > 0) {
+ // leftovers. Append any of the multiple node entries to the tail of
+ // the active list (so they get chosen last)
+ active.addAll(multiple);
+ // all the entries in the list have exactly one node
+ // so ask for as many as are needed
+ int ask = Math.min(nodesRemaining, active.size());
+ log.debug("load=1 nodes to select={} multiples={} available={} ask={}",
+ nodesRemaining, multiple.size(),active.size(), ask);
+ targets.addAll(active.subList(0, ask));
+ }
+ return targets;
+ }
+
+
+ /**
+ * Clone point
+ * @return
+ */
+ @Override
+ public Object clone() {
+ return super.clone();
+ }
+
+ /**
+ * Insert a list of nodes into the map; overwrite any with that name
+ * This is a bulk operation for testing.
+ * @param nodes collection of nodes.
+ */
+ @VisibleForTesting
+ public void insert(Collection<NodeInstance> nodes) {
+ for (NodeInstance node : nodes) {
+ put(node.hostname, node);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
new file mode 100644
index 0000000..f75df56
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tracks an outstanding request. This is used to correlate an allocation response
+ * with the
+ * node and role used in the request.
+ *
+ * The node identifier may be null -which indicates that a request was made without
+ * a specific target node
+ *
+ * Equality and the hash code are based <i>only</i> on the role and hostname,
+ * which are fixed in the constructor. This means that a simple
+ * instance constructed with (role, hostname) can be used to look up
+ * a complete request instance in the {@link OutstandingRequestTracker} map
+ */
+public final class OutstandingRequest {
+ protected static final Logger log =
+ LoggerFactory.getLogger(OutstandingRequest.class);
+
+ /**
+ * requested role
+ */
+ public final int roleId;
+
+ /**
+ * Node the request is for -may be null
+ */
+ public final NodeInstance node;
+ /**
+ * hostname -will be null if node==null
+ */
+ public final String hostname;
+
+ /**
+ * requested time -only valid after {@link #buildContainerRequest(Resource, long)}
+ */
+ public long requestedTime;
+
+ /**
+ * Create a request
+ * @param roleId role
+ * @param node node -can be null
+ */
+ public OutstandingRequest(int roleId,
+ NodeInstance node) {
+ this.roleId = roleId;
+ this.node = node;
+ this.hostname = node != null ? node.hostname : null;
+ }
+
+ /**
+ * Create an outsanding request with the given role and hostname
+ * Important: this is useful only for map lookups -the other constructor
+ * with the NodeInstance parameter is needed to generate node-specific
+ * container requests
+ * @param roleId role
+ * @param hostname hostname
+ */
+ public OutstandingRequest(int roleId, String hostname) {
+ this.node = null;
+ this.roleId = roleId;
+ this.hostname = hostname;
+ }
+
+ public boolean isLocated() {
+ return node != null;
+ }
+ /**
+ * Build a container request.
+ * If the request has an address, it is set in the container request
+ * (with a flag to enable relaxed priorities)
+ * @param resource resource
+ * @param role role
+ * @param time: time to record
+ * @return the request to raise
+ */
+ public AMRMClient.ContainerRequest buildContainerRequest(Resource resource,
+ RoleStatus role, long time) {
+ String[] hosts;
+ boolean relaxLocality;
+ boolean locationSpecified;
+ requestedTime = time;
+ if (node != null) {
+ hosts = new String[1];
+ hosts[0] = node.hostname;
+ relaxLocality = true;
+ locationSpecified = true;
+ // tell the node it is in play
+ node.getOrCreate(roleId);
+ log.info("Submitting request for container on {}", hosts[0]);
+ } else {
+ hosts = null;
+ relaxLocality = true;
+ locationSpecified = false;
+ }
+ Priority pri = ContainerPriority.createPriority(roleId,
+ locationSpecified);
+ AMRMClient.ContainerRequest request =
+ new AMRMClient.ContainerRequest(resource,
+ hosts,
+ null,
+ pri,
+ relaxLocality);
+
+ return request;
+ }
+
+ /**
+ * Mark the request as completed (or canceled).
+ */
+ public void completed() {
+ assert node != null : "null node";
+ node.getOrCreate(roleId).requestCompleted();
+ }
+
+ /**
+ * Equality is on hostname and role
+ * @param o other
+ * @return true on a match
+ */
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ OutstandingRequest request = (OutstandingRequest) o;
+
+ if (roleId != request.roleId) {
+ return false;
+ }
+ if (hostname != null
+ ? !hostname.equals(request.hostname)
+ : request.hostname != null) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * hash on hostname and role
+ * @return hash code
+ */
+ @Override
+ public int hashCode() {
+ int result = roleId;
+ result = 31 * result + (hostname != null ? hostname.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("OutstandingRequest{");
+ sb.append("roleId=").append(roleId);
+ sb.append(", node='").append(node).append('\'');
+ sb.append(", requestedTime=").append(requestedTime);
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
new file mode 100644
index 0000000..d847962
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Tracks outstanding requests made with a specific placement option.
+ * If an allocation comes in that is not in the map: either the allocation
+ * was unplaced, or the placed allocation could not be met on the specified
+ * host, and the RM/scheduler fell back to another location.
+ */
+
+public class OutstandingRequestTracker {
+ protected static final Logger log =
+ LoggerFactory.getLogger(OutstandingRequestTracker.class);
+
+ private Map<OutstandingRequest, OutstandingRequest> requests =
+ new HashMap<OutstandingRequest, OutstandingRequest>();
+
+ /**
+ * Create a new request for the specific role. If a
+ * location is set, the request is added to the list of requests to track.
+ * if it isn't -it isn't.
+ * This does not update the node instance's role's request count
+ * @param instance node instance to manager
+ * @param role role index
+ * @return a new request
+ */
+ public synchronized OutstandingRequest addRequest(NodeInstance instance, int role) {
+ OutstandingRequest request =
+ new OutstandingRequest(role, instance);
+ if (request.isLocated()) {
+ requests.put(request, request);
+ }
+ return request;
+ }
+
+ /**
+ * Look up any oustanding request to a (role, hostname).
+ * @param role role index
+ * @param hostname hostname
+ * @return the request or null if there was no outstanding one
+ */
+ public synchronized OutstandingRequest lookup(int role, String hostname) {
+ return requests.get(new OutstandingRequest(role, hostname));
+ }
+
+ /**
+ * Remove a request
+ * @param request matching request to find
+ * @return the request
+ */
+ public synchronized OutstandingRequest remove(OutstandingRequest request) {
+ return requests.remove(request);
+ }
+
+ /**
+ * Notification that a container has been allocated -drop it
+ * from the list of outstanding roles if need be
+ * @param role role index
+ * @param hostname hostname
+ * @return true if an entry was found and dropped
+ */
+ public synchronized boolean onContainerAllocated(int role, String hostname) {
+ OutstandingRequest request =
+ requests.remove(new OutstandingRequest(role, hostname));
+ if (request == null) {
+ return false;
+ } else {
+ //satisfied request
+ request.completed();
+ }
+ return true;
+ }
+
+ static class newerThan implements Comparator<Container>, Serializable {
+ private RoleHistory rh;
+
+ public newerThan(RoleHistory rh) {
+ this.rh = rh;
+ }
+
+ @Override
+ public int compare(Container c1, Container c2) {
+ int role1 = ContainerPriority.extractRole(c1);
+ int role2 = ContainerPriority.extractRole(c2);
+ if (role1 < role2) return -1;
+ if (role1 > role2) return 1;
+
+ NodeInstance o1 = rh.getOrCreateNodeInstance(c1), o2 = rh.getOrCreateNodeInstance(c2);
+ long age = o1.getOrCreate(role1).getLastUsed();
+ long age2 = o2.getOrCreate(role1).getLastUsed();
+
+ if (age > age2) {
+ return -1;
+ } else if (age < age2) {
+ return 1;
+ }
+ // equal
+ return 0;
+ }
+ }
+ /**
+ * Take a list of requests and split them into specific host requests and
+ * generic assignments. This is to give requested hosts priority
+ * in container assignments if more come back than expected
+ * @param rh RoleHistory instance
+ * @param allocatedContainers the list of allocated containers
+ * @param requested empty list of requested locations
+ * @param unrequested empty list of unrequested hosts
+ */
+ public synchronized void partitionRequests(RoleHistory rh, List<Container> allocatedContainers,
+ List<Container> requested,
+ List<Container> unrequested) {
+ Collections.sort(allocatedContainers, new newerThan(rh));
+ for (Container container : allocatedContainers) {
+ int role = ContainerPriority.extractRole(container);
+ String hostname = RoleHistoryUtils.hostnameOf(container);
+ if (requests.containsKey(new OutstandingRequest(role, hostname))) {
+ requested.add(container);
+ } else {
+ unrequested.add(container);
+ }
+ }
+ }
+
+
+ /**
+ * Cancel all outstanding requests for a role: return the hostnames
+ * of any canceled requests
+ *
+ * @param role role to cancel
+ * @return possibly empty list of hostnames
+ */
+ public synchronized List<NodeInstance> cancelOutstandingRequests(int role) {
+ List<NodeInstance> hosts = new ArrayList<NodeInstance>();
+ Iterator<Map.Entry<OutstandingRequest,OutstandingRequest>> iterator =
+ requests.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<OutstandingRequest, OutstandingRequest> next =
+ iterator.next();
+ OutstandingRequest request = next.getKey();
+ if (request.roleId == role) {
+ iterator.remove();
+ request.completed();
+ hosts.add(request.node);
+ }
+ }
+ return hosts;
+ }
+
+ public synchronized List<OutstandingRequest> listOutstandingRequests() {
+ return new ArrayList<OutstandingRequest>(requests.values());
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RMOperationHandler.java
new file mode 100644
index 0000000..4106b16
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RMOperationHandler.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+
+import java.util.List;
+
+public abstract class RMOperationHandler {
+
+
+ public abstract void releaseAssignedContainer(ContainerId containerId);
+
+ public abstract void addContainerRequest(AMRMClient.ContainerRequest req);
+
+
+ /**
+ * Execute an entire list of operations
+ * @param operations ops
+ */
+ public void execute(List<AbstractRMOperation> operations) {
+ for (AbstractRMOperation operation : operations) {
+ operation.execute(this);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
new file mode 100644
index 0000000..fa22676
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -0,0 +1,758 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.avro.RoleHistoryHeader;
+import org.apache.slider.server.avro.RoleHistoryWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The Role History.
+ *
+ * Synchronization policy: all public operations are synchronized.
+ * Protected methods are in place for testing -no guarantees are made.
+ *
+ * Inner classes have no synchronization guarantees; they should be manipulated
+ * in these classes and not externally.
+ *
+ * Note that as well as some methods marked visible for testing, there
+ * is the option for the time generator method, {@link #now()} to
+ * be overridden so that a repeatable time series can be used.
+ *
+ */
+public class RoleHistory {
+ protected static final Logger log =
+ LoggerFactory.getLogger(RoleHistory.class);
+ private final List<ProviderRole> providerRoles;
+ private final Map<String, ProviderRole> providerRoleMap =
+ new HashMap<String, ProviderRole>();
+ private long startTime;
+ /**
+ * Time when saved
+ */
+ private long saveTime;
+ /**
+ * If the history was loaded, the time at which the history was saved
+ *
+ */
+ private long thawedDataTime;
+
+ private NodeMap nodemap;
+ private int roleSize;
+ private boolean dirty;
+ private FileSystem filesystem;
+ private Path historyPath;
+ private RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+
+ private OutstandingRequestTracker outstandingRequests =
+ new OutstandingRequestTracker();
+
+ /**
+ * For each role, lists nodes that are available for data-local allocation,
+ ordered by more recently released - To accelerate node selection
+ */
+ private Map<Integer, LinkedList<NodeInstance>> availableNodes;
+
+ public RoleHistory(List<ProviderRole> providerRoles) throws
+ BadConfigException {
+ this.providerRoles = providerRoles;
+ roleSize = providerRoles.size();
+ for (ProviderRole providerRole : providerRoles) {
+ providerRoleMap.put(providerRole.name, providerRole);
+ }
+ reset();
+ }
+
+ /**
+ * Reset the variables -this does not adjust the fixed attributes
+ * of the history
+ */
+ protected synchronized void reset() throws BadConfigException {
+
+ nodemap = new NodeMap(roleSize);
+ resetAvailableNodeLists();
+
+ resetAvailableNodeLists();
+ outstandingRequests = new OutstandingRequestTracker();
+ Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
+
+
+ for (ProviderRole providerRole : providerRoles) {
+ addProviderRole(roleStats, providerRole);
+ }
+ }
+
+
+ private void addProviderRole(Map<Integer, RoleStatus> roleStats,
+ ProviderRole providerRole)
+ throws ArrayIndexOutOfBoundsException, BadConfigException {
+ int index = providerRole.id;
+ if (index < 0) {
+ throw new BadConfigException("Provider " + providerRole
+ + " id is out of range");
+ }
+ if (roleStats.get(index) != null) {
+ throw new BadConfigException(
+ providerRole.toString() + " id duplicates that of " +
+ roleStats.get(index));
+ }
+ roleStats.put(index, new RoleStatus(providerRole));
+ }
+
+
+ /**
+ * Add a new provider role to the map
+ * @param providerRole new provider role
+ */
+ public void addNewProviderRole(ProviderRole providerRole)
+ throws BadConfigException {
+ Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
+
+
+ for (ProviderRole role : providerRoles) {
+ roleStats.put(role.id, new RoleStatus(role));
+ }
+
+ addProviderRole(roleStats, providerRole);
+ }
+
+ /**
+ * Clear the lists of available nodes
+ */
+ private synchronized void resetAvailableNodeLists() {
+ availableNodes = new HashMap<Integer, LinkedList<NodeInstance>>(roleSize);
+ }
+
+ /**
+ * Reset the variables -this does not adjust the fixed attributes
+ * of the history.
+ * This intended for use by the RoleWriter logic.
+ */
+ public synchronized void prepareForReading(RoleHistoryHeader header) throws
+ IOException,
+ BadConfigException {
+ reset();
+
+ int roleCountInSource = header.getRoles();
+ if (roleCountInSource != roleSize) {
+ throw new IOException("Number of roles in source " + roleCountInSource
+ + " does not match the expected number of " +
+ roleSize);
+ }
+ //record when the data was loaded
+ setThawedDataTime(header.getSaved());
+ }
+
+ public synchronized long getStartTime() {
+ return startTime;
+ }
+
+ public synchronized long getSaveTime() {
+ return saveTime;
+ }
+
+ public long getThawedDataTime() {
+ return thawedDataTime;
+ }
+
+ public void setThawedDataTime(long thawedDataTime) {
+ this.thawedDataTime = thawedDataTime;
+ }
+
+ public synchronized int getRoleSize() {
+ return roleSize;
+ }
+
+ /**
+ * Get the total size of the cluster -the number of NodeInstances
+ * @return a count
+ */
+ public synchronized int getClusterSize() {
+ return nodemap.size();
+ }
+
+ public synchronized boolean isDirty() {
+ return dirty;
+ }
+
+ public synchronized void setDirty(boolean dirty) {
+ this.dirty = dirty;
+ }
+
+ /**
+ * Tell the history that it has been saved; marks itself as clean
+ * @param timestamp timestamp -updates the savetime field
+ */
+ public synchronized void saved(long timestamp) {
+ dirty = false;
+ saveTime = timestamp;
+ }
+
+ /**
+ * Get a clone of the nodemap.
+ * The instances inside are not cloned
+ * @return the map
+ */
+ public synchronized NodeMap cloneNodemap() {
+ return (NodeMap) nodemap.clone();
+ }
+
+ /**
+ * Get the node instance for the specific node -creating it if needed
+ * @param nodeAddr node address
+ * @return the instance
+ */
+ public synchronized NodeInstance getOrCreateNodeInstance(String hostname) {
+ //convert to a string
+ return nodemap.getOrCreate(hostname);
+ }
+
+ /**
+ * Insert a list of nodes into the map; overwrite any with that name.
+ * This is a bulk operation for testing.
+ * Important: this does not update the available node lists, these
+ * must be rebuilt afterwards.
+ * @param nodes collection of nodes.
+ */
+ @VisibleForTesting
+ public synchronized void insert(Collection<NodeInstance> nodes) {
+ nodemap.insert(nodes);
+ }
+
+ /**
+ * Get current time. overrideable for test subclasses
+ * @return current time in millis
+ */
+ protected long now() {
+ return System.currentTimeMillis();
+ }
+
+ /**
+ * Garbage collect the structure -this will dropp
+ * all nodes that have been inactive since the (relative) age
+ * @param age relative age
+ */
+ public void gc(long age) {
+ long absoluteTime = now() - age;
+ purgeUnusedEntries(absoluteTime);
+ }
+
+ /**
+ * Mark ourselves as dirty
+ */
+ public void touch() {
+ setDirty(true);
+ try {
+ saveHistoryIfDirty();
+ } catch (IOException e) {
+ log.warn("Failed to save history file ", e);
+ }
+ }
+
+ /**
+ * purge the history of
+ * all nodes that have been inactive since the absolute time
+ * @param absoluteTime time
+ */
+ public synchronized void purgeUnusedEntries(long absoluteTime) {
+ nodemap.purgeUnusedEntries(absoluteTime);
+ }
+
+ /**
+ * Get the path used for history files
+ * @return the directory used for history files
+ */
+ public Path getHistoryPath() {
+ return historyPath;
+ }
+
+ /**
+ * Save the history to its location using the timestamp as part of
+ * the filename. The saveTime and dirty fields are updated
+ * @param time timestamp timestamp to use as the save time
+ * @return the path saved to
+ * @throws IOException IO problems
+ */
+ @VisibleForTesting
+ public synchronized Path saveHistory(long time) throws IOException {
+ Path filename = historyWriter.createHistoryFilename(historyPath, time);
+ historyWriter.write(filesystem, filename, true, this, time);
+ saved(time);
+ return filename;
+ }
+
+ /**
+ * Save the history with the current timestamp if it is dirty;
+ * return the path saved to if this is the case
+ * @return the path or null if the history was not saved
+ * @throws IOException failed to save for some reason
+ */
+ public synchronized Path saveHistoryIfDirty() throws IOException {
+ if (isDirty()) {
+ long time = now();
+ return saveHistory(time);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Start up
+ * @param fs filesystem
+ * @param historyDir path in FS for history
+ * @return true if the history was thawed
+ */
+ public boolean onStart(FileSystem fs, Path historyDir) throws
+ BadConfigException {
+ assert filesystem == null;
+ filesystem = fs;
+ historyPath = historyDir;
+ startTime = now();
+ //assume the history is being thawed; this will downgrade as appropriate
+ return onThaw();
+ }
+
+ /**
+ * Handler for bootstrap event
+ */
+ public void onBootstrap() {
+ log.info("Role history bootstrapped");
+ }
+
+ /**
+ * Handle the thaw process <i>after the history has been rebuilt</i>,
+ * and after any gc/purge
+ */
+ @VisibleForTesting
+ public synchronized boolean onThaw() throws BadConfigException {
+ assert filesystem != null;
+ assert historyPath != null;
+ boolean thawSuccessful = false;
+ //load in files from data dir
+ Path loaded = null;
+ try {
+ loaded = historyWriter.loadFromHistoryDir(filesystem, historyPath, this);
+ } catch (IOException e) {
+ log.warn("Exception trying to load history from {}", historyPath, e);
+ }
+ if (loaded != null) {
+ thawSuccessful = true;
+ log.info("loaded history from {}", loaded);
+ // delete any old entries
+ try {
+ int count = historyWriter.purgeOlderHistoryEntries(filesystem, loaded);
+ log.debug("Deleted {} old history entries", count);
+ } catch (IOException e) {
+ log.info("Ignoring exception raised while trying to delete old entries",
+ e);
+ }
+
+ //thaw is then completed
+ buildAvailableNodeLists();
+ } else {
+ //fallback to bootstrap procedure
+ onBootstrap();
+ }
+ return thawSuccessful;
+ }
+
+
+ /**
+ * (After the thaw), rebuild the availability datastructures
+ */
+ @VisibleForTesting
+ public synchronized void buildAvailableNodeLists() {
+ resetAvailableNodeLists();
+ // build the list of available nodes
+ for (Map.Entry<String, NodeInstance> entry : nodemap
+ .entrySet()) {
+ NodeInstance ni = entry.getValue();
+ for (int i = 0; i < roleSize; i++) {
+ NodeEntry nodeEntry = ni.get(i);
+ if (nodeEntry != null && nodeEntry.isAvailable()) {
+ getOrCreateNodesForRoleId(i).add(ni);
+ }
+ }
+ }
+ // sort the resulting arrays
+ for (int i = 0; i < roleSize; i++) {
+ sortAvailableNodeList(i);
+ }
+ }
+
+ /**
+ * Get the nodes for an ID -may be null
+ * @param id role ID
+ * @return list
+ */
+ private LinkedList<NodeInstance> getNodesForRoleId(int id) {
+ return availableNodes.get(id);
+ }
+
+ /**
+ * Get the nodes for an ID -may be null
+ * @param id role ID
+ * @return list
+ */
+ private LinkedList<NodeInstance> getOrCreateNodesForRoleId(int id) {
+ LinkedList<NodeInstance> instances = availableNodes.get(id);
+ if (instances==null) {
+ instances = new LinkedList<NodeInstance>();
+ availableNodes.put(id, instances);
+ }
+ return instances;
+ }
+
+
+ /**
+ * Sort an available node list
+ * @param role role to sort
+ */
+ private void sortAvailableNodeList(int role) {
+ List<NodeInstance> nodesForRoleId = getNodesForRoleId(role);
+ if (nodesForRoleId != null) {
+ Collections.sort(nodesForRoleId, new NodeInstance.newerThan(role));
+ }
+ }
+
+ public synchronized void onAMRestart() {
+ //TODO once AM restart is implemented and we know what to expect
+ }
+
+ /**
+ * Find a node for use
+ * @param role role
+ * @return the instance, or null for none
+ */
+ @VisibleForTesting
+ public synchronized NodeInstance findNodeForNewInstance(RoleStatus role) {
+ if (role.getNoDataLocality()) {
+ return null;
+ }
+ int roleKey = role.getKey();
+ NodeInstance nodeInstance = null;
+
+ List<NodeInstance> targets = getNodesForRoleId(roleKey);
+ while (targets != null && !targets.isEmpty() && nodeInstance == null) {
+ NodeInstance head = targets.remove(0);
+ if (head.getActiveRoleInstances(roleKey) == 0) {
+ nodeInstance = head;
+ }
+ }
+ return nodeInstance;
+ }
+
+ /**
+ * Request an instance on a given node.
+ * An outstanding request is created & tracked, with the
+ * relevant node entry for that role updated.
+ *
+ * The role status entries will also be tracked
+ *
+ * Returns the request that is now being tracked.
+ * If the node instance is not null, it's details about the role is incremented
+ *
+ *
+ * @param node node to target or null for "any"
+ * @param role role to request
+ * @return the container priority
+ */
+ public synchronized AMRMClient.ContainerRequest requestInstanceOnNode(
+ NodeInstance node, RoleStatus role, Resource resource) {
+ OutstandingRequest outstanding = outstandingRequests.addRequest(node, role.getKey());
+ return outstanding.buildContainerRequest(resource, role, now());
+ }
+
+ /**
+ * Find a node for a role and request an instance on that (or a location-less
+ * instance)
+ * @param role role status
+ * @param resource resource capabilities
+ * @return a request ready to go
+ */
+ public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role,
+ Resource resource) {
+ NodeInstance node = findNodeForNewInstance(role);
+ return requestInstanceOnNode(node, role, resource);
+ }
+
+
+ /**
+ * Find a list of node for release; algorithm may make its own
+ * decisions on which to release.
+ * @param role role index
+ * @param count number of nodes to release
+ * @return a possibly empty list of nodes.
+ */
+ public synchronized List<NodeInstance> findNodesForRelease(int role, int count) {
+ return nodemap.findNodesForRelease(role, count);
+ }
+
+ /**
+ * Get the node entry of a container
+ * @param container container to look up
+ * @return the entry
+ */
+ public NodeEntry getOrCreateNodeEntry(Container container) {
+ NodeInstance node = getOrCreateNodeInstance(container);
+ return node.getOrCreate(ContainerPriority.extractRole(container));
+ }
+
+ /**
+ * Get the node instance of a container -always returns something
+ * @param container container to look up
+ * @return a (possibly new) node instance
+ */
+ public synchronized NodeInstance getOrCreateNodeInstance(Container container) {
+ String hostname = RoleHistoryUtils.hostnameOf(container);
+ return nodemap.getOrCreate(hostname);
+ }
+
+ /**
+ * Get the node instance of a an address if defined
+ * @param addr address
+ * @return a node instance or null
+ */
+ public synchronized NodeInstance getExistingNodeInstance(String hostname) {
+ return nodemap.get(hostname);
+ }
+
+ /**
+ * Perform any pre-allocation operations on the list of allocated containers
+ * based on knowledge of system state.
+ * Currently this places requested hosts ahead of unrequested ones.
+ * @param allocatedContainers list of allocated containers
+ * @return list of containers potentially reordered
+ */
+ public synchronized List<Container> prepareAllocationList(List<Container> allocatedContainers) {
+
+ //partition into requested and unrequested
+ List<Container> requested =
+ new ArrayList<Container>(allocatedContainers.size());
+ List<Container> unrequested =
+ new ArrayList<Container>(allocatedContainers.size());
+ outstandingRequests.partitionRequests(this, allocatedContainers, requested, unrequested);
+
+ //give the unrequested ones lower priority
+ requested.addAll(unrequested);
+ return requested;
+ }
+
+ /**
+ * A container has been allocated on a node -update the data structures
+ * @param container container
+ * @param desiredCount desired #of instances
+ * @param actualCount current count of instances
+ * @return true if an entry was found and dropped
+ */
+ public synchronized boolean onContainerAllocated(Container container, int desiredCount, int actualCount) {
+ int role = ContainerPriority.extractRole(container);
+ String hostname = RoleHistoryUtils.hostnameOf(container);
+ boolean requestFound =
+ outstandingRequests.onContainerAllocated(role, hostname);
+ if (desiredCount <= actualCount) {
+ //cancel the nodes
+ List<NodeInstance>
+ hosts = outstandingRequests.cancelOutstandingRequests(role);
+ if (!hosts.isEmpty()) {
+ //add the list
+ getOrCreateNodesForRoleId(role).addAll(hosts);
+ sortAvailableNodeList(role);
+ }
+ }
+ return requestFound;
+ }
+
+ /**
+ * A container has been assigned to a role instance on a node -update the data structures
+ * @param container container
+ */
+ public void onContainerAssigned(Container container) {
+ NodeEntry nodeEntry = getOrCreateNodeEntry(container);
+ nodeEntry.onStarting();
+ }
+
+ /**
+ * Event: a container start has been submitter
+ * @param container container being started
+ * @param instance instance bound to the container
+ */
+ public void onContainerStartSubmitted(Container container,
+ RoleInstance instance) {
+ NodeEntry nodeEntry = getOrCreateNodeEntry(container);
+ int role = ContainerPriority.extractRole(container);
+ // any actions we want here
+ }
+
+ /**
+ * Container start event
+ * @param container
+ */
+ public void onContainerStarted(Container container) {
+ NodeEntry nodeEntry = getOrCreateNodeEntry(container);
+ nodeEntry.onStartCompleted();
+ touch();
+ }
+
+ /**
+ * A container failed to start: update the node entry state
+ * and return the container to the queue
+ * @param container container that failed
+ * @return true if the node was queued
+ */
+ public boolean onNodeManagerContainerStartFailed(Container container) {
+ return markContainerFinished(container, false, true);
+ }
+
+ /**
+ * A container release request was issued
+ * @param container container submitted
+ */
+ public void onContainerReleaseSubmitted(Container container) {
+ NodeEntry nodeEntry = getOrCreateNodeEntry(container);
+ nodeEntry.release();
+ }
+
+ /**
+ * App state notified of a container completed
+ * @param container completed container
+ * @return true if the node was queued
+ */
+ public boolean onReleaseCompleted(Container container) {
+ return markContainerFinished(container, true, false);
+ }
+
+ /**
+ * App state notified of a container completed -but as
+ * it wasn't being released it is marked as failed
+ *
+ * @param container completed container
+ * @param shortLived was the container short lived?
+ * @return true if the node is considered available for work
+ */
+ public boolean onFailedContainer(Container container, boolean shortLived) {
+ return markContainerFinished(container, false, shortLived);
+ }
+
+ /**
+ * Mark a container finished; if it was released then that is treated
+ * differently. history is touch()ed
+ *
+ *
+ * @param container completed container
+ * @param wasReleased was the container released?
+ * @param shortLived was the container short lived?
+ * @return true if the node was queued
+ */
+ protected synchronized boolean markContainerFinished(Container container,
+ boolean wasReleased,
+ boolean shortLived) {
+ NodeEntry nodeEntry = getOrCreateNodeEntry(container);
+ boolean available;
+ if (shortLived) {
+ nodeEntry.onStartFailed();
+ available = false;
+ } else {
+ available = nodeEntry.containerCompleted(wasReleased);
+ maybeQueueNodeForWork(container, nodeEntry, available);
+ }
+ touch();
+ return available;
+ }
+
+ /**
+ * If the node is marked as available; queue it for assignments.
+ * Unsynced: expects caller to be in a sync block.
+ * @param container completed container
+ * @param nodeEntry node
+ * @param available available flag
+ * @return true if the node was queued
+ */
+ private boolean maybeQueueNodeForWork(Container container,
+ NodeEntry nodeEntry,
+ boolean available) {
+ if (available) {
+ //node is free
+ nodeEntry.setLastUsed(now());
+ NodeInstance ni = getOrCreateNodeInstance(container);
+ int roleId = ContainerPriority.extractRole(container);
+ log.debug("Node {} is now available for role id {}", ni, roleId);
+ getOrCreateNodesForRoleId(roleId).addFirst(ni);
+ }
+ return available;
+ }
+
+ /**
+ * Print the history to the log. This is for testing and diagnostics
+ */
+ public synchronized void dump() {
+ for (ProviderRole role : providerRoles) {
+ log.info(role.toString());
+ List<NodeInstance> instances =
+ getOrCreateNodesForRoleId(role.id);
+ log.info(" available: " + instances.size()
+ + " " + SliderUtils.joinWithInnerSeparator(", ", instances));
+ }
+
+ log.info("Nodes in Cluster: {}", getClusterSize());
+ for (NodeInstance node : nodemap.values()) {
+ log.info(node.toFullString());
+ }
+ }
+
+ /**
+ * Get a clone of the available list
+ * @param role role index
+ * @return a clone of the list
+ */
+ @VisibleForTesting
+ public List<NodeInstance> cloneAvailableList(int role) {
+ return new LinkedList<NodeInstance>(getOrCreateNodesForRoleId(role));
+ }
+
+ /**
+ * Get a snapshot of the outstanding request list
+ * @return a list of the requests outstanding at the time of requesting
+ */
+ @VisibleForTesting
+ public synchronized List<OutstandingRequest> getOutstandingRequestList() {
+ return outstandingRequests.listOutstandingRequests();
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistoryUtils.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistoryUtils.java
new file mode 100644
index 0000000..ea6197b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistoryUtils.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.slider.common.tools.SliderUtils;
+
+public class RoleHistoryUtils {
+
+ public static String hostnameOf(Container container) {
+ NodeId nodeId = container.getNodeId();
+ if (nodeId== null) {
+ throw new RuntimeException("Container has no node ID: %s" +
+ SliderUtils.containerToString(container));
+ }
+ return nodeId.getHost();
+ }
+
+ /**
+ * Decrement a value but hold it at zero. Usually a sanity check
+ * on counters tracking outstanding operations
+ * @param val value
+ * @return decremented value
+ */
+ public static int decToFloor(int val) {
+ int v = val-1;
+ if (v < 0) {
+ v = 0;
+ }
+ return v;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
new file mode 100644
index 0000000..962606e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.common.tools.SliderUtils;
+
+import java.util.Arrays;
+
+/**
+ * Tracking information about a container
+ */
+public final class RoleInstance implements Cloneable {
+
+ public Container container;
+ /**
+ * UUID of container used in Slider RPC to refer to instances.
+ * The string value of the container ID is used here.
+ */
+ public final String id;
+ public long createTime;
+ public long startTime;
+ /**
+ * flag set when it is released, to know if it has
+ * already been targeted for termination
+ */
+ public boolean released;
+ public String role;
+ public int roleId;
+ /**
+ * state from {@link ClusterDescription}
+ */
+ public int state;
+
+ /**
+ * Exit code: only valid if the state >= STOPPED
+ */
+ public int exitCode;
+
+ /**
+ * what was the command executed?
+ */
+ public String command;
+
+ /**
+ * Any diagnostics
+ */
+ public String diagnostics;
+
+ /**
+ * What is the tail output from the executed process (or [] if not started
+ * or the log cannot be picked up
+ */
+ public String[] output;
+
+ /**
+ * Any environment details
+ */
+ public String[] environment;
+
+ public String host;
+ public String hostURL;
+
+ /**
+ * Any information the provider wishes to retain on the state of
+ * an instance
+ */
+ public Object providerInfo;
+
+ public RoleInstance(Container container) {
+ this.container = container;
+ if (container == null) {
+ throw new NullPointerException("Null container");
+ }
+ if (container.getId() == null) {
+ throw new NullPointerException("Null container ID");
+ }
+ id = container.getId().toString();
+ if (container.getNodeId() != null) {
+ host = container.getNodeId().getHost();
+ }
+ if (container.getNodeHttpAddress() != null) {
+ hostURL = "http://" + container.getNodeHttpAddress();
+ }
+ }
+
+ public ContainerId getId() {
+ return container.getId();
+ }
+
+ public NodeId getHost() {
+ return container.getNodeId();
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("RoleInstance{");
+ sb.append("container=").append(SliderUtils.containerToString(container));
+ sb.append(", id='").append(id).append('\'');
+ sb.append(", createTime=").append(createTime);
+ sb.append(", startTime=").append(startTime);
+ sb.append(", released=").append(released);
+ sb.append(", role='").append(role).append('\'');
+ sb.append(", roleId=").append(roleId);
+ sb.append(", host=").append(host);
+ sb.append(", hostURL=").append(hostURL);
+ sb.append(", state=").append(state);
+ sb.append(", exitCode=").append(exitCode);
+ sb.append(", command='").append(command).append('\'');
+ sb.append(", diagnostics='").append(diagnostics).append('\'');
+ sb.append(", output=").append(Arrays.toString(output));
+ sb.append(", environment=").append(Arrays.toString(environment));
+ sb.append('}');
+ return sb.toString();
+ }
+
+ public ContainerId getContainerId() {
+ return container != null ? container.getId() : null;
+ }
+
+ /**
+ * Generate the protobuf format of a request
+ * @return protobuf format. This excludes the Container info
+ */
+ public Messages.RoleInstanceState toProtobuf() {
+ Messages.RoleInstanceState.Builder builder =
+ Messages.RoleInstanceState.newBuilder();
+ if (container != null) {
+ builder.setName(container.getId().toString());
+ } else {
+ builder.setName("unallocated instance");
+ }
+ if (command != null) {
+ builder.setCommand(command);
+ }
+ if (environment != null) {
+ builder.addAllEnvironment(Arrays.asList(environment));
+ }
+ if (diagnostics != null) {
+ builder.setDiagnostics(diagnostics);
+ }
+ builder.setExitCode(exitCode);
+
+ if (output != null) {
+ builder.addAllOutput(Arrays.asList(output));
+ }
+ if (role != null) {
+ builder.setRole(role);
+ }
+ builder.setRoleId(roleId);
+ builder.setState(state);
+
+ builder.setReleased(released);
+ builder.setCreateTime(createTime);
+ builder.setStartTime(startTime);
+ builder.setHost(host);
+ builder.setHostURL(hostURL);
+ return builder.build();
+ }
+
+ /**
+ * Clone operation clones all the simple values but shares the
+ * Container object into the cloned copy -same with the output,
+ * diagnostics and env arrays.
+ * @return a clone of the object
+ * @throws CloneNotSupportedException
+ */
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ RoleInstance cloned = (RoleInstance) super.clone();
+ return cloned;
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
new file mode 100644
index 0000000..169253b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * Models the ongoing status of all nodes in
+ * Nothing here is synchronized: grab the whole instance to update.
+ */
+public final class RoleStatus implements Cloneable {
+
+
+ private final String name;
+
+
+ /**
+ * Role key in the container details stored in the AM,
+ * currently mapped to priority
+ */
+ private final int key;
+
+ private final ProviderRole providerRole;
+
+ private int desired, actual, requested, releasing;
+ private int failed, started, startFailed, completed, totalRequested;
+
+ private String failureMessage = "";
+
+ public RoleStatus(ProviderRole providerRole) {
+ this.providerRole = providerRole;
+ this.name = providerRole.name;
+ this.key = providerRole.id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getKey() {
+ return key;
+ }
+
+ public int getPriority() {
+ return getKey();
+ }
+
+ /**
+ * Get the placement policy enum, from the values in
+ * {@link PlacementPolicy}
+ * @return the placement policy for this role
+ */
+ public int getPlacementPolicy() {
+ return providerRole.placementPolicy;
+ }
+
+ public boolean getExcludeFromFlexing() {
+ return 0 != (getPlacementPolicy() & PlacementPolicy.EXCLUDE_FROM_FLEXING);
+ }
+
+ public boolean getNoDataLocality() {
+ return 0 != (getPlacementPolicy() & PlacementPolicy.NO_DATA_LOCALITY);
+ }
+
+ public int getDesired() {
+ return desired;
+ }
+
+ public void setDesired(int desired) {
+ this.desired = desired;
+ }
+
+ public int getActual() {
+ return actual;
+ }
+
+ public int incActual() {
+ return ++actual;
+ }
+
+ public int decActual() {
+ if (0 > --actual) {
+ actual = 0;
+ }
+ return actual;
+ }
+
+ public synchronized int getRequested() {
+ return requested;
+ }
+
+ public synchronized int incRequested() {
+ totalRequested++;
+ return ++requested;
+ }
+
+ public synchronized int decRequested() {
+ if (0 > --requested) {
+ requested = 0;
+ }
+ return requested;
+ }
+
+ public int getReleasing() {
+ return releasing;
+ }
+
+ public int incReleasing() {
+ return ++releasing;
+ }
+
+ public int decReleasing() {
+ if (0 > --releasing) {
+ releasing = 0;
+ }
+ return releasing;
+ }
+
+ public int getFailed() {
+ return failed;
+ }
+
+ /**
+ * Note that a role failed, text will
+ * be used in any diagnostics if an exception
+ * is later raised.
+ * @param text text about the failure
+ */
+ public void noteFailed(String text) {
+ failed++;
+ if (text != null) {
+ failureMessage = text;
+ }
+ }
+
+ public int getStartFailed() {
+ return startFailed;
+ }
+
+ public void incStartFailed() {
+ startFailed++;
+ }
+
+ public String getFailureMessage() {
+ return failureMessage;
+ }
+
+ public int getCompleted() {
+ return completed;
+ }
+
+ public void setCompleted(int completed) {
+ this.completed = completed;
+ }
+
+ public void incCompleted() {
+ completed ++;
+ }
+ public int getStarted() {
+ return started;
+ }
+
+ public void incStarted() {
+ started++;
+ }
+
+ public int getTotalRequested() {
+ return totalRequested;
+ }
+
+ /**
+ * Get the number of roles we are short of.
+ * nodes released are ignored.
+ * @return the positive or negative number of roles to add/release.
+ * 0 means "do nothing".
+ */
+ public synchronized int getDelta() {
+ int inuse = actual + requested;
+ //don't know how to view these. Are they in-use or not?
+ //inuse += releasing;
+ int delta = desired - inuse;
+ if (delta < 0) {
+ //if we are releasing, remove the number that are already released.
+ delta += releasing;
+ //but never switch to a positive
+ delta = Math.min(delta, 0);
+ }
+ return delta;
+ }
+
+ @Override
+ public String toString() {
+ return "RoleStatus{" +
+ "name='" + name + '\'' +
+ ", key=" + key +
+ ", desired=" + desired +
+ ", actual=" + actual +
+ ", requested=" + requested +
+ ", releasing=" + releasing +
+ ", failed=" + failed +
+ ", started=" + started +
+ ", startFailed=" + startFailed +
+ ", completed=" + completed +
+ ", failureMessage='" + failureMessage + '\'' +
+
+ '}';
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return super.clone();
+ }
+
+ /**
+ * Get the provider role
+ * @return
+ */
+ public ProviderRole getProviderRole() {
+ return providerRole;
+ }
+
+ /**
+ * Build the statistics map from the current data
+ * @return a map for use in statistics reports
+ */
+ public Map<String, Integer> buildStatistics() {
+ Map<String, Integer> stats = new HashMap<String, Integer>();
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, getRequested());
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, getCompleted());
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, getDesired());
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, getFailed());
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, getActual());
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_REQUESTED, getTotalRequested());
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, getStarted());
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, getStartFailed());
+ return stats;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
new file mode 100644
index 0000000..4713dcd
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The methods to offer state access to the providers
+ */
+public interface StateAccessForProviders {
+ Map<Integer, RoleStatus> getRoleStatusMap();
+
+ /**
+ * Get the published configurations
+ * @return the configuration set
+ */
+ PublishedConfigSet getPublishedConfigurations();
+
+ Map<ContainerId, RoleInstance> getFailedNodes();
+
+ Map<ContainerId, RoleInstance> getLiveNodes();
+
+ /**
+ * Get the current cluster description
+ * @return the actual state of the cluster
+ */
+ ClusterDescription getClusterStatus();
+
+ /**
+ * Get at the snapshot of the resource config
+ * Changes here do not affect the application state.
+ * @return the most recent settings
+ */
+ ConfTreeOperations getResourcesSnapshot();
+
+ /**
+ * Get at the snapshot of the appconf config
+ * Changes here do not affect the application state.
+ * @return the most recent settings
+ */
+ ConfTreeOperations getAppConfSnapshot();
+
+ /**
+ * Get at the snapshot of the internals config.
+ * Changes here do not affect the application state.
+ * @return the internals settings
+ */
+
+ ConfTreeOperations getInternalsSnapshot();
+
+ /**
+ * Flag set to indicate the application is live -this only happens
+ * after the buildInstance operation
+ */
+ boolean isApplicationLive();
+
+ long getSnapshotTime();
+
+ AggregateConf getInstanceDefinitionSnapshot();
+
+ /**
+ * Look up a role from its key -or fail
+ *
+ * @param key key to resolve
+ * @return the status
+ * @throws YarnRuntimeException on no match
+ */
+ RoleStatus lookupRoleStatus(int key);
+
+ /**
+ * Look up a role from its key -or fail
+ *
+ * @param c container in a role
+ * @return the status
+ * @throws YarnRuntimeException on no match
+ */
+ RoleStatus lookupRoleStatus(Container c) throws YarnRuntimeException;
+
+ /**
+ * Look up a role from its key -or fail
+ *
+ * @param name container in a role
+ * @return the status
+ * @throws YarnRuntimeException on no match
+ */
+ RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException;
+
+ /**
+ * Clone a list of active containers
+ * @return the active containers at the time
+ * the call was made
+ */
+ List<RoleInstance> cloneActiveContainerList();
+
+ /**
+ * Get the number of active containers
+ * @return the number of active containers the time the call was made
+ */
+ int getNumActiveContainers();
+
+ /**
+ * Get any active container with the given ID
+ * @param id container Id
+ * @return the active container or null if it is not found
+ */
+ RoleInstance getActiveContainer(ContainerId id);
+
+ /**
+ * Create a clone of the list of live cluster nodes.
+ * @return the list of nodes, may be empty
+ */
+ List<RoleInstance> cloneLiveContainerInfoList();
+
+ /**
+ * Get the {@link RoleInstance} details on a container.
+ * This is an O(n) operation
+ * @param containerId the container ID
+ * @return null if there is no such node
+ * @throws NoSuchNodeException if the node cannot be found
+ */
+ RoleInstance getLiveInstanceByContainerID(String containerId)
+ throws NoSuchNodeException;
+
+ /**
+ * Get the details on a list of instaces referred to by ID.
+ * Unknown nodes are not returned
+ * <i>Important: the order of the results are undefined</i>
+ * @param containerIDs the containers
+ * @return list of instances
+ */
+ List<RoleInstance> getLiveInstancesByContainerIDs(
+ Collection<String> containerIDs);
+
+ /**
+ * Update the cluster description with anything interesting
+ * @param providerStatus status from the provider for the cluster info section
+ */
+ void refreshClusterStatus();
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMController.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMController.java
new file mode 100644
index 0000000..c3c6e60
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMController.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web;
+
+import com.google.inject.Inject;
+import org.apache.hadoop.yarn.webapp.Controller;
+import org.apache.slider.server.appmaster.web.layout.AppLayout;
+import org.apache.slider.server.appmaster.web.layout.ClusterSpecificationView;
+import org.apache.slider.server.appmaster.web.layout.ContainerStatsView;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class SliderAMController extends Controller {
+ private static final Logger log = LoggerFactory.getLogger(SliderAMController.class);
+
+ private final WebAppApi slider;
+
+ @Inject
+ public SliderAMController(WebAppApi slider, RequestContext ctx) {
+ super(ctx);
+ this.slider = slider;
+ }
+
+ @Override
+ public void index() {
+ setTitle("Slider App Master");
+
+ updateAppState();
+
+ render(AppLayout.class);
+ }
+
+ public void containerStats() {
+ setTitle("Container Statistics");
+
+ updateAppState();
+
+ render(ContainerStatsView.class);
+ }
+
+ public void specification() {
+ setTitle("Cluster Specification");
+
+ render(ClusterSpecificationView.class);
+ }
+
+ private void updateAppState() {
+ //TODO don't do this on every request?
+ slider.getAppState().refreshClusterStatus();
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
new file mode 100644
index 0000000..6e75d80
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter;
+import com.sun.jersey.api.core.ResourceConfig;
+import com.sun.jersey.core.util.FeaturesAndProperties;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.spi.container.servlet.ServletContainer;
+import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.hadoop.yarn.webapp.Dispatcher;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig;
+import org.apache.slider.server.appmaster.web.rest.AMWebServices;
+import org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider;
+import org.apache.slider.server.services.curator.CuratorHelper;
+import org.apache.slider.server.services.curator.RegistryBinderService;
+import org.apache.slider.server.services.curator.RegistryDiscoveryContext;
+import org.apache.slider.server.services.curator.RegistryRestResources;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ *
+ */
+public class SliderAMWebApp extends WebApp {
+ public static final String BASE_PATH = "slideram";
+ public static final String CONTAINER_STATS = "/stats";
+ public static final String CLUSTER_SPEC = "/spec";
+
+ public final RegistryBinderService<ServiceInstanceData> registry;
+
+ public SliderAMWebApp(RegistryBinderService<ServiceInstanceData> registry) {
+ Preconditions.checkNotNull(registry);
+ this.registry = registry;
+ }
+
+ @Override
+ public void setup() {
+ Logger.getLogger("com.sun.jersey").setLevel(Level.FINEST);
+ // Make one of these to ensure that the jax-b annotations
+ // are properly picked up.
+ bind(SliderJacksonJaxbJsonProvider.class);
+
+ // Get exceptions printed to the screen
+ bind(GenericExceptionHandler.class);
+ // bind the REST interface
+ bind(AMWebServices.class);
+ //bind(AMAgentWebServices.class);
+
+ CuratorHelper curatorHelper = registry.getCuratorHelper();
+ ServiceDiscovery<ServiceInstanceData> discovery = registry.getDiscovery();
+ RegistryDiscoveryContext discoveryContext = curatorHelper
+ .createDiscoveryContext(
+ discovery);
+
+ bind(RegistryDiscoveryContext.class).toInstance(discoveryContext);
+ RegistryRestResources registryRestResources =
+ new RegistryRestResources(discoveryContext, registry);
+ bind(RegistryRestResources.class).toInstance(registryRestResources);
+
+ route("/", SliderAMController.class);
+ route(CONTAINER_STATS, SliderAMController.class, "containerStats");
+ route(CLUSTER_SPEC, SliderAMController.class, "specification");
+ }
+
+ @Override
+ public void configureServlets() {
+ setup();
+
+ serve("/", "/__stop").with(Dispatcher.class);
+
+ for (String path : this.getServePathSpecs()) {
+ serve(path).with(Dispatcher.class);
+ }
+
+ String regex = "(?!/ws)";
+ serveRegex(regex).with(SliderDefaultWrapperServlet.class);
+
+ Map<String, String> params = new HashMap<String, String>();
+ params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true");
+ params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true");
+ params.put(FeaturesAndProperties.FEATURE_XMLROOTELEMENT_PROCESSING, "true");
+ params.put(ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS, GZIPContentEncodingFilter.class.getName());
+ params.put(ResourceConfig.PROPERTY_CONTAINER_RESPONSE_FILTERS, GZIPContentEncodingFilter.class.getName());
+ //params.put("com.sun.jersey.spi.container.ContainerRequestFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
+ //params.put("com.sun.jersey.spi.container.ContainerResponseFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
+ //params.put("com.sun.jersey.config.feature.Trace", "true");
+ params.put("com.sun.jersey.config.property.WadlGeneratorConfig",
+ AMWadlGeneratorConfig.CLASSNAME);
+ filter("/*").through(GuiceContainer.class, params);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
new file mode 100644
index 0000000..5fffa4a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
@@ -0,0 +1,71 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.slider.server.appmaster.web;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.FilterContainer;
+import org.apache.hadoop.http.FilterInitializer;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SliderAmFilterInitializer extends FilterInitializer {
+ private static final String FILTER_NAME = "AM_PROXY_FILTER";
+ private static final String FILTER_CLASS = SliderAmIpFilter.class.getCanonicalName();
+ private static final String HTTPS_PREFIX = "https://";
+ private static final String HTTP_PREFIX = "http://";
+ private Configuration configuration;
+
+ public static final String NAME =
+ "org.apache.slider.server.appmaster.web.SliderAmFilterInitializer";
+
+ @Override
+ public void initFilter(FilterContainer container, Configuration conf) {
+ configuration = conf;
+ Map<String, String> params = new HashMap<String, String>();
+ String proxy = WebAppUtils.getProxyHostAndPort(conf);
+ String[] parts = proxy.split(":");
+ params.put(SliderAmIpFilter.PROXY_HOST, parts[0]);
+ // todo: eventually call WebAppUtils.getHttpSchemePrefix
+ params.put(SliderAmIpFilter.PROXY_URI_BASE, getHttpSchemePrefix()
+ + proxy + getApplicationWebProxyBase());
+ params.put(SliderAmIpFilter.WS_CONTEXT_ROOT,
+ conf.get(SliderAmIpFilter.WS_CONTEXT_ROOT));
+ container.addFilter(FILTER_NAME, FILTER_CLASS, params);
+ }
+
+ @VisibleForTesting
+ protected String getApplicationWebProxyBase() {
+ return System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV);
+ }
+
+ private String getHttpSchemePrefix() {
+ return HttpConfig.Policy.HTTPS_ONLY ==
+ HttpConfig.Policy.fromString(configuration
+ .get(
+ YarnConfiguration.YARN_HTTP_POLICY_KEY,
+ YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))
+ ? HTTPS_PREFIX : HTTP_PREFIX;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
new file mode 100644
index 0000000..3893699
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
@@ -0,0 +1,131 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.slider.server.appmaster.web;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
+import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpPrincipal;
+import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.HashSet;
+import java.util.Set;
+
+public class SliderAmIpFilter implements Filter {
+ private static final Log LOG = LogFactory.getLog(SliderAmIpFilter.class);
+
+ public static final String PROXY_HOST = "PROXY_HOST";
+ public static final String PROXY_URI_BASE = "PROXY_URI_BASE";
+ //update the proxy IP list about every 5 min
+ private static final long updateInterval = 5 * 60 * 1000;
+ public static final String WS_CONTEXT_ROOT = "slider.rest.context.root";
+
+ private String proxyHost;
+ private Set<String> proxyAddresses = null;
+ private long lastUpdate;
+ private String proxyUriBase;
+ private String wsContextRoot;
+
+ @Override
+ public void init(FilterConfig conf) throws ServletException {
+ proxyHost = conf.getInitParameter(PROXY_HOST);
+ proxyUriBase = conf.getInitParameter(PROXY_URI_BASE);
+ wsContextRoot = conf.getInitParameter(WS_CONTEXT_ROOT);
+ }
+
+ protected Set<String> getProxyAddresses() throws ServletException {
+ long now = System.currentTimeMillis();
+ synchronized(this) {
+ if(proxyAddresses == null || (lastUpdate + updateInterval) >= now) {
+ try {
+ proxyAddresses = new HashSet<String>();
+ for(InetAddress add : InetAddress.getAllByName(proxyHost)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("proxy address is: " + add.getHostAddress());
+ }
+ proxyAddresses.add(add.getHostAddress());
+ }
+ lastUpdate = now;
+ } catch (UnknownHostException e) {
+ throw new ServletException("Could not locate "+proxyHost, e);
+ }
+ }
+ return proxyAddresses;
+ }
+ }
+
+ @Override
+ public void destroy() {
+ //Empty
+ }
+
+ @Override
+ public void doFilter(ServletRequest req, ServletResponse resp,
+ FilterChain chain) throws IOException, ServletException {
+ if(!(req instanceof HttpServletRequest)) {
+ throw new ServletException("This filter only works for HTTP/HTTPS");
+ }
+
+ HttpServletRequest httpReq = (HttpServletRequest)req;
+ HttpServletResponse httpResp = (HttpServletResponse)resp;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Remote address for request is: " + httpReq.getRemoteAddr());
+ }
+ if(!httpReq.getRequestURI().startsWith(wsContextRoot) &&
+ !getProxyAddresses().contains(httpReq.getRemoteAddr())) {
+ String redirectUrl = httpResp.encodeRedirectURL(proxyUriBase +
+ httpReq.getRequestURI());
+ httpResp.sendRedirect(redirectUrl);
+ return;
+ }
+
+ String user = null;
+
+ if (httpReq.getCookies() != null) {
+ for(Cookie c: httpReq.getCookies()) {
+ if(WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())){
+ user = c.getValue();
+ break;
+ }
+ }
+ }
+ if(user == null) {
+ LOG.warn("Could not find "+WebAppProxyServlet.PROXY_USER_COOKIE_NAME
+ +" cookie, so user will not be set");
+ chain.doFilter(req, resp);
+ } else {
+ final AmIpPrincipal principal = new AmIpPrincipal(user);
+ ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq,
+ principal);
+ chain.doFilter(requestWrapper, resp);
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderDefaultWrapperServlet.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderDefaultWrapperServlet.java
new file mode 100644
index 0000000..12c41ac
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderDefaultWrapperServlet.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web;
+
+import com.google.inject.Singleton;
+import org.apache.hadoop.yarn.webapp.DefaultWrapperServlet;
+
+import javax.servlet.RequestDispatcher;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+/**
+ *
+ */
+@Singleton
+public class SliderDefaultWrapperServlet extends DefaultWrapperServlet {
+ @Override
+ protected void doPost(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ RequestDispatcher rd = getServletContext().getNamedDispatcher("default");
+
+ HttpServletRequest wrapped = new HttpServletRequestWrapper(req) {
+ public String getServletPath() {
+ return "";
+ }
+ };
+
+ rd.forward(wrapped, resp);
+
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
new file mode 100644
index 0000000..4fac962
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web;
+
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
+
+import java.util.Map;
+
+/**
+ * Interface to pass information from the Slider AppMaster to the WebApp
+ */
+public interface WebAppApi {
+
+ /**
+ * The {@link AppState} for the current cluster
+ */
+ public StateAccessForProviders getAppState();
+
+ /**
+ * The {@link ProviderService} for the current cluster
+ */
+ public ProviderService getProviderService();
+
+ /**
+ * The {@link SliderClusterProtocol} for the current cluster
+ */
+ public SliderClusterProtocol getClusterProtocol();
+
+ /**
+ * Generate a mapping from role name to its {@link RoleStatus}. Be aware that this
+ * is a computed value and not just a getter
+ */
+ public Map<String,RoleStatus> getRoleStatusByName();
+
+ /**
+ * Returns an interface that can support the agent-based REST operations.
+ */
+ public AgentRestOperations getAgentRestOperations();
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
new file mode 100644
index 0000000..5c1c206
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web;
+
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ *
+ */
+public class WebAppApiImpl implements WebAppApi {
+ private static final Logger log = LoggerFactory.getLogger(WebAppApiImpl.class);
+
+ protected static final ProviderRole AM_ROLE_NAME = new ProviderRole("Slider Application Master", SliderKeys.ROLE_AM_PRIORITY_INDEX);
+
+ protected final SliderClusterProtocol clusterProto;
+ protected final StateAccessForProviders appState;
+ protected final ProviderService provider;
+
+ public WebAppApiImpl(SliderClusterProtocol clusterProto,
+ StateAccessForProviders appState, ProviderService provider) {
+ checkNotNull(clusterProto);
+ checkNotNull(appState);
+ checkNotNull(provider);
+
+ this.clusterProto = clusterProto;
+ this.appState = appState;
+ this.provider = provider;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.slider.server.appmaster.web.WebAppApi#getAppState()
+ */
+ @Override
+ public StateAccessForProviders getAppState() {
+ return appState;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.slider.server.appmaster.web.WebAppApi#getProviderService()
+ */
+ @Override
+ public ProviderService getProviderService() {
+ return provider;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.slider.server.appmaster.web.WebAppApi#getClusterProtocol()
+ */
+ @Override
+ public SliderClusterProtocol getClusterProtocol() {
+ return clusterProto;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.slider.server.appmaster.web.WebAppApi#getRoleStatusByName()
+ */
+ @Override
+ public TreeMap<String,RoleStatus> getRoleStatusByName() {
+ Map<Integer,ProviderRole> rolesById = rolesById(provider.getRoles());
+ Map<Integer,RoleStatus> status = appState.getRoleStatusMap();
+
+ return getRoleStatusesByName(rolesById, status);
+ }
+
+ /**
+ * Get the ProviderRoles by their index
+ * @param roles
+ * @return
+ */
+ private Map<Integer,ProviderRole> rolesById(List<ProviderRole> roles) {
+ Map<Integer,ProviderRole> rolesById = new HashMap<Integer,ProviderRole>();
+ rolesById.put(SliderKeys.ROLE_AM_PRIORITY_INDEX, AM_ROLE_NAME);
+
+ for (ProviderRole role : roles) {
+ rolesById.put(role.id, role);
+ }
+
+ return rolesById;
+ }
+
+ /**
+ * Join the ProviderRole by their ID with the RoleStatus by their ID, to get the RoleStatus by role name.
+ * @param rolesById
+ * @param statusById
+ * @return A Map of RoleStatus by the role name
+ */
+ private TreeMap<String,RoleStatus> getRoleStatusesByName(Map<Integer,ProviderRole> rolesById, Map<Integer,RoleStatus> statusById) {
+ TreeMap<String,RoleStatus> statusByName = new TreeMap<String,RoleStatus>();
+ for (Entry<Integer,ProviderRole> role : rolesById.entrySet()) {
+ final RoleStatus status = statusById.get(role.getKey());
+
+ if (null == status) {
+ log.error("Found ID ({}) which has no known ProviderRole", role.getKey());
+ } else {
+ statusByName.put(role.getValue().name, status);
+ }
+ }
+
+ return statusByName;
+ }
+
+ @Override
+ public AgentRestOperations getAgentRestOperations() {
+ return provider.getAgentRestOperations();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/AppLayout.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/AppLayout.java
new file mode 100644
index 0000000..d9a2cda
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/AppLayout.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.layout;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.slider.server.appmaster.web.view.IndexBlock;
+
+/**
+ *
+ */
+public class AppLayout extends WebUILayout {
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return IndexBlock.class;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ClusterSpecificationView.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ClusterSpecificationView.java
new file mode 100644
index 0000000..b54ca71
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ClusterSpecificationView.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.layout;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.slider.server.appmaster.web.view.ClusterSpecificationBlock;
+
+/**
+ *
+ */
+public class ClusterSpecificationView extends WebUILayout {
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return ClusterSpecificationBlock.class;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ContainerStatsView.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ContainerStatsView.java
new file mode 100644
index 0000000..39ba0ad
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/ContainerStatsView.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.layout;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.slider.server.appmaster.web.view.ContainerStatsBlock;
+
+
+
+/**
+ *
+ */
+public class ContainerStatsView extends WebUILayout {
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return ContainerStatsBlock.class;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/WebUILayout.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/WebUILayout.java
new file mode 100644
index 0000000..1681f59
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/layout/WebUILayout.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.layout;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
+import org.apache.slider.server.appmaster.web.view.NavBlock;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+
+/**
+ *
+ */
+public class WebUILayout extends TwoColumnLayout {
+
+ @Override
+ protected void preHead(Page.HTML<_> html) {
+ set(ACCORDION_ID, "nav");
+ set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
+ }
+
+ @Override
+ protected Class<? extends SubView> nav() {
+ return NavBlock.class;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGenerator.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGenerator.java
new file mode 100644
index 0000000..05aaa5b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGenerator.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest;
+
+import com.sun.jersey.server.wadl.ApplicationDescription;
+import com.sun.jersey.server.wadl.WadlGenerator;
+import com.sun.jersey.server.wadl.WadlGeneratorImpl;
+import com.sun.research.ws.wadl.Application;
+import com.sun.research.ws.wadl.Resource;
+import com.sun.research.ws.wadl.Resources;
+
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ *
+ */
+public class AMWadlGenerator extends WadlGeneratorImpl {
+ @Override
+ /**
+ * This method is called once the WADL application has been assembled, so it
+ * affords an opportunity to edit the resources presented by the WADL. In
+ * this case, we're removing the internal "/agents" resources.
+ */
+ public void attachTypes(ApplicationDescription egd) {
+ super.attachTypes(egd);
+
+ Application application = egd.getApplication();
+ List<Resources> resources = application.getResources();
+
+ for (Resources appResources : resources) {
+ List<Resource> resourceList = appResources.getResource();
+ for (Resource appResource : resourceList) {
+ String path = appResource.getPath();
+ if (RestPaths.SLIDER_CONTEXT_ROOT.equals(path)) {
+ List<Object> sliderResources = appResource.getMethodOrResource();
+ Iterator<Object> itor = sliderResources.iterator();
+ while (itor.hasNext()) {
+ Object sliderRes = itor.next();
+ if (sliderRes instanceof Resource) {
+ Resource res = (Resource) sliderRes;
+ if (RestPaths.SLIDER_SUBPATH_AGENTS.equals(res.getPath())) {
+ // assuming I'll get a list modification issue if I remove at this
+ // point
+ itor.remove();
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void setWadlGeneratorDelegate(WadlGenerator delegate) {
+ // do nothing
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java
new file mode 100644
index 0000000..4ae7490
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWadlGeneratorConfig.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest;
+
+import com.sun.jersey.api.wadl.config.WadlGeneratorConfig;
+import com.sun.jersey.api.wadl.config.WadlGeneratorDescription;
+
+import java.util.List;
+
+/**
+ *
+ */
+public class AMWadlGeneratorConfig extends WadlGeneratorConfig{
+
+ public static final String CLASSNAME = "org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig";
+ @Override
+ public List<WadlGeneratorDescription> configure() {
+ return generator(AMWadlGenerator.class).descriptions();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
new file mode 100644
index 0000000..91c83f2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.rest.agent.AgentResource;
+import org.apache.slider.server.appmaster.web.rest.management.ManagementResource;
+import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource;
+
+import javax.ws.rs.Path;
+
+/** The available REST services exposed by a slider AM. */
+@Singleton
+@Path(RestPaths.SLIDER_CONTEXT_ROOT)
+public class AMWebServices {
+ /** AM/WebApp info object */
+ private WebAppApi slider;
+
+ @Inject
+ public AMWebServices(WebAppApi slider) {
+ this.slider = slider;
+ }
+
+ @Path(RestPaths.SLIDER_SUBPATH_MANAGEMENT)
+ public ManagementResource getManagementResource() {
+ return new ManagementResource(slider);
+ }
+
+ @Path(RestPaths.SLIDER_SUBPATH_AGENTS)
+ public AgentResource getAgentResource () {
+ return new AgentResource(slider);
+ }
+
+ @Path(RestPaths.SLIDER_SUBPATH_PUBLISHER)
+ public PublisherResource getPublisherResource() {
+ return new PublisherResource(slider);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
new file mode 100644
index 0000000..29f4c5a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest;
+
+/**
+ * Paths in the REST App
+ */
+public class RestPaths {
+
+ public static final String WS_CONTEXT_ROOT = "/ws";
+ public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT +"/v1/slider";
+ public static final String SLIDER_SUBPATH_MANAGEMENT = "/mgmt";
+ public static final String SLIDER_SUBPATH_AGENTS = "/agents";
+ public static final String SLIDER_SUBPATH_PUBLISHER = "/publisher";
+
+ public static final String SLIDER_PATH_MANAGEMENT = SLIDER_CONTEXT_ROOT
+ + SLIDER_SUBPATH_MANAGEMENT;
+ public static final String SLIDER_PATH_AGENTS = SLIDER_CONTEXT_ROOT
+ + SLIDER_SUBPATH_AGENTS;
+
+ public static final String SLIDER_PATH_PUBLISHER = SLIDER_CONTEXT_ROOT
+ + SLIDER_SUBPATH_PUBLISHER;
+
+ public static final String SLIDER_SUBPATH_REGISTRY = "/registry";
+ public static final String SLIDER_PATH_REGISTRY = WS_CONTEXT_ROOT
+ + SLIDER_SUBPATH_REGISTRY;
+
+ public static final String REGISTRY_SERVICE = "v1/service";
+ public static final String REGISTRY_ANYSERVICE = "v1/anyservice";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java
new file mode 100644
index 0000000..95b0aa1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/SliderJacksonJaxbJsonProvider.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest;
+
+import com.google.inject.Singleton;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider;
+import org.codehaus.jackson.map.AnnotationIntrospector;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.introspect.JacksonAnnotationIntrospector;
+import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.ext.Provider;
+
+/**
+ * Implementation of JAX-RS abstractions based on {@link
+ * JacksonJaxbJsonProvider} needed to deserialize JSON content to, or serialize
+ * it from, POJO objects.
+ */
+@Singleton
+@Provider
+@Unstable
+@Private
+public class SliderJacksonJaxbJsonProvider extends JacksonJaxbJsonProvider {
+
+ public SliderJacksonJaxbJsonProvider() {
+ super();
+ }
+
+ @Override
+ public ObjectMapper locateMapper(Class<?> type, MediaType mediaType) {
+ ObjectMapper mapper = super.locateMapper(type, mediaType);
+ AnnotationIntrospector introspector = new AnnotationIntrospector.Pair(
+ new JaxbAnnotationIntrospector(),
+ new JacksonAnnotationIntrospector()
+ );
+ mapper.setAnnotationIntrospector(introspector);
+ //mapper.setSerializationInclusion(Inclusion.NON_NULL);
+ return mapper;
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
new file mode 100644
index 0000000..17cd8f2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+public enum AgentCommandType {
+ EXECUTION_COMMAND,
+ STATUS_COMMAND,
+ REGISTRATION_COMMAND
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
new file mode 100644
index 0000000..781ae00
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import com.google.gson.annotations.SerializedName;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class AgentEnv {
+
+ /**
+ * Various directories, configurable in <code>ambari-agent.ini</code>
+ */
+ private Directory[] stackFoldersAndFiles = new Directory[0];
+
+ /**
+ * Directories that match name <code>/etc/alternatives/*conf</code>
+ */
+ private Alternative[] alternatives = new Alternative[0];
+
+ /**
+ * List of existing users
+ */
+ private ExistingUser[] existingUsers = new ExistingUser[0];
+
+ /**
+ * List of repos
+ */
+ private String[] existingRepos = new String[0];
+
+ /**
+ * List of packages
+ */
+ private PackageDetail[] installedPackages = new PackageDetail[0];
+
+ /**
+ * The host health report
+ */
+ private HostHealth hostHealth = new HostHealth();
+
+ private Integer umask;
+
+ private Boolean iptablesIsRunning;
+
+ public Integer getUmask() {
+ return umask;
+ }
+
+ public void setUmask(Integer umask) {
+ this.umask = umask;
+ }
+
+ public Directory[] getStackFoldersAndFiles() {
+ return stackFoldersAndFiles;
+ }
+
+ public void setStackFoldersAndFiles(Directory[] dirs) {
+ stackFoldersAndFiles = dirs;
+ }
+
+ public void setExistingUsers(ExistingUser[] users) {
+ existingUsers = users;
+ }
+
+ public ExistingUser[] getExistingUsers() {
+ return existingUsers;
+ }
+
+ public void setAlternatives(Alternative[] dirs) {
+ alternatives = dirs;
+ }
+
+ public Alternative[] getAlternatives() {
+ return alternatives;
+ }
+
+ public void setExistingRepos(String[] repos) {
+ existingRepos = repos;
+ }
+
+ public String[] getExistingRepos() {
+ return existingRepos;
+ }
+
+ public void setInstalledPackages(PackageDetail[] packages) {
+ installedPackages = packages;
+ }
+
+ public PackageDetail[] getInstalledPackages() {
+ return installedPackages;
+ }
+
+ public void setHostHealth(HostHealth healthReport) {
+ hostHealth = healthReport;
+ }
+
+ public HostHealth getHostHealth() {
+ return hostHealth;
+ }
+
+ public Boolean getIptablesIsRunning() {
+ return iptablesIsRunning;
+ }
+
+ public void setIptablesIsRunning(Boolean iptablesIsRunning) {
+ this.iptablesIsRunning = iptablesIsRunning;
+ }
+
+ public static class HostHealth {
+ /**
+ * Java processes running on the system. Default empty array.
+ */
+ @SerializedName("activeJavaProcs")
+ private JavaProc[] activeJavaProcs = new JavaProc[0];
+
+ /**
+ * The current time when agent send the host check report
+ */
+ @SerializedName("agentTimeStampAtReporting")
+ private long agentTimeStampAtReporting = 0;
+
+ /**
+ * The current time when host check report was received
+ */
+ @SerializedName("serverTimeStampAtReporting")
+ private long serverTimeStampAtReporting = 0;
+
+ /**
+ * Live services running on the agent
+ */
+ @SerializedName("liveServices")
+ private LiveService[] liveServices = new LiveService[0];
+
+ public void setAgentTimeStampAtReporting(long currentTime) {
+ agentTimeStampAtReporting = currentTime;
+ }
+
+ public long getAgentTimeStampAtReporting() {
+ return agentTimeStampAtReporting;
+ }
+
+ public void setServerTimeStampAtReporting(long currentTime) {
+ serverTimeStampAtReporting = currentTime;
+ }
+
+ public long getServerTimeStampAtReporting() {
+ return serverTimeStampAtReporting;
+ }
+
+ public void setActiveJavaProcs(JavaProc[] procs) {
+ activeJavaProcs = procs;
+ }
+
+ public JavaProc[] getActiveJavaProcs() {
+ return activeJavaProcs;
+ }
+
+ public void setLiveServices(LiveService[] services) {
+ liveServices = services;
+ }
+
+ public LiveService[] getLiveServices() {
+ return liveServices;
+ }
+ }
+
+ public static class PackageDetail {
+ @SerializedName("name")
+ private String pkgName;
+ @SerializedName("version")
+ private String pkgVersion;
+ @SerializedName("repoName")
+ private String pkgRepoName;
+
+ public void setName(String name) {
+ pkgName = name;
+ }
+
+ public String getName() {
+ return pkgName;
+ }
+
+ public void setVersion(String version) {
+ pkgVersion = version;
+ }
+
+ public String getVersion() {
+ return pkgVersion;
+ }
+
+ public void setRepoName(String repoName) {
+ pkgRepoName = repoName;
+ }
+
+ public String getRepoName() {
+ return pkgRepoName;
+ }
+ }
+
+ /**
+ * Represents information about a directory of interest.
+ */
+ public static class Directory {
+ @SerializedName("name")
+ private String dirName;
+ @SerializedName("type")
+ private String dirType;
+
+ public void setName(String name) {
+ dirName = name;
+ }
+
+ public String getName() {
+ return dirName;
+ }
+
+ public void setType(String type) {
+ dirType = type;
+ }
+
+ public String getType() {
+ return dirType;
+ }
+ }
+
+ /**
+ * Represents information about running java processes.
+ */
+ public static class JavaProc {
+ @SerializedName("user")
+ private String user;
+ @SerializedName("pid")
+ private int pid = 0;
+ @SerializedName("hadoop")
+ private boolean is_hadoop = false;
+ @SerializedName("command")
+ private String command;
+
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public void setPid(int pid) {
+ this.pid = pid;
+ }
+
+ public int getPid() {
+ return pid;
+ }
+
+ public void setHadoop(boolean hadoop) {
+ is_hadoop = hadoop;
+ }
+
+ public boolean isHadoop() {
+ return is_hadoop;
+ }
+
+ public void setCommand(String cmd) {
+ command = cmd;
+ }
+
+ public String getCommand() {
+ return command;
+ }
+ }
+
+ public static class Alternative {
+ @SerializedName("name")
+ private String altName;
+ @SerializedName("target")
+ private String altTarget;
+
+ public void setName(String name) {
+ altName = name;
+ }
+
+ public String getName() {
+ return altName;
+ }
+
+ public void setTarget(String target) {
+ altTarget = target;
+ }
+
+ public String getTarget() {
+ return altTarget;
+ }
+ }
+
+ public static class LiveService {
+ @SerializedName("name")
+ private String svcName;
+ @SerializedName("status")
+ private String svcStatus;
+ @SerializedName("desc")
+ private String svcDesc;
+
+ public void setName(String name) {
+ svcName = name;
+ }
+
+ public String getName() {
+ return svcName;
+ }
+
+ public void setStatus(String status) {
+ svcStatus = status;
+ }
+
+ public String getStatus() {
+ return svcStatus;
+ }
+
+ public void setDesc(String desc) {
+ svcDesc = desc;
+ }
+
+ public String getDesc() {
+ return svcDesc;
+ }
+ }
+
+ public static class ExistingUser {
+ @SerializedName("name")
+ private String name;
+ @SerializedName("homeDir")
+ private String homeDir;
+ @SerializedName("status")
+ private String status;
+
+ public void setUserName(String userName) {
+ name = userName;
+ }
+
+ public String getUserName() {
+ return name;
+ }
+
+ public void setUserHomeDir(String userHomeDir) {
+ homeDir = userHomeDir;
+ }
+
+ public String getUserHomeDir() {
+ return homeDir;
+ }
+
+ public void setUserStatus(String userStatus) {
+ status = userStatus;
+ }
+
+ public String getUserStatus() {
+ return status;
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
new file mode 100644
index 0000000..96b7b47
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+/**
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class AgentResource {
+
+ private final WebAppApi slider;
+ private String agent_name;
+
+ public AgentResource(WebAppApi slider) {
+ this.slider = slider;
+ }
+
+ private void init(HttpServletResponse res) {
+ res.setContentType(null);
+ }
+
+ @GET
+ @Path("/agents/register")
+ public Response endpointAgentRegister() {
+ Response response = Response.status(200).entity("/agent/register").build();
+ return response;
+ }
+
+ @GET
+ @Path("/agents")
+ public Response endpointAgent() {
+ Response response = Response.status(200).entity("/agent").build();
+ return response;
+ }
+ @GET
+ @Path("/")
+ public Response endpointRoot() {
+ Response response = Response.status(200).entity("/").build();
+ return response;
+ }
+
+ @POST
+ @Path("/{agent_name: [a-zA-Z][a-zA-Z_0-9]*}/register")
+ @Consumes({MediaType.APPLICATION_JSON})
+ @Produces({MediaType.APPLICATION_JSON})
+ public RegistrationResponse register(Register registration,
+ @Context HttpServletResponse res,
+ @PathParam("agent_name") String agent_name) {
+ init(res);
+ this.agent_name = agent_name;
+ AgentRestOperations ops = slider.getAgentRestOperations();
+ return ops.handleRegistration(registration);
+
+ }
+
+ @POST
+ @Path("/{agent_name: [a-zA-Z][a-zA-Z_0-9]*}/heartbeat")
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces({MediaType.APPLICATION_JSON})
+ public HeartBeatResponse heartbeat(HeartBeat message,
+ @Context HttpServletResponse res,
+ @PathParam("agent_name") String agent_name) {
+ init(res);
+ AgentRestOperations ops = slider.getAgentRestOperations();
+ return ops.handleHeartBeat(message);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java
new file mode 100644
index 0000000..2891be8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+/**
+ *
+ */
+public interface AgentRestOperations {
+
+ RegistrationResponse handleRegistration(Register registration);
+
+ HeartBeatResponse handleHeartBeat(HeartBeat heartBeat);
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java
new file mode 100644
index 0000000..2a4961f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.Map;
+
+/**
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class CommandReport {
+
+ private String role;
+ private String actionId;
+ private String stdout;
+ private String stderr;
+ private String structuredOut;
+ private String status;
+ int exitCode;
+ private String clusterName;
+ private String serviceName;
+ private long taskId;
+ private String roleCommand;
+
+ private Map<String, Map<String, String>> configurationTags;
+
+ @JsonProperty("taskId")
+ public long getTaskId() {
+ return taskId;
+ }
+
+ @JsonProperty("taskId")
+ public void setTaskId(long taskId) {
+ this.taskId = taskId;
+ }
+
+ @JsonProperty("clusterName")
+ public void setClusterName(String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ @JsonProperty("clusterName")
+ public String getClusterName() {
+ return this.clusterName;
+ }
+
+ @JsonProperty("actionId")
+ public String getActionId() {
+ return this.actionId;
+ }
+
+ @JsonProperty("actionId")
+ public void setActionId(String actionId) {
+ this.actionId = actionId;
+ }
+
+ @JsonProperty("stderr")
+ public String getStdErr() {
+ return this.stderr;
+ }
+
+ @JsonProperty("stderr")
+ public void setStdErr(String stderr) {
+ this.stderr = stderr;
+ }
+
+ @JsonProperty("exitcode")
+ public int getExitCode() {
+ return this.exitCode;
+ }
+
+ @JsonProperty("exitcode")
+ public void setExitCode(int exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ @JsonProperty("stdout")
+ public String getStdOut() {
+ return this.stdout;
+ }
+
+ @JsonProperty("stdout")
+ public void setStdOut(String stdout) {
+ this.stdout = stdout;
+ }
+
+ @JsonProperty("structuredOut")
+ public String getStructuredOut() {
+ return this.structuredOut;
+ }
+
+
+ @JsonProperty("structuredOut")
+ public void setStructuredOut(String structuredOut) {
+ this.structuredOut = structuredOut;
+ }
+
+ @JsonProperty("roleCommand")
+ public String getRoleCommand() {
+ return this.roleCommand;
+ }
+
+ @JsonProperty("roleCommand")
+ public void setRoleCommand(String roleCommand) {
+ this.roleCommand = roleCommand;
+ }
+
+ @JsonProperty("role")
+ public String getRole() {
+ return role;
+ }
+
+ @JsonProperty("role")
+ public void setRole(String role) {
+ this.role = role;
+ }
+
+ @JsonProperty("status")
+ public String getStatus() {
+ return status;
+ }
+
+ @JsonProperty("status")
+ public void setStatus(String status) {
+ this.status = status;
+ }
+
+ @JsonProperty("serviceName")
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ @JsonProperty("serviceName")
+ public void setServiceName(String serviceName) {
+ this.serviceName = serviceName;
+ }
+
+ /**
+ * @param tags the config tags that match this command
+ */
+ @JsonProperty("configurationTags")
+ public void setConfigurationTags(Map<String, Map<String,String>> tags) {
+ configurationTags = tags;
+ }
+
+ /**
+ * @return the config tags that match this command, or <code>null</code>
+ * if none are present
+ */
+ @JsonProperty("configurationTags")
+ public Map<String, Map<String,String>> getConfigurationTags() {
+ return configurationTags;
+ }
+
+ @Override
+ public String toString() {
+ return "CommandReport{" +
+ "role='" + role + '\'' +
+ ", actionId='" + actionId + '\'' +
+ ", status='" + status + '\'' +
+ ", exitCode=" + exitCode +
+ ", clusterName='" + clusterName + '\'' +
+ ", serviceName='" + serviceName + '\'' +
+ ", taskId=" + taskId +
+ ", roleCommand=" + roleCommand +
+ ", configurationTags=" + configurationTags +
+ '}';
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
new file mode 100644
index 0000000..071432b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.Map;
+
+/**
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ComponentStatus {
+ String componentName;
+ String msg;
+ String status;
+ String serviceName;
+ String clusterName;
+ private Map<String, Map<String, String>> configurations;
+
+ public String getComponentName() {
+ return this.componentName;
+ }
+
+ public void setComponentName(String componentName) {
+ this.componentName = componentName;
+ }
+
+ public String getMessage() {
+ return this.msg;
+ }
+
+ public void setMessage(String msg) {
+ this.msg = msg;
+ }
+
+ public String getStatus() {
+ return this.status;
+ }
+
+ public void setStatus(String status) {
+ this.status = status;
+ }
+
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ public void setServiceName(String serviceName) {
+ this.serviceName = serviceName;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public void setClusterName(String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ /** @return the config tags that match this command, or <code>null</code> if none are present */
+ public Map<String, Map<String, String>> getConfiguration() {
+ return configurations;
+ }
+
+ /** @param configurations the config tags that match this status */
+ public void setConfiguration(Map<String, Map<String, String>> configurations) {
+ this.configurations = configurations;
+ }
+
+ @Override
+ public String toString() {
+ return "ComponentStatus{" +
+ "componentName='" + componentName + '\'' +
+ ", msg='" + msg + '\'' +
+ ", status='" + status + '\'' +
+ ", serviceName='" + serviceName + '\'' +
+ ", clusterName='" + clusterName + '\'' +
+ '}';
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java
new file mode 100644
index 0000000..27c4d54
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class DiskInfo {
+ String available;
+ String mountpoint;
+ String device;
+ String used;
+ String percent;
+ String size;
+ String type;
+
+ /**
+ * DiskInfo object that tracks information about a disk.
+ * @param mountpoint
+ * @param available
+ * @param used
+ * @param percent
+ * @param size
+ */
+ public DiskInfo(String device, String mountpoint, String available,
+ String used, String percent, String size, String type) {
+ this.device = device;
+ this.mountpoint = mountpoint;
+ this.available = available;
+ this.used = used;
+ this.percent = percent;
+ this.size = size;
+ this.type = type;
+ }
+
+ /**
+ * Needed for Serialization
+ */
+ public DiskInfo() {}
+
+ @JsonProperty("available")
+ public void setAvailable(String available) {
+ this.available = available;
+ }
+
+ @JsonProperty("available")
+ public String getAvailable() {
+ return this.available;
+ }
+
+ @JsonProperty("mountpoint")
+ public String getMountPoint() {
+ return this.mountpoint;
+ }
+
+ @JsonProperty("mountpoint")
+ public void setMountPoint(String mountpoint) {
+ this.mountpoint = mountpoint;
+ }
+
+ @JsonProperty("type")
+ public String getType() {
+ return this.type;
+ }
+
+ @JsonProperty("type")
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ @JsonProperty("used")
+ public String getUsed() {
+ return this.used;
+ }
+
+ @JsonProperty("used")
+ public void setUsed(String used) {
+ this.used = used;
+ }
+
+ @JsonProperty("percent")
+ public String getPercent() {
+ return this.percent;
+ }
+
+ @JsonProperty("percent")
+ public void setPercent(String percent) {
+ this.percent = percent;
+ }
+
+ @JsonProperty("size")
+ public String getSize() {
+ return this.size;
+ }
+
+ @JsonProperty("size")
+ public void setSize(String size) {
+ this.size = size;
+ }
+
+ @Override
+ public String toString() {
+ return "available=" + this.available + ",mountpoint=" + this.mountpoint
+ + ",used=" + this.used + ",percent=" + this.percent + ",size=" +
+ this.size + ",device=" + this.device +
+ ",type=" + this.type;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
new file mode 100644
index 0000000..c184e63
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ExecutionCommand {
+ private static Log LOG = LogFactory.getLog(ExecutionCommand.class);
+ private AgentCommandType commandType = AgentCommandType.EXECUTION_COMMAND;
+ private String clusterName;
+ private long taskId;
+ private String commandId;
+ private String hostname;
+ private String role;
+ private Map<String, String> hostLevelParams = new HashMap<String, String>();
+ private Map<String, String> roleParams = null;
+ private String roleCommand;
+ private Map<String, Map<String, String>> configurations;
+ private Map<String, String> commandParams;
+ private String serviceName;
+ private String componentName;
+
+ public ExecutionCommand(AgentCommandType commandType) {
+ this.commandType = commandType;
+ }
+
+ @JsonProperty("commandType")
+ public AgentCommandType getCommandType() {
+ return commandType;
+ }
+
+ @JsonProperty("commandType")
+ public void setCommandType(AgentCommandType commandType) {
+ this.commandType = commandType;
+ }
+
+ @JsonProperty("commandId")
+ public String getCommandId() {
+ return this.commandId;
+ }
+
+ @JsonProperty("commandId")
+ public void setCommandId(String commandId) {
+ this.commandId = commandId;
+ }
+
+ @JsonProperty("taskId")
+ public long getTaskId() {
+ return taskId;
+ }
+
+ @JsonProperty("taskId")
+ public void setTaskId(long taskId) {
+ this.taskId = taskId;
+ }
+
+ @JsonProperty("role")
+ public String getRole() {
+ return role;
+ }
+
+ @JsonProperty("role")
+ public void setRole(String role) {
+ this.role = role;
+ }
+
+ @JsonProperty("roleParams")
+ public Map<String, String> getRoleParams() {
+ return roleParams;
+ }
+
+ @JsonProperty("roleParams")
+ public void setRoleParams(Map<String, String> roleParams) {
+ this.roleParams = roleParams;
+ }
+
+ @JsonProperty("roleCommand")
+ public String getRoleCommand() {
+ return roleCommand;
+ }
+
+ @JsonProperty("roleCommand")
+ public void setRoleCommand(String cmd) {
+ this.roleCommand = cmd;
+ }
+
+ @JsonProperty("clusterName")
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ @JsonProperty("clusterName")
+ public void setClusterName(String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ @JsonProperty("hostname")
+ public String getHostname() {
+ return hostname;
+ }
+
+ @JsonProperty("hostname")
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ @JsonProperty("hostLevelParams")
+ public Map<String, String> getHostLevelParams() {
+ return hostLevelParams;
+ }
+
+ @JsonProperty("hostLevelParams")
+ public void setHostLevelParams(Map<String, String> params) {
+ this.hostLevelParams = params;
+ }
+
+ @JsonProperty("configurations")
+ public Map<String, Map<String, String>> getConfigurations() {
+ return configurations;
+ }
+
+ @JsonProperty("configurations")
+ public void setConfigurations(Map<String, Map<String, String>> configurations) {
+ this.configurations = configurations;
+ }
+
+ @JsonProperty("commandParams")
+ public Map<String, String> getCommandParams() {
+ return commandParams;
+ }
+
+ @JsonProperty("commandParams")
+ public void setCommandParams(Map<String, String> commandParams) {
+ this.commandParams = commandParams;
+ }
+
+ @JsonProperty("serviceName")
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ @JsonProperty("serviceName")
+ public void setServiceName(String serviceName) {
+ this.serviceName = serviceName;
+ }
+
+ @JsonProperty("componentName")
+ public String getComponentName() {
+ return componentName;
+ }
+
+ @JsonProperty("componentName")
+ public void setComponentName(String componentName) {
+ this.componentName = componentName;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
new file mode 100644
index 0000000..fa149b8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ *
+ * Data model for agent heartbeat for server (ambari or app master).
+ *
+ */
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class HeartBeat {
+ private long responseId = -1;
+ private long timestamp;
+ private String hostname;
+ List<CommandReport> reports = new ArrayList<CommandReport>();
+ List<ComponentStatus> componentStatus = new ArrayList<ComponentStatus>();
+ private List<DiskInfo> mounts = new ArrayList<DiskInfo>();
+ HostStatus nodeStatus;
+ private AgentEnv agentEnv = null;
+
+ public long getResponseId() {
+ return responseId;
+ }
+
+ public void setResponseId(long responseId) {
+ this.responseId=responseId;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ @JsonProperty("reports")
+ public List<CommandReport> getReports() {
+ return this.reports;
+ }
+
+ @JsonProperty("reports")
+ public void setReports(List<CommandReport> reports) {
+ this.reports = reports;
+ }
+
+ public HostStatus getNodeStatus() {
+ return nodeStatus;
+ }
+
+ public void setNodeStatus(HostStatus nodeStatus) {
+ this.nodeStatus = nodeStatus;
+ }
+
+ public AgentEnv getAgentEnv() {
+ return agentEnv;
+ }
+
+ public void setAgentEnv(AgentEnv env) {
+ agentEnv = env;
+ }
+
+ @JsonProperty("componentStatus")
+ public List<ComponentStatus> getComponentStatus() {
+ return componentStatus;
+ }
+
+ @JsonProperty("componentStatus")
+ public void setComponentStatus(List<ComponentStatus> componentStatus) {
+ this.componentStatus = componentStatus;
+ }
+
+ @JsonProperty("mounts")
+ public List<DiskInfo> getMounts() {
+ return this.mounts;
+ }
+
+ @JsonProperty("mounts")
+ public void setMounts(List<DiskInfo> mounts) {
+ this.mounts = mounts;
+ }
+
+ @Override
+ public String toString() {
+ return "HeartBeat{" +
+ "responseId=" + responseId +
+ ", timestamp=" + timestamp +
+ ", hostname='" + hostname + '\'' +
+ ", reports=" + reports +
+ ", componentStatus=" + componentStatus +
+ ", nodeStatus=" + nodeStatus +
+ '}';
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
new file mode 100644
index 0000000..ca2db32
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ * Controller to Agent response data model.
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class HeartBeatResponse {
+
+ private long responseId;
+
+ List<ExecutionCommand> executionCommands = new ArrayList<ExecutionCommand>();
+ List<StatusCommand> statusCommands = new ArrayList<StatusCommand>();
+
+ RegistrationCommand registrationCommand;
+
+ boolean restartAgent = false;
+ boolean hasMappedComponents = false;
+
+ @JsonProperty("responseId")
+ public long getResponseId() {
+ return responseId;
+ }
+
+ @JsonProperty("responseId")
+ public void setResponseId(long responseId) {
+ this.responseId=responseId;
+ }
+
+ @JsonProperty("executionCommands")
+ public List<ExecutionCommand> getExecutionCommands() {
+ return executionCommands;
+ }
+
+ @JsonProperty("executionCommands")
+ public void setExecutionCommands(List<ExecutionCommand> executionCommands) {
+ this.executionCommands = executionCommands;
+ }
+
+ @JsonProperty("statusCommands")
+ public List<StatusCommand> getStatusCommands() {
+ return statusCommands;
+ }
+
+ @JsonProperty("statusCommands")
+ public void setStatusCommands(List<StatusCommand> statusCommands) {
+ this.statusCommands = statusCommands;
+ }
+
+ @JsonProperty("registrationCommand")
+ public RegistrationCommand getRegistrationCommand() {
+ return registrationCommand;
+ }
+
+ @JsonProperty("registrationCommand")
+ public void setRegistrationCommand(RegistrationCommand registrationCommand) {
+ this.registrationCommand = registrationCommand;
+ }
+
+ @JsonProperty("restartAgent")
+ public boolean isRestartAgent() {
+ return restartAgent;
+ }
+
+ @JsonProperty("restartAgent")
+ public void setRestartAgent(boolean restartAgent) {
+ this.restartAgent = restartAgent;
+ }
+
+ @JsonProperty("hasMappedComponents")
+ public boolean hasMappedComponents() {
+ return hasMappedComponents;
+ }
+
+ @JsonProperty("hasMappedComponents")
+ public void setHasMappedComponents(boolean hasMappedComponents) {
+ this.hasMappedComponents = hasMappedComponents;
+ }
+
+ public void addExecutionCommand(ExecutionCommand execCmd) {
+ executionCommands.add(execCmd);
+ }
+
+ public void addStatusCommand(StatusCommand statCmd) {
+ statusCommands.add(statCmd);
+ }
+
+ @Override
+ public String toString() {
+ return "HeartBeatResponse{" +
+ "responseId=" + responseId +
+ ", executionCommands=" + executionCommands +
+ ", statusCommands=" + statusCommands +
+ ", registrationCommand=" + registrationCommand +
+ ", restartAgent=" + restartAgent +
+ '}';
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java
new file mode 100644
index 0000000..bef7b07
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class HostInfo {
+ private String architecture;
+ private String domain;
+ private String fqdn;
+ private String hardwareisa;
+ private String hardwaremodel;
+ private String hostname;
+ private String id;
+ private String interfaces;
+ private String ipaddress;
+ private String kernel;
+ private String kernelmajversion;
+ private String kernelrelease;
+ private String kernelversion;
+ private String macaddress;
+ private long memoryfree;
+ private long memorysize;
+ private List<DiskInfo> mounts = new ArrayList<DiskInfo>();
+ private long memorytotal;
+ private String netmask;
+ private String operatingsystem;
+ private String operatingsystemrelease;
+ private String osfamily;
+ private int physicalprocessorcount;
+ private int processorcount;
+ private boolean selinux;
+ private String swapfree;
+ private String swapsize;
+ private String timezone;
+ private String uptime;
+ private long uptime_days;
+ private long uptime_hours;
+
+
+ @JsonProperty("architecture")
+ public String getArchitecture() {
+ return this.architecture;
+ }
+
+ @JsonProperty("architecture")
+ public void setArchitecture(String architecture) {
+ this.architecture = architecture;
+ }
+
+ @JsonProperty("domain")
+ public String getDomain() {
+ return this.domain;
+ }
+
+ @JsonProperty("domain")
+ public void setDomain(String domain) {
+ this.domain = domain;
+ }
+
+ @JsonProperty("fqdn")
+ public String getFQDN() {
+ return this.fqdn;
+ }
+
+ @JsonProperty("fqdn")
+ public void setFQDN(String fqdn) {
+ this.fqdn = fqdn;
+ }
+
+ @JsonProperty("hardwareisa")
+ public String getHardwareIsa() {
+ return hardwareisa;
+ }
+
+ @JsonProperty("hardwareisa")
+ public void setHardwareIsa(String hardwareisa) {
+ this.hardwareisa = hardwareisa;
+ }
+
+ @JsonProperty("hardwaremodel")
+ public String getHardwareModel() {
+ return this.hardwaremodel;
+ }
+
+ @JsonProperty("hardwaremodel")
+ public void setHardwareModel(String hardwaremodel) {
+ this.hardwaremodel = hardwaremodel;
+ }
+
+ @JsonProperty("hostname")
+ public String getHostName() {
+ return this.hostname;
+ }
+
+ @JsonProperty("hostname")
+ public void setHostName(String hostname) {
+ this.hostname = hostname;
+ }
+
+ @JsonProperty("id")
+ public String getAgentUserId() {
+ return id;
+ }
+
+ @JsonProperty("id")
+ public void setAgentUserId(String id) {
+ this.id = id;
+ }
+
+ @JsonProperty("interfaces")
+ public String getInterfaces() {
+ return this.interfaces;
+ }
+
+ @JsonProperty("interfaces")
+ public void setInterfaces(String interfaces) {
+ this.interfaces = interfaces;
+ }
+
+ @JsonProperty("ipaddress")
+ public String getIPAddress() {
+ return this.ipaddress;
+ }
+
+ @JsonProperty("ipaddress")
+ public void setIPAddress(String ipaddress) {
+ this.ipaddress = ipaddress;
+ }
+
+ @JsonProperty("kernel")
+ public String getKernel() {
+ return this.kernel;
+ }
+
+ @JsonProperty("kernel")
+ public void setKernel(String kernel) {
+ this.kernel = kernel;
+ }
+
+ @JsonProperty("kernelmajversion")
+ public String getKernelMajVersion() {
+ return this.kernelmajversion;
+ }
+
+ @JsonProperty("kernelmajversion")
+ public void setKernelMajVersion(String kernelmajversion) {
+ this.kernelmajversion = kernelmajversion;
+ }
+
+ @JsonProperty("kernelrelease")
+ public String getKernelRelease() {
+ return this.kernelrelease;
+ }
+
+ @JsonProperty("kernelrelease")
+ public void setKernelRelease(String kernelrelease) {
+ this.kernelrelease = kernelrelease;
+ }
+
+ @JsonProperty("kernelversion")
+ public String getKernelVersion() {
+ return this.kernelversion;
+ }
+
+ @JsonProperty("kernelversion")
+ public void setKernelVersion(String kernelversion) {
+ this.kernelversion = kernelversion;
+ }
+
+ @JsonProperty("macaddress")
+ public String getMacAddress() {
+ return this.macaddress;
+ }
+
+ @JsonProperty("macaddress")
+ public void setMacAddress(String macaddress) {
+ this.macaddress = macaddress;
+ }
+
+ @JsonProperty("memoryfree")
+ public long getFreeMemory() {
+ return this.memoryfree;
+ }
+
+ @JsonProperty("memoryfree")
+ public void setFreeMemory(long memoryfree) {
+ this.memoryfree = memoryfree;
+ }
+
+ @JsonProperty("memorysize")
+ public long getMemorySize() {
+ return this.memorysize;
+ }
+
+ @JsonProperty("memorysize")
+ public void setMemorySize(long memorysize) {
+ this.memorysize = memorysize;
+ }
+
+ @JsonProperty("mounts")
+ public List<DiskInfo> getMounts() {
+ return this.mounts;
+ }
+
+ @JsonProperty("mounts")
+ public void setMounts(List<DiskInfo> mounts) {
+ this.mounts = mounts;
+ }
+
+ @JsonProperty("memorytotal")
+ public long getMemoryTotal() {
+ return this.memorytotal;
+ }
+
+ @JsonProperty("memorytotal")
+ public void setMemoryTotal(long memorytotal) {
+ this.memorytotal = memorytotal;
+ }
+
+ @JsonProperty("netmask")
+ public String getNetMask() {
+ return this.netmask;
+ }
+
+ @JsonProperty("netmask")
+ public void setNetMask(String netmask) {
+ this.netmask = netmask;
+ }
+
+ @JsonProperty("operatingsystem")
+ public String getOS() {
+ return this.operatingsystem;
+ }
+
+ @JsonProperty("operatingsystem")
+ public void setOS(String operatingsystem) {
+ this.operatingsystem = operatingsystem;
+ }
+
+ @JsonProperty("operatingsystemrelease")
+ public String getOSRelease() {
+ return this.operatingsystemrelease;
+ }
+
+ @JsonProperty("operatingsystemrelease")
+ public void setOSRelease(String operatingsystemrelease) {
+ this.operatingsystemrelease = operatingsystemrelease;
+ }
+
+ @JsonProperty("osfamily")
+ public String getOSFamily() {
+ return this.osfamily;
+ }
+
+ @JsonProperty("osfamily")
+ public void setOSFamily(String osfamily) {
+ this.osfamily = osfamily;
+ }
+
+ @JsonProperty("physicalprocessorcount")
+ public int getPhysicalProcessorCount() {
+ return this.physicalprocessorcount;
+ }
+
+ @JsonProperty("physicalprocessorcount")
+ public void setPhysicalProcessorCount(int physicalprocessorcount) {
+ this.physicalprocessorcount = physicalprocessorcount;
+ }
+
+ @JsonProperty("processorcount")
+ public int getProcessorCount() {
+ return this.processorcount;
+ }
+
+ @JsonProperty("processorcount")
+ public void setProcessorCount(int processorcount) {
+ this.processorcount = processorcount;
+ }
+
+ @JsonProperty("selinux")
+ public boolean getSeLinux() {
+ return selinux;
+ }
+
+ @JsonProperty("selinux")
+ public void setSeLinux(boolean selinux) {
+ this.selinux = selinux;
+ }
+
+ @JsonProperty("swapfree")
+ public String getSwapFree() {
+ return this.swapfree;
+ }
+
+ @JsonProperty("swapfree")
+ public void setSwapFree(String swapfree) {
+ this.swapfree = swapfree;
+ }
+
+ @JsonProperty("swapsize")
+ public String getSwapSize() {
+ return swapsize;
+ }
+
+ @JsonProperty("swapsize")
+ public void setSwapSize(String swapsize) {
+ this.swapsize = swapsize;
+ }
+
+ @JsonProperty("timezone")
+ public String getTimeZone() {
+ return this.timezone;
+ }
+
+ @JsonProperty("timezone")
+ public void setTimeZone(String timezone) {
+ this.timezone = timezone;
+ }
+
+ @JsonProperty("uptime")
+ public String getUptime() {
+ return this.uptime;
+ }
+
+ @JsonProperty("uptime")
+ public void setUpTime(String uptime) {
+ this.uptime = uptime;
+ }
+
+ @JsonProperty("uptime_hours")
+ public long getUptimeHours() {
+ return this.uptime_hours;
+ }
+
+ @JsonProperty("uptime_hours")
+ public void setUpTimeHours(long uptime_hours) {
+ this.uptime_hours = uptime_hours;
+ }
+
+ @JsonProperty("uptime_days")
+ public long getUpTimeDays() {
+ return this.uptime_days;
+ }
+
+ @JsonProperty("uptime_days")
+ public void setUpTimeDays(long uptime_days) {
+ this.uptime_days = uptime_days;
+ }
+
+ private String getDiskString() {
+ if (mounts == null) {
+ return null;
+ }
+ StringBuilder ret = new StringBuilder();
+ for (DiskInfo diskInfo : mounts) {
+ ret.append("(").append(diskInfo.toString()).append(")");
+ }
+ return ret.toString();
+ }
+
+ public String toString() {
+ return "[" +
+ "hostname=" + this.hostname + "," +
+ "fqdn=" + this.fqdn + "," +
+ "domain=" + this.domain + "," +
+ "architecture=" + this.architecture + "," +
+ "processorcount=" + this.processorcount + "," +
+ "physicalprocessorcount=" + this.physicalprocessorcount + "," +
+ "osname=" + this.operatingsystem + "," +
+ "osversion=" + this.operatingsystemrelease + "," +
+ "osfamily=" + this.osfamily + "," +
+ "memory=" + this.memorytotal + "," +
+ "uptime_hours=" + this.uptime_hours + "," +
+ "mounts=" + getDiskString() + "]\n";
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java
new file mode 100644
index 0000000..c584149
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class HostStatus {
+ public HostStatus(Status status, String cause) {
+ super();
+ this.status = status;
+ this.cause = cause;
+ }
+ public HostStatus() {
+ super();
+ }
+
+ public enum Status {
+ HEALTHY,
+ UNHEALTHY
+ }
+ Status status;
+ String cause;
+ public Status getStatus() {
+ return status;
+ }
+ public void setStatus(Status status) {
+ this.status = status;
+ }
+ public String getCause() {
+ return cause;
+ }
+ public void setCause(String cause) {
+ this.cause = cause;
+ }
+
+ @Override
+ public String toString() {
+ return "HostStatus{" +
+ "status=" + status +
+ ", cause='" + cause + '\'' +
+ '}';
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
new file mode 100644
index 0000000..9299a16
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ *
+ * Data model for agent to send heartbeat to ambari and/or app master.
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class Register {
+ private int responseId = -1;
+ private long timestamp;
+ private String hostname;
+ private int currentPingPort;
+ private HostInfo hardwareProfile;
+ private String publicHostname;
+ private AgentEnv agentEnv;
+ private String agentVersion;
+
+ @JsonProperty("responseId")
+ public int getResponseId() {
+ return responseId;
+ }
+
+ @JsonProperty("responseId")
+ public void setResponseId(int responseId) {
+ this.responseId=responseId;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ public HostInfo getHardwareProfile() {
+ return hardwareProfile;
+ }
+
+ public void setHardwareProfile(HostInfo hardwareProfile) {
+ this.hardwareProfile = hardwareProfile;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public String getPublicHostname() {
+ return publicHostname;
+ }
+
+ public void setPublicHostname(String name) {
+ this.publicHostname = name;
+ }
+
+ public AgentEnv getAgentEnv() {
+ return agentEnv;
+ }
+
+ public void setAgentEnv(AgentEnv env) {
+ this.agentEnv = env;
+ }
+
+ public String getAgentVersion() {
+ return agentVersion;
+ }
+
+ public void setAgentVersion(String agentVersion) {
+ this.agentVersion = agentVersion;
+ }
+
+ public int getCurrentPingPort() {
+ return currentPingPort;
+ }
+
+ public void setCurrentPingPort(int currentPingPort) {
+ this.currentPingPort = currentPingPort;
+ }
+
+ @Override
+ public String toString() {
+ String ret = "responseId=" + responseId + "\n" +
+ "timestamp=" + timestamp + "\n" +
+ "hostname=" + hostname + "\n" +
+ "currentPingPort=" + currentPingPort + "\n";
+
+ if (hardwareProfile != null)
+ ret = ret + "hardwareprofile=" + this.hardwareProfile.toString();
+ return ret;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java
new file mode 100644
index 0000000..4b87dd2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class RegistrationCommand {
+
+ private String command;
+
+ public String getCommand() {
+ return command;
+ }
+
+ public void setCommand(String command) {
+ this.command = command;
+ }
+
+ public RegistrationCommand(String command) {
+
+ this.command = command;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
new file mode 100644
index 0000000..734119d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class RegistrationResponse {
+
+ @JsonProperty("response")
+ private RegistrationStatus response;
+
+ /**
+ * exitstatus is a code of error which was rised on server side. exitstatus
+ * = 0 (OK - Default) exitstatus = 1 (Registration failed because different
+ * version of agent and server)
+ */
+ @JsonProperty("exitstatus")
+ private int exitstatus;
+
+ /** log - message, which will be printed to agents log */
+ @JsonProperty("log")
+ private String log;
+
+ //Response id to start with, usually zero.
+ @JsonProperty("responseId")
+ private long responseId;
+
+ @JsonProperty("statusCommands")
+ private List<StatusCommand> statusCommands = null;
+
+ public RegistrationResponse() {
+ }
+
+ public RegistrationStatus getResponseStatus() {
+ return response;
+ }
+
+ public void setResponseStatus(RegistrationStatus response) {
+ this.response = response;
+ }
+
+ public List<StatusCommand> getStatusCommands() {
+ return statusCommands;
+ }
+
+ public void setStatusCommands(List<StatusCommand> statusCommands) {
+ this.statusCommands = statusCommands;
+ }
+
+ public long getResponseId() {
+ return responseId;
+ }
+
+ public void setResponseId(long responseId) {
+ this.responseId = responseId;
+ }
+
+ public void setExitstatus(int exitstatus) {
+ this.exitstatus = exitstatus;
+ }
+
+ public void setLog(String log) {
+ this.log = log;
+ }
+
+ @Override
+ public String toString() {
+ return "RegistrationResponse{" +
+ "response=" + response +
+ ", responseId=" + responseId +
+ ", statusCommands=" + statusCommands +
+ '}';
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java
new file mode 100644
index 0000000..8374710
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+public enum RegistrationStatus {
+ OK,
+ FAILED
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
new file mode 100644
index 0000000..0610cac
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Command to report the status of a list of services in roles.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class StatusCommand {
+ public static String STATUS_COMMAND = "STATUS";
+ public static String GET_CONFIG_COMMAND = "GET_CONFIG";
+
+ AgentCommandType agentCommandType;
+
+ private String clusterName;
+ private String serviceName;
+ private String componentName;
+ private Map<String, Map<String, String>> configurations;
+ private Map<String, String> commandParams = new HashMap<String, String>();
+ private Map<String, String> hostLevelParams = new HashMap<String, String>();
+ private String roleCommand;
+
+ public StatusCommand() {
+ this.agentCommandType = AgentCommandType.STATUS_COMMAND;
+ }
+
+ @JsonProperty("clusterName")
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ @JsonProperty("clusterName")
+ public void setClusterName(String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ @JsonProperty("serviceName")
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ @JsonProperty("serviceName")
+ public void setServiceName(String serviceName) {
+ this.serviceName = serviceName;
+ }
+
+ @JsonProperty("componentName")
+ public String getComponentName() {
+ return componentName;
+ }
+
+ @JsonProperty("componentName")
+ public void setComponentName(String componentName) {
+ this.componentName = componentName;
+ }
+
+ @JsonProperty("configurations")
+ public Map<String, Map<String, String>> getConfigurations() {
+ return configurations;
+ }
+
+ @JsonProperty("configurations")
+ public void setConfigurations(Map<String, Map<String, String>> configurations) {
+ this.configurations = configurations;
+ }
+
+ @JsonProperty("hostLevelParams")
+ public Map<String, String> getHostLevelParams() {
+ return hostLevelParams;
+ }
+
+ @JsonProperty("hostLevelParams")
+ public void setHostLevelParams(Map<String, String> params) {
+ this.hostLevelParams = params;
+ }
+
+ @JsonProperty("commandParams")
+ public Map<String, String> getCommandParams() {
+ return commandParams;
+ }
+
+ @JsonProperty("commandParams")
+ public void setCommandParams(Map<String, String> commandParams) {
+ this.commandParams = commandParams;
+ }
+
+ @JsonProperty("commandType")
+ public AgentCommandType getCommandType() {
+ return agentCommandType;
+ }
+
+ @JsonProperty("commandType")
+ public void setCommandType(AgentCommandType commandType) {
+ this.agentCommandType = commandType;
+ }
+
+ @JsonProperty("roleCommand")
+ public String getRoleCommand() {
+ return roleCommand;
+ }
+
+ @JsonProperty("roleCommand")
+ public void setRoleCommand(String roleCommand) {
+ this.roleCommand = roleCommand;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
new file mode 100644
index 0000000..94db409
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.management;
+
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+import org.apache.slider.server.appmaster.web.rest.management.resources.AggregateConfResource;
+import org.apache.slider.server.appmaster.web.rest.management.resources.ConfTreeResource;
+import org.apache.slider.server.appmaster.web.rest.management.resources.ResourceFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriInfo;
+import java.net.URL;
+
+/**
+ *
+ */
+public class ManagementResource {
+ protected static final Logger log =
+ LoggerFactory.getLogger(ManagementResource.class);
+ private final WebAppApi slider;
+
+ public ManagementResource(WebAppApi slider) {
+ this.slider = slider;
+ }
+
+ private void init(HttpServletResponse res) {
+ res.setContentType(null);
+ }
+
+ @GET
+ public Response getWadl (@Context HttpServletRequest request) {
+ try {
+ java.net.URI location = new URL(request.getScheme(),
+ request.getServerName(),
+ request.getServerPort(),
+ "/application.wadl").toURI();
+ return Response.temporaryRedirect(location).build();
+ } catch (Exception e) {
+ log.error("Error during redirect to WADL", e);
+ throw new WebApplicationException(Response.serverError().build());
+ }
+ }
+
+ @GET
+ @Path("/app")
+ @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
+ public AggregateConfResource getAggregateConfiguration(@Context UriInfo uriInfo,
+ @Context HttpServletResponse res) {
+ init(res);
+ return ResourceFactory.createAggregateConfResource(getAggregateConf(),
+ uriInfo.getAbsolutePathBuilder());
+ }
+
+ @GET
+ @Path("/app/configurations/{config}")
+ @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
+ public ConfTreeResource getConfTreeResource(@PathParam("config") String config,
+ @Context UriInfo uriInfo,
+ @Context HttpServletResponse res) {
+ init(res);
+ AggregateConfResource aggregateConf =
+ ResourceFactory.createAggregateConfResource(getAggregateConf(),
+ uriInfo.getBaseUriBuilder()
+ .path(RestPaths.SLIDER_CONTEXT_ROOT).path(
+ "mgmt/app"));
+ return aggregateConf.getConfTree(config);
+ }
+
+ protected AggregateConf getAggregateConf() {
+ return slider.getAppState().getInstanceDefinitionSnapshot();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
new file mode 100644
index 0000000..9df692a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.management.resources;
+
+import org.apache.slider.core.conf.AggregateConf;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.ws.rs.core.UriBuilder;
+import java.util.HashMap;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class AggregateConfResource {
+ private String href;
+ private final ConfTreeResource resources;
+ private final ConfTreeResource internal;
+ private final ConfTreeResource appConf;
+ @JsonIgnore
+ private Map<String, ConfTreeResource> confMap;
+
+ public AggregateConfResource(AggregateConf conf, UriBuilder uriBuilder) {
+ if (uriBuilder != null) {
+ this.href =
+ uriBuilder.build(null).toASCIIString();
+ resources =
+ ResourceFactory.createConfTreeResource(conf.getAppConf(),
+ uriBuilder.clone().path(
+ "configurations").path(
+ "resources"));
+ internal =
+ ResourceFactory.createConfTreeResource(conf.getInternal(),
+ uriBuilder.clone().path(
+ "configurations").path(
+ "internal"));
+ appConf =
+ ResourceFactory.createConfTreeResource(conf.getAppConf(),
+ uriBuilder.clone().path(
+ "configurations").path(
+ "appConf"));
+ initConfMap();
+ } else {
+ resources = null;
+ internal = null;
+ appConf = null;
+ }
+ }
+
+ private void initConfMap() {
+ confMap = new HashMap<String, ConfTreeResource>();
+ confMap.put("internal", internal);
+ confMap.put("resources", resources);
+ confMap.put("appConf", appConf);
+ }
+
+ public AggregateConfResource() {
+ this(null, null);
+ }
+
+ public ConfTreeResource getConfTree(String name) {
+ return confMap.get(name);
+ }
+
+ public String getHref() {
+ return href;
+ }
+
+ public void setHref(String href) {
+ this.href = href;
+ }
+
+ public ConfTreeResource getResources() {
+ return resources;
+ }
+
+ public ConfTreeResource getInternal() {
+ return internal;
+ }
+
+ public ConfTreeResource getAppConf() {
+ return appConf;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java
new file mode 100644
index 0000000..a44448e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.management.resources;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.ws.rs.core.UriBuilder;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ComponentResource {
+ private final Map<String, String> props;
+ private String href;
+
+ public ComponentResource() {
+ this(null, null, null, null);
+ }
+
+ public ComponentResource(String name,
+ Map<String, String> props,
+ UriBuilder uriBuilder,
+ Map<String, Object> pathElems) {
+ this.props = props;
+ }
+
+ public Map<String, String> getProps() {
+ return props;
+ }
+
+ public String getHref() {
+ return href;
+ }
+
+ public void setHref(String href) {
+ this.href = href;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
new file mode 100644
index 0000000..79f5399
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.management.resources;
+
+import org.apache.slider.core.conf.ConfTree;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.ws.rs.core.UriBuilder;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ConfTreeResource {
+
+ private final String href;
+ private final Map<String, Object> metadata;
+ private final Map<String, String> global;
+ private final Map<String, Map<String, String>> components;
+
+ public ConfTreeResource() {
+ this(null, null);
+ }
+
+ public ConfTreeResource(ConfTree confTree,
+ UriBuilder uriBuilder) {
+ if (uriBuilder != null && confTree != null) {
+ metadata = confTree.metadata;
+ global = confTree.global;
+ components = confTree.components;
+ uriBuilder = uriBuilder.clone();
+ this.href = uriBuilder.build(null).toASCIIString();
+ } else {
+ this.href = null;
+ this.metadata = null;
+ this.global = null;
+ this.components = null;
+ }
+ }
+
+ public Map<String, Object> getMetadata() {
+ return metadata;
+ }
+
+ public Map<String, String> getGlobal() {
+ return global;
+ }
+
+ public Map<String, Map<String, String>> getComponents() {
+ return components;
+ }
+
+ public String getHref() {
+ return href;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ResourceFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ResourceFactory.java
new file mode 100644
index 0000000..9876412
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ResourceFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.management.resources;
+
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.ws.rs.core.UriBuilder;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ResourceFactory {
+
+ public static AggregateConfResource createAggregateConfResource(AggregateConf conf,
+ UriBuilder uriBuilder) {
+ return new AggregateConfResource(conf, uriBuilder);
+ }
+
+ public static ConfTreeResource createConfTreeResource(ConfTree confTree,
+ UriBuilder uriBuilder) {
+ return new ConfTreeResource(confTree, uriBuilder);
+ }
+
+ public static ComponentResource createComponentResource(String name,
+ Map<String, String> props,
+ UriBuilder uriBuilder,
+ Map<String, Object> pathElems) {
+ return new ComponentResource(name, props, uriBuilder, pathElems);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
new file mode 100644
index 0000000..0545a34
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.publisher;
+
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.io.IOException;
+
+public class PublisherResource {
+ protected static final Logger log =
+ LoggerFactory.getLogger(PublisherResource.class);
+ private final WebAppApi slider;
+
+ public PublisherResource(WebAppApi slider) {
+ this.slider = slider;
+ }
+
+ private void init(HttpServletResponse res, UriInfo uriInfo) {
+ res.setContentType(null);
+ log.debug(uriInfo.getRequestUri().toString());
+ }
+
+ private PublishedConfigSet getContent() {
+ return slider.getAppState().getPublishedConfigurations();
+ }
+
+ @GET
+ @Path("/")
+ @Produces({MediaType.APPLICATION_JSON})
+ public PublishedConfigSet getPublishedConfiguration(
+ @Context UriInfo uriInfo,
+ @Context HttpServletResponse res) {
+ init(res, uriInfo);
+
+ PublishedConfigSet publishedConfigSet = getContent();
+ log.debug("number of avaiable configurations: {}", publishedConfigSet.size());
+ return publishedConfigSet;
+ }
+
+ private void logRequest(UriInfo uriInfo) {
+ log.debug(uriInfo.getRequestUri().toString());
+ }
+
+ @GET
+ @Path("/{config}")
+ @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
+ public PublishedConfiguration getConfigurationInstance(
+ @PathParam("config") String config,
+ @Context UriInfo uriInfo,
+ @Context HttpServletResponse res) {
+ init(res, uriInfo);
+
+ PublishedConfiguration publishedConfig = getContent().get(config);
+ if (publishedConfig == null) {
+ log.info("Configuration {} not found", config);
+ throw new NotFoundException("Not found: " + uriInfo.getAbsolutePath());
+ }
+ return publishedConfig;
+ }
+
+
+ @GET
+ @Path("/{config}/json")
+ @Produces({MediaType.APPLICATION_JSON})
+ public String getConfigurationContentJson(
+ @PathParam("config") String config,
+ @Context UriInfo uriInfo,
+ @Context HttpServletResponse res) throws IOException {
+
+ // delegate (including init)
+ PublishedConfiguration publishedConfig =
+ getConfigurationInstance(config, uriInfo, res);
+ return publishedConfig.asJson();
+ }
+
+
+ @GET
+ @Path("/{config}/xml")
+ @Produces({MediaType.APPLICATION_XML})
+ public String getConfigurationContentXML(
+ @PathParam("config") String config,
+ @Context UriInfo uriInfo,
+ @Context HttpServletResponse res) throws IOException {
+
+ // delegate (including init)
+ PublishedConfiguration publishedConfig =
+ getConfigurationInstance(config, uriInfo, res);
+ return publishedConfig.asConfigurationXML();
+ }
+
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
new file mode 100644
index 0000000..137c476
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.view;
+
+import com.google.inject.Inject;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class ClusterSpecificationBlock extends HtmlBlock {
+ private static final Logger log = LoggerFactory.getLogger(ClusterSpecificationBlock.class);
+
+ private StateAccessForProviders appState;
+
+ @Inject
+ public ClusterSpecificationBlock(WebAppApi slider) {
+ this.appState = slider.getAppState();
+ }
+
+ @Override
+ protected void render(Block html) {
+ doRender(html);
+ }
+
+ // An extra method to make testing easier since you can't make an instance of Block
+ protected void doRender(Hamlet html) {
+ html.
+ div("cluster_json").
+ h2("JSON Cluster Specification").
+ pre().
+ _(getJson())._()._();
+ }
+
+ /**
+ * Get the JSON, catching any exceptions and returning error text instead
+ * @return
+ */
+ private String getJson() {
+ return appState.getClusterStatus().toString();
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
new file mode 100644
index 0000000..5645e0e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.view;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.inject.Inject;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.client.SliderClusterOperations;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ *
+ */
+public class ContainerStatsBlock extends HtmlBlock {
+ private static final Logger log = LoggerFactory.getLogger(ContainerStatsBlock.class);
+
+ private static final String EVEN = "even", ODD = "odd", BOLD = "bold", SCHEME = "http://", PATH = "/node/container/";
+
+ // Some functions that help transform the data into an object we can use to abstract presentation specifics
+ protected static final Function<Entry<String,Integer>,Entry<TableContent,Integer>> stringIntPairFunc = toTableContentFunction();
+ protected static final Function<Entry<String,String>,Entry<TableContent,String>> stringStringPairFunc = toTableContentFunction();
+
+ private WebAppApi slider;
+ private SliderClusterOperations clusterOps;
+
+ @Inject
+ public ContainerStatsBlock(WebAppApi slider) {
+ this.slider = slider;
+ clusterOps = new SliderClusterOperations(slider.getClusterProtocol());
+ }
+
+ /**
+ * Sort a collection of ClusterNodes by name
+ */
+ protected static class ClusterNodeNameComparator implements Comparator<ClusterNode> {
+
+ @Override
+ public int compare(ClusterNode node1, ClusterNode node2) {
+ if (null == node1 && null != node2) {
+ return -1;
+ } else if (null != node1 && null == node2) {
+ return 1;
+ } else if (null == node1 && null == node2) {
+ return 0;
+ }
+
+ final String name1 = node1.name, name2 = node2.name;
+ if (null == name1 && null != name2) {
+ return -1;
+ } else if (null != name1 && null == name2) {
+ return 1;
+ } else if (null == name1 && null == name2) {
+ return 0;
+ }
+
+ return name1.compareTo(name2);
+ }
+
+ }
+
+ @Override
+ protected void render(Block html) {
+ // TODO Probably better to just get a copy of this list for us to avoid the repeated synchronization?
+ // does this change if we have 50 node, 100node, 500 node clusters?
+ final Map<String,RoleInstance> containerInstances = getContainerInstances(slider.getAppState().cloneActiveContainerList());
+
+ for (Entry<String,RoleStatus> entry : slider.getRoleStatusByName().entrySet()) {
+ final String name = entry.getKey();
+ final RoleStatus roleStatus = entry.getValue();
+
+ DIV<Hamlet> div = html.div("role-info ui-widget-content ui-corner-all");
+
+ List<ClusterNode> nodesInRole;
+ try {
+ nodesInRole = clusterOps.listClusterNodesInRole(name);
+ } catch (Exception e) {
+ log.error("Could not fetch containers for role: " + name, e);
+ nodesInRole = Collections.emptyList();
+ }
+
+ div.h2(BOLD, StringUtils.capitalize(name));
+
+ // Generate the details on this role
+ Iterable<Entry<String,Integer>> stats = roleStatus.buildStatistics().entrySet();
+ generateRoleDetails(div,"role-stats-wrap", "Specifications", Iterables.transform(stats, stringIntPairFunc));
+
+ // Sort the ClusterNodes by their name (containerid)
+ Collections.sort(nodesInRole, new ClusterNodeNameComparator());
+
+ // Generate the containers running this role
+ generateRoleDetails(div, "role-stats-containers", "Containers",
+ Iterables.transform(nodesInRole, new Function<ClusterNode,Entry<TableContent,String>>() {
+
+ @Override
+ public Entry<TableContent,String> apply(ClusterNode input) {
+ final String containerId = input.name;
+
+ if (containerInstances.containsKey(containerId)) {
+ RoleInstance roleInst = containerInstances.get(containerId);
+ if (roleInst.container.getNodeHttpAddress() != null) {
+ return Maps.<TableContent,String> immutableEntry(
+ new TableAnchorContent(containerId, buildNodeUrlForContainer(roleInst.container.getNodeHttpAddress(), containerId)), null);
+ }
+ }
+ return Maps.immutableEntry(new TableContent(input.name), null);
+ }
+
+ }));
+
+ ClusterDescription desc = slider.getAppState().getClusterStatus();
+ Map<String,String> options = desc.getRole(name);
+ Iterable<Entry<TableContent,String>> tableContent;
+
+ // Generate the pairs of data in the expected form
+ if (null != options) {
+ tableContent = Iterables.transform(options.entrySet(), stringStringPairFunc);
+ } else {
+ // Or catch that we have no options and provide "empty"
+ tableContent = Collections.<Entry<TableContent,String>> emptySet();
+ }
+
+ // Generate the options used by this role
+ generateRoleDetails(div, "role-options-wrap", "Role Options", tableContent);
+
+ // Close the div for this role
+ div._();
+ }
+ }
+
+ protected static <T> Function<Entry<String,T>,Entry<TableContent,T>> toTableContentFunction() {
+ return new Function<Entry<String,T>,Entry<TableContent,T>>() {
+ @Override
+ public Entry<TableContent,T> apply(Entry<String,T> input) {
+ return Maps.immutableEntry(new TableContent(input.getKey()), input.getValue());
+ }
+ };
+ }
+
+ protected Map<String,RoleInstance> getContainerInstances(List<RoleInstance> roleInstances) {
+ Map<String,RoleInstance> map = Maps.newHashMapWithExpectedSize(roleInstances.size());
+ for (RoleInstance roleInstance : roleInstances) {
+ // UUID is the containerId
+ map.put(roleInstance.id, roleInstance);
+ }
+ return map;
+ }
+
+ /**
+ * Given a div, a name for this data, and some pairs of data, generate a nice HTML table. If contents is empty (of size zero), then a mesage will be printed
+ * that there were no items instead of an empty table.
+ *
+ * @param div
+ * @param detailsName
+ * @param contents
+ */
+ protected <T1 extends TableContent,T2> void generateRoleDetails(DIV<Hamlet> parent, String divSelector, String detailsName, Iterable<Entry<T1,T2>> contents) {
+ final DIV<DIV<Hamlet>> div = parent.div(divSelector).h3(BOLD, detailsName);
+
+ int offset = 0;
+ TABLE<DIV<DIV<Hamlet>>> table = null;
+ TBODY<TABLE<DIV<DIV<Hamlet>>>> tbody = null;
+ for (Entry<T1,T2> content : contents) {
+ if (null == table) {
+ table = div.table("ui-widget-content ui-corner-bottom");
+ tbody = table.tbody();
+ }
+
+ TR<TBODY<TABLE<DIV<DIV<Hamlet>>>>> row = tbody.tr(offset % 2 == 0 ? EVEN : ODD);
+
+ // Defer to the implementation of the TableContent for what the cell should contain
+ content.getKey().printCell(row);
+
+ // Only add the second column if the element is non-null
+ // This also lets us avoid making a second method if we're only making a one-column table
+ if (null != content.getValue()) {
+ row.td(content.getValue().toString());
+ }
+
+ row._();
+
+ offset++;
+ }
+
+ // If we made a table, close it out
+ if (null != table) {
+ tbody._()._();
+ } else {
+ // Otherwise, throw in a nice "no content" message
+ div.p("no-table-contents")._("None")._();
+ }
+
+ // Close out the initial div
+ div._();
+ }
+
+ /**
+ * Build a URL from the address:port and container ID directly to the NodeManager service
+ * @param nodeAddress
+ * @param containerId
+ * @return
+ */
+ protected String buildNodeUrlForContainer(String nodeAddress, String containerId) {
+ StringBuilder sb = new StringBuilder(SCHEME.length() + nodeAddress.length() + PATH.length() + containerId.length());
+
+ sb.append(SCHEME).append(nodeAddress).append(PATH).append(containerId);
+
+ return sb.toString();
+ }
+
+ /**
+ * Creates a table cell with the provided String as content.
+ */
+ protected static class TableContent {
+ private String cell;
+
+ public TableContent(String cell) {
+ this.cell = cell;
+ }
+
+ public String getCell() {
+ return cell;
+ }
+
+ /**
+ * Adds a td to the given tr. The tr is not closed
+ * @param tableRow
+ */
+ public void printCell(TR<?> tableRow) {
+ tableRow.td(this.cell);
+ }
+ }
+
+ /**
+ * Creates a table cell with an anchor to the given URL with the provided String as content.
+ */
+ protected static class TableAnchorContent extends TableContent {
+ private String anchorUrl;
+
+ public TableAnchorContent(String cell, String anchorUrl) {
+ super(cell);
+ this.anchorUrl = anchorUrl;
+ }
+
+ /* (non-javadoc)
+ * @see org.apache.slider.server.appmaster.web.view.ContainerStatsBlock$TableContent#printCell()
+ */
+ @Override
+ public void printCell(TR<?> tableRow) {
+ tableRow.td().a(anchorUrl, getCell())._();
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
new file mode 100644
index 0000000..75be354
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.view;
+
+import com.google.inject.Inject;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ *
+ */
+public class IndexBlock extends HtmlBlock {
+ private static final String HBASE = "HBase";
+ private static final Logger log = LoggerFactory.getLogger(IndexBlock.class);
+
+ private StateAccessForProviders appState;
+ private ProviderService providerService;
+
+ @Inject
+ public IndexBlock(WebAppApi slider) {
+ this.appState = slider.getAppState();
+ this.providerService = slider.getProviderService();
+ }
+
+ @Override
+ protected void render(Block html) {
+ final String providerName = getProviderName();
+
+ doIndex(html, providerName);
+ }
+
+ // An extra method to make testing easier since you can't make an instance of Block
+ protected void doIndex(Hamlet html, String providerName) {
+ DIV<Hamlet> div = html.div("general_info").h1("index_header", providerName + " cluster: '" + appState.getClusterStatus().name + "'");
+
+ UL<DIV<Hamlet>> ul = div.ul();
+
+ ul.li("Total number of containers for cluster: " + appState.getNumActiveContainers());
+ ul.li("Cluster created: " + getInfoAvoidingNulls(StatusKeys.INFO_CREATE_TIME_HUMAN));
+ ul.li("Cluster last flexed: " + getInfoAvoidingNulls(StatusKeys.INFO_FLEX_TIME_HUMAN));
+ ul.li("Cluster running since: " + getInfoAvoidingNulls(StatusKeys.INFO_LIVE_TIME_HUMAN));
+ ul.li("Cluster HDFS storage path: " + appState.getClusterStatus().dataPath);
+ ul.li("Cluster configuration path: " + appState.getClusterStatus().originConfigurationPath);
+
+ ul._()._();
+
+ html.div("provider_info").h3(providerName + " specific information");
+ ul = div.ul();
+ addProviderServiceOptions(providerService, ul);
+ ul._()._();
+ }
+
+ private String getProviderName() {
+ String providerServiceName = providerService.getName().toLowerCase();
+
+ // Get HBase properly capitalized
+ if (providerServiceName.contains("hbase")) {
+ return HBASE;
+ }
+
+ return StringUtils.capitalize(providerServiceName);
+ }
+
+ private String getInfoAvoidingNulls(String key) {
+ String createTime = appState.getClusterStatus().getInfo(key);
+
+ return null == createTime ? "N/A" : createTime;
+ }
+
+ protected void addProviderServiceOptions(ProviderService providerService, UL<DIV<Hamlet>> ul) {
+ Map<String,URL> details = providerService.buildMonitorDetails(appState.getClusterStatus());
+ if (null == details) {
+ return;
+ }
+
+ // Loop over each entry, placing the text in the UL, adding an anchor when the URL is non-null
+ for (Entry<String,URL> entry : details.entrySet()) {
+ if (null != entry.getValue()) {
+ String url = entry.getValue().toString();
+ ul.li()._(entry.getKey()).a(url, url)._();
+ } else {
+ ul.li(entry.getKey());
+ }
+ }
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
new file mode 100644
index 0000000..d367d18
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.view;
+
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.slider.server.appmaster.web.SliderAMWebApp;
+
+/**
+ *
+ */
+public class NavBlock extends HtmlBlock {
+
+ @Override
+ protected void render(Block html) {
+ html.
+ div("#nav").
+ h3("Slider").
+ ul().
+ li().a(this.prefix(), "Overview")._().
+ li().a(this.prefix() + SliderAMWebApp.CONTAINER_STATS, "Statistics")._().
+ li().a(this.prefix() + SliderAMWebApp.CLUSTER_SPEC, "Specification")._()._()._();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
new file mode 100644
index 0000000..74ab927
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.avro;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.avro.AvroTypeException;
+import org.apache.avro.Schema;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.Encoder;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Locale;
+
+/**
+ * Write out the role history to an output stream
+ */
+public class RoleHistoryWriter {
+ protected static final Logger log =
+ LoggerFactory.getLogger(RoleHistoryWriter.class);
+
+ /**
+ * Although Avro is designed to handle some changes, we still keep a version
+ * marker in the file to catch changes that are fundamentally incompatible
+ * at the semantic level -changes that require either a different
+ * parser or get rejected outright.
+ */
+ public static final int ROLE_HISTORY_VERSION = 0x01;
+
+ /**
+ * Write out the history.
+ * This does not update the history's dirty/savetime fields
+ *
+ * @param out outstream
+ * @param history history
+ * @param savetime time in millis for the save time to go in as a record
+ * @return no of records written
+ * @throws IOException IO failures
+ */
+ public long write(OutputStream out, RoleHistory history, long savetime)
+ throws IOException {
+ try {
+ DatumWriter<RoleHistoryRecord> writer =
+ new SpecificDatumWriter<RoleHistoryRecord>(RoleHistoryRecord.class);
+
+ int roles = history.getRoleSize();
+ RoleHistoryHeader header = new RoleHistoryHeader();
+ header.setVersion(ROLE_HISTORY_VERSION);
+ header.setSaved(savetime);
+ header.setSavedx(Long.toHexString(savetime));
+ header.setSavedate(SliderUtils.toGMTString(savetime));
+ header.setRoles(roles);
+ RoleHistoryRecord record = new RoleHistoryRecord(header);
+ Schema schema = record.getSchema();
+ Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
+ writer.write(record, encoder);
+ long count = 0;
+ //now for every role history entry, write out its record
+ Collection<NodeInstance> instances = history.cloneNodemap().values();
+ for (NodeInstance instance : instances) {
+ for (int role = 0; role < roles; role++) {
+ NodeEntry nodeEntry = instance.get(role);
+
+ if (nodeEntry != null) {
+ NodeEntryRecord ner = build(nodeEntry, role, instance.hostname);
+ record = new RoleHistoryRecord(ner);
+ writer.write(record, encoder);
+ count++;
+ }
+ }
+ }
+ // footer
+ RoleHistoryFooter footer = new RoleHistoryFooter();
+ footer.setCount(count);
+ writer.write(new RoleHistoryRecord(footer), encoder);
+ encoder.flush();
+ out.close();
+ return count;
+ } finally {
+ out.close();
+ }
+ }
+
+ /**
+ * Write write the file
+ *
+ *
+ * @param fs filesystem
+ * @param path path
+ * @param overwrite overwrite flag
+ * @param history history
+ * @param savetime time in millis for the save time to go in as a record
+ * @return no of records written
+ * @throws IOException IO failures
+ */
+ public long write(FileSystem fs, Path path, boolean overwrite,
+ RoleHistory history, long savetime) throws IOException {
+ FSDataOutputStream out = fs.create(path, overwrite);
+ return write(out, history, savetime);
+ }
+
+
+ /**
+ * Create the filename for a history file
+ * @param time time value
+ * @return a filename such that later filenames sort later in the directory
+ */
+ public Path createHistoryFilename(Path historyPath, long time) {
+ String filename = String.format(Locale.ENGLISH,
+ SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,
+ time);
+ Path path = new Path(historyPath, filename);
+ return path;
+ }
+
+ private NodeEntryRecord build(NodeEntry entry, int role, String hostname) {
+ NodeEntryRecord record = new NodeEntryRecord(
+ hostname, role, entry.getLive() > 0, entry.getLastUsed()
+ );
+ return record;
+ }
+
+ /**
+ * Read a history, returning one that is ready to have its onThaw()
+ * method called
+ * @param in input source
+ * @param history a history set up with the expected roles;
+ * this will be built up with a node map configured with the node instances
+ * and entries loaded from the source
+ * @return no. of entries read
+ * @throws IOException problems
+ */
+ public int read(InputStream in, RoleHistory history) throws
+ IOException,
+ BadConfigException {
+ try {
+ DatumReader<RoleHistoryRecord> reader =
+ new SpecificDatumReader<RoleHistoryRecord>(RoleHistoryRecord.class);
+ Decoder decoder =
+ DecoderFactory.get().jsonDecoder(RoleHistoryRecord.getClassSchema(),
+ in);
+
+ //read header : no entry -> EOF
+ RoleHistoryRecord record = reader.read(null, decoder);
+ Object entry = record.getEntry();
+ if (!(entry instanceof RoleHistoryHeader)) {
+ throw new IOException("Role History Header not found at start of file");
+ }
+ RoleHistoryHeader header = (RoleHistoryHeader) entry;
+ Long saved = header.getSaved();
+ if (header.getVersion() != ROLE_HISTORY_VERSION) {
+ throw new IOException(
+ String.format("Can't read role file version %04x -need %04x",
+ header.getVersion(),
+ ROLE_HISTORY_VERSION));
+ }
+ history.prepareForReading(header);
+ RoleHistoryFooter footer = null;
+ int records = 0;
+ //go through reading data
+ try {
+ while (true) {
+ record = reader.read(null, decoder);
+ entry = record.getEntry();
+
+ if (entry instanceof RoleHistoryHeader) {
+ throw new IOException("Duplicate Role History Header found");
+ }
+ if (entry instanceof RoleHistoryFooter) {
+ //tail end of the file
+ footer = (RoleHistoryFooter) entry;
+ break;
+ }
+ records++;
+ NodeEntryRecord nodeEntryRecord = (NodeEntryRecord) entry;
+ Integer roleId = nodeEntryRecord.getRole();
+ NodeEntry nodeEntry = new NodeEntry(roleId);
+ nodeEntry.setLastUsed(nodeEntryRecord.getLastUsed());
+ if (nodeEntryRecord.getActive()) {
+ //if active at the time of save, make the last used time the save time
+ nodeEntry.setLastUsed(saved);
+ }
+
+ String hostname =
+ SliderUtils.sequenceToString(nodeEntryRecord.getHost());
+ NodeInstance instance = history.getOrCreateNodeInstance(hostname);
+ instance.set(roleId, nodeEntry);
+ }
+ } catch (EOFException e) {
+ EOFException ex = new EOFException(
+ "End of file reached after " + records + " records");
+ ex.initCause(e);
+ throw ex;
+ }
+ //at this point there should be no data left.
+ if (in.read() > 0) {
+ // footer is in stream before the last record
+ throw new EOFException(
+ "File footer reached before end of file -after " + records +
+ " records");
+ }
+ if (records != footer.getCount()) {
+ log.warn("mismatch between no of records saved {} and number read {}",
+ footer.getCount(), records);
+ }
+ return records;
+ } finally {
+ in.close();
+ }
+
+ }
+
+ /**
+ * Read a role history from a path in a filesystem
+ * @param fs filesystem
+ * @param path path to the file
+ * @param roleHistory history to build
+ * @return the number of records read
+ * @throws IOException any problem
+ */
+ public int read(FileSystem fs, Path path, RoleHistory roleHistory) throws
+ IOException,
+ BadConfigException {
+ FSDataInputStream instream = fs.open(path);
+ return read(instream, roleHistory);
+ }
+
+ /**
+ * Read a role history from local file
+ * @param file path to the file
+ * @param roleHistory history to build
+ * @return the number of records read
+ * @throws IOException any problem
+ */
+ public int read(File file, RoleHistory roleHistory) throws
+ IOException,
+ BadConfigException {
+
+
+ return read(new FileInputStream(file), roleHistory);
+ }
+
+ /**
+ * Read from a resource in the classpath -used for testing
+ * @param resource resource
+ * @param roleHistory history to build
+ * @return the number of records read
+ * @throws IOException any problem
+ */
+ public int read(String resource, RoleHistory roleHistory) throws
+ IOException,
+ BadConfigException {
+
+ return read(this.getClass().getClassLoader().getResourceAsStream(resource),
+ roleHistory);
+ }
+
+
+ /**
+ * Find all history entries in a dir. The dir is created if it is
+ * not already defined.
+ *
+ * The scan uses the match pattern {@link SliderKeys#HISTORY_FILENAME_MATCH_PATTERN}
+ * while dropping empty files and directories which match the pattern.
+ * The list is then sorted with a comparator that sorts on filename,
+ * relying on the filename of newer created files being later than the old ones.
+ *
+ *
+ *
+ * @param fs filesystem
+ * @param dir dir to scan
+ * @param includeEmptyFiles should empty files be included in the result?
+ * @return a possibly empty list
+ * @throws IOException IO problems
+ * @throws FileNotFoundException if the target dir is actually a path
+ */
+ public List<Path> findAllHistoryEntries(FileSystem fs,
+ Path dir,
+ boolean includeEmptyFiles) throws IOException {
+ assert fs != null;
+ assert dir != null;
+ if (!fs.exists(dir)) {
+ fs.mkdirs(dir);
+ } else if (!fs.isDirectory(dir)) {
+ throw new FileNotFoundException("Not a directory " + dir.toString());
+ }
+
+ PathFilter filter = new GlobFilter(SliderKeys.HISTORY_FILENAME_GLOB_PATTERN);
+ FileStatus[] stats = fs.listStatus(dir, filter);
+ List<Path> paths = new ArrayList<Path>(stats.length);
+ for (FileStatus stat : stats) {
+ log.debug("Possible entry: {}", stat.toString());
+ if (stat.isFile() && (includeEmptyFiles || stat.getLen() > 0)) {
+ paths.add(stat.getPath());
+ }
+ }
+ sortHistoryPaths(paths);
+ return paths;
+ }
+
+ @VisibleForTesting
+ public static void sortHistoryPaths(List<Path> paths) {
+ Collections.sort(paths, new NewerFilesFirst());
+ }
+
+
+ /**
+ * Iterate through the paths until one can be loaded
+ * @param roleHistory role history
+ * @param paths paths to load
+ * @return the path of any loaded history -or null if all failed to load
+ */
+ public Path attemptToReadHistory(RoleHistory roleHistory, FileSystem fileSystem, List<Path> paths) throws
+ BadConfigException {
+ ListIterator<Path> pathIterator = paths.listIterator();
+ boolean success = false;
+ Path path = null;
+ while (!success && pathIterator.hasNext()) {
+ path = pathIterator.next();
+ try {
+ read(fileSystem, path, roleHistory);
+ //success
+ success = true;
+ } catch (IOException e) {
+ log.info("Failed to read {}", path, e);
+ } catch (AvroTypeException e) {
+ log.warn("Failed to parse {}", path, e);
+ }
+ }
+ return success ? path : null;
+ }
+
+ /**
+ * Try to load the history from a directory -a failure to load a specific
+ * file is downgraded to a log and the next older path attempted instead
+ * @param fs filesystem
+ * @param dir dir to load from
+ * @param roleHistory role history to build up
+ * @return the path loaded
+ * @throws IOException if indexing the history directory fails.
+ */
+ public Path loadFromHistoryDir(FileSystem fs, Path dir,
+ RoleHistory roleHistory) throws
+ IOException,
+ BadConfigException {
+ assert fs != null: "null filesystem";
+ List<Path> entries = findAllHistoryEntries(fs, dir, false);
+ return attemptToReadHistory(roleHistory, fs, entries);
+ }
+
+ /**
+ * Delete all old history entries older than the one we want to keep. This
+ * uses the filename ordering to determine age, not timestamps
+ * @param fileSystem filesystem
+ * @param keep path to keep -used in thresholding the files
+ * @return the number of files deleted
+ * @throws FileNotFoundException if the path to keep is not present (safety
+ * check to stop the entire dir being purged)
+ * @throws IOException IO problems
+ */
+ public int purgeOlderHistoryEntries(FileSystem fileSystem, Path keep) throws
+ IOException {
+ assert fileSystem != null : "null filesystem";
+ if (!fileSystem.exists(keep)) {
+ throw new FileNotFoundException(keep.toString());
+ }
+ Path dir = keep.getParent();
+ log.debug("Purging entries in {} up to {}", dir, keep);
+ List<Path> paths = findAllHistoryEntries(fileSystem, dir, true);
+ Collections.sort(paths, new OlderFilesFirst());
+ int deleteCount = 0;
+ for (Path path : paths) {
+ if (path.equals(keep)) {
+ break;
+ } else {
+ log.debug("Deleting {}", path);
+ deleteCount++;
+ fileSystem.delete(path, false);
+ }
+ }
+ return deleteCount;
+ }
+
+ /**
+ * Compare two filenames by name; the more recent one comes first
+ */
+ public static class NewerFilesFirst implements Comparator<Path> ,
+ Serializable {
+
+ /**
+ * Takes the ordering of path names from the normal string comparison
+ * and negates it, so that names that come after other names in
+ * the string sort come before here
+ * @param o1 leftmost
+ * @param o2 rightmost
+ * @return positive if o1 > o2
+ */
+ @Override
+ public int compare(Path o1, Path o2) {
+ return (o2.getName().compareTo(o1.getName()));
+ }
+ }
+ /**
+ * Compare two filenames by name; the older ones comes first
+ */
+ public static class OlderFilesFirst implements Comparator<Path> ,
+ Serializable {
+
+ /**
+ * Takes the ordering of path names from the normal string comparison
+ * and negates it, so that names that come after other names in
+ * the string sort come before here
+ * @param o1 leftmost
+ * @param o2 rightmost
+ * @return positive if o1 > o2
+ */
+ @Override
+ public int compare(Path o1, Path o2) {
+ return (o1.getName().compareTo(o2.getName()));
+ }
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/exec/ApplicationEventHandler.java b/slider-core/src/main/java/org/apache/slider/server/exec/ApplicationEventHandler.java
new file mode 100644
index 0000000..1fc0bf1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/exec/ApplicationEventHandler.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.exec;
+
+/**
+ * Callback when a long-lived application exits
+ */
+public interface ApplicationEventHandler {
+
+ void onApplicationStarted(RunLongLivedApp application);
+
+ void onApplicationExited(RunLongLivedApp application, int exitCode);
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/exec/RunLongLivedApp.java b/slider-core/src/main/java/org/apache/slider/server/exec/RunLongLivedApp.java
new file mode 100644
index 0000000..b251e65
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/exec/RunLongLivedApp.java
@@ -0,0 +1,439 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.exec;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.SliderInternalStateException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Execute an application.
+ *
+ * Hadoop's Shell class isn't used because it assumes it is executing
+ * a short lived application:
+ */
+public class RunLongLivedApp implements Runnable {
+ public static final int STREAM_READER_SLEEP_TIME = 200;
+ public static final int RECENT_LINE_LOG_LIMIT = 64;
+ /**
+ * Class log
+ */
+ static final Logger LOG = LoggerFactory.getLogger(RunLongLivedApp.class);
+ /**
+ * Log supplied in the constructor for the spawned process
+ */
+ final Logger processLog;
+ private final ProcessBuilder builder;
+ private Process process;
+ private Exception exception;
+ private Integer exitCode = null;
+ volatile boolean done;
+ private Thread execThread;
+ private Thread logThread;
+ private ProcessStreamReader processStreamReader;
+ //list of recent lines, recorded for extraction into reports
+ private final List<String> recentLines = new LinkedList<String>();
+ private final int recentLineLimit = RECENT_LINE_LOG_LIMIT;
+
+ private ApplicationEventHandler applicationEventHandler;
+
+ public RunLongLivedApp(Logger processLog, String... commands) {
+ this.processLog = processLog;
+ builder = new ProcessBuilder(commands);
+ initBuilder();
+ }
+
+ public RunLongLivedApp(Logger processLog, List<String> commands) {
+ this.processLog = processLog;
+ builder = new ProcessBuilder(commands);
+ initBuilder();
+ }
+
+ private void initBuilder() {
+ builder.redirectErrorStream(false);
+ }
+
+ public ProcessBuilder getBuilder() {
+ return builder;
+ }
+
+ /**
+ * Set an optional application exit callback
+ * @param applicationEventHandler callback to notify on application exit
+ */
+ public void setApplicationEventHandler(ApplicationEventHandler applicationEventHandler) {
+ this.applicationEventHandler = applicationEventHandler;
+ }
+
+ /**
+ * Add an entry to the environment
+ * @param key key -must not be null
+ * @param val value
+ */
+ public void putEnv(String key, String val) {
+ if (val == null) {
+ throw new RuntimeException("Null value for key " + key);
+ }
+ builder.environment().put(key, val);
+ }
+
+ /**
+ * Bulk set the environment from a map. This does
+ * not replace the existing environment, just extend it/overwrite single
+ * entries.
+ * @param map map to add
+ */
+ public void putEnvMap(Map<String, String> map) {
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ String val = entry.getValue();
+ String key = entry.getKey();
+ putEnv(key, val);
+ }
+ }
+
+ /**
+ * Get the process environment
+ * @param key
+ * @return
+ */
+ public String getEnv(String key) {
+ return builder.environment().get(key);
+ }
+
+ /**
+ * Get the process reference
+ * @return the process -null if the process is not started
+ */
+ public Process getProcess() {
+ return process;
+ }
+
+ /**
+ * Get any exception raised by the process
+ * @return an exception or null
+ */
+ public Exception getException() {
+ return exception;
+ }
+
+ public List<String> getCommands() {
+ return builder.command();
+ }
+
+ public String getCommand() {
+ return getCommands().get(0);
+ }
+
+ /**
+ * probe to see if the process is running
+ * @return true iff the process has been started and is not yet finished
+ */
+ public boolean isRunning() {
+ return process != null && !done;
+ }
+
+ /**
+ * Get the exit code: null until the process has finished
+ * @return the exit code or null
+ */
+ public Integer getExitCode() {
+ return exitCode;
+ }
+
+ /**
+ * Stop the process if it is running.
+ * This will trigger an application completion event with the given exit code
+ */
+ public void stop() {
+ if (!isRunning()) {
+ return;
+ }
+ process.destroy();
+ }
+
+ /**
+ * Get a text description of the builder suitable for log output
+ * @return a multiline string
+ */
+ protected String describeBuilder() {
+ StringBuilder buffer = new StringBuilder();
+ for (String arg : builder.command()) {
+ buffer.append('"').append(arg).append("\" ");
+ }
+ return buffer.toString();
+ }
+
+ private void dumpEnv(StringBuilder buffer) {
+ buffer.append("\nEnvironment\n-----------");
+ Map<String, String> env = builder.environment();
+ Set<String> keys = env.keySet();
+ List<String> sortedKeys = new ArrayList<String>(keys);
+ Collections.sort(sortedKeys);
+ for (String key : sortedKeys) {
+ buffer.append(key).append("=").append(env.get(key)).append('\n');
+ }
+ }
+
+ /**
+ * Exec the process
+ * @return the process
+ * @throws IOException
+ */
+ private Process spawnChildProcess() throws IOException, SliderException {
+ if (process != null) {
+ throw new SliderInternalStateException("Process already started");
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Spawning process:\n " + describeBuilder());
+ }
+ process = builder.start();
+ return process;
+ }
+
+ /**
+ * Entry point for waiting for the program to finish
+ */
+ @Override // Runnable
+ public void run() {
+ LOG.debug("Application callback thread running");
+ //notify the callback that the process has started
+ if (applicationEventHandler != null) {
+ applicationEventHandler.onApplicationStarted(this);
+ }
+ try {
+ exitCode = process.waitFor();
+ } catch (InterruptedException e) {
+ LOG.debug("Process wait interrupted -exiting thread");
+ } finally {
+ //here the process has finished
+ LOG.info("process has finished");
+ //tell the logger it has to finish too
+ done = true;
+
+ //now call the callback if it is set
+ if (applicationEventHandler != null) {
+ applicationEventHandler.onApplicationExited(this, exitCode);
+ }
+ try {
+ logThread.join();
+ } catch (InterruptedException ignored) {
+ //ignored
+ }
+ }
+ }
+
+ /**
+ * Create a thread to wait for this command to complete.
+ * THE THREAD IS NOT STARTED.
+ * @return the thread
+ * @throws IOException Execution problems
+ */
+ private Thread spawnIntoThread() throws IOException, SliderException {
+ spawnChildProcess();
+ return new Thread(this, getCommand());
+ }
+
+ /**
+ * Spawn the application
+ * @throws IOException IO problems
+ * @throws SliderException internal state of this class is wrong
+ */
+ public void spawnApplication() throws IOException, SliderException {
+ execThread = spawnIntoThread();
+ execThread.start();
+ processStreamReader =
+ new ProcessStreamReader(processLog, STREAM_READER_SLEEP_TIME);
+ logThread = new Thread(processStreamReader, "IO");
+ logThread.start();
+ }
+
+ /**
+ * Get the lines of recent output
+ * @return the last few lines of output; an empty list if there are none
+ * or the process is not actually running
+ */
+ public synchronized List<String> getRecentOutput() {
+ return new ArrayList<String>(recentLines);
+ }
+
+
+ /**
+ * add the recent line to the list of recent lines; deleting
+ * an earlier on if the limit is reached.
+ *
+ * Implementation note: yes, a circular array would be more
+ * efficient, especially with some power of two as the modulo,
+ * but is it worth the complexity and risk of errors for
+ * something that is only called once per line of IO?
+ * @param line line to record
+ * @param isErrorStream is the line from the error stream
+ */
+ private synchronized void recordRecentLine(String line,
+ boolean isErrorStream) {
+ if (line == null) {
+ return;
+ }
+ String entry = (isErrorStream ? "[ERR] " : "[OUT] ") + line;
+ recentLines.add(entry);
+ if (recentLines.size() > recentLineLimit) {
+ recentLines.remove(0);
+ }
+ }
+
+ /**
+ * Class to read data from the two process streams, and, when run in a thread
+ * to keep running until the <code>done</code> flag is set.
+ * Lines are fetched from stdout and stderr and logged at info and error
+ * respectively.
+ */
+
+ private class ProcessStreamReader implements Runnable {
+ private final Logger streamLog;
+ private final int sleepTime;
+
+ private ProcessStreamReader(Logger streamLog, int sleepTime) {
+ this.streamLog = streamLog;
+ this.sleepTime = sleepTime;
+ }
+
+ private int readCharNonBlocking(BufferedReader reader) throws IOException {
+ if (reader.ready()) {
+ return reader.read();
+ } else {
+ return -1;
+ }
+ }
+
+ /**
+ * Read in a line, or, if the limit has been reached, the buffer
+ * so far
+ * @param reader source of data
+ * @param line line to build
+ * @param limit limit of line length
+ * @return true if the line can be printed
+ * @throws IOException IO trouble
+ */
+ private boolean readAnyLine(BufferedReader reader,
+ StringBuilder line,
+ int limit)
+ throws IOException {
+ int next;
+ while ((-1 != (next = readCharNonBlocking(reader)))) {
+ if (next != '\n') {
+ line.append((char) next);
+ limit--;
+ if (line.length() > limit) {
+ //enough has been read in to print it any
+ return true;
+ }
+ } else {
+ //line end return flag to say so
+ return true;
+ }
+ }
+ //here the end of the stream is hit, or the limit
+ return false;
+ }
+
+
+ @Override //Runnable
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+ public void run() {
+ BufferedReader errReader = null;
+ BufferedReader outReader = null;
+ StringBuilder outLine = new StringBuilder(256);
+ StringBuilder errorLine = new StringBuilder(256);
+ try {
+ errReader = new BufferedReader(new InputStreamReader(process
+ .getErrorStream()));
+ outReader = new BufferedReader(new InputStreamReader(process
+ .getInputStream()));
+ while (!done) {
+ boolean processed = false;
+ if (readAnyLine(errReader, errorLine, 256)) {
+ String line = errorLine.toString();
+ recordRecentLine(line, true);
+ streamLog.warn(line);
+ errorLine.setLength(0);
+ processed = true;
+ }
+ if (readAnyLine(outReader, outLine, 256)) {
+ String line = outLine.toString();
+ recordRecentLine(line, false);
+ streamLog.info(line);
+ outLine.setLength(0);
+ processed |= true;
+ }
+ if (!processed) {
+ //nothing processed: wait a bit for data.
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ //ignore this, rely on the done flag
+ LOG.debug("Ignoring ", e);
+ }
+ }
+ }
+ //get here, done time
+
+ //print the current error line then stream through the rest
+ streamLog.error(errorLine.toString());
+ String line = errReader.readLine();
+ while (line != null) {
+ streamLog.error(line);
+ if (Thread.interrupted()) {
+ break;
+ }
+ line = errReader.readLine();
+ recordRecentLine(line, true);
+ }
+ //now do the info line
+ streamLog.info(outLine.toString());
+ line = outReader.readLine();
+ while (line != null) {
+ streamLog.info(line);
+ if (Thread.interrupted()) {
+ break;
+ }
+ line = outReader.readLine();
+ recordRecentLine(line, false);
+ }
+
+ } catch (Exception ignored) {
+ LOG.warn("encountered ", ignored);
+ //process connection has been torn down
+ } finally {
+ IOUtils.closeStream(errReader);
+ IOUtils.closeStream(outReader);
+ }
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
new file mode 100644
index 0000000..9c14ca7
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+public class HttpProbe extends Probe {
+ protected static final Logger log = LoggerFactory.getLogger(HttpProbe.class);
+
+ private final URL url;
+ private final int timeout;
+ private final int min, max;
+
+
+ public HttpProbe(URL url, int timeout, int min, int max, Configuration conf) throws IOException {
+ super("Http probe of " + url + " [" + min + "-" + max + "]", conf);
+ this.url = url;
+ this.timeout = timeout;
+ this.min = min;
+ this.max = max;
+ }
+
+ public static HttpURLConnection getConnection(URL url, int timeout) throws IOException {
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setInstanceFollowRedirects(true);
+ connection.setConnectTimeout(timeout);
+ return connection;
+ }
+
+ @Override
+ public ProbeStatus ping(boolean livePing) {
+ ProbeStatus status = new ProbeStatus();
+ HttpURLConnection connection = null;
+ try {
+ if (log.isDebugEnabled()) {
+ // LOG.debug("Fetching " + url + " with timeout " + timeout);
+ }
+ connection = getConnection(url, this.timeout);
+ int rc = connection.getResponseCode();
+ if (rc < min || rc > max) {
+ String error = "Probe " + url + " error code: " + rc;
+ log.info(error);
+ status.fail(this,
+ new IOException(error));
+ } else {
+ status.succeed(this);
+ }
+ } catch (IOException e) {
+ String error = "Probe " + url + " failed: " + e;
+ log.info(error, e);
+ status.fail(this,
+ new IOException(error, e));
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
+ return status;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
new file mode 100644
index 0000000..a1ad44f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+/**
+ * Build up log entries for ease of splunk
+ */
+public class LogEntryBuilder {
+
+ private final StringBuilder builder = new StringBuilder();
+
+ public LogEntryBuilder() {
+ }
+
+ public LogEntryBuilder(String text) {
+ elt(text);
+ }
+
+
+ public LogEntryBuilder(String name, Object value) {
+ entry(name, value);
+ }
+
+ public LogEntryBuilder elt(String text) {
+ addComma();
+ builder.append(text);
+ return this;
+ }
+
+ public LogEntryBuilder elt(String name, Object value) {
+ addComma();
+ entry(name, value);
+ return this;
+ }
+
+ private void addComma() {
+ if (!isEmpty()) {
+ builder.append(", ");
+ }
+ }
+
+ private void entry(String name, Object value) {
+ builder.append(name).append('=');
+ if (value != null) {
+ builder.append('"').append(value.toString()).append('"');
+ } else {
+ builder.append("null");
+ }
+ }
+
+ @Override
+ public String toString() {
+ return builder.toString();
+ }
+
+ private boolean isEmpty() {
+ return builder.length() == 0;
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorKeys.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorKeys.java
new file mode 100644
index 0000000..f7bdd4a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorKeys.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+/**
+ * Config keys for monitoring
+ */
+public interface MonitorKeys {
+
+ /**
+ * Prefix of all other configuration options: {@value}
+ */
+ String MONITOR_KEY_PREFIX = "service.monitor.";
+
+
+ /**
+ * Classname of the reporter Key: {@value}
+ */
+ String MONITOR_REPORTER =
+ MONITOR_KEY_PREFIX + "report.classname";
+
+ /**
+ * Interval in milliseconds between reporting health status to the reporter
+ * Key: {@value}
+ */
+ String MONITOR_REPORT_INTERVAL =
+ MONITOR_KEY_PREFIX + "report.interval";
+
+ /**
+ * Time in millis between the last probing cycle ending and the new one
+ * beginning. Key: {@value}
+ */
+ String MONITOR_PROBE_INTERVAL =
+ MONITOR_KEY_PREFIX + "probe.interval";
+
+ /**
+ * How long in milliseconds does the probing loop have to be blocked before
+ * that is considered a liveness failure Key: {@value}
+ */
+ String MONITOR_PROBE_TIMEOUT =
+ MONITOR_KEY_PREFIX + "probe.timeout";
+
+ /**
+ * How long in milliseconds does the probing loop have to be blocked before
+ * that is considered a liveness failure Key: {@value}
+ */
+ String MONITOR_BOOTSTRAP_TIMEOUT =
+ MONITOR_KEY_PREFIX + "bootstrap.timeout";
+
+
+ /**
+ * does the monitor depend on DFS being live
+ */
+ String MONITOR_DEPENDENCY_DFSLIVE =
+ MONITOR_KEY_PREFIX + "dependency.dfslive";
+
+
+ /**
+ * default timeout for the entire bootstrap phase {@value}
+ */
+
+ int BOOTSTRAP_TIMEOUT_DEFAULT = 60000;
+
+
+ /**
+ * Default value if the key is not in the config file: {@value}
+ */
+ int REPORT_INTERVAL_DEFAULT = 10000;
+ /**
+ * Default value if the key is not in the config file: {@value}
+ */
+ int PROBE_INTERVAL_DEFAULT = 10000;
+ /**
+ * Default value if the key is not in the config file: {@value}
+ */
+ int PROBE_TIMEOUT_DEFAULT = 60000;
+
+ /**
+ * Port probe enabled/disabled flag Key: {@value}
+ */
+ String PORT_PROBE_ENABLED =
+ MONITOR_KEY_PREFIX + "portprobe.enabled";
+
+
+ /**
+ * Port probing key : port to attempt to create a TCP connection to {@value}
+ */
+ String PORT_PROBE_PORT =
+ MONITOR_KEY_PREFIX + "portprobe.port";
+
+ /**
+ * Port probing key : port to attempt to create a TCP connection to {@value}
+ */
+ String PORT_PROBE_HOST =
+ MONITOR_KEY_PREFIX + "portprobe.host";
+
+
+ /**
+ * Port probing key : timeout of the connection attempt {@value}
+ */
+ String PORT_PROBE_CONNECT_TIMEOUT =
+ MONITOR_KEY_PREFIX + "portprobe.connect.timeout";
+
+ /**
+ * Port probing key : bootstrap timeout -how long in milliseconds should the
+ * port probing take to connect before the failure to connect is considered a
+ * liveness failure. That is: how long should the IPC port take to come up?
+ * {@value}
+ */
+ String PORT_PROBE_BOOTSTRAP_TIMEOUT =
+ MONITOR_KEY_PREFIX + "portprobe.bootstrap.timeout";
+
+
+ /**
+ * default timeout for port probes {@value}
+ */
+ int PORT_PROBE_BOOTSTRAP_TIMEOUT_DEFAULT = 60000;
+
+ /**
+ * default value for port probe connection attempts {@value}
+ */
+
+ int PORT_PROBE_CONNECT_TIMEOUT_DEFAULT = 1000;
+
+
+ /**
+ * default port for probes {@value}
+ */
+ int DEFAULT_PROBE_PORT = 8020;
+
+
+ /**
+ * default host for probes {@value}
+ */
+ String DEFAULT_PROBE_HOST = "localhost";
+
+
+ /**
+ * Probe enabled/disabled flag Key: {@value}
+ */
+ String LS_PROBE_ENABLED =
+ MONITOR_KEY_PREFIX + "lsprobe.enabled";
+
+ /**
+ * Probe path for LS operation Key: {@value}
+ */
+ String LS_PROBE_PATH =
+ MONITOR_KEY_PREFIX + "lsprobe.path";
+
+ /**
+ * Default path for LS operation Key: {@value}
+ */
+ String LS_PROBE_DEFAULT = "/";
+
+ /**
+ * Port probing key : bootstrap timeout -how long in milliseconds should the
+ * port probing take to connect before the failure to connect is considered a
+ * liveness failure. That is: how long should the IPC port take to come up?
+ * {@value}
+ */
+ String LS_PROBE_BOOTSTRAP_TIMEOUT =
+ MONITOR_KEY_PREFIX + "lsprobe.bootstrap.timeout";
+
+
+ /**
+ * default timeout for port probes {@value}
+ */
+
+ int LS_PROBE_BOOTSTRAP_TIMEOUT_DEFAULT = PORT_PROBE_BOOTSTRAP_TIMEOUT_DEFAULT;
+
+
+ /**
+ * Probe enabled/disabled flag Key: {@value}
+ */
+ String WEB_PROBE_ENABLED =
+ MONITOR_KEY_PREFIX + "webprobe.enabled";
+
+ /**
+ * Probe URL Key: {@value}
+ */
+ String WEB_PROBE_URL =
+ MONITOR_KEY_PREFIX + "webprobe.url";
+
+ /**
+ * Default path for web probe Key: {@value}
+ */
+ String WEB_PROBE_DEFAULT_URL = "http://localhost:50070/";
+
+ /**
+ * min error code Key: {@value}
+ */
+ String WEB_PROBE_MIN =
+ MONITOR_KEY_PREFIX + "webprobe.min";
+ /**
+ * min error code Key: {@value}
+ */
+ String WEB_PROBE_MAX =
+ MONITOR_KEY_PREFIX + "webprobe.max";
+
+
+ /**
+ * Port probing key : timeout of the connection attempt {@value}
+ */
+ String WEB_PROBE_CONNECT_TIMEOUT =
+ MONITOR_KEY_PREFIX + "webprobe.connect.timeout";
+
+ /**
+ * Default HTTP response code expected from the far end for
+ * the endpoint to be considered live.
+ */
+ int WEB_PROBE_DEFAULT_CODE = 200;
+
+ /**
+ * Port probing key : bootstrap timeout -how long in milliseconds should the
+ * port probing take to connect before the failure to connect is considered a
+ * liveness failure. That is: how long should the IPC port take to come up?
+ * {@value}
+ */
+ String WEB_PROBE_BOOTSTRAP_TIMEOUT =
+ MONITOR_KEY_PREFIX + "webprobe.bootstrap.timeout";
+
+
+ /**
+ * default timeout for port probes {@value}
+ */
+
+ int WEB_PROBE_BOOTSTRAP_TIMEOUT_DEFAULT = PORT_PROBE_BOOTSTRAP_TIMEOUT_DEFAULT;
+
+ /**
+ * Probe enabled/disabled flag Key: {@value}
+ */
+ String JT_PROBE_ENABLED =
+ MONITOR_KEY_PREFIX + "jtprobe.enabled";
+
+ /**
+ * Port probing key : bootstrap timeout -how long in milliseconds should the
+ * port probing take to connect before the failure to connect is considered a
+ * liveness failure. That is: how long should the IPC port take to come up?
+ * {@value}
+ */
+ String JT_PROBE_BOOTSTRAP_TIMEOUT =
+ MONITOR_KEY_PREFIX + "jtprobe.bootstrap.timeout";
+
+
+ /**
+ * default timeout for port probes {@value}
+ */
+
+ int JT_PROBE_BOOTSTRAP_TIMEOUT_DEFAULT = PORT_PROBE_BOOTSTRAP_TIMEOUT_DEFAULT;
+
+
+ /**
+ * Probe enabled/disabled flag Key: {@value}
+ */
+ String PID_PROBE_ENABLED =
+ MONITOR_KEY_PREFIX + "pidprobe.enabled";
+
+ /**
+ * PID probing key : pid to attempt to create a TCP connection to {@value}
+ */
+ String PID_PROBE_PIDFILE =
+ MONITOR_KEY_PREFIX + "pidprobe.pidfile";
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
new file mode 100644
index 0000000..9e59974
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Formatter;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TreeSet;
+
+/**
+ * Various utils to work with the monitor
+ */
+public final class MonitorUtils {
+ protected static final Logger log = LoggerFactory.getLogger(MonitorUtils.class);
+
+ private MonitorUtils() {
+ }
+
+ public static String toPlural(int val) {
+ return val != 1 ? "s" : "";
+ }
+
+ /**
+ * Convert the arguments -including dropping any empty strings that creep in
+ * @param args arguments
+ * @return a list view with no empty strings
+ */
+ public static List<String> prepareArgs(String[] args) {
+ List<String> argsList = new ArrayList<String>(args.length);
+ StringBuilder argsStr = new StringBuilder("Arguments: [");
+ for (String arg : args) {
+ argsStr.append('"').append(arg).append("\" ");
+ if (!arg.isEmpty()) {
+ argsList.add(arg);
+ }
+ }
+ argsStr.append(']');
+ log.debug(argsStr.toString());
+ return argsList;
+ }
+
+ /**
+ * convert the system properties to a buffer for printing
+ * @return a multi-line list of key-value pairs
+ */
+ public static String dumpSystemProperties() {
+ TreeSet<String> keys = new TreeSet<String>();
+ for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
+ keys.add(entry.getKey().toString());
+ }
+ StringBuilder builder = new StringBuilder();
+ for (String key : keys) {
+ builder
+ .append(" ")
+ .append(key)
+ .append('=')
+ .append(System.getProperty(key))
+ .append('\n');
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Dump the system environment
+ * @return the environment
+ */
+ public static String dumpEnv() {
+ Map<String, String> env = System.getenv();
+ TreeSet<String> keys = new TreeSet<String>();
+ for (Map.Entry<String, String> entry : env.entrySet()) {
+ keys.add(entry.getKey());
+ }
+ StringBuilder builder = new StringBuilder();
+ for (String key : keys) {
+ builder
+ .append(" ")
+ .append(key)
+ .append('=')
+ .append(env.get(key))
+ .append('\n');
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Convert milliseconds to human time -the exact format is unspecified
+ * @param milliseconds a time in milliseconds
+ * @return a time that is converted to human intervals
+ */
+ public static String millisToHumanTime(long milliseconds) {
+ StringBuilder sb = new StringBuilder();
+ // Send all output to the Appendable object sb
+ Formatter formatter = new Formatter(sb, Locale.US);
+
+ long s = Math.abs(milliseconds / 1000);
+ long m = Math.abs(milliseconds % 1000);
+ if (milliseconds > 0) {
+ formatter.format("%d.%03ds", s, m);
+ } else if (milliseconds == 0) {
+ formatter.format("0");
+ } else {
+ formatter.format("-%d.%03ds", s, m);
+ }
+ return sb.toString();
+ }
+
+ public static InetSocketAddress getURIAddress(URI uri) {
+ String host = uri.getHost();
+ int port = uri.getPort();
+ return new InetSocketAddress(host, port);
+ }
+
+
+ /**
+ * Get the localhost -may be null
+ * @return the localhost if known
+ */
+ public static InetAddress getLocalHost() {
+ InetAddress localHost;
+ try {
+ localHost = InetAddress.getLocalHost();
+ } catch (UnknownHostException e) {
+ localHost = null;
+ }
+ return localHost;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
new file mode 100644
index 0000000..b1ff792
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+
+/**
+ * Probe for a port being open
+ */
+public class PortProbe extends Probe {
+ protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
+ private final String host;
+ private final int port;
+ private final int timeout;
+
+ public PortProbe(String host, int port, int timeout, String name, Configuration conf)
+ throws IOException {
+ super("Port probe " + name + " " + host + ":" + port + " for " + timeout + "ms",
+ conf);
+ this.host = host;
+ this.port = port;
+ this.timeout = timeout;
+ }
+
+ public static PortProbe createPortProbe(Configuration conf,
+ String hostname,
+ int port) throws IOException {
+ PortProbe portProbe = new PortProbe(hostname,
+ port,
+ conf.getInt(
+ PORT_PROBE_CONNECT_TIMEOUT,
+ PORT_PROBE_CONNECT_TIMEOUT_DEFAULT),
+ "",
+ conf);
+
+ return portProbe;
+ }
+
+ @Override
+ public void init() throws IOException {
+ if (port >= 65536) {
+ throw new IOException("Port is out of range: " + port);
+ }
+ InetAddress target;
+ if (host != null) {
+ log.debug("looking up host " + host);
+ target = InetAddress.getByName(host);
+ } else {
+ log.debug("Host is null, retrieving localhost address");
+ target = InetAddress.getLocalHost();
+ }
+ log.info("Checking " + target + ":" + port);
+ }
+
+ /**
+ * Try to connect to the (host,port); a failure to connect within
+ * the specified timeout is a failure
+ * @param livePing is the ping live: true for live; false for boot time
+ * @return the outcome
+ */
+ @Override
+ public ProbeStatus ping(boolean livePing) {
+ ProbeStatus status = new ProbeStatus();
+ InetSocketAddress sockAddr = new InetSocketAddress(host, port);
+ Socket socket = new Socket();
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug("Connecting to " + sockAddr.toString() + " connection-timeout=" +
+ MonitorUtils.millisToHumanTime(timeout));
+ }
+ socket.connect(sockAddr, timeout);
+ status.succeed(this);
+ } catch (IOException e) {
+ String error = "Probe " + sockAddr + " failed: " + e;
+ log.debug(error, e);
+ status.fail(this,
+ new IOException(error, e));
+ } finally {
+ IOUtils.closeSocket(socket);
+ }
+ return status;
+
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java
new file mode 100644
index 0000000..be4b5ef
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.IOException;
+
+/**
+ * Base class of all probes.
+ */
+public abstract class Probe implements MonitorKeys {
+
+ protected final Configuration conf;
+ private String name;
+
+ // =======================================================
+ /*
+ * These fields are all used by the probe loops
+ * to maintain state. Please Leave them alone.
+ */
+ public int successCount;
+ public int failureCount;
+ public long bootstrapStarted;
+ public long bootstrapFinished;
+ private boolean booted = false;
+
+ // =======================================================
+
+ /**
+ * Create a probe of a specific name
+ *
+ * @param name probe name
+ * @param conf configuration being stored.
+ */
+ public Probe(String name, Configuration conf) {
+ this.name = name;
+ this.conf = conf;
+ }
+
+
+ protected void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+
+ @Override
+ public String toString() {
+ return getName() +
+ " {" +
+ "successCount=" + successCount +
+ ", failureCount=" + failureCount +
+ '}';
+ }
+
+ /**
+ * perform any prelaunch initialization
+ */
+ public void init() throws IOException {
+
+ }
+
+ /**
+ * Ping the endpoint. All exceptions must be caught and included in the
+ * (failure) status.
+ *
+ * @param livePing is the ping live: true for live; false for boot time
+ * @return the status
+ */
+ public abstract ProbeStatus ping(boolean livePing);
+
+ public void beginBootstrap() {
+ bootstrapStarted = System.currentTimeMillis();
+ }
+
+ public void endBootstrap() {
+ setBooted(true);
+ bootstrapFinished = System.currentTimeMillis();
+ }
+
+ public boolean isBooted() {
+ return booted;
+ }
+
+ public void setBooted(boolean booted) {
+ this.booted = booted;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeFailedException.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeFailedException.java
new file mode 100644
index 0000000..5389eac
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeFailedException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+/**
+ * An exception to raise on a probe failure
+ */
+public class ProbeFailedException extends Exception {
+
+ public final ProbeStatus status;
+
+ public ProbeFailedException(String text, ProbeStatus status) {
+ super((text == null ? "" : (text + ": ")) + status, status.getThrown());
+ this.status = status;
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeInterruptedException.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeInterruptedException.java
new file mode 100644
index 0000000..dee0e09
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeInterruptedException.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+/**
+ * This exception is raised when the probe loop detects that it has been requested to stop
+ *
+ */
+public class ProbeInterruptedException extends Exception {
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbePhase.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbePhase.java
new file mode 100644
index 0000000..d87c81b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbePhase.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+/**
+ * Probe phases. The names are for strings; the index is the order in which things happen;
+ * -any state can got to terminating directly.
+ */
+public enum ProbePhase {
+ INIT("Initializing", 0),
+ DEPENDENCY_CHECKING("Dependencies", 1),
+ BOOTSTRAPPING("Bootstrapping", 2),
+ LIVE("Live", 3),
+ TERMINATING("Terminating", 4);
+
+ private final String name;
+ private final int index;
+
+ ProbePhase(String name, int index) {
+ this.name = name;
+ this.index = index;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ /**
+ * How many phases are there?
+ */
+ public static final int PHASE_COUNT = TERMINATING.index + 1;
+
+ @Override
+ public String toString() {
+ return name;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeReportHandler.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeReportHandler.java
new file mode 100644
index 0000000..36c20c8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeReportHandler.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+/**
+ * This interface is for use by the Poll Workers to send events to the reporters.
+ *
+ * It is up the reporters what to do with the specific events.
+ */
+public interface ProbeReportHandler {
+
+ /**
+ * The probe process has changed state.
+ * @param probePhase the new process phrase
+ */
+ void probeProcessStateChange(ProbePhase probePhase);
+
+ /**
+ * Report a probe outcome
+ * @param phase the current phase of probing
+ * @param status the probe status
+ */
+ void probeResult(ProbePhase phase, ProbeStatus status);
+
+ /**
+ * A probe has failed
+ */
+ void probeFailure(ProbeFailedException exception);
+
+ /**
+ * A probe has just booted
+ * @param status probe status
+ */
+ void probeBooted(ProbeStatus status);
+
+ boolean commence(String name, String description);
+
+ void unregister();
+
+ /**
+ * A heartbeat event should be raised
+ * @param status the probe status
+ */
+ void heartbeat(ProbeStatus status);
+
+ /**
+ * A probe has timed out
+ * @param currentPhase the current execution phase
+ * @param probe the probe that timed out
+ * @param lastStatus the last status that was successfully received -which is implicitly
+ * not the status of the timed out probe
+ * @param currentTime the current time
+ */
+ void probeTimedOut(ProbePhase currentPhase,
+ Probe probe,
+ ProbeStatus lastStatus,
+ long currentTime);
+
+ /**
+ * Event to say that the live probe cycle completed so the entire
+ * system can be considered functional.
+ */
+ void liveProbeCycleCompleted();
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java
new file mode 100644
index 0000000..653f479
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * Status message of a probe. This is designed to be sent over the wire, though the exception
+ * Had better be unserializable at the far end if that is to work.
+ */
+public final class ProbeStatus implements Serializable {
+
+ private long timestamp;
+ private String timestampText;
+ private boolean success;
+ private boolean realOutcome;
+ private String message;
+ private Throwable thrown;
+ private transient Probe originator;
+ private ProbePhase probePhase;
+
+ public ProbeStatus() {
+ }
+
+ public ProbeStatus(long timestamp, String message, Throwable thrown) {
+ this.success = false;
+ this.message = message;
+ this.thrown = thrown;
+ setTimestamp(timestamp);
+ }
+
+ public ProbeStatus(long timestamp, String message) {
+ this.success = true;
+ setTimestamp(timestamp);
+ this.message = message;
+ this.thrown = null;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ timestampText = new Date(timestamp).toString();
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ /**
+ * Set both the success and the real outcome bits to the same value
+ * @param success the new value
+ */
+ public void setSuccess(boolean success) {
+ this.success = success;
+ realOutcome = success;
+ }
+
+ public String getTimestampText() {
+ return timestampText;
+ }
+
+ public boolean getRealOutcome() {
+ return realOutcome;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public Throwable getThrown() {
+ return thrown;
+ }
+
+ public void setThrown(Throwable thrown) {
+ this.thrown = thrown;
+ }
+
+ public ProbePhase getProbePhase() {
+ return probePhase;
+ }
+
+ public void setProbePhase(ProbePhase probePhase) {
+ this.probePhase = probePhase;
+ }
+
+ /**
+ * Get the probe that generated this result. May be null
+ * @return a possibly null reference to a probe
+ */
+ public Probe getOriginator() {
+ return originator;
+ }
+
+ /**
+ * The probe has succeeded -capture the current timestamp, set
+ * success to true, and record any other data needed.
+ * @param probe probe
+ */
+ public void succeed(Probe probe) {
+ finish(probe, true, probe.getName(), null);
+ }
+
+ /**
+ * A probe has failed either because the test returned false, or an exception
+ * was thrown. The {@link #success} field is set to false, any exception
+ * thrown is recorded.
+ * @param probe probe that failed
+ * @param thrown an exception that was thrown.
+ */
+ public void fail(Probe probe, Throwable thrown) {
+ finish(probe, false, "Failure in " + probe, thrown);
+ }
+
+ public void finish(Probe probe, boolean succeeded, String text, Throwable thrown) {
+ setTimestamp(System.currentTimeMillis());
+ setSuccess(succeeded);
+ originator = probe;
+ message = text;
+ this.thrown = thrown;
+ }
+
+ @Override
+ public String toString() {
+ LogEntryBuilder builder = new LogEntryBuilder("Probe Status");
+ builder.elt("time", timestampText)
+ .elt("phase", probePhase)
+ .elt("outcome", (success ? "success" : "failure"));
+
+ if (success != realOutcome) {
+ builder.elt("originaloutcome", (realOutcome ? "success" : "failure"));
+ }
+ builder.elt("message", message);
+ if (thrown != null) {
+ builder.elt("exception", thrown);
+ }
+
+ return builder.toString();
+ }
+
+ public boolean inPhase(ProbePhase phase) {
+ return getProbePhase().equals(phase);
+ }
+
+ /**
+ * Flip the success bit on while the real outcome bit is kept false
+ */
+ public void markAsSuccessful() {
+ success = true;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeWorker.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeWorker.java
new file mode 100644
index 0000000..f64ec8d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeWorker.java
@@ -0,0 +1,446 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This is the entry point to do work. A list of probes is taken in, in order of
+ * booting. Once live they go to the live probes list.
+ *
+ * The dependency probes are a set of probes for dependent services, all of which
+ * must be live before boot probes commence.
+ *
+ * The boot probes are executed and are allowed to fail; failure is interpreted as "not yet live"
+ *
+ * Once all boot probes are live, the live list is used for probes; these must not fail.
+ *
+ * There is no timeout on dependency probe bootstrap time, because of the notion that
+ * restarting this service will have no effect on the dependencies.
+ */
+
+public class ProbeWorker implements Runnable {
+ protected static final Logger log = LoggerFactory.getLogger(ProbeWorker.class);
+
+ public static final String FAILED_TO_BOOT = "Monitored service failed to bootstrap after ";
+ public static final String FAILURE_OF_A_LIVE_PROBE_DURING_BOOTSTRAPPING = "Failure of a live probe during bootstrapping";
+ private final List<Probe> monitorProbes;
+ private final List<Probe> dependencyProbes;
+ public final int interval;
+ protected volatile ProbeStatus lastStatus;
+ protected volatile ProbeStatus lastFailingBootstrapProbe;
+ protected volatile Probe currentProbe;
+ private volatile boolean mustExit;
+ private final int bootstrapTimeout;
+ private long bootstrapEndtime;
+
+ private ProbeReportHandler reportHandler;
+ private volatile ProbePhase probePhase = ProbePhase.INIT;
+
+ /**
+ * Create a probe worker
+ * @param monitorProbes list of probes that must boot and then go live -after which
+ * they must stay live.
+ * @param dependencyProbes the list of dependency probes that must all succeed before
+ * any attempt to probe the direct probe list is performed. Once the
+ * dependency phase has completed, these probes are never checked again.
+ * @param interval probe interval in milliseconds.
+ * @param bootstrapTimeout timeout for bootstrap in milliseconds
+ */
+ public ProbeWorker(List<Probe> monitorProbes, List<Probe> dependencyProbes, int interval, int bootstrapTimeout) {
+ this.monitorProbes = monitorProbes;
+ this.dependencyProbes = dependencyProbes != null ? dependencyProbes : new ArrayList<Probe>(0);
+ this.interval = interval;
+ lastStatus = new ProbeStatus(now(),
+ "Initial status");
+ lastStatus.setProbePhase(ProbePhase.INIT);
+ this.bootstrapTimeout = bootstrapTimeout;
+ }
+
+ public void init() throws IOException {
+ for (Probe probe : monitorProbes) {
+ probe.init();
+ }
+ for (Probe probe : dependencyProbes) {
+ probe.init();
+ }
+ }
+
+ public void setReportHandler(ProbeReportHandler reportHandler) {
+ this.reportHandler = reportHandler;
+ }
+
+ public void setMustExit() {
+ this.mustExit = true;
+ }
+
+ public ProbeStatus getLastStatus() {
+ return lastStatus;
+ }
+
+ public synchronized Probe getCurrentProbe() {
+ return currentProbe;
+ }
+
+ public ProbePhase getProbePhase() {
+ return probePhase;
+ }
+
+ /**
+ * Enter the new process state, and report it to the report handler.
+ * This is synchronized just to make sure there isn't more than one
+ * invocation at the same time.
+ * @param status the new process status
+ */
+ private synchronized void enterProbePhase(ProbePhase status) {
+ this.probePhase = status;
+ if (reportHandler != null) {
+ reportHandler.probeProcessStateChange(status);
+ }
+ }
+
+ /**
+ * Report the probe status to the listener -setting the probe phase field
+ * before doing so.
+ * The value is also stored in the {@link #lastStatus} field
+ * @param status the new status
+ */
+ private void reportProbeStatus(ProbeStatus status) {
+ ProbePhase phase = getProbePhase();
+ status.setProbePhase(phase);
+ lastStatus = status;
+ reportHandler.probeResult(phase, status);
+ }
+
+ /**
+ * Ping one probe. Logs the operation at debug level; sets the field <code>currentProbe</code>
+ * to the probe for the duration of the operation -this is used when identifying the
+ * cause of a hung reporting loop
+ * @param probe probe to ping
+ * @param live flag to indicate whether or not the operation is live or bootstrapping
+ * @return the status of the ping
+ * @throws ProbeInterruptedException if the probe has been told to exit
+ */
+ private ProbeStatus ping(Probe probe, boolean live) throws ProbeInterruptedException {
+ if (log.isDebugEnabled()) {
+ log.debug("Executing " + probe);
+ }
+ checkForExitRequest();
+ currentProbe = probe;
+ try {
+ return probe.ping(live);
+ } finally {
+ currentProbe = null;
+ }
+ }
+
+ /**
+ * Check for an exit request -and convert it to an exception if made
+ * @throws ProbeInterruptedException iff {@link #mustExit} is true
+ */
+ private void checkForExitRequest() throws ProbeInterruptedException {
+ if (mustExit) {
+ throw new ProbeInterruptedException();
+ }
+ }
+
+ /**
+ * Check the dependencies.
+ * The moment a failing test is reached the call returns without
+ * any reporting.
+ *
+ * All successful probes are reported, so as to keep the heartbeats happy.
+ *
+ * @return the status of the last dependency check. If this is a success
+ * them every probe passed.
+ */
+ private ProbeStatus checkDependencyProbes() throws ProbeInterruptedException {
+ ProbeStatus status = null;
+ for (Probe dependency : dependencyProbes) {
+ //ping them, making clear they are not to run any bootstrap logic
+ status = ping(dependency, true);
+
+ if (!status.isSuccess()) {
+ //the first failure means the rest of the list can be skipped
+ break;
+ }
+ reportProbeStatus(status);
+ }
+ //return the last status
+ return status;
+ }
+
+ /**
+ * Run through all the dependency probes and report their outcomes successes (even if they fail)
+ * @return true iff all the probes have succeeded.
+ * @throws ProbeInterruptedException if the process was interrupted.
+ */
+ public boolean checkAndReportDependencyProbes() throws ProbeInterruptedException {
+ ProbeStatus status;
+ status = checkDependencyProbes();
+ if (status != null && !status.isSuccess()) {
+ //during dependency checking, a failure is still reported as a success
+ status.markAsSuccessful();
+ reportProbeStatus(status);
+ //then return without checking anything else
+ return false;
+ }
+ //all dependencies are done.
+ return true;
+ }
+
+ /**
+ * Begin bootstrapping by telling each probe that they have started.
+ * This sets the timeouts up, as well as permits any other set-up actions
+ * to begin.
+ */
+ private void beginBootstrapProbes() {
+ synchronized (this) {
+ bootstrapEndtime = now() + bootstrapTimeout;
+ }
+ for (Probe probe : monitorProbes) {
+ probe.beginBootstrap();
+ }
+ }
+
+ private long now() {
+ return System.currentTimeMillis();
+ }
+
+
+ /**
+ * Check the bootstrap probe list. All successful probes get reported.
+ * The first unsuccessful probe will be returned and not reported (left for policy upstream).
+ * If the failing probe has timed out, that is turned into a {@link ProbeFailedException}
+ * @return the last (unsuccessful) probe, or null if they all succeeded
+ * @throws ProbeInterruptedException interrupts
+ * @throws ProbeFailedException on a boot timeout
+ */
+ private boolean checkBootstrapProbes() throws ProbeInterruptedException, ProbeFailedException {
+ verifyBootstrapHasNotTimedOut();
+
+ boolean probeFailed = false;
+ //now run through all the bootstrap probes
+ for (Probe probe : monitorProbes) {
+ //ping them
+ ProbeStatus status = ping(probe, false);
+ if (!status.isSuccess()) {
+ probeFailed = true;
+ lastFailingBootstrapProbe = status;
+ probe.failureCount++;
+ if (log.isDebugEnabled()) {
+ log.debug("Booting probe failed: " + status);
+ }
+ //at this point check to see if the timeout has occurred -and if so, force in the last probe status.
+
+ //this is a failure but not a timeout
+ //during boot, a failure of a probe that hasn't booted is still reported as a success
+ if (!probe.isBooted()) {
+ //so the success bit is flipped
+ status.markAsSuccessful();
+ reportProbeStatus(status);
+ } else {
+ //the probe had booted but then it switched to failing
+
+ //update the status unedited
+ reportProbeStatus(status);
+ //then fail
+ throw raiseProbeFailure(status, FAILURE_OF_A_LIVE_PROBE_DURING_BOOTSTRAPPING);
+ }
+ } else {
+ //this probe is working
+ if (!probe.isBooted()) {
+ //if it is new, mark it as live
+ if (log.isDebugEnabled()) {
+ log.debug("Booting probe is now live: " + probe);
+ }
+ probe.endBootstrap();
+ //tell the report handler that another probe has booted
+ reportHandler.probeBooted(status);
+ }
+ //push out its status
+ reportProbeStatus(status);
+ probe.successCount++;
+ }
+ }
+ return !probeFailed;
+ }
+
+
+ public int getBootstrapTimeout() {
+ return bootstrapTimeout;
+ }
+
+ /**
+ * This checks that bootstrap operations have not timed out
+ * @throws ProbeFailedException if the bootstrap has failed
+ */
+ public void verifyBootstrapHasNotTimedOut() throws ProbeFailedException {
+ //first step -look for a timeout
+ if (isBootstrapTimeExceeded()) {
+ String text = FAILED_TO_BOOT
+ + MonitorUtils.millisToHumanTime(bootstrapTimeout);
+
+ ProbeStatus status;
+ if (lastFailingBootstrapProbe != null) {
+ status = lastFailingBootstrapProbe;
+ status.setSuccess(false);
+ } else {
+ status = new ProbeStatus();
+ status.finish(null, false, text, null);
+ }
+
+ throw raiseProbeFailure(status,
+ text);
+ }
+ }
+
+ /**
+ * predicate that gets current time and checks for its time being exceeded.
+ * @return true iff the current time is > the end time
+ */
+ public synchronized boolean isBootstrapTimeExceeded() {
+ return now() > bootstrapEndtime;
+ }
+
+ /**
+ * run through all the bootstrap probes and see if they are live.
+ * @return true iff all boot probes succeeded
+ * @throws ProbeInterruptedException the probe interruption flags
+ * @throws ProbeFailedException if a probe failed.
+ */
+ public boolean checkAndReportBootstrapProbes() throws ProbeInterruptedException,
+ ProbeFailedException {
+ if (bootstrapTimeout <= 0) {
+ //there is no period of grace for bootstrapping probes, so return true saying
+ //this phase is complete
+ return true;
+ }
+ //now the bootstrapping probes
+ return checkBootstrapProbes();
+ }
+
+
+ /**
+ * run through all the live probes, pinging and reporting them.
+ * A single probe failure is turned into an exception
+ * @throws ProbeFailedException a probe failed
+ * @throws ProbeInterruptedException the probe process was explicitly interrupted
+ */
+ protected void checkAndReportLiveProbes() throws ProbeFailedException, ProbeInterruptedException {
+ ProbeStatus status = null;
+ //go through the live list
+ if (log.isDebugEnabled()) {
+ log.debug("Checking live probes");
+ }
+ for (Probe probe : monitorProbes) {
+ status = ping(probe, true);
+ reportProbeStatus(status);
+ if (!status.isSuccess()) {
+ throw raiseProbeFailure(status, "Failure of probe in \"live\" monitor");
+ }
+ probe.successCount++;
+ }
+ //here all is well, so notify the reporter
+ reportHandler.liveProbeCycleCompleted();
+ }
+
+ /**
+ * Run the set of probes relevant for this phase of the probe lifecycle.
+ * @throws ProbeFailedException a probe failed
+ * @throws ProbeInterruptedException the probe process was explicitly interrupted
+ */
+ protected void executeProbePhases() throws ProbeFailedException, ProbeInterruptedException {
+ switch (probePhase) {
+ case INIT:
+ enterProbePhase(ProbePhase.DEPENDENCY_CHECKING);
+ //fall through straight into the dependency check
+ case DEPENDENCY_CHECKING:
+ if (checkAndReportDependencyProbes()) {
+ enterProbePhase(ProbePhase.BOOTSTRAPPING);
+ beginBootstrapProbes();
+ }
+ break;
+ case BOOTSTRAPPING:
+ if (checkAndReportBootstrapProbes()) {
+ enterProbePhase(ProbePhase.LIVE);
+ }
+ break;
+ case LIVE:
+ checkAndReportLiveProbes();
+ break;
+
+ case TERMINATING:
+ default:
+ //do nothing.
+ break;
+ }
+ }
+
+
+ /**
+ * Raise a probe failure; injecting the phase into the status result first
+ *
+ * @param status ping result
+ * @param text optional text -null or "" means "none"
+ * @return an exception ready to throw
+ */
+ private ProbeFailedException raiseProbeFailure(ProbeStatus status, String text) {
+ status.setProbePhase(probePhase);
+ log.info("Probe failed: " + status);
+ return new ProbeFailedException(text, status);
+ }
+
+ @Override
+ public void run() {
+ int size = monitorProbes.size();
+ log.info("Probe Worker Starting; " + size + " probe" + MonitorUtils.toPlural(size) + ":");
+ enterProbePhase(ProbePhase.DEPENDENCY_CHECKING);
+ for (Probe probe : monitorProbes) {
+ log.info(probe.getName());
+ }
+ while (!mustExit) {
+ try {
+ Thread.sleep(interval);
+ executeProbePhases();
+ } catch (ProbeFailedException e) {
+ //relay to the inner loop handler
+ probeFailed(e);
+ } catch (InterruptedException interrupted) {
+ break;
+ } catch (ProbeInterruptedException e) {
+ //exit raised.
+ //this will be true, just making extra-sure
+ break;
+ }
+ }
+ log.info("Probe Worker Exiting");
+ enterProbePhase(ProbePhase.TERMINATING);
+ }
+
+
+ protected void probeFailed(ProbeFailedException e) {
+ reportHandler.probeFailure(e);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java
new file mode 100644
index 0000000..096838d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/ReportingLoop.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * This is the monitor service
+ */
+public final class ReportingLoop implements Runnable, ProbeReportHandler, MonitorKeys, Closeable {
+ protected static final Logger log = LoggerFactory.getLogger(ReportingLoop.class);
+ private final ProbeWorker worker;
+ private final Thread workerThread;
+ private final int reportInterval;
+ private final int probeTimeout;
+ private final int bootstrapTimeout;
+ private ProbeReportHandler reporter;
+ private final String name;
+ private volatile boolean mustExit;
+
+ public ReportingLoop(String name,
+ ProbeReportHandler reporter,
+ List<Probe> probes,
+ List<Probe> dependencyProbes,
+ int probeInterval,
+ int reportInterval,
+ int probeTimeout,
+ int bootstrapTimeout) throws IOException {
+ this(name,
+ reporter,
+ new ProbeWorker(probes, dependencyProbes, probeInterval, bootstrapTimeout),
+ reportInterval,
+ probeTimeout);
+ }
+
+ /**
+ * Create a new reporting loop -and bond the worker's ProbeReportHandler
+ * to us
+ * @param name
+ * @param reporter
+ * @param worker
+ * @param reportInterval
+ * @param probeTimeout
+ */
+ public ReportingLoop(String name,
+ ProbeReportHandler reporter,
+ ProbeWorker worker,
+ int reportInterval,
+ int probeTimeout) throws IOException {
+ this.name = name;
+ this.reporter = reporter;
+ this.reportInterval = reportInterval;
+ this.probeTimeout = probeTimeout;
+ this.worker = worker;
+ this.bootstrapTimeout = worker.getBootstrapTimeout();
+ worker.setReportHandler(this);
+ workerThread = new Thread(worker, "probe thread - " + name);
+ worker.init();
+ }
+
+ public int getBootstrapTimeout() {
+ return bootstrapTimeout;
+ }
+
+ public ReportingLoop withReporter(ProbeReportHandler reporter) {
+ assert this.reporter == null : "attempting to reassign reporter ";
+ assert reporter != null : "new reporter is null";
+ this.reporter = reporter;
+ return this;
+ }
+
+ /**
+ * Start the monitoring.
+ *
+ * @return false if the monitoring did not start and that the worker threads
+ * should be run up.
+ */
+ public boolean startReporting() {
+ String description = "Service Monitor for " + name + ", probe-interval= "
+ + MonitorUtils.millisToHumanTime(worker.interval)
+ + ", report-interval=" + MonitorUtils.millisToHumanTime(reportInterval)
+ + ", probe-timeout=" + timeoutToStr(probeTimeout)
+ + ", bootstrap-timeout=" + timeoutToStr(bootstrapTimeout);
+ log.info("Starting reporting"
+ + " to " + reporter
+ + description);
+ return reporter.commence(name, description);
+ }
+
+ private String timeoutToStr(int timeout) {
+ return timeout >= 0 ? MonitorUtils.millisToHumanTime(timeout) : "not set";
+ }
+
+ private void startWorker() {
+ log.info("Starting reporting worker thread ");
+ workerThread.setDaemon(true);
+ workerThread.start();
+ }
+
+
+ /**
+ * This exits the process cleanly
+ */
+ @Override
+ public void close() {
+ log.info("Stopping reporting");
+ mustExit = true;
+ if (worker != null) {
+ worker.setMustExit();
+ workerThread.interrupt();
+ }
+ if (reporter != null) {
+ reporter.unregister();
+ }
+ }
+
+ @Override
+ public void probeFailure(ProbeFailedException exception) {
+ reporter.probeFailure(exception);
+ }
+
+ @Override
+ public void probeProcessStateChange(ProbePhase probePhase) {
+ reporter.probeProcessStateChange(probePhase);
+ }
+
+ @Override
+ public void probeBooted(ProbeStatus status) {
+ reporter.probeBooted(status);
+ }
+
+ private long now() {
+ return System.currentTimeMillis();
+ }
+
+ @Override
+ public void probeResult(ProbePhase phase, ProbeStatus status) {
+ reporter.probeResult(phase, status);
+ }
+
+ @Override
+ public boolean commence(String n, String description) {
+ return true;
+ }
+
+ @Override
+ public void unregister() {
+ }
+
+ @Override
+ public void heartbeat(ProbeStatus status) {
+ }
+
+ @Override
+ public void probeTimedOut(ProbePhase currentPhase, Probe probe, ProbeStatus lastStatus,
+ long currentTime) {
+ }
+
+ @Override
+ public void liveProbeCycleCompleted() {
+ //delegate to the reporter
+ reporter.liveProbeCycleCompleted();
+ }
+
+ /**
+ * The reporting loop
+ */
+ void reportingLoop() {
+
+ while (!mustExit) {
+ try {
+ ProbeStatus workerStatus = worker.getLastStatus();
+ long now = now();
+ long lastStatusIssued = workerStatus.getTimestamp();
+ long timeSinceLastStatusIssued = now - lastStatusIssued;
+ //two actions can occur here: a heartbeat is issued or a timeout reported.
+ //this flag decides which
+ boolean heartbeat;
+
+ //based on phase, decide whether to heartbeat or timeout
+ ProbePhase probePhase = worker.getProbePhase();
+ switch (probePhase) {
+ case DEPENDENCY_CHECKING:
+ //no timeouts in dependency phase
+ heartbeat = true;
+ break;
+
+ case BOOTSTRAPPING:
+ //the timeout here is fairly straightforward: heartbeats are
+ //raised while the worker hasn't timed out
+ heartbeat = bootstrapTimeout < 0 || timeSinceLastStatusIssued < bootstrapTimeout;
+
+ break;
+
+ case LIVE:
+ //use the probe timeout interval between the current time
+ //and the time the last status event was received.
+ heartbeat = timeSinceLastStatusIssued < probeTimeout;
+ break;
+
+ case INIT:
+ case TERMINATING:
+ default:
+ //send a heartbeat, because this isn't the time to be failing
+ heartbeat = true;
+ }
+ if (heartbeat) {
+ //a heartbeat is sent to the reporter
+ reporter.heartbeat(workerStatus);
+ } else {
+ //no response from the worker -it is hung.
+ reporter.probeTimedOut(probePhase,
+ worker.getCurrentProbe(),
+ workerStatus,
+ now
+ );
+ }
+
+ //now sleep
+ Thread.sleep(reportInterval);
+
+ } catch (InterruptedException e) {
+ //interrupted -always exit the loop.
+ break;
+ }
+ }
+ //this point is reached if and only if a clean exit was requested or something failed.
+ }
+
+ /**
+ * This can be run in a separate thread, or it can be run directly from the caller.
+ * Test runs do the latter, HAM runs multiple reporting threads.
+ */
+ @Override
+ public void run() {
+ try {
+ startWorker();
+ reportingLoop();
+ } catch (RuntimeException e) {
+ log.warn("Failure in the reporting loop: " + e, e);
+ //rethrow so that inline code can pick it up (e.g. test runs)
+ throw e;
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
new file mode 100644
index 0000000..8bc6dd0
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Probe for YARN application
+ */
+public class YarnApplicationProbe extends Probe {
+ protected static final Logger log = LoggerFactory.getLogger(
+ YarnApplicationProbe.class);
+
+ /**
+ * Yarn client service
+ */
+ private SliderYarnClientImpl yarnClient;
+ private final String clustername;
+ private final String username;
+
+ public YarnApplicationProbe(String clustername,
+ SliderYarnClientImpl yarnClient,
+ String name,
+ Configuration conf, String username)
+ throws IOException {
+ super("Port probe " + name + " " + clustername,
+ conf);
+ this.clustername = clustername;
+ this.yarnClient = yarnClient;
+ this.username = username;
+ }
+
+
+ @Override
+ public void init() throws IOException {
+
+ log.info("Checking " + clustername );
+ }
+
+ /**
+ * Try to connect to the (host,port); a failure to connect within
+ * the specified timeout is a failure
+ * @param livePing is the ping live: true for live; false for boot time
+ * @return the outcome
+ */
+ @Override
+ public ProbeStatus ping(boolean livePing) {
+
+ ProbeStatus status = new ProbeStatus();
+ try {
+
+ List<ApplicationReport> instances =
+ yarnClient.listInstances(username);
+ ApplicationReport instance =
+ yarnClient.findClusterInInstanceList(instances, clustername);
+ if (null == instance) {
+ throw UnknownApplicationInstanceException.unknownInstance(clustername);
+ }
+
+ status.succeed(this);
+ } catch (Exception e) {
+ status.fail(this, e);
+ }
+ return status;
+
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
new file mode 100644
index 0000000..8c44aea
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.curator;
+
+import com.google.common.base.Preconditions;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
+import org.apache.curator.x.discovery.strategies.RandomStrategy;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+
+/**
+ * This class creates a curator -but does not start or close it. That
+ * is the responsbility of the owner
+ */
+public class CuratorHelper extends Configured {
+
+ private final CuratorFramework curator;
+ private final String connectionString;
+
+ public CuratorHelper(Configuration conf, String connectionString) {
+ super(conf);
+ this.connectionString = connectionString;
+ curator = createCurator(this.connectionString);
+ }
+
+
+ public CuratorFramework getCurator() {
+ return curator;
+ }
+
+ public String getConnectionString() {
+ return connectionString;
+ }
+
+ /**
+ * Create a retry policy for this configuration
+ * @param conf
+ * @return
+ */
+ public RetryPolicy createRetryPolicy() {
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ return retryPolicy;
+ }
+
+ private CuratorFramework createCurator(String connectionString) {
+ CuratorFramework curator =
+ CuratorFrameworkFactory.newClient(connectionString,
+ createRetryPolicy());
+ return curator;
+ }
+
+ /**
+ * Create an (united) curator client service
+ * @param connectionString ZK binding
+ * @return the service
+ */
+ public CuratorService createCuratorClientService() {
+ CuratorService curatorService =
+ new CuratorService("Curator ", curator, connectionString);
+ return curatorService;
+ }
+
+ /**
+ * Create a discovery builder bonded to this curator
+ * @return
+ */
+ public ServiceDiscoveryBuilder<ServiceInstanceData> createDiscoveryBuilder() {
+ ServiceDiscoveryBuilder<ServiceInstanceData> discoveryBuilder =
+ ServiceDiscoveryBuilder.builder(ServiceInstanceData.class);
+ discoveryBuilder.client(curator);
+ return discoveryBuilder;
+ }
+
+ /**
+ * Create an instance
+ * @param discoveryBuilder builder to create the discovery from
+ */
+
+ public RegistryBinderService<ServiceInstanceData> createRegistryBinderService(
+ String basePath,
+ ServiceDiscoveryBuilder<ServiceInstanceData> discoveryBuilder) {
+ discoveryBuilder.basePath(basePath);
+ return new RegistryBinderService<ServiceInstanceData>(curator,
+ basePath,
+ discoveryBuilder.build());
+ }
+
+
+ /**
+ * Create an instance -including the initial binder
+ * @param basePath base path
+ * @return the binder service
+ */
+ public RegistryBinderService<ServiceInstanceData> createRegistryBinderService(
+ String basePath) {
+ ServiceDiscoveryBuilder<ServiceInstanceData> discoveryBuilder =
+ createDiscoveryBuilder();
+ //registry will start curator as well as the binder, in the correct order
+ RegistryBinderService<ServiceInstanceData> registryBinderService =
+ createRegistryBinderService(basePath, discoveryBuilder);
+ return registryBinderService;
+ }
+
+ public RegistryDiscoveryContext createDiscoveryContext(
+ ServiceDiscovery<ServiceInstanceData> discovery) {
+ Preconditions.checkNotNull(discovery);
+ return new RegistryDiscoveryContext(discovery,
+ new RandomStrategy<ServiceInstanceData>(),
+ RegistryConsts.INSTANCE_REFRESH_MS,
+ ServiceInstanceData.class);
+
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
new file mode 100644
index 0000000..f92ee4e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.curator;
+
+import com.google.common.base.Preconditions;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.utils.ZKPaths;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.service.AbstractService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+
+public class CuratorService extends AbstractService {
+ protected static final Logger log =
+ LoggerFactory.getLogger(CuratorService.class);
+ protected final String basePath;
+
+ private final CuratorFramework curator;
+ private CuratorHelper curatorHelper;
+
+
+ public CuratorService(String name,
+ CuratorFramework curator,
+ String basePath) {
+ super(name);
+ this.curator = Preconditions.checkNotNull(curator, "null client");
+ this.basePath = basePath;
+ }
+
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ curatorHelper = new CuratorHelper(conf, basePath);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ curator.start();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ closeCuratorComponent(curator);
+ }
+
+ public CuratorFramework getCurator() {
+ return curator;
+ }
+
+ protected void closeCuratorComponent(Closeable closeable) {
+ try {
+ IOUtils.closeStream(closeable);
+ } catch (Throwable ignored) {
+ //triggered on an attempt to close before started
+ log.debug("Error when closing {}", ignored);
+ }
+ }
+
+ public String pathForName(String name) {
+ return ZKPaths.makePath(getBasePath(), name);
+ }
+
+ protected String pathForInstance(String name, String id) {
+ return ZKPaths.makePath(pathForName(name), id);
+ }
+
+ public String getBasePath() {
+ return basePath;
+ }
+
+ public CuratorHelper getCuratorHelper() {
+ return curatorHelper;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
new file mode 100644
index 0000000..61efde2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.curator;
+
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.apache.curator.x.discovery.ServiceType;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonTypeInfo;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CuratorServiceInstance<T> {
+
+ public String name;
+ public String id;
+ public String address;
+ public Integer port;
+ public Integer sslPort;
+ public T payload;
+ public long registrationTimeUTC;
+ public ServiceType serviceType;
+ public CuratorUriSpec uriSpec;
+
+ public CuratorServiceInstance() {
+ }
+
+ public CuratorServiceInstance(ServiceInstance<T> si) {
+ this.name = si.getName();
+ this.id = si.getId();
+ this.address = si.getAddress();
+ this.port = si.getPort();
+ this.sslPort = si.getSslPort();
+ this.payload = si.getPayload();
+ this.registrationTimeUTC = si.getRegistrationTimeUTC();
+ this.serviceType = si.getServiceType();
+ this.uriSpec = new CuratorUriSpec();
+ this.uriSpec.addAll(si.getUriSpec().getParts());
+ }
+
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("CuratorServiceInstance{");
+ sb.append("name='").append(name).append('\'');
+ sb.append(", id='").append(id).append('\'');
+ sb.append(", address='").append(address).append('\'');
+ sb.append(", port=").append(port);
+ sb.append(", sslPort=").append(sslPort);
+ sb.append(", payload=").append(payload);
+ sb.append(", registrationTimeUTC=").append(registrationTimeUTC);
+ sb.append(", serviceType=").append(serviceType);
+ sb.append(", uriSpec=").append(uriSpec);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS)
+ public T getPayload() {
+ return payload;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
new file mode 100644
index 0000000..8923e63
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.curator;
+
+import com.google.common.collect.Lists;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ *
+ */
+public class CuratorServiceInstances<T> {
+ private final List<CuratorServiceInstance<T>> services;
+
+ public CuratorServiceInstances() {
+ services = Lists.newArrayList();
+ }
+
+ public CuratorServiceInstances(Collection<? extends CuratorServiceInstance<T>> c) {
+ services = Lists.newArrayList(c);
+ }
+
+ public List<CuratorServiceInstance<T>> getServices() {
+ return Lists.newArrayList(services);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
new file mode 100644
index 0000000..b2a877a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.curator;
+
+import com.google.common.collect.Lists;
+import org.apache.curator.x.discovery.UriSpec;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+
+import java.util.List;
+
+/**
+ *
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CuratorUriSpec extends UriSpec{
+
+ private final List<Part> parts = Lists.newArrayList();
+
+ public CuratorUriSpec() {
+ super();
+ }
+
+ @Override
+ public List<Part> getParts() {
+ return Lists.newArrayList(parts);
+ }
+
+ @Override
+ public void add(Part part) {
+ super.add(part);
+ parts.add(part);
+ }
+
+ public void addAll(List<Part> parts) {
+ parts.addAll(parts);
+ }
+
+ @Override
+ public void remove(Part part) {
+ super.remove(part);
+ parts.remove(part);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
new file mode 100644
index 0000000..2fa8ac5
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.curator;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.apache.curator.x.discovery.ServiceInstanceBuilder;
+import org.apache.curator.x.discovery.ServiceType;
+import org.apache.curator.x.discovery.UriSpec;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.persist.JsonSerDeser;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * YARN service for Curator service discovery; the discovery instance's
+ * start/close methods are tied to the lifecycle of this service
+ * @param <Payload> the payload of the operation
+ */
+public class RegistryBinderService<Payload> extends CuratorService {
+ protected static final Logger log =
+ LoggerFactory.getLogger(RegistryBinderService.class);
+
+ private final ServiceDiscovery<Payload> discovery;
+
+ private final Map<String, ServiceInstance<Payload>> entries =
+ new HashMap<String, ServiceInstance<Payload>>();
+
+ JsonSerDeser<CuratorServiceInstance<Payload>> deser =
+ new JsonSerDeser<CuratorServiceInstance<Payload>>(
+ CuratorServiceInstance.class);
+
+ /**
+ * Create an instance
+ * @param curator. Again, does not need to be started
+ * @param discovery discovery instance -not yet started
+ */
+ public RegistryBinderService(CuratorFramework curator,
+ String basePath,
+ ServiceDiscovery<Payload> discovery) {
+ super("RegistryBinderService", curator, basePath);
+
+ this.discovery =
+ Preconditions.checkNotNull(discovery, "null discovery arg");
+ }
+
+
+ public ServiceDiscovery<Payload> getDiscovery() {
+ return discovery;
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ discovery.start();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ closeCuratorComponent(discovery);
+ super.serviceStop();
+ }
+
+ /**
+ * register an instance -only valid once the service is started
+ * @param id ID -must be unique
+ * @param name name
+ * @param url URL
+ * @param payload payload (may be null)
+ * @return the instance
+ * @throws Exception on registration problems
+ */
+ public ServiceInstance<Payload> register(String name,
+ String id,
+ URL url,
+ Payload payload) throws Exception {
+ Preconditions.checkNotNull(id, "null `id` arg");
+ Preconditions.checkNotNull(name, "null `name` arg");
+ Preconditions.checkNotNull(url, "null `url` arg");
+ Preconditions.checkState(isInState(STATE.STARTED), "Not started: " + this);
+
+ if (lookup(id) != null) {
+ throw new BadClusterStateException(
+ "existing entry for service id %s name %s %s",
+ id, name, url);
+ }
+ int port = url.getPort();
+ if (port == 0) {
+ throw new IOException("Port undefined in " + url);
+ }
+ UriSpec uriSpec = new UriSpec(url.toString());
+ ServiceInstance<Payload> instance = builder()
+ .name(name)
+ .id(id)
+ .payload(payload)
+ .port(port)
+ .serviceType(ServiceType.DYNAMIC)
+ .uriSpec(uriSpec)
+ .build();
+ log.info("registering {}", instance.toString());
+ discovery.registerService(instance);
+ log.info("registration completed {}", instance.toString());
+ synchronized (this) {
+ entries.put(id, instance);
+ }
+ return instance;
+ }
+
+ /**
+ * Get the registered instance by its ID
+ * @param id ID
+ * @return instance or null
+ */
+ public synchronized ServiceInstance<Payload> lookup(String id) {
+ Preconditions.checkNotNull(id, "null `id` arg");
+ return entries.get(id);
+ }
+
+ /**
+ * Create a builder. This is already pre-prepared with address, registration
+ * time and a (random) UUID
+ * @return a builder
+ * @throws IOException IO problems, including enumerating network ports
+ */
+ public ServiceInstanceBuilder<Payload> builder() throws
+ IOException {
+ try {
+ return ServiceInstance.builder();
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+
+ public List<String> instanceIDs(String servicename) throws Exception {
+ List<String> instanceIds;
+
+ try {
+ instanceIds =
+ getCurator().getChildren().forPath(pathForName(servicename));
+ } catch (KeeperException.NoNodeException e) {
+ instanceIds = Lists.newArrayList();
+ }
+ return instanceIds;
+ }
+
+
+ /**
+ * Return a service instance POJO
+ *
+ * @param name name of the service
+ * @param id ID of the instance
+ * @return the instance or <code>null</code> if not found
+ * @throws Exception errors
+ */
+ public CuratorServiceInstance<Payload> queryForInstance(String name, String id) throws
+ Exception {
+ String path = pathForInstance(name, id);
+ try {
+ byte[] bytes = getCurator().getData().forPath(path);
+ return deser.fromBytes(bytes);
+ } catch (KeeperException.NoNodeException ignore) {
+ // ignore
+ }
+ return null;
+ }
+
+
+
+ /**
+ * List all the instances
+ * @param name name of the service
+ * @return a list of instances and their payloads
+ * @throws IOException any problem
+ */
+ public List<CuratorServiceInstance<Payload>> listInstances(String name) throws
+ IOException {
+ try {
+ List<String> instanceIDs = instanceIDs(name);
+ List<CuratorServiceInstance<Payload>> instances =
+ new ArrayList<CuratorServiceInstance<Payload>>(
+ instanceIDs.size());
+ for (String instanceID : instanceIDs) {
+ CuratorServiceInstance<Payload> instance =
+ queryForInstance(name, instanceID);
+ instances.add(instance);
+ }
+ return instances;
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+
+ public Collection<String> queryForNames() throws IOException {
+ try {
+ return getDiscovery().queryForNames();
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryConsts.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryConsts.java
new file mode 100644
index 0000000..37c09ff
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryConsts.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.curator;
+
+public class RegistryConsts {
+ public static final String REGISTRY_RESOURCE_PATH = "/ws/registry";
+ public static final int INSTANCE_REFRESH_MS = 1000;
+ public static String SLIDER_INSTANCE_NAME_FORMAT = "%s-%s";
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryDiscoveryContext.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryDiscoveryContext.java
new file mode 100644
index 0000000..3979163
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryDiscoveryContext.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.curator;
+
+import org.apache.curator.x.discovery.ProviderStrategy;
+import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.curator.x.discovery.server.contexts.GenericDiscoveryContext;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+
+public class RegistryDiscoveryContext extends GenericDiscoveryContext<ServiceInstanceData> {
+
+ public RegistryDiscoveryContext(ServiceDiscovery<ServiceInstanceData> serviceDiscovery,
+ ProviderStrategy<ServiceInstanceData> providerStrategy,
+ int instanceRefreshMs,
+ Class<ServiceInstanceData> payloadType) {
+ super(serviceDiscovery, providerStrategy, instanceRefreshMs, payloadType);
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryNaming.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryNaming.java
new file mode 100644
index 0000000..ce6c4af
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryNaming.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.curator;
+
+import java.util.Locale;
+
+public class RegistryNaming {
+
+ public static String createRegistryName(String instanceName,
+ String userName,
+ String serviceName) {
+ return serviceName;
+ }
+
+ public static String createUniqueInstanceId(String instanceName,
+ String userName,
+ String serviceName,
+ int appId) {
+ return String.format(Locale.ENGLISH,
+ RegistryConsts.SLIDER_INSTANCE_NAME_FORMAT, userName,
+ instanceName,
+ serviceName,
+ appId);
+
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryRestResources.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryRestResources.java
new file mode 100644
index 0000000..d897370
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryRestResources.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.curator;
+
+import com.google.inject.Singleton;
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.apache.curator.x.discovery.server.rest.DiscoveryContext;
+import org.apache.curator.x.discovery.server.rest.DiscoveryResource;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.net.URL;
+import java.util.List;
+import java.util.Random;
+
+@Singleton
+@Path(RegistryConsts.REGISTRY_RESOURCE_PATH)
+public class RegistryRestResources extends DiscoveryResource<ServiceInstanceData> {
+ protected static final Logger log =
+ LoggerFactory.getLogger(RegistryRestResources.class);
+ private final RegistryBinderService<ServiceInstanceData> registry;
+ private DiscoveryContext<ServiceInstanceData> context;
+ private Random randomizer = new Random();
+
+ public RegistryRestResources(@Context DiscoveryContext<ServiceInstanceData> context,
+ RegistryBinderService<ServiceInstanceData> registry) {
+ super(context);
+ this.context = context;
+ this.registry = registry;
+ }
+
+ @GET
+ public Response getWadl (@Context HttpServletRequest request) {
+ try {
+ java.net.URI location = new URL(request.getScheme(),
+ request.getServerName(),
+ request.getServerPort(),
+ "/application.wadl").toURI();
+ return Response.temporaryRedirect(location).build();
+ } catch (Exception e) {
+ log.error("Error during redirect to WADL", e);
+ throw new WebApplicationException(Response.serverError().build());
+ }
+
+ }
+
+ @Override
+ @javax.ws.rs.GET
+ @javax.ws.rs.Path("v1/service/{name}")
+ @javax.ws.rs.Produces({"application/json"})
+ public Response getAll(@PathParam("name") String name) {
+ try {
+ List<CuratorServiceInstance<ServiceInstanceData>>
+ instances = registry.listInstances(name);
+ return Response.ok(new CuratorServiceInstances<ServiceInstanceData>(instances)).build();
+ } catch (Exception e) {
+ log.error("Error during generation of response", e);
+ return Response.serverError().build();
+ }
+ }
+
+ @Override
+ @GET
+ @Path("v1/service/{name}/{id}")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response get(@PathParam("name") String name,
+ @PathParam("id") String id) {
+ try {
+ CuratorServiceInstance<ServiceInstanceData> instance = registry.queryForInstance(name, id);
+ if ( instance == null )
+ {
+ return Response.status(Response.Status.NOT_FOUND).build();
+ }
+ Response.ResponseBuilder builder = Response.ok(instance);
+ return builder.build();
+ } catch (Exception e) {
+ log.error(String.format("Trying to get instance (%s) from service (%s)",
+ id,
+ name), e);
+ return Response.serverError().build();
+ }
+ }
+
+ @Override
+ @GET
+ @Path("v1/anyservice/{name}")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getAny(@PathParam("name") String name) {
+ try {
+ List<CuratorServiceInstance<ServiceInstanceData>>
+ instances = registry.listInstances(name);
+ if (instances == null || instances.size() == 0) {
+ return Response.status(Response.Status.NOT_FOUND).build();
+ }
+
+ CuratorServiceInstance<ServiceInstanceData> randomInstance =
+ instances.get(randomizer.nextInt(instances.size()));
+ if ( randomInstance == null )
+ {
+ return Response.status(Response.Status.NOT_FOUND).build();
+ }
+ return Response.ok(randomInstance).build();
+ } catch (Exception e) {
+ log.error(String.format("Trying to get any instance from service (%s)", name), e);
+ return Response.serverError().build();
+ }
+ }
+
+ @Override
+ @PUT
+ @Path("v1/service/{name}/{id}")
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response putService(ServiceInstance<ServiceInstanceData> instance,
+ @PathParam("name") String name,
+ @PathParam("id") String id) {
+ throw new UnsupportedOperationException("putService not supported");
+ }
+
+ @Override
+ @DELETE
+ @Path("v1/service/{name}/{id}")
+ public Response removeService(@PathParam("name") String name,
+ @PathParam("id") String id) {
+ throw new UnsupportedOperationException("removeService not supported");
+ }
+}
+
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/AbstractSliderLaunchedService.java
new file mode 100644
index 0000000..a39437e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/AbstractSliderLaunchedService.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.core.registry.zk.ZookeeperUtils;
+import org.apache.slider.server.services.curator.CuratorHelper;
+import org.apache.slider.server.services.curator.RegistryBinderService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_REGISTRY_PATH;
+import static org.apache.slider.common.SliderXmlConfKeys.REGISTRY_PATH;
+
+/**
+ * Base service for the standard slider client/server services
+ */
+public abstract class AbstractSliderLaunchedService extends
+ CompoundLaunchedService {
+ private static final Logger log =
+ LoggerFactory.getLogger(AbstractSliderLaunchedService.class);
+
+ public AbstractSliderLaunchedService(String name) {
+ super(name);
+ // make sure all the yarn configs get loaded
+ new YarnConfiguration();
+ }
+
+ /**
+ * Start the registration service
+ * @return the instance
+ * @throws BadConfigException
+ */
+ protected RegistryBinderService<ServiceInstanceData> startRegistrationService()
+ throws BadConfigException {
+
+ String registryQuorum = lookupZKQuorum();
+ String zkPath = getConfig().get(REGISTRY_PATH, DEFAULT_REGISTRY_PATH);
+ return startRegistrationService(registryQuorum, zkPath);
+ }
+
+ /**
+ * look up the registry quorum from the config
+ * @return the quorum string
+ * @throws BadConfigException if it is not there or invalid
+ */
+ public String lookupZKQuorum() throws BadConfigException {
+ String registryQuorum = getConfig().get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM);
+ if (SliderUtils.isUnset(registryQuorum)) {
+ throw new BadConfigException(
+ "No Zookeeper quorum provided in the"
+ + " configuration property " + SliderXmlConfKeys.REGISTRY_ZK_QUORUM
+ );
+ }
+ ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
+ return registryQuorum;
+ }
+
+ /**
+ * Start the registration service
+ * @param zkConnection
+ * @param zkPath
+ * @return
+ */
+ public RegistryBinderService<ServiceInstanceData> startRegistrationService(
+ String zkConnection, String zkPath) {
+ CuratorHelper curatorHelper =
+ new CuratorHelper(getConfig(), zkConnection);
+
+ //registry will start curator as well as the binder, in the correct order
+ RegistryBinderService<ServiceInstanceData> registryBinderService =
+ curatorHelper.createRegistryBinderService(zkPath);
+ boolean started = deployChildService(registryBinderService);
+ return registryBinderService;
+ }
+
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/ClosingService.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/ClosingService.java
new file mode 100644
index 0000000..7642336
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/ClosingService.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.service.AbstractService;
+
+import java.io.Closeable;
+
+/**
+ * Service that closes the closeable supplied during shutdown, if not null.
+ */
+public class ClosingService extends AbstractService {
+
+ private Closeable closeable;
+
+
+ public ClosingService(String name,
+ Closeable closeable) {
+ super(name);
+ this.closeable = closeable;
+ }
+
+
+ public Closeable getCloseable() {
+ return closeable;
+ }
+
+ public void setCloseable(Closeable closeable) {
+ this.closeable = closeable;
+ }
+
+ /**
+ * Stop routine will close the closeable -if not null - and set the
+ * reference to null afterwards
+ * @throws Exception
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ IOUtils.closeStream(closeable);
+ closeable = null;
+
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/CompoundLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/CompoundLaunchedService.java
new file mode 100644
index 0000000..80d857d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/CompoundLaunchedService.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.core.main.RunService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CompoundLaunchedService extends CompoundService
+ implements RunService {
+ private static final Logger log = LoggerFactory.getLogger(
+ CompoundLaunchedService.class);
+ private String[] argv;
+
+ public CompoundLaunchedService(String name) {
+ super(name);
+ }
+
+ public CompoundLaunchedService() {
+ super("CompoundLaunchedService");
+ }
+
+ public CompoundLaunchedService(Service... children) {
+ super(children);
+ }
+
+ /**
+ * Implementation of set-ness, groovy definition of true/false for a string
+ * @param s
+ * @return
+ */
+ protected static boolean isUnset(String s) {
+ return SliderUtils.isUnset(s);
+ }
+
+ protected static boolean isSet(String s) {
+ return SliderUtils.isSet(s);
+ }
+
+ protected String[] getArgv() {
+ return argv;
+ }
+
+ /**
+ * Pre-init argument binding
+ * @param config the initial configuration build up by the
+ * service launcher.
+ * @param args argument list list of arguments passed to the command line
+ * after any launcher-specific commands have been stripped.
+ * @return the configuration
+ * @throws Exception
+ */
+ @Override
+ public Configuration bindArgs(Configuration config, String... args) throws
+ Exception {
+ this.argv = args;
+ if (log.isDebugEnabled()) {
+ log.debug("Binding {} Arguments:", args.length);
+
+ StringBuilder builder = new StringBuilder();
+ for (String arg : args) {
+ builder.append('"').append(arg).append("\" ");
+ }
+ log.debug(builder.toString());
+ }
+ return config;
+ }
+
+ @Override
+ public int runService() throws Throwable {
+ return LauncherExitCodes.EXIT_SUCCESS;
+ }
+
+ @Override
+ public void addService(Service service) {
+ Preconditions.checkNotNull(service, "null service");
+ super.addService(service);
+ }
+
+ /**
+ * Run a child service -initing and starting it if this
+ * service has already passed those parts of its own lifecycle
+ * @param service the service to start
+ */
+ protected boolean deployChildService(Service service) {
+ service.init(getConfig());
+ addService(service);
+ if (isInState(STATE.STARTED)) {
+ service.start();
+ return true;
+ }
+ return false;
+ }
+
+ protected void requireArgumentSet(String argname, String argfield)
+ throws BadCommandArgumentsException {
+ if (isUnset(argfield)) {
+ throw new BadCommandArgumentsException("Required argument "
+ + argname
+ + " missing");
+ }
+ }
+
+ protected void requireArgumentSet(String argname, Object argfield) throws
+ BadCommandArgumentsException {
+ if (argfield == null) {
+ throw new BadCommandArgumentsException("Required argument "
+ + argname
+ + " missing");
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/CompoundService.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/CompoundService.java
new file mode 100644
index 0000000..00a1988
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/CompoundService.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceStateChangeListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * An extended composite service which not only makes the
+ * addService method public, it auto-registers
+ * itself as a listener for state change events.
+ *
+ * When all child services has stopped, this service stops itself.
+ */
+public class CompoundService extends CompositeService implements Parent,
+ ServiceStateChangeListener {
+
+ private static final Logger log =
+ LoggerFactory.getLogger(CompoundService.class);
+
+ public CompoundService(String name) {
+ super(name);
+ }
+
+
+ public CompoundService() {
+ super("CompoundService");
+ }
+
+ /**
+ * Varargs constructor
+ * @param children children
+ */
+ public CompoundService(Service ... children) {
+ this();
+ for (Service child : children) {
+ addService(child);
+ }
+ }
+
+ /**
+ * Add a service, and register it
+ * @param service the {@link Service} to be added.
+ * Important: do not add a service to a parent during your own serviceInit/start,
+ * in Hadoop 2.2; you will trigger a ConcurrentModificationException.
+ */
+ @Override
+ public void addService(Service service) {
+ service.registerServiceListener(this);
+ super.addService(service);
+ }
+
+ /**
+ * When this service is started, any service stopping with a failure
+ * exception is converted immediately into a failure of this service,
+ * storing the failure and stopping ourselves.
+ * @param child the service that has changed.
+ */
+ @Override
+ public void stateChanged(Service child) {
+ //if that child stopped while we are running:
+ if (isInState(STATE.STARTED) && child.isInState(STATE.STOPPED)) {
+ // a child service has stopped
+ //did the child fail? if so: propagate
+ Throwable failureCause = child.getFailureCause();
+ if (failureCause != null) {
+ log.info("Child service " + child + " failed", failureCause);
+ //failure. Convert to an exception
+ Exception e = SliderServiceUtils.convertToException(failureCause);
+ //flip ourselves into the failed state
+ noteFailure(e);
+ stop();
+ } else {
+ log.info("Child service completed {}", child);
+ if (areAllChildrenStopped()) {
+ log.info("All children are halted: stopping");
+ stop();
+ }
+ }
+ }
+ }
+
+ private boolean areAllChildrenStopped() {
+ List<Service> children = getServices();
+ boolean stopped = true;
+ for (Service child : children) {
+ if (!child.isInState(STATE.STOPPED)) {
+ stopped = false;
+ break;
+ }
+ }
+ return stopped;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/EventCallback.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/EventCallback.java
new file mode 100644
index 0000000..ec4a1e3
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/EventCallback.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+public interface EventCallback {
+
+ public void eventCallbackEvent();
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/EventNotifyingService.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/EventNotifyingService.java
new file mode 100644
index 0000000..28027fc
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/EventNotifyingService.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+import org.apache.hadoop.service.AbstractService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A service that calls the supplied callback when it is started -after the
+ * given delay, then stops itself.
+ * Because it calls in on a different thread, it can be used for callbacks
+ * that don't
+ */
+public class EventNotifyingService extends AbstractService implements Runnable {
+ protected static final Logger log =
+ LoggerFactory.getLogger(EventNotifyingService.class);
+ private final EventCallback callback;
+ private final int delay;
+
+ public EventNotifyingService(EventCallback callback, int delay) {
+ super("EventNotifyingService");
+ assert callback != null;
+ this.callback = callback;
+ this.delay = delay;
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ log.debug("Notifying {} after a delay of {} millis", callback, delay);
+ new Thread(this, "event").start();
+ }
+
+ @Override
+ public void run() {
+ if (delay > 0) {
+ try {
+ Thread.sleep(delay);
+ } catch (InterruptedException e) {
+
+ }
+ }
+ log.debug("Notifying {}", callback);
+ callback.eventCallbackEvent();
+ stop();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/ForkedProcessService.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/ForkedProcessService.java
new file mode 100644
index 0000000..f90bb7b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/ForkedProcessService.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.main.ExitCodeProvider;
+import org.apache.slider.core.main.ServiceLaunchException;
+import org.apache.slider.server.exec.ApplicationEventHandler;
+import org.apache.slider.server.exec.RunLongLivedApp;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Service wrapper for an external program that is launched and can/will terminate.
+ * This service is notified when the subprocess terminates, and stops itself
+ * and converts a non-zero exit code into a failure exception
+ */
+public class ForkedProcessService extends AbstractService implements
+ ApplicationEventHandler,
+ ExitCodeProvider,
+ Runnable {
+
+ /**
+ * Log for the forked master process
+ */
+ protected static final Logger log =
+ LoggerFactory.getLogger(ForkedProcessService.class);
+
+ private final String name;
+ private final AtomicBoolean processTerminated = new AtomicBoolean(false);
+ ;
+ private boolean processStarted = false;
+ private RunLongLivedApp process;
+ private Map<String, String> environment;
+ private List<String> commands;
+ private String commandLine;
+ private int executionTimeout = -1;
+ private int timeoutCode = 1;
+
+ /**
+ * Exit code set when the spawned process exits
+ */
+ private AtomicInteger exitCode = new AtomicInteger(0);
+ private Thread timeoutThread;
+
+ public ForkedProcessService(String name) {
+ super(name);
+ this.name = name;
+ }
+
+ @Override //AbstractService
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ }
+
+ @Override //AbstractService
+ protected void serviceStart() throws Exception {
+ if (process == null) {
+ throw new ServiceStateException("Subprocess not yet configured");
+ }
+ //now spawn the process -expect updates via callbacks
+ process.spawnApplication();
+ }
+
+ @Override //AbstractService
+ protected void serviceStop() throws Exception {
+ completed(0);
+ if (process != null) {
+ process.stop();
+ }
+ }
+
+ /**
+ * Set the timeout by which time a process must have finished -or -1 for forever
+ * @param timeout timeout in milliseconds
+ */
+ public void setTimeout(int timeout, int code) {
+ this.executionTimeout = timeout;
+ this.timeoutCode = code;
+ }
+
+ /**
+ * Build the process to execute when the service is started
+ * @param commands list of commands is inserted on the front
+ * @param env environment variables above those generated by
+ * @throws IOException IO problems
+ * @throws SliderException anything internal
+ */
+ public void build(Map<String, String> environment,
+ List<String> commands) throws
+ IOException,
+ SliderException {
+ assert process == null;
+ this.commands = commands;
+ this.commandLine = SliderUtils.join(commands, " ", false);
+ this.environment = environment;
+ process = new RunLongLivedApp(log, commands);
+ process.setApplicationEventHandler(this);
+ //set the env variable mapping
+ process.putEnvMap(environment);
+ }
+
+ @Override // ApplicationEventHandler
+ public synchronized void onApplicationStarted(RunLongLivedApp application) {
+ log.info("Process has started");
+ processStarted = true;
+ if (executionTimeout > 0) {
+ timeoutThread = new Thread(this);
+ timeoutThread.start();
+ }
+ }
+
+ @Override // ApplicationEventHandler
+ public void onApplicationExited(RunLongLivedApp application,
+ int exitC) {
+ synchronized (this) {
+ completed(exitC);
+ //note whether or not the service had already stopped
+ log.info("Process has exited with exit code {}", exitC);
+ if (exitC != 0) {
+ reportFailure(exitC, name + " failed with code " +
+ exitC);
+ }
+ }
+ //now stop itself
+ if (!isInState(STATE.STOPPED)) {
+ stop();
+ }
+ }
+
+ private void reportFailure(int exitC, String text) {
+ this.exitCode.set(exitC);
+ //error
+ ServiceLaunchException execEx =
+ new ServiceLaunchException(exitC,
+ text);
+ log.debug("Noting failure", execEx);
+ noteFailure(execEx);
+ }
+
+ /**
+ * handle timeout response by escalating it to a failure
+ */
+ @Override
+ public void run() {
+ try {
+ synchronized (processTerminated) {
+ if (!processTerminated.get()) {
+ processTerminated.wait(executionTimeout);
+ }
+ }
+
+ } catch (InterruptedException e) {
+ //assume signalled; exit
+ }
+ //check the status; if the marker isn't true, bail
+ if (!processTerminated.getAndSet(true)) {
+ log.info("process timeout: reporting error code {}", timeoutCode);
+
+ //timeout
+ if (isInState(STATE.STARTED)) {
+ //trigger a failure
+ process.stop();
+ }
+ reportFailure(timeoutCode, name + ": timeout after " + executionTimeout
+ + " millis: exit code =" + timeoutCode);
+ }
+ }
+
+ protected void completed(int exitCode) {
+ this.exitCode.set(exitCode);
+ processTerminated.set(true);
+ synchronized (processTerminated) {
+ processTerminated.notify();
+ }
+ }
+
+ public boolean isProcessTerminated() {
+ return processTerminated.get();
+ }
+
+ public synchronized boolean isProcessStarted() {
+ return processStarted;
+ }
+
+
+ @Override // ExitCodeProvider
+ public int getExitCode() {
+ return exitCode.get();
+ }
+
+ public String getCommandLine() {
+ return commandLine;
+ }
+
+ /**
+ * Get the recent output from the process, or [] if not defined
+ * @return a possibly empty list
+ */
+ public List<String> getRecentOutput() {
+ return process != null
+ ? process.getRecentOutput()
+ : new LinkedList<String>();
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/Parent.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/Parent.java
new file mode 100644
index 0000000..1d83477
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/Parent.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+import org.apache.hadoop.service.Service;
+
+import java.util.List;
+
+/**
+ * Interface that services with public methods to manipulate child services
+ * should implement
+ */
+public interface Parent extends Service {
+
+ void addService(Service service);
+
+ /**
+ * Get an unmodifiable list of services
+ * @return a list of child services at the time of invocation -
+ * added services will not be picked up.
+ */
+ List<Service> getServices();
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/RpcService.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/RpcService.java
new file mode 100644
index 0000000..bccfba2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/RpcService.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.service.AbstractService;
+
+import java.net.InetSocketAddress;
+
+/**
+ * A YARN service that maps the start/stop lifecycle of an RPC server
+ * to the YARN service lifecycle
+ */
+public class RpcService extends AbstractService {
+
+ /** RPC server*/
+ private final Server server;
+
+ /**
+ * Construct an instance
+ * @param server server to manger
+ */
+ public RpcService(Server server) {
+ super("RpcService");
+ this.server = server;
+ }
+
+ public Server getServer() {
+ return server;
+ }
+
+ public InetSocketAddress getConnectAddress() {
+ return NetUtils.getConnectAddress(server);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ server.start();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (server != null) {
+ server.stop();
+ }
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/SecurityCheckerService.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/SecurityCheckerService.java
new file mode 100644
index 0000000..76a5761
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/SecurityCheckerService.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.slider.common.tools.SliderUtils;
+
+/**
+ * A security checker service, which validates that the service
+ * is running with security in its init() operation.
+ */
+public class SecurityCheckerService extends AbstractService {
+
+ public SecurityCheckerService() {
+ super("Security Checker");
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ SliderUtils.initProcessSecurity(conf);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/SequenceService.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/SequenceService.java
new file mode 100644
index 0000000..0725fe6
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/SequenceService.java
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceStateChangeListener;
+import org.apache.hadoop.service.ServiceStateException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * This resembles the YARN CompositeService, except that it
+ * starts one service after another: it's init & start operations
+ * only work with one service
+ */
+
+public class SequenceService extends AbstractService implements Parent,
+ ServiceStateChangeListener {
+
+ private static final Logger log =
+ LoggerFactory.getLogger(SequenceService.class);
+
+ /**
+ * list of services
+ */
+ private final List<Service> serviceList = new ArrayList<Service>();
+
+ /**
+ * The current service.
+ * Volatile -may change & so should be read into a
+ * local variable before working with
+ */
+ private volatile Service currentService;
+ /*
+ the previous service -the last one that finished.
+ Null if one did not finish yet
+ */
+ private volatile Service previousService;
+
+ /**
+ * Create a service sequence with the given list of services
+ * @param name service name
+ * @param offspring initial sequence
+ */
+ public SequenceService(String name, Service...offspring) {
+ super(name);
+ for (Service service : offspring) {
+ addService(service);
+ }
+ }
+
+ /**
+ * Get the current service -which may be null
+ * @return service running
+ */
+ public Service getCurrentService() {
+ return currentService;
+ }
+
+ public Service getPreviousService() {
+ return previousService;
+ }
+
+ /**
+ * When started
+ * @throws Exception
+ */
+ @Override
+ protected void serviceStart() throws Exception {
+ startNextService();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ //stop current service.
+ //this triggers a callback that is caught and ignored
+ Service current = currentService;
+ previousService = current;
+ currentService = null;
+ if (current != null) {
+ current.stop();
+ }
+ }
+
+
+ /**
+ * Start the next service in the list.
+ * Return false if there are no more services to run, or this
+ * service has stopped
+ * @return true if a service was started
+ * @throws RuntimeException from any init or start failure
+ * @throws ServiceStateException if this call is made before
+ * the service is started
+ */
+ public synchronized boolean startNextService() {
+ if (isInState(STATE.STOPPED)) {
+ //downgrade to a failed
+ log.debug("Not starting next service -{} is stopped", this);
+ return false;
+ }
+ if (!isInState(STATE.STARTED)) {
+ //reject attempts to start a service too early
+ throw new ServiceStateException(
+ "Cannot start a child service when not started");
+ }
+ if (serviceList.isEmpty()) {
+ //nothing left to run
+ return false;
+ }
+ if (currentService != null && currentService.getFailureCause() != null) {
+ //did the last service fail? Is this caused by some premature callback?
+ log.debug("Not starting next service due to a failure of {}",
+ currentService);
+ return false;
+ }
+ //bear in mind that init & start can fail, which
+ //can trigger re-entrant calls into the state change listener.
+ //by setting the current service to null
+ //the start-next-service logic is skipped.
+ //now, what does that mean w.r.t exit states?
+
+ currentService = null;
+ Service head = serviceList.remove(0);
+
+ try {
+ head.init(getConfig());
+ head.registerServiceListener(this);
+ head.start();
+ } catch (RuntimeException e) {
+ noteFailure(e);
+ throw e;
+ }
+ //at this point the service must have explicitly started & not failed,
+ //else an exception would have been raised
+ currentService = head;
+ return true;
+ }
+
+ /**
+ * State change event relays service stop events to
+ * {@link #onServiceCompleted(Service)}. Subclasses can
+ * extend that with extra logic
+ * @param service the service that has changed.
+ */
+ @Override
+ public void stateChanged(Service service) {
+ if (service == currentService && service.isInState(STATE.STOPPED)) {
+ onServiceCompleted(service);
+ }
+ }
+
+ /**
+ * handler for service completion: base class starts the next service
+ * @param service service that has completed
+ */
+ protected synchronized void onServiceCompleted(Service service) {
+ log.info("Running service stopped: {}", service);
+ previousService = currentService;
+
+
+ //start the next service if we are not stopped ourselves
+ if (isInState(STATE.STARTED)) {
+
+ //did the service fail? if so: propagate
+ Throwable failureCause = service.getFailureCause();
+ if (failureCause != null) {
+ Exception e = SliderServiceUtils.convertToException(failureCause);
+ noteFailure(e);
+ stop();
+ }
+
+ //start the next service
+ boolean started;
+ try {
+ started = startNextService();
+ } catch (Exception e) {
+ //something went wrong here
+ noteFailure(e);
+ started = false;
+ }
+ if (!started) {
+ //no start because list is empty
+ //stop and expect the notification to go upstream
+ stop();
+ }
+ } else {
+ //not started, so just note that the current service
+ //has gone away
+ currentService = null;
+ }
+ }
+
+ /**
+ * Add the passed {@link Service} to the list of services managed by this
+ * {@link SequenceService}
+ * @param service the {@link Service} to be added
+ */
+ @Override //Parent
+ public synchronized void addService(Service service) {
+ log.debug("Adding service {} ", service.getName());
+ synchronized (serviceList) {
+ serviceList.add(service);
+ }
+ }
+
+ /**
+ * Get an unmodifiable list of services
+ * @return a list of child services at the time of invocation -
+ * added services will not be picked up.
+ */
+ @Override //Parent
+ public synchronized List<Service> getServices() {
+ return Collections.unmodifiableList(serviceList);
+ }
+
+ @Override // Object
+ public synchronized String toString() {
+ return super.toString() + "; current service " + currentService
+ + "; queued service count=" + serviceList.size();
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/SliderServiceUtils.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/SliderServiceUtils.java
new file mode 100644
index 0000000..2f080f7
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/SliderServiceUtils.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+public class SliderServiceUtils {
+
+ public static Exception convertToException(Throwable failureCause) {
+ return (failureCause instanceof Exception) ?
+ (Exception)failureCause
+ : new Exception(failureCause);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/WebAppService.java b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/WebAppService.java
new file mode 100644
index 0000000..b98643b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/docstore/utility/WebAppService.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.webapp.WebApp;
+
+/**
+ * Contains a webapp reference and stops it in teardown if non-null
+ *
+ * It does not start the application.
+ * Access to the field is not synchronized across threads; it is the
+ * responsibility of the caller.
+ */
+public class WebAppService<T extends WebApp> extends AbstractService {
+
+ private T webApp;
+
+ public WebAppService(String name) {
+ super(name);
+ }
+
+ public WebAppService(String name, T app) {
+ super(name);
+ webApp = app;
+ }
+
+ public T getWebApp() {
+ return webApp;
+ }
+
+ public void setWebApp(T webApp) {
+ this.webApp = webApp;
+ }
+
+
+ @Override
+ protected void serviceStart() throws Exception {
+
+ }
+
+ /**
+ * Stop operation stops the webapp; sets the reference to null
+ * @throws Exception
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ if (webApp != null) {
+ webApp.stop();
+ webApp = null;
+ }
+ }
+}
diff --git a/slider-core/src/main/proto/SliderClusterMessages.proto b/slider-core/src/main/proto/SliderClusterMessages.proto
new file mode 100644
index 0000000..00da2b8
--- /dev/null
+++ b/slider-core/src/main/proto/SliderClusterMessages.proto
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and stable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *stable* .proto interface.
+ */
+
+// This file contains protocol buffers that are used throughout HDFS -- i.e.
+// by the client, server, and data transfer protocols.
+
+option java_package = "org.apache.slider.api.proto";
+option java_outer_classname = "Messages";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package org.apache.slider.api;
+
+//import "Security.proto";
+
+message RoleInstanceState {
+ required string name = 1;
+ optional string role = 2;
+ required uint32 state = 4;
+ required uint32 exitCode = 5;
+ optional string command = 6;
+ optional string diagnostics = 7;
+ repeated string output = 8;
+ repeated string environment = 9;
+ required uint32 roleId = 10;
+ required bool released = 11;
+ required int64 createTime = 12;
+ required int64 startTime = 13;
+ required string host = 14;
+ required string hostURL = 15;
+}
+
+/**
+ * stop the cluster
+ */
+message StopClusterRequestProto {
+ /**
+ message to include
+ */
+ required string message = 1;
+}
+/**
+ * stop the cluster
+ */
+message StopClusterResponseProto {
+}
+
+/**
+ * flex the cluster
+ */
+message FlexClusterRequestProto {
+ required string clusterSpec = 1;
+}
+
+
+/**
+ * stop the cluster
+ */
+message FlexClusterResponseProto {
+ required bool response = 1;
+}
+
+
+/**
+ * void request
+ */
+message GetJSONClusterStatusRequestProto {
+}
+
+/**
+ * response
+ */
+message GetJSONClusterStatusResponseProto {
+ required string clusterSpec = 1;
+}
+
+/**
+ * list the nodes in a role
+ */
+message ListNodeUUIDsByRoleRequestProto {
+ required string role = 1;
+}
+
+/**
+ * list the nodes in a role
+ */
+message ListNodeUUIDsByRoleResponseProto {
+ repeated string uuid = 1 ;
+}
+
+/**
+ * get a node
+ */
+message GetNodeRequestProto {
+ required string uuid = 1;
+}
+
+
+/**
+ * response on a node
+ */
+message GetNodeResponseProto {
+ required RoleInstanceState clusterNode = 1 ;
+}
+
+/**
+ * list the nodes for the UUDs
+ */
+message GetClusterNodesRequestProto {
+ repeated string uuid = 1 ;
+}
+
+/**
+ * list the nodes in a role
+ */
+message GetClusterNodesResponseProto {
+ repeated RoleInstanceState clusterNode = 1 ;
+}
+
+/**
+ * Echo
+ */
+message EchoRequestProto {
+ required string text = 1;
+}
+
+/**
+ * Echo reply
+ */
+message EchoResponseProto {
+ required string text = 1;
+}
+
+
+/**
+ * Kill a container
+ */
+message KillContainerRequestProto {
+ required string id = 1;
+}
+
+/**
+ * Kill reply
+ */
+message KillContainerResponseProto {
+ required bool success = 1;
+}
+
+/**
+ * AM suicide
+ */
+message AMSuicideRequestProto {
+ required string text = 1;
+ required int32 signal = 2;
+ required int32 delay = 3;
+}
+
+/**
+ * AM suicide reply. For this to be returned implies
+ * a failure of the AM to kill itself
+ */
+message AMSuicideResponseProto {
+
+}
+
+
+/**
+ * Ask for the instance definition details
+ */
+message GetInstanceDefinitionRequestProto {
+
+}
+
+/**
+ * Get the definition back as three separate JSON strings
+ */
+message GetInstanceDefinitionResponseProto {
+ required string internal = 1;
+ required string resources = 2;
+ required string application = 3;
+}
diff --git a/slider-core/src/main/proto/SliderClusterProtocol.proto b/slider-core/src/main/proto/SliderClusterProtocol.proto
new file mode 100644
index 0000000..a2a1001
--- /dev/null
+++ b/slider-core/src/main/proto/SliderClusterProtocol.proto
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and stable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *stable* .proto interface.
+ */
+
+// This file contains protocol buffers that are used throughout HDFS -- i.e.
+// by the client, server, and data transfer protocols.
+
+option java_package = "org.apache.slider.api.proto";
+option java_outer_classname = "SliderClusterAPI";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package org.apache.slider.api;
+
+//import "Security.proto";
+import "SliderClusterMessages.proto";
+
+
+/**
+ * Protocol used from between Slider Client and AM
+ */
+service SliderClusterProtocolPB {
+
+ /**
+ * Stop the cluster
+ */
+
+ rpc stopCluster(StopClusterRequestProto)
+ returns(StopClusterResponseProto);
+
+ /**
+ * Flex the cluster.
+ */
+ rpc flexCluster(FlexClusterRequestProto)
+ returns(FlexClusterResponseProto);
+
+ /**
+ * Get the current cluster status
+ */
+ rpc getJSONClusterStatus(GetJSONClusterStatusRequestProto)
+ returns(GetJSONClusterStatusResponseProto);
+
+ /**
+ * Get the instance definition
+ */
+ rpc getInstanceDefinition(GetInstanceDefinitionRequestProto)
+ returns(GetInstanceDefinitionResponseProto);
+
+ /**
+ * List all running nodes in a role
+ */
+ rpc listNodeUUIDsByRole(ListNodeUUIDsByRoleRequestProto)
+ returns(ListNodeUUIDsByRoleResponseProto);
+
+ /**
+ * Get the details on a node
+ */
+ rpc getNode(GetNodeRequestProto)
+ returns(GetNodeResponseProto);
+
+ /**
+ * Get the
+ * details on a list of nodes.
+ * Unknown nodes are not returned
+ * <i>Important: the order of the results are undefined</i>
+ */
+ rpc getClusterNodes(GetClusterNodesRequestProto)
+ returns(GetClusterNodesResponseProto);
+
+ /**
+ * echo some text
+ */
+ rpc echo(EchoRequestProto)
+ returns(EchoResponseProto);
+
+ /**
+ * kill a container
+ */
+ rpc killContainer(KillContainerRequestProto)
+ returns(KillContainerResponseProto);
+
+ /**
+ * kill the AM
+ */
+ rpc amSuicide(AMSuicideRequestProto)
+ returns(AMSuicideResponseProto);
+
+}
diff --git a/slider-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/slider-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
new file mode 100644
index 0000000..9e67c15
--- /dev/null
+++ b/slider-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.slider.server.appmaster.rpc.SliderRPCSecurityInfo
diff --git a/slider-core/src/main/resources/org/apache/slider/log4j.properties b/slider-core/src/main/resources/org/apache/slider/log4j.properties
new file mode 100644
index 0000000..65a7ad0
--- /dev/null
+++ b/slider-core/src/main/resources/org/apache/slider/log4j.properties
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is a log4j config for slider
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{3} (%F:%M(%L)) - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=%c{1}: %m%n
+
+
+#at debug this provides details on what is going on
+log4j.logger.org.apache.slider=DEBUG
+#log4j.logger.org.apache.slider.exec.RunLongLivedApp=ERROR
+
+log4j.logger.org.apache.hadoop.security=DEBUG
+log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+#crank back on some noise
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.hadoop.yarn.client.RMProxy=WARN
+
+# for test runs we don't care about native code
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+# HDFS is noise on tets
+log4j.logger.org.apache.hadoop.hdfs.server.datanode=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.namenode=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+log4j.logger.org.apache.zookeeper=WARN
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/agent.txt b/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/agent.txt
new file mode 100644
index 0000000..79c1972
--- /dev/null
+++ b/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/agent.txt
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+This is the conf directory for the python agent
\ No newline at end of file
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json b/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json
new file mode 100644
index 0000000..1bc8381
--- /dev/null
+++ b/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command.json
@@ -0,0 +1,169 @@
+{
+ "roleCommand": "START",
+ "clusterName": "c1",
+ "hostname": "c6402.ambari.apache.org",
+ "hostLevelParams": {
+ "java_home": "/usr/jdk64/jdk1.7.0_45"
+ },
+ "commandType": "EXECUTION_COMMAND",
+ "roleParams": {},
+ "serviceName": "HBASE",
+ "role": "HBASE_MASTER",
+ "commandParams": {},
+ "taskId": 24,
+ "public_hostname": "c6402.ambari.apache.org",
+ "configurations": {
+ "hbase-log4j": {
+ "log4j.threshold": "ALL",
+ "log4j.rootLogger": "${hbase.root.logger}",
+ "log4j.logger.org.apache.zookeeper": "INFO",
+ "log4j.logger.org.apache.hadoop.hbase": "DEBUG",
+ "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher": "INFO",
+ "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil": "INFO",
+ "log4j.category.SecurityLogger": "${hbase.security.logger}",
+ "log4j.appender.console": "org.apache.log4j.ConsoleAppender",
+ "log4j.appender.console.target": "System.err",
+ "log4j.appender.console.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.console.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "log4j.appender.RFAS": "org.apache.log4j.RollingFileAppender",
+ "log4j.appender.RFAS.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.RFAS.layout.ConversionPattern": "%d{ISO8601} %p %c: %m%n",
+ "log4j.appender.RFAS.MaxFileSize": "${hbase.security.log.maxfilesize}",
+ "log4j.appender.RFAS.MaxBackupIndex": "${hbase.security.log.maxbackupindex}",
+ "log4j.appender.RFAS.File": "${hbase.log.dir}/${hbase.security.log.file}",
+ "log4j.appender.RFA": "org.apache.log4j.RollingFileAppender",
+ "log4j.appender.RFA.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.RFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "log4j.appender.RFA.MaxFileSize": "${hbase.log.maxfilesize}",
+ "log4j.appender.RFA.MaxBackupIndex": "${hbase.log.maxbackupindex}",
+ "log4j.appender.RFA.File": "${hbase.log.dir}/${hbase.log.file}",
+ "log4j.appender.NullAppender": "org.apache.log4j.varia.NullAppender",
+ "log4j.appender.DRFA": "org.apache.log4j.DailyRollingFileAppender",
+ "log4j.appender.DRFA.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.DRFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "log4j.appender.DRFA.File": "${hbase.log.dir}/${hbase.log.file}",
+ "log4j.appender.DRFA.DatePattern": ".yyyy-MM-dd",
+ "log4j.additivity.SecurityLogger": "false",
+ "hbase.security.logger": "INFO,console",
+ "hbase.security.log.maxfilesize": "256MB",
+ "hbase.security.log.maxbackupindex": "20",
+ "hbase.security.log.file": "SecurityAuth.audit",
+ "hbase.root.logger": "INFO,console",
+ "hbase.log.maxfilesize": "256MB",
+ "hbase.log.maxbackupindex": "20",
+ "hbase.log.file": "hbase.log",
+ "hbase.log.dir": "."
+ },
+ "global": {
+ "hbase_root": "/share/hbase/hbase-0.96.1-hadoop2",
+ "security_enabled": "false",
+ "hbase_pid_dir": "/var/run/hbase",
+ "proxyuser_group": "users",
+ "syncLimit": "5",
+ "hbase_regionserver_heapsize": "1024m",
+ "rca_enabled": "false",
+ "tickTime": "2000",
+ "hbase_master_heapsize": "1024m",
+ "initLimit": "10",
+ "user_group": "hadoop",
+ "hbase_user": "hbase",
+ "hbase_log_dir": "/var/log/hbase"
+ },
+ "hdfs-site": {
+ "dfs.namenode.checkpoint.period": "21600",
+ "dfs.namenode.avoid.write.stale.datanode": "true",
+ "dfs.namenode.checkpoint.txns": "1000000",
+ "dfs.block.access.token.enable": "true",
+ "dfs.support.append": "true",
+ "dfs.datanode.address": "0.0.0.0:${ambari.dfs.datanode.port}",
+ "dfs.cluster.administrators": " hdfs",
+ "dfs.replication": "3",
+ "ambari.dfs.datanode.http.port": "50075",
+ "dfs.datanode.balance.bandwidthPerSec": "6250000",
+ "dfs.namenode.safemode.threshold-pct": "1.0f",
+ "dfs.namenode.checkpoint.edits.dir": "${dfs.namenode.checkpoint.dir}",
+ "dfs.permissions.enabled": "true",
+ "dfs.client.read.shortcircuit": "true",
+ "dfs.namenode.https-address": "c6402.ambari.apache.org:50470",
+ "dfs.journalnode.edits.dir": "/grid/0/hdfs/journal",
+ "dfs.blocksize": "134217728",
+ "dfs.datanode.max.transfer.threads": "1024",
+ "dfs.datanode.du.reserved": "1073741824",
+ "dfs.webhdfs.enabled": "true",
+ "dfs.namenode.handler.count": "100",
+ "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary",
+ "fs.permissions.umask-mode": "022",
+ "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.ipc.address": "0.0.0.0:8010",
+ "dfs.datanode.data.dir": "/hadoop/hdfs/data",
+ "dfs.namenode.http-address": "c6402.ambari.apache.org:50070",
+ "dfs.blockreport.initialDelay": "120",
+ "dfs.datanode.failed.volumes.tolerated": "0",
+ "dfs.namenode.accesstime.precision": "0",
+ "ambari.dfs.datanode.port": "50010",
+ "dfs.namenode.avoid.read.stale.datanode": "true",
+ "dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090",
+ "dfs.namenode.stale.datanode.interval": "30000",
+ "dfs.heartbeat.interval": "3",
+ "dfs.client.read.shortcircuit.streams.cache.size": "4096",
+ "dfs.permissions.superusergroup": "hdfs",
+ "dfs.https.port": "50470",
+ "dfs.journalnode.http-address": "0.0.0.0:8480",
+ "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket",
+ "dfs.namenode.write.stale.datanode.ratio": "1.0f",
+ "dfs.hosts.exclude": "/etc/hadoop/conf/dfs.exclude",
+ "dfs.datanode.data.dir.perm": "750",
+ "dfs.namenode.name.dir.restore": "true",
+ "dfs.replication.max": "50",
+ "dfs.namenode.name.dir": "/hadoop/hdfs/namenode"
+ },
+ "hbase-site": {
+ "hbase.hstore.flush.retries.number": "120",
+ "hbase.client.keyvalue.maxsize": "10485760",
+ "hbase.hstore.compactionThreshold": "3",
+ "hbase.rootdir": "hdfs://c6402.ambari.apache.org:8020/apps/hbase/data",
+ "hbase.stagingdir": "hdfs://c6402.ambari.apache.org:8020/apps/hbase/staging",
+ "hbase.regionserver.handler.count": "60",
+ "hbase.regionserver.global.memstore.lowerLimit": "0.38",
+ "hbase.hregion.memstore.block.multiplier": "2",
+ "hbase.hregion.memstore.flush.size": "134217728",
+ "hbase.superuser": "hbase",
+ "hbase.zookeeper.property.clientPort": "2181",
+ "hbase.regionserver.global.memstore.upperLimit": "0.4",
+ "zookeeper.session.timeout": "30000",
+ "hbase.tmp.dir": "/hadoop/hbase",
+ "hbase.hregion.max.filesize": "10737418240",
+ "hfile.block.cache.size": "0.40",
+ "hbase.security.authentication": "simple",
+ "hbase.defaults.for.version.skip": "true",
+ "hbase.zookeeper.quorum": "c6402.ambari.apache.org",
+ "zookeeper.znode.parent": "/hbase-unsecure",
+ "hbase.hstore.blockingStoreFiles": "10",
+ "hbase.hregion.majorcompaction": "86400000",
+ "hbase.security.authorization": "false",
+ "hbase.cluster.distributed": "true",
+ "hbase.hregion.memstore.mslab.enabled": "true",
+ "hbase.client.scanner.caching": "100",
+ "hbase.zookeeper.useMulti": "true",
+ "hbase.regionserver.info.port": "0",
+ "hbase.master.info.port": "60010"
+ },
+ "core-site": {
+ "io.serializations": "org.apache.hadoop.io.serializer.WritableSerialization",
+ "gluster.daemon.user": "null",
+ "fs.trash.interval": "360",
+ "hadoop.security.authentication": "simple",
+ "io.compression.codecs": "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec",
+ "mapreduce.jobtracker.webinterface.trusted": "false",
+ "fs.AbstractFileSystem.glusterfs.impl": "null",
+ "fs.defaultFS": "hdfs://c6402.ambari.apache.org:8020",
+ "ipc.client.connect.max.retries": "50",
+ "ipc.client.idlethreshold": "8000",
+ "io.file.buffer.size": "131072",
+ "hadoop.security.authorization": "false",
+ "hadoop.security.auth_to_local": "\n RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\n RULE:[2:$1@$0](jhs@.*)s/.*/mapred/\n RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\n RULE:[2:$1@$0](hm@.*)s/.*/hbase/\n RULE:[2:$1@$0](rs@.*)s/.*/hbase/\n DEFAULT",
+ "ipc.client.connection.maxidletime": "30000"
+ }
+ },
+ "commandId": "2-2"
+}
\ No newline at end of file
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json b/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
new file mode 100644
index 0000000..cab952e
--- /dev/null
+++ b/slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
@@ -0,0 +1,169 @@
+{
+ "roleCommand": "{{COMMAND}}",
+ "clusterName": "{{CLUSTER_NAME}}",
+ "hostname": "{{HOST_NAME}}",
+ "hostLevelParams": {
+ "java_home": "/usr/jdk64/jdk1.7.0_45"
+ },
+ "commandType": "EXECUTION_COMMAND",
+ "roleParams": {},
+ "serviceName": "{{SERVICE_NAME}}",
+ "role": "{{ROLE_NAME}}",
+ "commandParams": {},
+ "taskId": "{{TASK_ID}}",
+ "public_hostname": "{{HOST_NAME}}",
+ "configurations": {
+ "hbase-log4j": {
+ "log4j.threshold": "ALL",
+ "log4j.rootLogger": "${hbase.root.logger}",
+ "log4j.logger.org.apache.zookeeper": "INFO",
+ "log4j.logger.org.apache.hadoop.hbase": "DEBUG",
+ "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher": "INFO",
+ "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil": "INFO",
+ "log4j.category.SecurityLogger": "${hbase.security.logger}",
+ "log4j.appender.console": "org.apache.log4j.ConsoleAppender",
+ "log4j.appender.console.target": "System.err",
+ "log4j.appender.console.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.console.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "log4j.appender.RFAS": "org.apache.log4j.RollingFileAppender",
+ "log4j.appender.RFAS.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.RFAS.layout.ConversionPattern": "%d{ISO8601} %p %c: %m%n",
+ "log4j.appender.RFAS.MaxFileSize": "${hbase.security.log.maxfilesize}",
+ "log4j.appender.RFAS.MaxBackupIndex": "${hbase.security.log.maxbackupindex}",
+ "log4j.appender.RFAS.File": "${hbase.log.dir}/${hbase.security.log.file}",
+ "log4j.appender.RFA": "org.apache.log4j.RollingFileAppender",
+ "log4j.appender.RFA.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.RFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "log4j.appender.RFA.MaxFileSize": "${hbase.log.maxfilesize}",
+ "log4j.appender.RFA.MaxBackupIndex": "${hbase.log.maxbackupindex}",
+ "log4j.appender.RFA.File": "${hbase.log.dir}/${hbase.log.file}",
+ "log4j.appender.NullAppender": "org.apache.log4j.varia.NullAppender",
+ "log4j.appender.DRFA": "org.apache.log4j.DailyRollingFileAppender",
+ "log4j.appender.DRFA.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.DRFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "log4j.appender.DRFA.File": "${hbase.log.dir}/${hbase.log.file}",
+ "log4j.appender.DRFA.DatePattern": ".yyyy-MM-dd",
+ "log4j.additivity.SecurityLogger": "false",
+ "hbase.security.logger": "INFO,console",
+ "hbase.security.log.maxfilesize": "256MB",
+ "hbase.security.log.maxbackupindex": "20",
+ "hbase.security.log.file": "SecurityAuth.audit",
+ "hbase.root.logger": "INFO,console",
+ "hbase.log.maxfilesize": "256MB",
+ "hbase.log.maxbackupindex": "20",
+ "hbase.log.file": "hbase.log",
+ "hbase.log.dir": "."
+ },
+ "global": {
+ "hbase_root": "{{HBASE_HOME}}",
+ "security_enabled": "false",
+ "hbase_pid_dir": "{{PID_DIR}}",
+ "proxyuser_group": "users",
+ "syncLimit": "5",
+ "hbase_regionserver_heapsize": "{{REGION_SERVER_HEAP_SIZE}}",
+ "rca_enabled": "false",
+ "tickTime": "2000",
+ "hbase_master_heapsize": "{{MASTER_HEAP_SIZE}}",
+ "initLimit": "10",
+ "user_group": "{{GROUP_NAME}}",
+ "hbase_user": "{{USER_NAME}}",
+ "hbase_log_dir": "{{LOG_DIR}}"
+ },
+ "hdfs-site": {
+ "dfs.namenode.checkpoint.period": "21600",
+ "dfs.namenode.avoid.write.stale.datanode": "true",
+ "dfs.namenode.checkpoint.txns": "1000000",
+ "dfs.block.access.token.enable": "true",
+ "dfs.support.append": "true",
+ "dfs.datanode.address": "0.0.0.0:${ambari.dfs.datanode.port}",
+ "dfs.cluster.administrators": " hdfs",
+ "dfs.replication": "3",
+ "ambari.dfs.datanode.http.port": "50075",
+ "dfs.datanode.balance.bandwidthPerSec": "6250000",
+ "dfs.namenode.safemode.threshold-pct": "1.0f",
+ "dfs.namenode.checkpoint.edits.dir": "${dfs.namenode.checkpoint.dir}",
+ "dfs.permissions.enabled": "true",
+ "dfs.client.read.shortcircuit": "true",
+ "dfs.namenode.https-address": "{{NAMENODE_HTTPS_ADDRESS}}",
+ "dfs.journalnode.edits.dir": "/grid/0/hdfs/journal",
+ "dfs.blocksize": "134217728",
+ "dfs.datanode.max.transfer.threads": "1024",
+ "dfs.datanode.du.reserved": "1073741824",
+ "dfs.webhdfs.enabled": "true",
+ "dfs.namenode.handler.count": "100",
+ "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary",
+ "fs.permissions.umask-mode": "022",
+ "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.ipc.address": "0.0.0.0:8010",
+ "dfs.datanode.data.dir": "/hadoop/hdfs/data",
+ "dfs.namenode.http-address": "{{NAMENODE_HTTP_ADDRESS}}",
+ "dfs.blockreport.initialDelay": "120",
+ "dfs.datanode.failed.volumes.tolerated": "0",
+ "dfs.namenode.accesstime.precision": "0",
+ "ambari.dfs.datanode.port": "50010",
+ "dfs.namenode.avoid.read.stale.datanode": "true",
+ "dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090",
+ "dfs.namenode.stale.datanode.interval": "30000",
+ "dfs.heartbeat.interval": "3",
+ "dfs.client.read.shortcircuit.streams.cache.size": "4096",
+ "dfs.permissions.superusergroup": "hdfs",
+ "dfs.https.port": "50470",
+ "dfs.journalnode.http-address": "0.0.0.0:8480",
+ "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket",
+ "dfs.namenode.write.stale.datanode.ratio": "1.0f",
+ "dfs.hosts.exclude": "/etc/hadoop/conf/dfs.exclude",
+ "dfs.datanode.data.dir.perm": "750",
+ "dfs.namenode.name.dir.restore": "true",
+ "dfs.replication.max": "50",
+ "dfs.namenode.name.dir": "/hadoop/hdfs/namenode"
+ },
+ "hbase-site": {
+ "hbase.hstore.flush.retries.number": "120",
+ "hbase.client.keyvalue.maxsize": "10485760",
+ "hbase.hstore.compactionThreshold": "3",
+ "hbase.rootdir": "{{HBASE_ROOT_DIR}}",
+ "hbase.stagingdir": "{{HBASE_STAGING_DIR}}",
+ "hbase.regionserver.handler.count": "60",
+ "hbase.regionserver.global.memstore.lowerLimit": "0.38",
+ "hbase.hregion.memstore.block.multiplier": "2",
+ "hbase.hregion.memstore.flush.size": "134217728",
+ "hbase.superuser": "{{HBASE_SUPERUSER}}",
+ "hbase.zookeeper.property.clientPort": "{{ZK_CLIENT_PORT}}",
+ "hbase.regionserver.global.memstore.upperLimit": "0.4",
+ "zookeeper.session.timeout": "30000",
+ "hbase.tmp.dir": "/hadoop/hbase",
+ "hbase.hregion.max.filesize": "10737418240",
+ "hfile.block.cache.size": "0.40",
+ "hbase.security.authentication": "simple",
+ "hbase.defaults.for.version.skip": "true",
+ "hbase.zookeeper.quorum": "{{ZK_HOSTS}}",
+ "zookeeper.znode.parent": "{{ZK_NODE_PARENT}}",
+ "hbase.hstore.blockingStoreFiles": "10",
+ "hbase.hregion.majorcompaction": "86400000",
+ "hbase.security.authorization": "false",
+ "hbase.cluster.distributed": "true",
+ "hbase.hregion.memstore.mslab.enabled": "true",
+ "hbase.client.scanner.caching": "100",
+ "hbase.zookeeper.useMulti": "true",
+ "hbase.regionserver.info.port": "{{REGION_SERVER_INFO_PORT}}",
+ "hbase.master.info.port": "{{MASTER_INFO_PORT}}"
+ },
+ "core-site": {
+ "io.serializations": "org.apache.hadoop.io.serializer.WritableSerialization",
+ "gluster.daemon.user": "null",
+ "fs.trash.interval": "360",
+ "hadoop.security.authentication": "simple",
+ "io.compression.codecs": "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec",
+ "mapreduce.jobtracker.webinterface.trusted": "false",
+ "fs.AbstractFileSystem.glusterfs.impl": "null",
+ "fs.defaultFS": "{{DEFAULT_FS}}",
+ "ipc.client.connect.max.retries": "50",
+ "ipc.client.idlethreshold": "8000",
+ "io.file.buffer.size": "131072",
+ "hadoop.security.authorization": "false",
+ "hadoop.security.auth_to_local": "\n RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\n RULE:[2:$1@$0](jhs@.*)s/.*/mapred/\n RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\n RULE:[2:$1@$0](hm@.*)s/.*/hbase/\n RULE:[2:$1@$0](rs@.*)s/.*/hbase/\n DEFAULT",
+ "ipc.client.connection.maxidletime": "30000"
+ }
+ },
+ "commandId": "{{COMMAND_ID}}"
+}
\ No newline at end of file
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/agent/role-node.xml b/slider-core/src/main/resources/org/apache/slider/providers/agent/role-node.xml
new file mode 100644
index 0000000..aff1e05
--- /dev/null
+++ b/slider-core/src/main/resources/org/apache/slider/providers/agent/role-node.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+ <!--
+ Role options for an agent-managed node
+ -->
+<configuration>
+ <property>
+ <name>role.name</name>
+ <value>node</value>
+ </property>
+
+ <property>
+ <name>role.instances</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>role.priority</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>role.placement.policy</name>
+ <value>2</value>
+ </property>
+
+ <property>
+ <name>yarn.memory</name>
+ <value>256</value>
+ </property>
+
+ <property>
+ <name>yarn.vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>jvm.heapsize</name>
+ <value>256M</value>
+ </property>
+
+ <property>
+ <name>env.MALLOC_ARENA_MAX</name>
+ <value>4</value>
+ </property>
+
+</configuration>
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/dynamic/application.properties b/slider-core/src/main/resources/org/apache/slider/providers/dynamic/application.properties
new file mode 100644
index 0000000..d9b42de
--- /dev/null
+++ b/slider-core/src/main/resources/org/apache/slider/providers/dynamic/application.properties
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# gets updated at build time
+application.name=${pom.name}
+application.version=${pom.version}
+application.build=${buildNumber}
+application.build.java.version=${java.version}
+application.build.user=${user.name}
+application.build.info=${pom.name}-${pom.version} Built against commit# ${buildNumber} on Java ${java.version} by ${user.name}
+hadoop.build.info=${hadoop.version}
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
new file mode 100644
index 0000000..f28159a
--- /dev/null
+++ b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
@@ -0,0 +1,19 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+
+
+ },
+
+ "global": {
+ "env.MALLOC_ARENA_MAX": "4"
+ },
+
+ "components": {
+ "slider-appmaster" : {
+ "jvm.heapsize": "256M"
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/internal.json b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/internal.json
new file mode 100644
index 0000000..2367d8f
--- /dev/null
+++ b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/internal.json
@@ -0,0 +1,17 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ },
+
+ "global": {
+ "internal.container.failure.shortlife": "60000",
+ "internal.container.failure.threshold": "5",
+ "slider.cluster.directory.permissions": "0770",
+ "slider.data.directory.permissions": "0770"
+ },
+
+ "components": {
+
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json
new file mode 100644
index 0000000..d76e5e7
--- /dev/null
+++ b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/resources.json
@@ -0,0 +1,18 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+
+ },
+
+ "global": {
+ },
+
+ "components": {
+ "slider-appmaster": {
+ "component.instances": "1",
+ "yarn.vcores": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/main/resources/org/apache/slider/slider.xml b/slider-core/src/main/resources/org/apache/slider/slider.xml
new file mode 100644
index 0000000..eb4ecc0
--- /dev/null
+++ b/slider-core/src/main/resources/org/apache/slider/slider.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <property>
+ <name>slider.config.loaded</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>slider.provider.hbase</name>
+ <value>org.apache.slider.providers.hbase.HBaseProviderFactory</value>
+ </property>
+ <property>
+ <name>slider.provider.accumulo</name>
+ <value>org.apache.slider.providers.accumulo.AccumuloProviderFactory</value>
+ </property>
+ <property>
+ <name>slider.provider.agent</name>
+ <value>org.apache.slider.providers.agent.AgentProviderFactory</value>
+ </property>
+</configuration>
diff --git a/slider-core/src/main/resources/webapps/slideram/.keep b/slider-core/src/main/resources/webapps/slideram/.keep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/slideram/.keep
diff --git a/slider-core/src/main/resources/webapps/static/busy.gif b/slider-core/src/main/resources/webapps/static/busy.gif
new file mode 100644
index 0000000..058ce1f
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/busy.gif
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/css/demo_page.css b/slider-core/src/main/resources/webapps/static/dt-1.9.4/css/demo_page.css
new file mode 100644
index 0000000..51c9448
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/css/demo_page.css
@@ -0,0 +1,93 @@
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * General page setup
+ */
+#dt_example {
+ font: 80%/1.45em "Lucida Grande", Verdana, Arial, Helvetica, sans-serif;
+ margin: 0;
+ padding: 0;
+ color: #333;
+ background-color: #fff;
+}
+
+
+#dt_example #container {
+ width: 800px;
+ margin: 30px auto;
+ padding: 0;
+}
+
+
+#dt_example #footer {
+ margin: 50px auto 0 auto;
+ padding: 0;
+}
+
+#dt_example #demo {
+ margin: 30px auto 0 auto;
+}
+
+#dt_example .demo_jui {
+ margin: 30px auto 0 auto;
+}
+
+#dt_example .big {
+ font-size: 1.3em;
+ font-weight: bold;
+ line-height: 1.6em;
+ color: #4E6CA3;
+}
+
+#dt_example .spacer {
+ height: 20px;
+ clear: both;
+}
+
+#dt_example .clear {
+ clear: both;
+}
+
+#dt_example pre {
+ padding: 15px;
+ background-color: #F5F5F5;
+ border: 1px solid #CCCCCC;
+}
+
+#dt_example h1 {
+ margin-top: 2em;
+ font-size: 1.3em;
+ font-weight: normal;
+ line-height: 1.6em;
+ color: #4E6CA3;
+ border-bottom: 1px solid #B0BED9;
+ clear: both;
+}
+
+#dt_example h2 {
+ font-size: 1.2em;
+ font-weight: normal;
+ line-height: 1.6em;
+ color: #4E6CA3;
+ clear: both;
+}
+
+#dt_example a {
+ color: #0063DC;
+ text-decoration: none;
+}
+
+#dt_example a:hover {
+ text-decoration: underline;
+}
+
+#dt_example ul {
+ color: #4E6CA3;
+}
+
+.css_right {
+ float: right;
+}
+
+.css_left {
+ float: left;
+}
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/css/demo_table.css b/slider-core/src/main/resources/webapps/static/dt-1.9.4/css/demo_table.css
new file mode 100644
index 0000000..3bc0433
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/css/demo_table.css
@@ -0,0 +1,538 @@
+/*
+ * File: demo_table.css
+ * CVS: $Id$
+ * Description: CSS descriptions for DataTables demo pages
+ * Author: Allan Jardine
+ * Created: Tue May 12 06:47:22 BST 2009
+ * Modified: $Date$ by $Author$
+ * Language: CSS
+ * Project: DataTables
+ *
+ * Copyright 2009 Allan Jardine. All Rights Reserved.
+ *
+ * ***************************************************************************
+ * DESCRIPTION
+ *
+ * The styles given here are suitable for the demos that are used with the standard DataTables
+ * distribution (see www.datatables.net). You will most likely wish to modify these styles to
+ * meet the layout requirements of your site.
+ *
+ * Common issues:
+ * 'full_numbers' pagination - I use an extra selector on the body tag to ensure that there is
+ * no conflict between the two pagination types. If you want to use full_numbers pagination
+ * ensure that you either have "example_alt_pagination" as a body class name, or better yet,
+ * modify that selector.
+ * Note that the path used for Images is relative. All images are by default located in
+ * ../images/ - relative to this CSS file.
+ */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables features
+ */
+
+.dataTables_wrapper {
+ position: relative;
+ min-height: 302px;
+ clear: both;
+ _height: 302px;
+ zoom: 1; /* Feeling sorry for IE */
+}
+
+.dataTables_processing {
+ position: absolute;
+ top: 50%;
+ left: 50%;
+ width: 250px;
+ height: 30px;
+ margin-left: -125px;
+ margin-top: -15px;
+ padding: 14px 0 2px 0;
+ border: 1px solid #ddd;
+ text-align: center;
+ color: #999;
+ font-size: 14px;
+ background-color: white;
+}
+
+.dataTables_length {
+ width: 40%;
+ float: left;
+}
+
+.dataTables_filter {
+ width: 50%;
+ float: right;
+ text-align: right;
+}
+
+.dataTables_info {
+ width: 60%;
+ float: left;
+}
+
+.dataTables_paginate {
+ width: 44px;
+ * width: 50px;
+ float: right;
+ text-align: right;
+}
+
+/* Pagination nested */
+.paginate_disabled_previous, .paginate_enabled_previous, .paginate_disabled_next, .paginate_enabled_next {
+ height: 19px;
+ width: 19px;
+ margin-left: 3px;
+ float: left;
+}
+
+.paginate_disabled_previous {
+ background-image: url('../images/back_disabled.jpg');
+}
+
+.paginate_enabled_previous {
+ background-image: url('../images/back_enabled.jpg');
+}
+
+.paginate_disabled_next {
+ background-image: url('../images/forward_disabled.jpg');
+}
+
+.paginate_enabled_next {
+ background-image: url('../images/forward_enabled.jpg');
+}
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables display
+ */
+table.display {
+ margin: 0 auto;
+ clear: both;
+ width: 100%;
+
+ /* Note Firefox 3.5 and before have a bug with border-collapse
+ * ( https://bugzilla.mozilla.org/show%5Fbug.cgi?id=155955 )
+ * border-spacing: 0; is one possible option. Conditional-css.com is
+ * useful for this kind of thing
+ *
+ * Further note IE 6/7 has problems when calculating widths with border width.
+ * It subtracts one px relative to the other browsers from the first column, and
+ * adds one to the end...
+ *
+ * If you want that effect I'd suggest setting a border-top/left on th/td's and
+ * then filling in the gaps with other borders.
+ */
+}
+
+table.display thead th {
+ padding: 3px 18px 3px 10px;
+ border-bottom: 1px solid black;
+ font-weight: bold;
+ cursor: pointer;
+ * cursor: hand;
+}
+
+table.display tfoot th {
+ padding: 3px 18px 3px 10px;
+ border-top: 1px solid black;
+ font-weight: bold;
+}
+
+table.display tr.heading2 td {
+ border-bottom: 1px solid #aaa;
+}
+
+table.display td {
+ padding: 3px 10px;
+}
+
+table.display td.center {
+ text-align: center;
+}
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables sorting
+ */
+
+.sorting_asc {
+ background: url('../images/sort_asc.png') no-repeat center right;
+}
+
+.sorting_desc {
+ background: url('../images/sort_desc.png') no-repeat center right;
+}
+
+.sorting {
+ background: url('../images/sort_both.png') no-repeat center right;
+}
+
+.sorting_asc_disabled {
+ background: url('../images/sort_asc_disabled.png') no-repeat center right;
+}
+
+.sorting_desc_disabled {
+ background: url('../images/sort_desc_disabled.png') no-repeat center right;
+}
+
+
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables row classes
+ */
+table.display tr.odd.gradeA {
+ background-color: #ddffdd;
+}
+
+table.display tr.even.gradeA {
+ background-color: #eeffee;
+}
+
+table.display tr.odd.gradeC {
+ background-color: #ddddff;
+}
+
+table.display tr.even.gradeC {
+ background-color: #eeeeff;
+}
+
+table.display tr.odd.gradeX {
+ background-color: #ffdddd;
+}
+
+table.display tr.even.gradeX {
+ background-color: #ffeeee;
+}
+
+table.display tr.odd.gradeU {
+ background-color: #ddd;
+}
+
+table.display tr.even.gradeU {
+ background-color: #eee;
+}
+
+
+tr.odd {
+ background-color: #E2E4FF;
+}
+
+tr.even {
+ background-color: white;
+}
+
+
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Misc
+ */
+.dataTables_scroll {
+ clear: both;
+}
+
+.dataTables_scrollBody {
+ *margin-top: -1px;
+}
+
+.top, .bottom {
+ padding: 15px;
+ background-color: #F5F5F5;
+ border: 1px solid #CCCCCC;
+}
+
+.top .dataTables_info {
+ float: none;
+}
+
+.clear {
+ clear: both;
+}
+
+.dataTables_empty {
+ text-align: center;
+}
+
+tfoot input {
+ margin: 0.5em 0;
+ width: 100%;
+ color: #444;
+}
+
+tfoot input.search_init {
+ color: #999;
+}
+
+td.group {
+ background-color: #d1cfd0;
+ border-bottom: 2px solid #A19B9E;
+ border-top: 2px solid #A19B9E;
+}
+
+td.details {
+ background-color: #d1cfd0;
+ border: 2px solid #A19B9E;
+}
+
+
+.example_alt_pagination div.dataTables_info {
+ width: 40%;
+}
+
+.paging_full_numbers {
+ width: 400px;
+ height: 22px;
+ line-height: 22px;
+}
+
+.paging_full_numbers span.paginate_button,
+ .paging_full_numbers span.paginate_active {
+ border: 1px solid #aaa;
+ -webkit-border-radius: 5px;
+ -moz-border-radius: 5px;
+ padding: 2px 5px;
+ margin: 0 3px;
+ cursor: pointer;
+ *cursor: hand;
+}
+
+.paging_full_numbers span.paginate_button {
+ background-color: #ddd;
+}
+
+.paging_full_numbers span.paginate_button:hover {
+ background-color: #ccc;
+}
+
+.paging_full_numbers span.paginate_active {
+ background-color: #99B3FF;
+}
+
+table.display tr.even.row_selected td {
+ background-color: #B0BED9;
+}
+
+table.display tr.odd.row_selected td {
+ background-color: #9FAFD1;
+}
+
+
+/*
+ * Sorting classes for columns
+ */
+/* For the standard odd/even */
+tr.odd td.sorting_1 {
+ background-color: #D3D6FF;
+}
+
+tr.odd td.sorting_2 {
+ background-color: #DADCFF;
+}
+
+tr.odd td.sorting_3 {
+ background-color: #E0E2FF;
+}
+
+tr.even td.sorting_1 {
+ background-color: #EAEBFF;
+}
+
+tr.even td.sorting_2 {
+ background-color: #F2F3FF;
+}
+
+tr.even td.sorting_3 {
+ background-color: #F9F9FF;
+}
+
+
+/* For the Conditional-CSS grading rows */
+/*
+ Colour calculations (based off the main row colours)
+ Level 1:
+ dd > c4
+ ee > d5
+ Level 2:
+ dd > d1
+ ee > e2
+ */
+tr.odd.gradeA td.sorting_1 {
+ background-color: #c4ffc4;
+}
+
+tr.odd.gradeA td.sorting_2 {
+ background-color: #d1ffd1;
+}
+
+tr.odd.gradeA td.sorting_3 {
+ background-color: #d1ffd1;
+}
+
+tr.even.gradeA td.sorting_1 {
+ background-color: #d5ffd5;
+}
+
+tr.even.gradeA td.sorting_2 {
+ background-color: #e2ffe2;
+}
+
+tr.even.gradeA td.sorting_3 {
+ background-color: #e2ffe2;
+}
+
+tr.odd.gradeC td.sorting_1 {
+ background-color: #c4c4ff;
+}
+
+tr.odd.gradeC td.sorting_2 {
+ background-color: #d1d1ff;
+}
+
+tr.odd.gradeC td.sorting_3 {
+ background-color: #d1d1ff;
+}
+
+tr.even.gradeC td.sorting_1 {
+ background-color: #d5d5ff;
+}
+
+tr.even.gradeC td.sorting_2 {
+ background-color: #e2e2ff;
+}
+
+tr.even.gradeC td.sorting_3 {
+ background-color: #e2e2ff;
+}
+
+tr.odd.gradeX td.sorting_1 {
+ background-color: #ffc4c4;
+}
+
+tr.odd.gradeX td.sorting_2 {
+ background-color: #ffd1d1;
+}
+
+tr.odd.gradeX td.sorting_3 {
+ background-color: #ffd1d1;
+}
+
+tr.even.gradeX td.sorting_1 {
+ background-color: #ffd5d5;
+}
+
+tr.even.gradeX td.sorting_2 {
+ background-color: #ffe2e2;
+}
+
+tr.even.gradeX td.sorting_3 {
+ background-color: #ffe2e2;
+}
+
+tr.odd.gradeU td.sorting_1 {
+ background-color: #c4c4c4;
+}
+
+tr.odd.gradeU td.sorting_2 {
+ background-color: #d1d1d1;
+}
+
+tr.odd.gradeU td.sorting_3 {
+ background-color: #d1d1d1;
+}
+
+tr.even.gradeU td.sorting_1 {
+ background-color: #d5d5d5;
+}
+
+tr.even.gradeU td.sorting_2 {
+ background-color: #e2e2e2;
+}
+
+tr.even.gradeU td.sorting_3 {
+ background-color: #e2e2e2;
+}
+
+
+/*
+ * Row highlighting example
+ */
+.ex_highlight #example tbody tr.even:hover, #example tbody tr.even td.highlighted {
+ background-color: #ECFFB3;
+}
+
+.ex_highlight #example tbody tr.odd:hover, #example tbody tr.odd td.highlighted {
+ background-color: #E6FF99;
+}
+
+.ex_highlight_row #example tr.even:hover {
+ background-color: #ECFFB3;
+}
+
+.ex_highlight_row #example tr.even:hover td.sorting_1 {
+ background-color: #DDFF75;
+}
+
+.ex_highlight_row #example tr.even:hover td.sorting_2 {
+ background-color: #E7FF9E;
+}
+
+.ex_highlight_row #example tr.even:hover td.sorting_3 {
+ background-color: #E2FF89;
+}
+
+.ex_highlight_row #example tr.odd:hover {
+ background-color: #E6FF99;
+}
+
+.ex_highlight_row #example tr.odd:hover td.sorting_1 {
+ background-color: #D6FF5C;
+}
+
+.ex_highlight_row #example tr.odd:hover td.sorting_2 {
+ background-color: #E0FF84;
+}
+
+.ex_highlight_row #example tr.odd:hover td.sorting_3 {
+ background-color: #DBFF70;
+}
+
+
+/*
+ * KeyTable
+ */
+table.KeyTable td {
+ border: 3px solid transparent;
+}
+
+table.KeyTable td.focus {
+ border: 3px solid #3366FF;
+}
+
+table.display tr.gradeA {
+ background-color: #eeffee;
+}
+
+table.display tr.gradeC {
+ background-color: #ddddff;
+}
+
+table.display tr.gradeX {
+ background-color: #ffdddd;
+}
+
+table.display tr.gradeU {
+ background-color: #ddd;
+}
+
+div.box {
+ height: 100px;
+ padding: 10px;
+ overflow: auto;
+ border: 1px solid #8080FF;
+ background-color: #E5E5FF;
+}
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/css/jui-dt.css b/slider-core/src/main/resources/webapps/static/dt-1.9.4/css/jui-dt.css
new file mode 100644
index 0000000..89bd2f0
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/css/jui-dt.css
@@ -0,0 +1,322 @@
+/*
+ * File: demo_table_jui.css
+ * CVS: $Id$
+ * Description: CSS descriptions for DataTables demo pages
+ * Author: Allan Jardine
+ * Created: Tue May 12 06:47:22 BST 2009
+ * Modified: $Date$ by $Author$
+ * Language: CSS
+ * Project: DataTables
+ *
+ * Copyright 2009 Allan Jardine. All Rights Reserved.
+ *
+ * ***************************************************************************
+ * DESCRIPTION
+ *
+ * The styles given here are suitable for the demos that are used with the standard DataTables
+ * distribution (see www.datatables.net). You will most likely wish to modify these styles to
+ * meet the layout requirements of your site.
+ *
+ * Common issues:
+ * 'full_numbers' pagination - I use an extra selector on the body tag to ensure that there is
+ * no conflict between the two pagination types. If you want to use full_numbers pagination
+ * ensure that you either have "example_alt_pagination" as a body class name, or better yet,
+ * modify that selector.
+ * Note that the path used for Images is relative. All images are by default located in
+ * ../images/ - relative to this CSS file.
+ */
+
+
+/*
+ * jQuery UI specific styling
+ */
+
+.paging_two_button .ui-button {
+ float: left;
+ cursor: pointer;
+ * cursor: hand;
+}
+
+.paging_full_numbers .ui-button {
+ padding: 2px 6px;
+ margin: 0;
+ cursor: pointer;
+ * cursor: hand;
+}
+
+.ui-buttonset .ui-button {
+ margin-right: -0.1em !important;
+}
+
+.paging_full_numbers {
+ width: 350px !important;
+}
+
+.ui-toolbar {
+ padding: 5px;
+}
+
+.dataTables_paginate {
+ width: auto;
+}
+
+.dataTables_info {
+ padding-top: 3px;
+}
+
+table.display thead th {
+ padding: 3px 0px 3px 10px;
+ cursor: pointer;
+ * cursor: hand;
+}
+
+div.dataTables_wrapper .ui-widget-header {
+ font-weight: normal;
+}
+
+
+/*
+ * Sort arrow icon positioning
+ */
+table.display thead th div.DataTables_sort_wrapper {
+ position: relative;
+ padding-right: 20px;
+ padding-right: 20px;
+}
+
+table.display thead th div.DataTables_sort_wrapper span {
+ position: absolute;
+ top: 50%;
+ margin-top: -8px;
+ right: 0;
+}
+
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ *
+ * Everything below this line is the same as demo_table.css. This file is
+ * required for 'cleanliness' of the markup
+ *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables features
+ */
+
+.dataTables_wrapper {
+ position: relative;
+ min-height: 302px;
+ _height: 302px;
+ clear: both;
+}
+
+.dataTables_processing {
+ position: absolute;
+ top: 0px;
+ left: 50%;
+ width: 250px;
+ margin-left: -125px;
+ border: 1px solid #ddd;
+ text-align: center;
+ color: #999;
+ font-size: 11px;
+ padding: 2px 0;
+}
+
+.dataTables_length {
+ width: 40%;
+ float: left;
+}
+
+.dataTables_filter {
+ width: 50%;
+ float: right;
+ text-align: right;
+}
+
+.dataTables_info {
+ width: 50%;
+ float: left;
+}
+
+.dataTables_paginate {
+ float: right;
+ text-align: right;
+}
+
+/* Pagination nested */
+.paginate_disabled_previous, .paginate_enabled_previous, .paginate_disabled_next, .paginate_enabled_next {
+ height: 19px;
+ width: 19px;
+ margin-left: 3px;
+ float: left;
+}
+
+.paginate_disabled_previous {
+ background-image: url('../images/back_disabled.jpg');
+}
+
+.paginate_enabled_previous {
+ background-image: url('../images/back_enabled.jpg');
+}
+
+.paginate_disabled_next {
+ background-image: url('../images/forward_disabled.jpg');
+}
+
+.paginate_enabled_next {
+ background-image: url('../images/forward_enabled.jpg');
+}
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables display
+ */
+table.display {
+ margin: 0 auto;
+ width: 100%;
+ clear: both;
+ border-collapse: collapse;
+}
+
+table.display tfoot th {
+ padding: 3px 0px 3px 10px;
+ font-weight: bold;
+ font-weight: normal;
+}
+
+table.display tr.heading2 td {
+ border-bottom: 1px solid #aaa;
+}
+
+table.display td {
+ padding: 3px 10px;
+}
+
+table.display td.center {
+ text-align: center;
+}
+
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * DataTables sorting
+ */
+
+.sorting_asc {
+ background: url('../images/sort_asc.jpg') no-repeat center right;
+}
+
+.sorting_desc {
+ background: url('../images/sort_desc.jpg') no-repeat center right;
+}
+
+.sorting {
+ background: url('../images/sort_both.jpg') no-repeat center right;
+}
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Misc
+ */
+.dataTables_scroll {
+ clear: both;
+}
+
+.top, .bottom {
+ padding: 15px;
+ background-color: #F5F5F5;
+ border: 1px solid #CCCCCC;
+}
+
+.top .dataTables_info {
+ float: none;
+}
+
+.clear {
+ clear: both;
+}
+
+.dataTables_empty {
+ text-align: center;
+}
+
+tfoot input {
+ margin: 0.5em 0;
+ width: 100%;
+ color: #444;
+}
+
+tfoot input.search_init {
+ color: #999;
+}
+
+td.group {
+ background-color: #d1cfd0;
+ border-bottom: 2px solid #A19B9E;
+ border-top: 2px solid #A19B9E;
+}
+
+td.details {
+ background-color: #d1cfd0;
+ border: 2px solid #A19B9E;
+}
+
+
+.example_alt_pagination div.dataTables_info {
+ width: 40%;
+}
+
+.paging_full_numbers span.paginate_button,
+ .paging_full_numbers span.paginate_active {
+ border: 1px solid #aaa;
+ -webkit-border-radius: 5px;
+ -moz-border-radius: 5px;
+ padding: 2px 5px;
+ margin: 0 3px;
+ cursor: pointer;
+ *cursor: hand;
+}
+
+.paging_full_numbers span.paginate_button {
+ background-color: #ddd;
+}
+
+.paging_full_numbers span.paginate_button:hover {
+ background-color: #ccc;
+}
+
+.paging_full_numbers span.paginate_active {
+ background-color: #99B3FF;
+}
+
+table.display tr.even.row_selected td {
+ background-color: #B0BED9;
+}
+
+table.display tr.odd.row_selected td {
+ background-color: #9FAFD1;
+}
+
+/* Striping */
+tr.odd { background: rgba(255, 255, 255, 0.1); }
+tr.even { background: rgba(0, 0, 255, 0.05); }
+
+
+/*
+ * Sorting classes for columns
+ */
+tr.odd td.sorting_1 { background: rgba(0, 0, 0, 0.03); }
+tr.odd td.sorting_2 { background: rgba(0, 0, 0, 0.02); }
+tr.odd td.sorting_3 { background: rgba(0, 0, 0, 0.02); }
+tr.even td.sorting_1 { background: rgba(0, 0, 0, 0.08); }
+tr.even td.sorting_2 { background: rgba(0, 0, 0, 0.06); }
+tr.even td.sorting_3 { background: rgba(0, 0, 0, 0.06); }
+
+.css_left { position: relative; float: left; }
+.css_right { position: relative; float: right; }
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/back_disabled.jpg b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/back_disabled.jpg
new file mode 100644
index 0000000..1e73a54
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/back_disabled.jpg
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/back_enabled.jpg b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/back_enabled.jpg
new file mode 100644
index 0000000..a6d764c
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/back_enabled.jpg
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/favicon.ico b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/favicon.ico
new file mode 100644
index 0000000..6eeaa2a
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/favicon.ico
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/forward_disabled.jpg b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/forward_disabled.jpg
new file mode 100644
index 0000000..28a9dc5
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/forward_disabled.jpg
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/forward_enabled.jpg b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/forward_enabled.jpg
new file mode 100644
index 0000000..598c075
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/forward_enabled.jpg
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_asc.png b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_asc.png
new file mode 100644
index 0000000..a56d0e2
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_asc.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_asc_disabled.png b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_asc_disabled.png
new file mode 100644
index 0000000..b7e621e
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_asc_disabled.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_both.png b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_both.png
new file mode 100644
index 0000000..839ac4b
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_both.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_desc.png b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_desc.png
new file mode 100644
index 0000000..90b2951
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_desc.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_desc_disabled.png b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_desc_disabled.png
new file mode 100644
index 0000000..2409653
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/images/sort_desc_disabled.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/dt-1.9.4/js/jquery.dataTables.min.js b/slider-core/src/main/resources/webapps/static/dt-1.9.4/js/jquery.dataTables.min.js
new file mode 100644
index 0000000..a2405bf
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/dt-1.9.4/js/jquery.dataTables.min.js
@@ -0,0 +1,157 @@
+/*
+ * File: jquery.dataTables.min.js
+ * Version: 1.9.4
+ * Author: Allan Jardine (www.sprymedia.co.uk)
+ * Info: www.datatables.net
+ *
+ * Copyright 2008-2012 Allan Jardine, all rights reserved.
+ *
+ * This source file is free software, under either the GPL v2 license or a
+ * BSD style license, available at:
+ * http://datatables.net/license_gpl2
+ * http://datatables.net/license_bsd
+ *
+ * This source file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
+ */
+(function(la,s,p){(function(i){if(typeof define==="function"&&define.amd)define(["jquery"],i);else jQuery&&!jQuery.fn.dataTable&&i(jQuery)})(function(i){var l=function(h){function n(a,b){var c=l.defaults.columns,d=a.aoColumns.length;b=i.extend({},l.models.oColumn,c,{sSortingClass:a.oClasses.sSortable,sSortingClassJUI:a.oClasses.sSortJUI,nTh:b?b:s.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.oDefaults:d});a.aoColumns.push(b);if(a.aoPreSearchCols[d]===
+p||a.aoPreSearchCols[d]===null)a.aoPreSearchCols[d]=i.extend({},l.models.oSearch);else{b=a.aoPreSearchCols[d];if(b.bRegex===p)b.bRegex=true;if(b.bSmart===p)b.bSmart=true;if(b.bCaseInsensitive===p)b.bCaseInsensitive=true}q(a,d,null)}function q(a,b,c){var d=a.aoColumns[b];if(c!==p&&c!==null){if(c.mDataProp&&!c.mData)c.mData=c.mDataProp;if(c.sType!==p){d.sType=c.sType;d._bAutoType=false}i.extend(d,c);r(d,c,"sWidth","sWidthOrig");if(c.iDataSort!==p)d.aDataSort=[c.iDataSort];r(d,c,"aDataSort")}var e=d.mRender?
+ca(d.mRender):null,f=ca(d.mData);d.fnGetData=function(g,j){var k=f(g,j);if(d.mRender&&j&&j!=="")return e(k,j,g);return k};d.fnSetData=Ja(d.mData);if(!a.oFeatures.bSort)d.bSortable=false;if(!d.bSortable||i.inArray("asc",d.asSorting)==-1&&i.inArray("desc",d.asSorting)==-1){d.sSortingClass=a.oClasses.sSortableNone;d.sSortingClassJUI=""}else if(i.inArray("asc",d.asSorting)==-1&&i.inArray("desc",d.asSorting)==-1){d.sSortingClass=a.oClasses.sSortable;d.sSortingClassJUI=a.oClasses.sSortJUI}else if(i.inArray("asc",
+d.asSorting)!=-1&&i.inArray("desc",d.asSorting)==-1){d.sSortingClass=a.oClasses.sSortableAsc;d.sSortingClassJUI=a.oClasses.sSortJUIAscAllowed}else if(i.inArray("asc",d.asSorting)==-1&&i.inArray("desc",d.asSorting)!=-1){d.sSortingClass=a.oClasses.sSortableDesc;d.sSortingClassJUI=a.oClasses.sSortJUIDescAllowed}}function o(a){if(a.oFeatures.bAutoWidth===false)return false;ta(a);for(var b=0,c=a.aoColumns.length;b<c;b++)a.aoColumns[b].nTh.style.width=a.aoColumns[b].sWidth}function v(a,b){a=A(a,"bVisible");
+return typeof a[b]==="number"?a[b]:null}function w(a,b){a=A(a,"bVisible");b=i.inArray(b,a);return b!==-1?b:null}function D(a){return A(a,"bVisible").length}function A(a,b){var c=[];i.map(a.aoColumns,function(d,e){d[b]&&c.push(e)});return c}function G(a){for(var b=l.ext.aTypes,c=b.length,d=0;d<c;d++){var e=b[d](a);if(e!==null)return e}return"string"}function E(a,b){b=b.split(",");for(var c=[],d=0,e=a.aoColumns.length;d<e;d++)for(var f=0;f<e;f++)if(a.aoColumns[d].sName==b[f]){c.push(f);break}return c}
+function Y(a){for(var b="",c=0,d=a.aoColumns.length;c<d;c++)b+=a.aoColumns[c].sName+",";if(b.length==d)return"";return b.slice(0,-1)}function ma(a,b,c,d){var e,f,g,j,k;if(b)for(e=b.length-1;e>=0;e--){var m=b[e].aTargets;i.isArray(m)||O(a,1,"aTargets must be an array of targets, not a "+typeof m);f=0;for(g=m.length;f<g;f++)if(typeof m[f]==="number"&&m[f]>=0){for(;a.aoColumns.length<=m[f];)n(a);d(m[f],b[e])}else if(typeof m[f]==="number"&&m[f]<0)d(a.aoColumns.length+m[f],b[e]);else if(typeof m[f]===
+"string"){j=0;for(k=a.aoColumns.length;j<k;j++)if(m[f]=="_all"||i(a.aoColumns[j].nTh).hasClass(m[f]))d(j,b[e])}}if(c){e=0;for(a=c.length;e<a;e++)d(e,c[e])}}function R(a,b){var c;c=i.isArray(b)?b.slice():i.extend(true,{},b);b=a.aoData.length;var d=i.extend(true,{},l.models.oRow);d._aData=c;a.aoData.push(d);var e;d=0;for(var f=a.aoColumns.length;d<f;d++){c=a.aoColumns[d];typeof c.fnRender==="function"&&c.bUseRendered&&c.mData!==null?S(a,b,d,da(a,b,d)):S(a,b,d,F(a,b,d));if(c._bAutoType&&c.sType!="string"){e=
+F(a,b,d,"type");if(e!==null&&e!==""){e=G(e);if(c.sType===null)c.sType=e;else if(c.sType!=e&&c.sType!="html")c.sType="string"}}}a.aiDisplayMaster.push(b);a.oFeatures.bDeferRender||ua(a,b);return b}function ea(a){var b,c,d,e,f,g,j;if(a.bDeferLoading||a.sAjaxSource===null)for(b=a.nTBody.firstChild;b;){if(b.nodeName.toUpperCase()=="TR"){c=a.aoData.length;b._DT_RowIndex=c;a.aoData.push(i.extend(true,{},l.models.oRow,{nTr:b}));a.aiDisplayMaster.push(c);f=b.firstChild;for(d=0;f;){g=f.nodeName.toUpperCase();
+if(g=="TD"||g=="TH"){S(a,c,d,i.trim(f.innerHTML));d++}f=f.nextSibling}}b=b.nextSibling}e=fa(a);d=[];b=0;for(c=e.length;b<c;b++)for(f=e[b].firstChild;f;){g=f.nodeName.toUpperCase();if(g=="TD"||g=="TH")d.push(f);f=f.nextSibling}c=0;for(e=a.aoColumns.length;c<e;c++){j=a.aoColumns[c];if(j.sTitle===null)j.sTitle=j.nTh.innerHTML;var k=j._bAutoType,m=typeof j.fnRender==="function",u=j.sClass!==null,x=j.bVisible,y,B;if(k||m||u||!x){g=0;for(b=a.aoData.length;g<b;g++){f=a.aoData[g];y=d[g*e+c];if(k&&j.sType!=
+"string"){B=F(a,g,c,"type");if(B!==""){B=G(B);if(j.sType===null)j.sType=B;else if(j.sType!=B&&j.sType!="html")j.sType="string"}}if(j.mRender)y.innerHTML=F(a,g,c,"display");else if(j.mData!==c)y.innerHTML=F(a,g,c,"display");if(m){B=da(a,g,c);y.innerHTML=B;j.bUseRendered&&S(a,g,c,B)}if(u)y.className+=" "+j.sClass;if(x)f._anHidden[c]=null;else{f._anHidden[c]=y;y.parentNode.removeChild(y)}j.fnCreatedCell&&j.fnCreatedCell.call(a.oInstance,y,F(a,g,c,"display"),f._aData,g,c)}}}if(a.aoRowCreatedCallback.length!==
+0){b=0;for(c=a.aoData.length;b<c;b++){f=a.aoData[b];K(a,"aoRowCreatedCallback",null,[f.nTr,f._aData,b])}}}function V(a,b){return b._DT_RowIndex!==p?b._DT_RowIndex:null}function va(a,b,c){b=W(a,b);var d=0;for(a=a.aoColumns.length;d<a;d++)if(b[d]===c)return d;return-1}function na(a,b,c,d){for(var e=[],f=0,g=d.length;f<g;f++)e.push(F(a,b,d[f],c));return e}function F(a,b,c,d){var e=a.aoColumns[c];if((c=e.fnGetData(a.aoData[b]._aData,d))===p){if(a.iDrawError!=a.iDraw&&e.sDefaultContent===null){O(a,0,"Requested unknown parameter "+
+(typeof e.mData=="function"?"{mData function}":"'"+e.mData+"'")+" from the data source for row "+b);a.iDrawError=a.iDraw}return e.sDefaultContent}if(c===null&&e.sDefaultContent!==null)c=e.sDefaultContent;else if(typeof c==="function")return c();if(d=="display"&&c===null)return"";return c}function S(a,b,c,d){a.aoColumns[c].fnSetData(a.aoData[b]._aData,d)}function ca(a){if(a===null)return function(){return null};else if(typeof a==="function")return function(c,d,e){return a(c,d,e)};else if(typeof a===
+"string"&&(a.indexOf(".")!==-1||a.indexOf("[")!==-1)){var b=function(c,d,e){var f=e.split("."),g;if(e!==""){var j=0;for(g=f.length;j<g;j++){if(e=f[j].match(ga)){f[j]=f[j].replace(ga,"");if(f[j]!=="")c=c[f[j]];g=[];f.splice(0,j+1);f=f.join(".");j=0;for(var k=c.length;j<k;j++)g.push(b(c[j],d,f));c=e[0].substring(1,e[0].length-1);c=c===""?g:g.join(c);break}if(c===null||c[f[j]]===p)return p;c=c[f[j]]}}return c};return function(c,d){return b(c,d,a)}}else return function(c){return c[a]}}function Ja(a){if(a===
+null)return function(){};else if(typeof a==="function")return function(c,d){a(c,"set",d)};else if(typeof a==="string"&&(a.indexOf(".")!==-1||a.indexOf("[")!==-1)){var b=function(c,d,e){e=e.split(".");var f,g,j=0;for(g=e.length-1;j<g;j++){if(f=e[j].match(ga)){e[j]=e[j].replace(ga,"");c[e[j]]=[];f=e.slice();f.splice(0,j+1);g=f.join(".");for(var k=0,m=d.length;k<m;k++){f={};b(f,d[k],g);c[e[j]].push(f)}return}if(c[e[j]]===null||c[e[j]]===p)c[e[j]]={};c=c[e[j]]}c[e[e.length-1].replace(ga,"")]=d};return function(c,
+d){return b(c,d,a)}}else return function(c,d){c[a]=d}}function oa(a){for(var b=[],c=a.aoData.length,d=0;d<c;d++)b.push(a.aoData[d]._aData);return b}function wa(a){a.aoData.splice(0,a.aoData.length);a.aiDisplayMaster.splice(0,a.aiDisplayMaster.length);a.aiDisplay.splice(0,a.aiDisplay.length);I(a)}function xa(a,b){for(var c=-1,d=0,e=a.length;d<e;d++)if(a[d]==b)c=d;else a[d]>b&&a[d]--;c!=-1&&a.splice(c,1)}function da(a,b,c){var d=a.aoColumns[c];return d.fnRender({iDataRow:b,iDataColumn:c,oSettings:a,
+aData:a.aoData[b]._aData,mDataProp:d.mData},F(a,b,c,"display"))}function ua(a,b){var c=a.aoData[b],d;if(c.nTr===null){c.nTr=s.createElement("tr");c.nTr._DT_RowIndex=b;if(c._aData.DT_RowId)c.nTr.id=c._aData.DT_RowId;if(c._aData.DT_RowClass)c.nTr.className=c._aData.DT_RowClass;for(var e=0,f=a.aoColumns.length;e<f;e++){var g=a.aoColumns[e];d=s.createElement(g.sCellType);d.innerHTML=typeof g.fnRender==="function"&&(!g.bUseRendered||g.mData===null)?da(a,b,e):F(a,b,e,"display");if(g.sClass!==null)d.className=
+g.sClass;if(g.bVisible){c.nTr.appendChild(d);c._anHidden[e]=null}else c._anHidden[e]=d;g.fnCreatedCell&&g.fnCreatedCell.call(a.oInstance,d,F(a,b,e,"display"),c._aData,b,e)}K(a,"aoRowCreatedCallback",null,[c.nTr,c._aData,b])}}function Ka(a){var b,c,d;if(i("th, td",a.nTHead).length!==0){b=0;for(d=a.aoColumns.length;b<d;b++){c=a.aoColumns[b].nTh;c.setAttribute("role","columnheader");if(a.aoColumns[b].bSortable){c.setAttribute("tabindex",a.iTabIndex);c.setAttribute("aria-controls",a.sTableId)}a.aoColumns[b].sClass!==
+null&&i(c).addClass(a.aoColumns[b].sClass);if(a.aoColumns[b].sTitle!=c.innerHTML)c.innerHTML=a.aoColumns[b].sTitle}}else{var e=s.createElement("tr");b=0;for(d=a.aoColumns.length;b<d;b++){c=a.aoColumns[b].nTh;c.innerHTML=a.aoColumns[b].sTitle;c.setAttribute("tabindex","0");a.aoColumns[b].sClass!==null&&i(c).addClass(a.aoColumns[b].sClass);e.appendChild(c)}i(a.nTHead).html("")[0].appendChild(e);ha(a.aoHeader,a.nTHead)}i(a.nTHead).children("tr").attr("role","row");if(a.bJUI){b=0;for(d=a.aoColumns.length;b<
+d;b++){c=a.aoColumns[b].nTh;e=s.createElement("div");e.className=a.oClasses.sSortJUIWrapper;i(c).contents().appendTo(e);var f=s.createElement("span");f.className=a.oClasses.sSortIcon;e.appendChild(f);c.appendChild(e)}}if(a.oFeatures.bSort)for(b=0;b<a.aoColumns.length;b++)a.aoColumns[b].bSortable!==false?ya(a,a.aoColumns[b].nTh,b):i(a.aoColumns[b].nTh).addClass(a.oClasses.sSortableNone);a.oClasses.sFooterTH!==""&&i(a.nTFoot).children("tr").children("th").addClass(a.oClasses.sFooterTH);if(a.nTFoot!==
+null){c=Z(a,null,a.aoFooter);b=0;for(d=a.aoColumns.length;b<d;b++)if(c[b]){a.aoColumns[b].nTf=c[b];a.aoColumns[b].sClass&&i(c[b]).addClass(a.aoColumns[b].sClass)}}}function ia(a,b,c){var d,e,f,g=[],j=[],k=a.aoColumns.length,m;if(c===p)c=false;d=0;for(e=b.length;d<e;d++){g[d]=b[d].slice();g[d].nTr=b[d].nTr;for(f=k-1;f>=0;f--)!a.aoColumns[f].bVisible&&!c&&g[d].splice(f,1);j.push([])}d=0;for(e=g.length;d<e;d++){if(a=g[d].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[d].length;f<b;f++){m=k=1;
+if(j[d][f]===p){a.appendChild(g[d][f].cell);for(j[d][f]=1;g[d+k]!==p&&g[d][f].cell==g[d+k][f].cell;){j[d+k][f]=1;k++}for(;g[d][f+m]!==p&&g[d][f].cell==g[d][f+m].cell;){for(c=0;c<k;c++)j[d+c][f+m]=1;m++}g[d][f].cell.rowSpan=k;g[d][f].cell.colSpan=m}}}}function H(a){var b=K(a,"aoPreDrawCallback","preDraw",[a]);if(i.inArray(false,b)!==-1)P(a,false);else{var c,d;b=[];var e=0,f=a.asStripeClasses.length;c=a.aoOpenRows.length;a.bDrawing=true;if(a.iInitDisplayStart!==p&&a.iInitDisplayStart!=-1){a._iDisplayStart=
+a.oFeatures.bServerSide?a.iInitDisplayStart:a.iInitDisplayStart>=a.fnRecordsDisplay()?0:a.iInitDisplayStart;a.iInitDisplayStart=-1;I(a)}if(a.bDeferLoading){a.bDeferLoading=false;a.iDraw++}else if(a.oFeatures.bServerSide){if(!a.bDestroying&&!La(a))return}else a.iDraw++;if(a.aiDisplay.length!==0){var g=a._iDisplayStart;d=a._iDisplayEnd;if(a.oFeatures.bServerSide){g=0;d=a.aoData.length}for(g=g;g<d;g++){var j=a.aoData[a.aiDisplay[g]];j.nTr===null&&ua(a,a.aiDisplay[g]);var k=j.nTr;if(f!==0){var m=a.asStripeClasses[e%
+f];if(j._sRowStripe!=m){i(k).removeClass(j._sRowStripe).addClass(m);j._sRowStripe=m}}K(a,"aoRowCallback",null,[k,a.aoData[a.aiDisplay[g]]._aData,e,g]);b.push(k);e++;if(c!==0)for(j=0;j<c;j++)if(k==a.aoOpenRows[j].nParent){b.push(a.aoOpenRows[j].nTr);break}}}else{b[0]=s.createElement("tr");if(a.asStripeClasses[0])b[0].className=a.asStripeClasses[0];c=a.oLanguage;f=c.sZeroRecords;if(a.iDraw==1&&a.sAjaxSource!==null&&!a.oFeatures.bServerSide)f=c.sLoadingRecords;else if(c.sEmptyTable&&a.fnRecordsTotal()===
+0)f=c.sEmptyTable;c=s.createElement("td");c.setAttribute("valign","top");c.colSpan=D(a);c.className=a.oClasses.sRowEmpty;c.innerHTML=za(a,f);b[e].appendChild(c)}K(a,"aoHeaderCallback","header",[i(a.nTHead).children("tr")[0],oa(a),a._iDisplayStart,a.fnDisplayEnd(),a.aiDisplay]);K(a,"aoFooterCallback","footer",[i(a.nTFoot).children("tr")[0],oa(a),a._iDisplayStart,a.fnDisplayEnd(),a.aiDisplay]);e=s.createDocumentFragment();c=s.createDocumentFragment();if(a.nTBody){f=a.nTBody.parentNode;c.appendChild(a.nTBody);
+if(!a.oScroll.bInfinite||!a._bInitComplete||a.bSorted||a.bFiltered)for(;c=a.nTBody.firstChild;)a.nTBody.removeChild(c);c=0;for(d=b.length;c<d;c++)e.appendChild(b[c]);a.nTBody.appendChild(e);f!==null&&f.appendChild(a.nTBody)}K(a,"aoDrawCallback","draw",[a]);a.bSorted=false;a.bFiltered=false;a.bDrawing=false;if(a.oFeatures.bServerSide){P(a,false);a._bInitComplete||pa(a)}}}function qa(a){if(a.oFeatures.bSort)$(a,a.oPreviousSearch);else if(a.oFeatures.bFilter)X(a,a.oPreviousSearch);else{I(a);H(a)}}function Ma(a){var b=
+i("<div></div>")[0];a.nTable.parentNode.insertBefore(b,a.nTable);a.nTableWrapper=i('<div id="'+a.sTableId+'_wrapper" class="'+a.oClasses.sWrapper+'" role="grid"></div>')[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var c=a.nTableWrapper,d=a.sDom.split(""),e,f,g,j,k,m,u,x=0;x<d.length;x++){f=0;g=d[x];if(g=="<"){j=i("<div></div>")[0];k=d[x+1];if(k=="'"||k=='"'){m="";for(u=2;d[x+u]!=k;){m+=d[x+u];u++}if(m=="H")m=a.oClasses.sJUIHeader;else if(m=="F")m=a.oClasses.sJUIFooter;if(m.indexOf(".")!=-1){k=
+m.split(".");j.id=k[0].substr(1,k[0].length-1);j.className=k[1]}else if(m.charAt(0)=="#")j.id=m.substr(1,m.length-1);else j.className=m;x+=u}c.appendChild(j);c=j}else if(g==">")c=c.parentNode;else if(g=="l"&&a.oFeatures.bPaginate&&a.oFeatures.bLengthChange){e=Na(a);f=1}else if(g=="f"&&a.oFeatures.bFilter){e=Oa(a);f=1}else if(g=="r"&&a.oFeatures.bProcessing){e=Pa(a);f=1}else if(g=="t"){e=Qa(a);f=1}else if(g=="i"&&a.oFeatures.bInfo){e=Ra(a);f=1}else if(g=="p"&&a.oFeatures.bPaginate){e=Sa(a);f=1}else if(l.ext.aoFeatures.length!==
+0){j=l.ext.aoFeatures;u=0;for(k=j.length;u<k;u++)if(g==j[u].cFeature){if(e=j[u].fnInit(a))f=1;break}}if(f==1&&e!==null){if(typeof a.aanFeatures[g]!=="object")a.aanFeatures[g]=[];a.aanFeatures[g].push(e);c.appendChild(e)}}b.parentNode.replaceChild(a.nTableWrapper,b)}function ha(a,b){b=i(b).children("tr");var c,d,e,f,g,j,k,m,u,x,y=function(B,T,M){for(B=B[T];B[M];)M++;return M};a.splice(0,a.length);e=0;for(j=b.length;e<j;e++)a.push([]);e=0;for(j=b.length;e<j;e++){c=b[e];for(d=c.firstChild;d;){if(d.nodeName.toUpperCase()==
+"TD"||d.nodeName.toUpperCase()=="TH"){m=d.getAttribute("colspan")*1;u=d.getAttribute("rowspan")*1;m=!m||m===0||m===1?1:m;u=!u||u===0||u===1?1:u;k=y(a,e,0);x=m===1?true:false;for(g=0;g<m;g++)for(f=0;f<u;f++){a[e+f][k+g]={cell:d,unique:x};a[e+f].nTr=c}}d=d.nextSibling}}}function Z(a,b,c){var d=[];if(!c){c=a.aoHeader;if(b){c=[];ha(c,b)}}b=0;for(var e=c.length;b<e;b++)for(var f=0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!d[f]||!a.bSortCellsTop))d[f]=c[b][f].cell;return d}function La(a){if(a.bAjaxDataGet){a.iDraw++;
+P(a,true);var b=Ta(a);Aa(a,b);a.fnServerData.call(a.oInstance,a.sAjaxSource,b,function(c){Ua(a,c)},a);return false}else return true}function Ta(a){var b=a.aoColumns.length,c=[],d,e,f,g;c.push({name:"sEcho",value:a.iDraw});c.push({name:"iColumns",value:b});c.push({name:"sColumns",value:Y(a)});c.push({name:"iDisplayStart",value:a._iDisplayStart});c.push({name:"iDisplayLength",value:a.oFeatures.bPaginate!==false?a._iDisplayLength:-1});for(f=0;f<b;f++){d=a.aoColumns[f].mData;c.push({name:"mDataProp_"+
+f,value:typeof d==="function"?"function":d})}if(a.oFeatures.bFilter!==false){c.push({name:"sSearch",value:a.oPreviousSearch.sSearch});c.push({name:"bRegex",value:a.oPreviousSearch.bRegex});for(f=0;f<b;f++){c.push({name:"sSearch_"+f,value:a.aoPreSearchCols[f].sSearch});c.push({name:"bRegex_"+f,value:a.aoPreSearchCols[f].bRegex});c.push({name:"bSearchable_"+f,value:a.aoColumns[f].bSearchable})}}if(a.oFeatures.bSort!==false){var j=0;d=a.aaSortingFixed!==null?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();
+for(f=0;f<d.length;f++){e=a.aoColumns[d[f][0]].aDataSort;for(g=0;g<e.length;g++){c.push({name:"iSortCol_"+j,value:e[g]});c.push({name:"sSortDir_"+j,value:d[f][1]});j++}}c.push({name:"iSortingCols",value:j});for(f=0;f<b;f++)c.push({name:"bSortable_"+f,value:a.aoColumns[f].bSortable})}return c}function Aa(a,b){K(a,"aoServerParams","serverParams",[b])}function Ua(a,b){if(b.sEcho!==p)if(b.sEcho*1<a.iDraw)return;else a.iDraw=b.sEcho*1;if(!a.oScroll.bInfinite||a.oScroll.bInfinite&&(a.bSorted||a.bFiltered))wa(a);
+a._iRecordsTotal=parseInt(b.iTotalRecords,10);a._iRecordsDisplay=parseInt(b.iTotalDisplayRecords,10);var c=Y(a);c=b.sColumns!==p&&c!==""&&b.sColumns!=c;var d;if(c)d=E(a,b.sColumns);b=ca(a.sAjaxDataProp)(b);for(var e=0,f=b.length;e<f;e++)if(c){for(var g=[],j=0,k=a.aoColumns.length;j<k;j++)g.push(b[e][d[j]]);R(a,g)}else R(a,b[e]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=false;H(a);a.bAjaxDataGet=true;P(a,false)}function Oa(a){var b=a.oPreviousSearch,c=a.oLanguage.sSearch;c=c.indexOf("_INPUT_")!==
+-1?c.replace("_INPUT_",'<input type="text" />'):c===""?'<input type="text" />':c+' <input type="text" />';var d=s.createElement("div");d.className=a.oClasses.sFilter;d.innerHTML="<label>"+c+"</label>";if(!a.aanFeatures.f)d.id=a.sTableId+"_filter";c=i('input[type="text"]',d);d._DT_Input=c[0];c.val(b.sSearch.replace('"',"""));c.bind("keyup.DT",function(){for(var e=a.aanFeatures.f,f=this.value===""?"":this.value,g=0,j=e.length;g<j;g++)e[g]!=i(this).parents("div.dataTables_filter")[0]&&i(e[g]._DT_Input).val(f);
+f!=b.sSearch&&X(a,{sSearch:f,bRegex:b.bRegex,bSmart:b.bSmart,bCaseInsensitive:b.bCaseInsensitive})});c.attr("aria-controls",a.sTableId).bind("keypress.DT",function(e){if(e.keyCode==13)return false});return d}function X(a,b,c){var d=a.oPreviousSearch,e=a.aoPreSearchCols,f=function(g){d.sSearch=g.sSearch;d.bRegex=g.bRegex;d.bSmart=g.bSmart;d.bCaseInsensitive=g.bCaseInsensitive};if(a.oFeatures.bServerSide)f(b);else{Va(a,b.sSearch,c,b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<a.aoPreSearchCols.length;b++)Wa(a,
+e[b].sSearch,b,e[b].bRegex,e[b].bSmart,e[b].bCaseInsensitive);Xa(a)}a.bFiltered=true;i(a.oInstance).trigger("filter",a);a._iDisplayStart=0;I(a);H(a);Ba(a,0)}function Xa(a){for(var b=l.ext.afnFiltering,c=A(a,"bSearchable"),d=0,e=b.length;d<e;d++)for(var f=0,g=0,j=a.aiDisplay.length;g<j;g++){var k=a.aiDisplay[g-f];if(!b[d](a,na(a,k,"filter",c),k)){a.aiDisplay.splice(g-f,1);f++}}}function Wa(a,b,c,d,e,f){if(b!==""){var g=0;b=Ca(b,d,e,f);for(d=a.aiDisplay.length-1;d>=0;d--){e=Ya(F(a,a.aiDisplay[d],c,
+"filter"),a.aoColumns[c].sType);if(!b.test(e)){a.aiDisplay.splice(d,1);g++}}}}function Va(a,b,c,d,e,f){d=Ca(b,d,e,f);e=a.oPreviousSearch;c||(c=0);if(l.ext.afnFiltering.length!==0)c=1;if(b.length<=0){a.aiDisplay.splice(0,a.aiDisplay.length);a.aiDisplay=a.aiDisplayMaster.slice()}else if(a.aiDisplay.length==a.aiDisplayMaster.length||e.sSearch.length>b.length||c==1||b.indexOf(e.sSearch)!==0){a.aiDisplay.splice(0,a.aiDisplay.length);Ba(a,1);for(b=0;b<a.aiDisplayMaster.length;b++)d.test(a.asDataSearch[b])&&
+a.aiDisplay.push(a.aiDisplayMaster[b])}else for(b=c=0;b<a.asDataSearch.length;b++)if(!d.test(a.asDataSearch[b])){a.aiDisplay.splice(b-c,1);c++}}function Ba(a,b){if(!a.oFeatures.bServerSide){a.asDataSearch=[];var c=A(a,"bSearchable");b=b===1?a.aiDisplayMaster:a.aiDisplay;for(var d=0,e=b.length;d<e;d++)a.asDataSearch[d]=Da(a,na(a,b[d],"filter",c))}}function Da(a,b){a=b.join(" ");if(a.indexOf("&")!==-1)a=i("<div>").html(a).text();return a.replace(/[\n\r]/g," ")}function Ca(a,b,c,d){if(c){a=b?a.split(" "):
+Ea(a).split(" ");a="^(?=.*?"+a.join(")(?=.*?")+").*$";return new RegExp(a,d?"i":"")}else{a=b?a:Ea(a);return new RegExp(a,d?"i":"")}}function Ya(a,b){if(typeof l.ext.ofnSearch[b]==="function")return l.ext.ofnSearch[b](a);else if(a===null)return"";else if(b=="html")return a.replace(/[\r\n]/g," ").replace(/<.*?>/g,"");else if(typeof a==="string")return a.replace(/[\r\n]/g," ");return a}function Ea(a){return a.replace(new RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),
+"\\$1")}function Ra(a){var b=s.createElement("div");b.className=a.oClasses.sInfo;if(!a.aanFeatures.i){a.aoDrawCallback.push({fn:Za,sName:"information"});b.id=a.sTableId+"_info"}a.nTable.setAttribute("aria-describedby",a.sTableId+"_info");return b}function Za(a){if(!(!a.oFeatures.bInfo||a.aanFeatures.i.length===0)){var b=a.oLanguage,c=a._iDisplayStart+1,d=a.fnDisplayEnd(),e=a.fnRecordsTotal(),f=a.fnRecordsDisplay(),g;g=f===0?b.sInfoEmpty:b.sInfo;if(f!=e)g+=" "+b.sInfoFiltered;g+=b.sInfoPostFix;g=za(a,
+g);if(b.fnInfoCallback!==null)g=b.fnInfoCallback.call(a.oInstance,a,c,d,e,f,g);a=a.aanFeatures.i;b=0;for(c=a.length;b<c;b++)i(a[b]).html(g)}}function za(a,b){var c=a.fnFormatNumber(a._iDisplayStart+1),d=a.fnDisplayEnd();d=a.fnFormatNumber(d);var e=a.fnRecordsDisplay();e=a.fnFormatNumber(e);var f=a.fnRecordsTotal();f=a.fnFormatNumber(f);if(a.oScroll.bInfinite)c=a.fnFormatNumber(1);return b.replace(/_START_/g,c).replace(/_END_/g,d).replace(/_TOTAL_/g,e).replace(/_MAX_/g,f)}function ra(a){var b,c,d=
+a.iInitDisplayStart;if(a.bInitialised===false)setTimeout(function(){ra(a)},200);else{Ma(a);Ka(a);ia(a,a.aoHeader);a.nTFoot&&ia(a,a.aoFooter);P(a,true);a.oFeatures.bAutoWidth&&ta(a);b=0;for(c=a.aoColumns.length;b<c;b++)if(a.aoColumns[b].sWidth!==null)a.aoColumns[b].nTh.style.width=t(a.aoColumns[b].sWidth);if(a.oFeatures.bSort)$(a);else if(a.oFeatures.bFilter)X(a,a.oPreviousSearch);else{a.aiDisplay=a.aiDisplayMaster.slice();I(a);H(a)}if(a.sAjaxSource!==null&&!a.oFeatures.bServerSide){c=[];Aa(a,c);a.fnServerData.call(a.oInstance,
+a.sAjaxSource,c,function(e){var f=a.sAjaxDataProp!==""?ca(a.sAjaxDataProp)(e):e;for(b=0;b<f.length;b++)R(a,f[b]);a.iInitDisplayStart=d;if(a.oFeatures.bSort)$(a);else{a.aiDisplay=a.aiDisplayMaster.slice();I(a);H(a)}P(a,false);pa(a,e)},a)}else if(!a.oFeatures.bServerSide){P(a,false);pa(a)}}}function pa(a,b){a._bInitComplete=true;K(a,"aoInitComplete","init",[a,b])}function Fa(a){var b=l.defaults.oLanguage;!a.sEmptyTable&&a.sZeroRecords&&b.sEmptyTable==="No data available in table"&&r(a,a,"sZeroRecords",
+"sEmptyTable");!a.sLoadingRecords&&a.sZeroRecords&&b.sLoadingRecords==="Loading..."&&r(a,a,"sZeroRecords","sLoadingRecords")}function Na(a){if(a.oScroll.bInfinite)return null;var b='<select size="1" '+('name="'+a.sTableId+'_length"')+">",c,d,e=a.aLengthMenu;if(e.length==2&&typeof e[0]==="object"&&typeof e[1]==="object"){c=0;for(d=e[0].length;c<d;c++)b+='<option value="'+e[0][c]+'">'+e[1][c]+"</option>"}else{c=0;for(d=e.length;c<d;c++)b+='<option value="'+e[c]+'">'+e[c]+"</option>"}b+="</select>";
+e=s.createElement("div");if(!a.aanFeatures.l)e.id=a.sTableId+"_length";e.className=a.oClasses.sLength;e.innerHTML="<label>"+a.oLanguage.sLengthMenu.replace("_MENU_",b)+"</label>";i('select option[value="'+a._iDisplayLength+'"]',e).attr("selected",true);i("select",e).bind("change.DT",function(){var f=i(this).val(),g=a.aanFeatures.l;c=0;for(d=g.length;c<d;c++)g[c]!=this.parentNode&&i("select",g[c]).val(f);a._iDisplayLength=parseInt(f,10);I(a);if(a.fnDisplayEnd()==a.fnRecordsDisplay()){a._iDisplayStart=
+a.fnDisplayEnd()-a._iDisplayLength;if(a._iDisplayStart<0)a._iDisplayStart=0}if(a._iDisplayLength==-1)a._iDisplayStart=0;H(a)});i("select",e).attr("aria-controls",a.sTableId);return e}function I(a){a._iDisplayEnd=a.oFeatures.bPaginate===false?a.aiDisplay.length:a._iDisplayStart+a._iDisplayLength>a.aiDisplay.length||a._iDisplayLength==-1?a.aiDisplay.length:a._iDisplayStart+a._iDisplayLength}function Sa(a){if(a.oScroll.bInfinite)return null;var b=s.createElement("div");b.className=a.oClasses.sPaging+
+a.sPaginationType;l.ext.oPagination[a.sPaginationType].fnInit(a,b,function(c){I(c);H(c)});a.aanFeatures.p||a.aoDrawCallback.push({fn:function(c){l.ext.oPagination[c.sPaginationType].fnUpdate(c,function(d){I(d);H(d)})},sName:"pagination"});return b}function Ga(a,b){var c=a._iDisplayStart;if(typeof b==="number"){a._iDisplayStart=b*a._iDisplayLength;if(a._iDisplayStart>a.fnRecordsDisplay())a._iDisplayStart=0}else if(b=="first")a._iDisplayStart=0;else if(b=="previous"){a._iDisplayStart=a._iDisplayLength>=
+0?a._iDisplayStart-a._iDisplayLength:0;if(a._iDisplayStart<0)a._iDisplayStart=0}else if(b=="next")if(a._iDisplayLength>=0){if(a._iDisplayStart+a._iDisplayLength<a.fnRecordsDisplay())a._iDisplayStart+=a._iDisplayLength}else a._iDisplayStart=0;else if(b=="last")if(a._iDisplayLength>=0){b=parseInt((a.fnRecordsDisplay()-1)/a._iDisplayLength,10)+1;a._iDisplayStart=(b-1)*a._iDisplayLength}else a._iDisplayStart=0;else O(a,0,"Unknown paging action: "+b);i(a.oInstance).trigger("page",a);return c!=a._iDisplayStart}
+function Pa(a){var b=s.createElement("div");if(!a.aanFeatures.r)b.id=a.sTableId+"_processing";b.innerHTML=a.oLanguage.sProcessing;b.className=a.oClasses.sProcessing;a.nTable.parentNode.insertBefore(b,a.nTable);return b}function P(a,b){if(a.oFeatures.bProcessing)for(var c=a.aanFeatures.r,d=0,e=c.length;d<e;d++)c[d].style.visibility=b?"visible":"hidden";i(a.oInstance).trigger("processing",[a,b])}function Qa(a){if(a.oScroll.sX===""&&a.oScroll.sY==="")return a.nTable;var b=s.createElement("div"),c=s.createElement("div"),
+d=s.createElement("div"),e=s.createElement("div"),f=s.createElement("div"),g=s.createElement("div"),j=a.nTable.cloneNode(false),k=a.nTable.cloneNode(false),m=a.nTable.getElementsByTagName("thead")[0],u=a.nTable.getElementsByTagName("tfoot").length===0?null:a.nTable.getElementsByTagName("tfoot")[0],x=a.oClasses;c.appendChild(d);f.appendChild(g);e.appendChild(a.nTable);b.appendChild(c);b.appendChild(e);d.appendChild(j);j.appendChild(m);if(u!==null){b.appendChild(f);g.appendChild(k);k.appendChild(u)}b.className=
+x.sScrollWrapper;c.className=x.sScrollHead;d.className=x.sScrollHeadInner;e.className=x.sScrollBody;f.className=x.sScrollFoot;g.className=x.sScrollFootInner;if(a.oScroll.bAutoCss){c.style.overflow="hidden";c.style.position="relative";f.style.overflow="hidden";e.style.overflow="auto"}c.style.border="0";c.style.width="100%";f.style.border="0";d.style.width=a.oScroll.sXInner!==""?a.oScroll.sXInner:"100%";j.removeAttribute("id");j.style.marginLeft="0";a.nTable.style.marginLeft="0";if(u!==null){k.removeAttribute("id");
+k.style.marginLeft="0"}d=i(a.nTable).children("caption");if(d.length>0){d=d[0];if(d._captionSide==="top")j.appendChild(d);else d._captionSide==="bottom"&&u&&k.appendChild(d)}if(a.oScroll.sX!==""){c.style.width=t(a.oScroll.sX);e.style.width=t(a.oScroll.sX);if(u!==null)f.style.width=t(a.oScroll.sX);i(e).scroll(function(){c.scrollLeft=this.scrollLeft;if(u!==null)f.scrollLeft=this.scrollLeft})}if(a.oScroll.sY!=="")e.style.height=t(a.oScroll.sY);a.aoDrawCallback.push({fn:$a,sName:"scrolling"});a.oScroll.bInfinite&&
+i(e).scroll(function(){if(!a.bDrawing&&i(this).scrollTop()!==0)if(i(this).scrollTop()+i(this).height()>i(a.nTable).height()-a.oScroll.iLoadGap)if(a.fnDisplayEnd()<a.fnRecordsDisplay()){Ga(a,"next");I(a);H(a)}});a.nScrollHead=c;a.nScrollFoot=f;return b}function $a(a){var b=a.nScrollHead.getElementsByTagName("div")[0],c=b.getElementsByTagName("table")[0],d=a.nTable.parentNode,e,f,g,j,k,m,u,x,y=[],B=[],T=a.nTFoot!==null?a.nScrollFoot.getElementsByTagName("div")[0]:null,M=a.nTFoot!==null?T.getElementsByTagName("table")[0]:
+null,L=a.oBrowser.bScrollOversize,ja=function(z){u=z.style;u.paddingTop="0";u.paddingBottom="0";u.borderTopWidth="0";u.borderBottomWidth="0";u.height=0};i(a.nTable).children("thead, tfoot").remove();e=i(a.nTHead).clone()[0];a.nTable.insertBefore(e,a.nTable.childNodes[0]);g=a.nTHead.getElementsByTagName("tr");j=e.getElementsByTagName("tr");if(a.nTFoot!==null){k=i(a.nTFoot).clone()[0];a.nTable.insertBefore(k,a.nTable.childNodes[1]);m=a.nTFoot.getElementsByTagName("tr");k=k.getElementsByTagName("tr")}if(a.oScroll.sX===
+""){d.style.width="100%";b.parentNode.style.width="100%"}var U=Z(a,e);e=0;for(f=U.length;e<f;e++){x=v(a,e);U[e].style.width=a.aoColumns[x].sWidth}a.nTFoot!==null&&N(function(z){z.style.width=""},k);if(a.oScroll.bCollapse&&a.oScroll.sY!=="")d.style.height=d.offsetHeight+a.nTHead.offsetHeight+"px";e=i(a.nTable).outerWidth();if(a.oScroll.sX===""){a.nTable.style.width="100%";if(L&&(i("tbody",d).height()>d.offsetHeight||i(d).css("overflow-y")=="scroll"))a.nTable.style.width=t(i(a.nTable).outerWidth()-
+a.oScroll.iBarWidth)}else if(a.oScroll.sXInner!=="")a.nTable.style.width=t(a.oScroll.sXInner);else if(e==i(d).width()&&i(d).height()<i(a.nTable).height()){a.nTable.style.width=t(e-a.oScroll.iBarWidth);if(i(a.nTable).outerWidth()>e-a.oScroll.iBarWidth)a.nTable.style.width=t(e)}else a.nTable.style.width=t(e);e=i(a.nTable).outerWidth();N(ja,j);N(function(z){y.push(t(i(z).width()))},j);N(function(z,Q){z.style.width=y[Q]},g);i(j).height(0);if(a.nTFoot!==null){N(ja,k);N(function(z){B.push(t(i(z).width()))},
+k);N(function(z,Q){z.style.width=B[Q]},m);i(k).height(0)}N(function(z,Q){z.innerHTML="";z.style.width=y[Q]},j);a.nTFoot!==null&&N(function(z,Q){z.innerHTML="";z.style.width=B[Q]},k);if(i(a.nTable).outerWidth()<e){g=d.scrollHeight>d.offsetHeight||i(d).css("overflow-y")=="scroll"?e+a.oScroll.iBarWidth:e;if(L&&(d.scrollHeight>d.offsetHeight||i(d).css("overflow-y")=="scroll"))a.nTable.style.width=t(g-a.oScroll.iBarWidth);d.style.width=t(g);a.nScrollHead.style.width=t(g);if(a.nTFoot!==null)a.nScrollFoot.style.width=
+t(g);if(a.oScroll.sX==="")O(a,1,"The table cannot fit into the current element which will cause column misalignment. The table has been drawn at its minimum possible width.");else a.oScroll.sXInner!==""&&O(a,1,"The table cannot fit into the current element which will cause column misalignment. Increase the sScrollXInner value or remove it to allow automatic calculation")}else{d.style.width=t("100%");a.nScrollHead.style.width=t("100%");if(a.nTFoot!==null)a.nScrollFoot.style.width=t("100%")}if(a.oScroll.sY===
+"")if(L)d.style.height=t(a.nTable.offsetHeight+a.oScroll.iBarWidth);if(a.oScroll.sY!==""&&a.oScroll.bCollapse){d.style.height=t(a.oScroll.sY);L=a.oScroll.sX!==""&&a.nTable.offsetWidth>d.offsetWidth?a.oScroll.iBarWidth:0;if(a.nTable.offsetHeight<d.offsetHeight)d.style.height=t(a.nTable.offsetHeight+L)}L=i(a.nTable).outerWidth();c.style.width=t(L);b.style.width=t(L);c=i(a.nTable).height()>d.clientHeight||i(d).css("overflow-y")=="scroll";b.style.paddingRight=c?a.oScroll.iBarWidth+"px":"0px";if(a.nTFoot!==
+null){M.style.width=t(L);T.style.width=t(L);T.style.paddingRight=c?a.oScroll.iBarWidth+"px":"0px"}i(d).scroll();if(a.bSorted||a.bFiltered)d.scrollTop=0}function N(a,b,c){for(var d=0,e=0,f=b.length,g,j;e<f;){g=b[e].firstChild;for(j=c?c[e].firstChild:null;g;){if(g.nodeType===1){c?a(g,j,d):a(g,d);d++}g=g.nextSibling;j=c?j.nextSibling:null}e++}}function ab(a,b){if(!a||a===null||a==="")return 0;if(!b)b=s.body;var c=s.createElement("div");c.style.width=t(a);b.appendChild(c);a=c.offsetWidth;b.removeChild(c);
+return a}function ta(a){var b=0,c,d=0,e=a.aoColumns.length,f,g,j=i("th",a.nTHead),k=a.nTable.getAttribute("width");g=a.nTable.parentNode;for(f=0;f<e;f++)if(a.aoColumns[f].bVisible){d++;if(a.aoColumns[f].sWidth!==null){c=ab(a.aoColumns[f].sWidthOrig,g);if(c!==null)a.aoColumns[f].sWidth=t(c);b++}}if(e==j.length&&b===0&&d==e&&a.oScroll.sX===""&&a.oScroll.sY==="")for(f=0;f<a.aoColumns.length;f++){c=i(j[f]).width();if(c!==null)a.aoColumns[f].sWidth=t(c)}else{b=a.nTable.cloneNode(false);f=a.nTHead.cloneNode(true);
+d=s.createElement("tbody");c=s.createElement("tr");b.removeAttribute("id");b.appendChild(f);if(a.nTFoot!==null){b.appendChild(a.nTFoot.cloneNode(true));N(function(u){u.style.width=""},b.getElementsByTagName("tr"))}b.appendChild(d);d.appendChild(c);d=i("thead th",b);if(d.length===0)d=i("tbody tr:eq(0)>td",b);j=Z(a,f);for(f=d=0;f<e;f++){var m=a.aoColumns[f];if(m.bVisible&&m.sWidthOrig!==null&&m.sWidthOrig!=="")j[f-d].style.width=t(m.sWidthOrig);else if(m.bVisible)j[f-d].style.width="";else d++}for(f=
+0;f<e;f++)if(a.aoColumns[f].bVisible){d=bb(a,f);if(d!==null){d=d.cloneNode(true);if(a.aoColumns[f].sContentPadding!=="")d.innerHTML+=a.aoColumns[f].sContentPadding;c.appendChild(d)}}g.appendChild(b);if(a.oScroll.sX!==""&&a.oScroll.sXInner!=="")b.style.width=t(a.oScroll.sXInner);else if(a.oScroll.sX!==""){b.style.width="";if(i(b).width()<g.offsetWidth)b.style.width=t(g.offsetWidth)}else if(a.oScroll.sY!=="")b.style.width=t(g.offsetWidth);else if(k)b.style.width=t(k);b.style.visibility="hidden";cb(a,
+b);e=i("tbody tr:eq(0)",b).children();if(e.length===0)e=Z(a,i("thead",b)[0]);if(a.oScroll.sX!==""){for(f=d=g=0;f<a.aoColumns.length;f++)if(a.aoColumns[f].bVisible){g+=a.aoColumns[f].sWidthOrig===null?i(e[d]).outerWidth():parseInt(a.aoColumns[f].sWidth.replace("px",""),10)+(i(e[d]).outerWidth()-i(e[d]).width());d++}b.style.width=t(g);a.nTable.style.width=t(g)}for(f=d=0;f<a.aoColumns.length;f++)if(a.aoColumns[f].bVisible){g=i(e[d]).width();if(g!==null&&g>0)a.aoColumns[f].sWidth=t(g);d++}e=i(b).css("width");
+a.nTable.style.width=e.indexOf("%")!==-1?e:t(i(b).outerWidth());b.parentNode.removeChild(b)}if(k)a.nTable.style.width=t(k)}function cb(a,b){if(a.oScroll.sX===""&&a.oScroll.sY!==""){i(b).width();b.style.width=t(i(b).outerWidth()-a.oScroll.iBarWidth)}else if(a.oScroll.sX!=="")b.style.width=t(i(b).outerWidth())}function bb(a,b){var c=db(a,b);if(c<0)return null;if(a.aoData[c].nTr===null){var d=s.createElement("td");d.innerHTML=F(a,c,b,"");return d}return W(a,c)[b]}function db(a,b){for(var c=-1,d=-1,e=
+0;e<a.aoData.length;e++){var f=F(a,e,b,"display")+"";f=f.replace(/<.*?>/g,"");if(f.length>c){c=f.length;d=e}}return d}function t(a){if(a===null)return"0px";if(typeof a=="number"){if(a<0)return"0px";return a+"px"}var b=a.charCodeAt(a.length-1);if(b<48||b>57)return a;return a+"px"}function eb(){var a=s.createElement("p"),b=a.style;b.width="100%";b.height="200px";b.padding="0px";var c=s.createElement("div");b=c.style;b.position="absolute";b.top="0px";b.left="0px";b.visibility="hidden";b.width="200px";
+b.height="150px";b.padding="0px";b.overflow="hidden";c.appendChild(a);s.body.appendChild(c);b=a.offsetWidth;c.style.overflow="scroll";a=a.offsetWidth;if(b==a)a=c.clientWidth;s.body.removeChild(c);return b-a}function $(a,b){var c,d,e,f,g,j,k=[],m=[],u=l.ext.oSort,x=a.aoData,y=a.aoColumns,B=a.oLanguage.oAria;if(!a.oFeatures.bServerSide&&(a.aaSorting.length!==0||a.aaSortingFixed!==null)){k=a.aaSortingFixed!==null?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(c=0;c<k.length;c++){d=k[c][0];
+e=w(a,d);f=a.aoColumns[d].sSortDataType;if(l.ext.afnSortData[f]){g=l.ext.afnSortData[f].call(a.oInstance,a,d,e);if(g.length===x.length){e=0;for(f=x.length;e<f;e++)S(a,e,d,g[e])}else O(a,0,"Returned data sort array (col "+d+") is the wrong length")}}c=0;for(d=a.aiDisplayMaster.length;c<d;c++)m[a.aiDisplayMaster[c]]=c;var T=k.length,M;c=0;for(d=x.length;c<d;c++)for(e=0;e<T;e++){M=y[k[e][0]].aDataSort;g=0;for(j=M.length;g<j;g++){f=y[M[g]].sType;f=u[(f?f:"string")+"-pre"];x[c]._aSortData[M[g]]=f?f(F(a,
+c,M[g],"sort")):F(a,c,M[g],"sort")}}a.aiDisplayMaster.sort(function(L,ja){var U,z,Q,aa,ka;for(U=0;U<T;U++){ka=y[k[U][0]].aDataSort;z=0;for(Q=ka.length;z<Q;z++){aa=y[ka[z]].sType;aa=u[(aa?aa:"string")+"-"+k[U][1]](x[L]._aSortData[ka[z]],x[ja]._aSortData[ka[z]]);if(aa!==0)return aa}}return u["numeric-asc"](m[L],m[ja])})}if((b===p||b)&&!a.oFeatures.bDeferRender)ba(a);c=0;for(d=a.aoColumns.length;c<d;c++){e=y[c].sTitle.replace(/<.*?>/g,"");b=y[c].nTh;b.removeAttribute("aria-sort");b.removeAttribute("aria-label");
+if(y[c].bSortable)if(k.length>0&&k[0][0]==c){b.setAttribute("aria-sort",k[0][1]=="asc"?"ascending":"descending");b.setAttribute("aria-label",e+((y[c].asSorting[k[0][2]+1]?y[c].asSorting[k[0][2]+1]:y[c].asSorting[0])=="asc"?B.sSortAscending:B.sSortDescending))}else b.setAttribute("aria-label",e+(y[c].asSorting[0]=="asc"?B.sSortAscending:B.sSortDescending));else b.setAttribute("aria-label",e)}a.bSorted=true;i(a.oInstance).trigger("sort",a);if(a.oFeatures.bFilter)X(a,a.oPreviousSearch,1);else{a.aiDisplay=
+a.aiDisplayMaster.slice();a._iDisplayStart=0;I(a);H(a)}}function ya(a,b,c,d){fb(b,{},function(e){if(a.aoColumns[c].bSortable!==false){var f=function(){var g,j;if(e.shiftKey){for(var k=false,m=0;m<a.aaSorting.length;m++)if(a.aaSorting[m][0]==c){k=true;g=a.aaSorting[m][0];j=a.aaSorting[m][2]+1;if(a.aoColumns[g].asSorting[j]){a.aaSorting[m][1]=a.aoColumns[g].asSorting[j];a.aaSorting[m][2]=j}else a.aaSorting.splice(m,1);break}k===false&&a.aaSorting.push([c,a.aoColumns[c].asSorting[0],0])}else if(a.aaSorting.length==
+1&&a.aaSorting[0][0]==c){g=a.aaSorting[0][0];j=a.aaSorting[0][2]+1;a.aoColumns[g].asSorting[j]||(j=0);a.aaSorting[0][1]=a.aoColumns[g].asSorting[j];a.aaSorting[0][2]=j}else{a.aaSorting.splice(0,a.aaSorting.length);a.aaSorting.push([c,a.aoColumns[c].asSorting[0],0])}$(a)};if(a.oFeatures.bProcessing){P(a,true);setTimeout(function(){f();a.oFeatures.bServerSide||P(a,false)},0)}else f();typeof d=="function"&&d(a)}})}function ba(a){var b,c,d,e,f,g=a.aoColumns.length,j=a.oClasses;for(b=0;b<g;b++)a.aoColumns[b].bSortable&&
+i(a.aoColumns[b].nTh).removeClass(j.sSortAsc+" "+j.sSortDesc+" "+a.aoColumns[b].sSortingClass);c=a.aaSortingFixed!==null?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(b=0;b<a.aoColumns.length;b++)if(a.aoColumns[b].bSortable){f=a.aoColumns[b].sSortingClass;e=-1;for(d=0;d<c.length;d++)if(c[d][0]==b){f=c[d][1]=="asc"?j.sSortAsc:j.sSortDesc;e=d;break}i(a.aoColumns[b].nTh).addClass(f);if(a.bJUI){f=i("span."+j.sSortIcon,a.aoColumns[b].nTh);f.removeClass(j.sSortJUIAsc+" "+j.sSortJUIDesc+" "+
+j.sSortJUI+" "+j.sSortJUIAscAllowed+" "+j.sSortJUIDescAllowed);f.addClass(e==-1?a.aoColumns[b].sSortingClassJUI:c[e][1]=="asc"?j.sSortJUIAsc:j.sSortJUIDesc)}}else i(a.aoColumns[b].nTh).addClass(a.aoColumns[b].sSortingClass);f=j.sSortColumn;if(a.oFeatures.bSort&&a.oFeatures.bSortClasses){a=W(a);e=[];for(b=0;b<g;b++)e.push("");b=0;for(d=1;b<c.length;b++){j=parseInt(c[b][0],10);e[j]=f+d;d<3&&d++}f=new RegExp(f+"[123]");var k;b=0;for(c=a.length;b<c;b++){j=b%g;d=a[b].className;k=e[j];j=d.replace(f,k);
+if(j!=d)a[b].className=i.trim(j);else if(k.length>0&&d.indexOf(k)==-1)a[b].className=d+" "+k}}}function Ha(a){if(!(!a.oFeatures.bStateSave||a.bDestroying)){var b,c;b=a.oScroll.bInfinite;var d={iCreate:(new Date).getTime(),iStart:b?0:a._iDisplayStart,iEnd:b?a._iDisplayLength:a._iDisplayEnd,iLength:a._iDisplayLength,aaSorting:i.extend(true,[],a.aaSorting),oSearch:i.extend(true,{},a.oPreviousSearch),aoSearchCols:i.extend(true,[],a.aoPreSearchCols),abVisCols:[]};b=0;for(c=a.aoColumns.length;b<c;b++)d.abVisCols.push(a.aoColumns[b].bVisible);
+K(a,"aoStateSaveParams","stateSaveParams",[a,d]);a.fnStateSave.call(a.oInstance,a,d)}}function gb(a,b){if(a.oFeatures.bStateSave){var c=a.fnStateLoad.call(a.oInstance,a);if(c){var d=K(a,"aoStateLoadParams","stateLoadParams",[a,c]);if(i.inArray(false,d)===-1){a.oLoadedState=i.extend(true,{},c);a._iDisplayStart=c.iStart;a.iInitDisplayStart=c.iStart;a._iDisplayEnd=c.iEnd;a._iDisplayLength=c.iLength;a.aaSorting=c.aaSorting.slice();a.saved_aaSorting=c.aaSorting.slice();i.extend(a.oPreviousSearch,c.oSearch);
+i.extend(true,a.aoPreSearchCols,c.aoSearchCols);b.saved_aoColumns=[];for(d=0;d<c.abVisCols.length;d++){b.saved_aoColumns[d]={};b.saved_aoColumns[d].bVisible=c.abVisCols[d]}K(a,"aoStateLoaded","stateLoaded",[a,c])}}}}function lb(a,b,c,d,e){var f=new Date;f.setTime(f.getTime()+c*1E3);c=la.location.pathname.split("/");a=a+"_"+c.pop().replace(/[\/:]/g,"").toLowerCase();var g;if(e!==null){g=typeof i.parseJSON==="function"?i.parseJSON(b):eval("("+b+")");b=e(a,g,f.toGMTString(),c.join("/")+"/")}else b=a+
+"="+encodeURIComponent(b)+"; expires="+f.toGMTString()+"; path="+c.join("/")+"/";a=s.cookie.split(";");e=b.split(";")[0].length;f=[];if(e+s.cookie.length+10>4096){for(var j=0,k=a.length;j<k;j++)if(a[j].indexOf(d)!=-1){var m=a[j].split("=");try{(g=eval("("+decodeURIComponent(m[1])+")"))&&g.iCreate&&f.push({name:m[0],time:g.iCreate})}catch(u){}}for(f.sort(function(x,y){return y.time-x.time});e+s.cookie.length+10>4096;){if(f.length===0)return;d=f.pop();s.cookie=d.name+"=; expires=Thu, 01-Jan-1970 00:00:01 GMT; path="+
+c.join("/")+"/"}}s.cookie=b}function mb(a){var b=la.location.pathname.split("/");a=a+"_"+b[b.length-1].replace(/[\/:]/g,"").toLowerCase()+"=";b=s.cookie.split(";");for(var c=0;c<b.length;c++){for(var d=b[c];d.charAt(0)==" ";)d=d.substring(1,d.length);if(d.indexOf(a)===0)return decodeURIComponent(d.substring(a.length,d.length))}return null}function C(a){for(var b=0;b<l.settings.length;b++)if(l.settings[b].nTable===a)return l.settings[b];return null}function fa(a){var b=[];a=a.aoData;for(var c=0,d=
+a.length;c<d;c++)a[c].nTr!==null&&b.push(a[c].nTr);return b}function W(a,b){var c=[],d,e,f,g,j;e=0;var k=a.aoData.length;if(b!==p){e=b;k=b+1}for(e=e;e<k;e++){j=a.aoData[e];if(j.nTr!==null){b=[];for(d=j.nTr.firstChild;d;){f=d.nodeName.toLowerCase();if(f=="td"||f=="th")b.push(d);d=d.nextSibling}f=d=0;for(g=a.aoColumns.length;f<g;f++)if(a.aoColumns[f].bVisible)c.push(b[f-d]);else{c.push(j._anHidden[f]);d++}}}return c}function O(a,b,c){a=a===null?"DataTables warning: "+c:"DataTables warning (table id = '"+
+a.sTableId+"'): "+c;if(b===0)if(l.ext.sErrMode=="alert")alert(a);else throw new Error(a);else la.console&&console.log&&console.log(a)}function r(a,b,c,d){if(d===p)d=c;if(b[c]!==p)a[d]=b[c]}function hb(a,b){var c;for(var d in b)if(b.hasOwnProperty(d)){c=b[d];if(typeof h[d]==="object"&&c!==null&&i.isArray(c)===false)i.extend(true,a[d],c);else a[d]=c}return a}function fb(a,b,c){i(a).bind("click.DT",b,function(d){a.blur();c(d)}).bind("keypress.DT",b,function(d){d.which===13&&c(d)}).bind("selectstart.DT",
+function(){return false})}function J(a,b,c,d){c&&a[b].push({fn:c,sName:d})}function K(a,b,c,d){b=a[b];for(var e=[],f=b.length-1;f>=0;f--)e.push(b[f].fn.apply(a.oInstance,d));c!==null&&i(a.oInstance).trigger(c,d);return e}function ib(a){var b=i('<div style="position:absolute; top:0; left:0; height:1px; width:1px; overflow:hidden"><div style="position:absolute; top:1px; left:1px; width:100px; overflow:scroll;"><div id="DT_BrowserTest" style="width:100%; height:10px;"></div></div></div>')[0];s.body.appendChild(b);
+a.oBrowser.bScrollOversize=i("#DT_BrowserTest",b)[0].offsetWidth===100?true:false;s.body.removeChild(b)}function jb(a){return function(){var b=[C(this[l.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return l.ext.oApi[a].apply(this,b)}}var ga=/\[.*?\]$/,kb=la.JSON?JSON.stringify:function(a){var b=typeof a;if(b!=="object"||a===null){if(b==="string")a='"'+a+'"';return a+""}var c,d,e=[],f=i.isArray(a);for(c in a){d=a[c];b=typeof d;if(b==="string")d='"'+d+'"';else if(b==="object"&&d!==
+null)d=kb(d);e.push((f?"":'"'+c+'":')+d)}return(f?"[":"{")+e+(f?"]":"}")};this.$=function(a,b){var c,d=[],e;c=C(this[l.ext.iApiIndex]);var f=c.aoData,g=c.aiDisplay,j=c.aiDisplayMaster;b||(b={});b=i.extend({},{filter:"none",order:"current",page:"all"},b);if(b.page=="current"){b=c._iDisplayStart;for(c=c.fnDisplayEnd();b<c;b++)(e=f[g[b]].nTr)&&d.push(e)}else if(b.order=="current"&&b.filter=="none"){b=0;for(c=j.length;b<c;b++)(e=f[j[b]].nTr)&&d.push(e)}else if(b.order=="current"&&b.filter=="applied"){b=
+0;for(c=g.length;b<c;b++)(e=f[g[b]].nTr)&&d.push(e)}else if(b.order=="original"&&b.filter=="none"){b=0;for(c=f.length;b<c;b++)(e=f[b].nTr)&&d.push(e)}else if(b.order=="original"&&b.filter=="applied"){b=0;for(c=f.length;b<c;b++){e=f[b].nTr;i.inArray(b,g)!==-1&&e&&d.push(e)}}else O(c,1,"Unknown selection options");f=i(d);d=f.filter(a);a=f.find(a);return i([].concat(i.makeArray(d),i.makeArray(a)))};this._=function(a,b){var c=[],d=this.$(a,b);a=0;for(b=d.length;a<b;a++)c.push(this.fnGetData(d[a]));return c};
+this.fnAddData=function(a,b){if(a.length===0)return[];var c=[],d,e=C(this[l.ext.iApiIndex]);if(typeof a[0]==="object"&&a[0]!==null)for(var f=0;f<a.length;f++){d=R(e,a[f]);if(d==-1)return c;c.push(d)}else{d=R(e,a);if(d==-1)return c;c.push(d)}e.aiDisplay=e.aiDisplayMaster.slice();if(b===p||b)qa(e);return c};this.fnAdjustColumnSizing=function(a){var b=C(this[l.ext.iApiIndex]);o(b);if(a===p||a)this.fnDraw(false);else if(b.oScroll.sX!==""||b.oScroll.sY!=="")this.oApi._fnScrollDraw(b)};this.fnClearTable=
+function(a){var b=C(this[l.ext.iApiIndex]);wa(b);if(a===p||a)H(b)};this.fnClose=function(a){for(var b=C(this[l.ext.iApiIndex]),c=0;c<b.aoOpenRows.length;c++)if(b.aoOpenRows[c].nParent==a){(a=b.aoOpenRows[c].nTr.parentNode)&&a.removeChild(b.aoOpenRows[c].nTr);b.aoOpenRows.splice(c,1);return 0}return 1};this.fnDeleteRow=function(a,b,c){var d=C(this[l.ext.iApiIndex]),e,f;a=typeof a==="object"?V(d,a):a;var g=d.aoData.splice(a,1);e=0;for(f=d.aoData.length;e<f;e++)if(d.aoData[e].nTr!==null)d.aoData[e].nTr._DT_RowIndex=
+e;e=i.inArray(a,d.aiDisplay);d.asDataSearch.splice(e,1);xa(d.aiDisplayMaster,a);xa(d.aiDisplay,a);typeof b==="function"&&b.call(this,d,g);if(d._iDisplayStart>=d.fnRecordsDisplay()){d._iDisplayStart-=d._iDisplayLength;if(d._iDisplayStart<0)d._iDisplayStart=0}if(c===p||c){I(d);H(d)}return g};this.fnDestroy=function(a){var b=C(this[l.ext.iApiIndex]),c=b.nTableWrapper.parentNode,d=b.nTBody,e,f;a=a===p?false:a;b.bDestroying=true;K(b,"aoDestroyCallback","destroy",[b]);if(!a){e=0;for(f=b.aoColumns.length;e<
+f;e++)b.aoColumns[e].bVisible===false&&this.fnSetColumnVis(e,true)}i(b.nTableWrapper).find("*").andSelf().unbind(".DT");i("tbody>tr>td."+b.oClasses.sRowEmpty,b.nTable).parent().remove();if(b.nTable!=b.nTHead.parentNode){i(b.nTable).children("thead").remove();b.nTable.appendChild(b.nTHead)}if(b.nTFoot&&b.nTable!=b.nTFoot.parentNode){i(b.nTable).children("tfoot").remove();b.nTable.appendChild(b.nTFoot)}b.nTable.parentNode.removeChild(b.nTable);i(b.nTableWrapper).remove();b.aaSorting=[];b.aaSortingFixed=
+[];ba(b);i(fa(b)).removeClass(b.asStripeClasses.join(" "));i("th, td",b.nTHead).removeClass([b.oClasses.sSortable,b.oClasses.sSortableAsc,b.oClasses.sSortableDesc,b.oClasses.sSortableNone].join(" "));if(b.bJUI){i("th span."+b.oClasses.sSortIcon+", td span."+b.oClasses.sSortIcon,b.nTHead).remove();i("th, td",b.nTHead).each(function(){var g=i("div."+b.oClasses.sSortJUIWrapper,this),j=g.contents();i(this).append(j);g.remove()})}if(!a&&b.nTableReinsertBefore)c.insertBefore(b.nTable,b.nTableReinsertBefore);
+else a||c.appendChild(b.nTable);e=0;for(f=b.aoData.length;e<f;e++)b.aoData[e].nTr!==null&&d.appendChild(b.aoData[e].nTr);if(b.oFeatures.bAutoWidth===true)b.nTable.style.width=t(b.sDestroyWidth);if(f=b.asDestroyStripes.length){a=i(d).children("tr");for(e=0;e<f;e++)a.filter(":nth-child("+f+"n + "+e+")").addClass(b.asDestroyStripes[e])}e=0;for(f=l.settings.length;e<f;e++)l.settings[e]==b&&l.settings.splice(e,1);h=b=null};this.fnDraw=function(a){var b=C(this[l.ext.iApiIndex]);if(a===false){I(b);H(b)}else qa(b)};
+this.fnFilter=function(a,b,c,d,e,f){var g=C(this[l.ext.iApiIndex]);if(g.oFeatures.bFilter){if(c===p||c===null)c=false;if(d===p||d===null)d=true;if(e===p||e===null)e=true;if(f===p||f===null)f=true;if(b===p||b===null){X(g,{sSearch:a+"",bRegex:c,bSmart:d,bCaseInsensitive:f},1);if(e&&g.aanFeatures.f){b=g.aanFeatures.f;c=0;for(d=b.length;c<d;c++)try{b[c]._DT_Input!=s.activeElement&&i(b[c]._DT_Input).val(a)}catch(j){i(b[c]._DT_Input).val(a)}}}else{i.extend(g.aoPreSearchCols[b],{sSearch:a+"",bRegex:c,bSmart:d,
+bCaseInsensitive:f});X(g,g.oPreviousSearch,1)}}};this.fnGetData=function(a,b){var c=C(this[l.ext.iApiIndex]);if(a!==p){var d=a;if(typeof a==="object"){var e=a.nodeName.toLowerCase();if(e==="tr")d=V(c,a);else if(e==="td"){d=V(c,a.parentNode);b=va(c,d,a)}}if(b!==p)return F(c,d,b,"");return c.aoData[d]!==p?c.aoData[d]._aData:null}return oa(c)};this.fnGetNodes=function(a){var b=C(this[l.ext.iApiIndex]);if(a!==p)return b.aoData[a]!==p?b.aoData[a].nTr:null;return fa(b)};this.fnGetPosition=function(a){var b=
+C(this[l.ext.iApiIndex]),c=a.nodeName.toUpperCase();if(c=="TR")return V(b,a);else if(c=="TD"||c=="TH"){c=V(b,a.parentNode);a=va(b,c,a);return[c,w(b,a),a]}return null};this.fnIsOpen=function(a){for(var b=C(this[l.ext.iApiIndex]),c=0;c<b.aoOpenRows.length;c++)if(b.aoOpenRows[c].nParent==a)return true;return false};this.fnOpen=function(a,b,c){var d=C(this[l.ext.iApiIndex]),e=fa(d);if(i.inArray(a,e)!==-1){this.fnClose(a);e=s.createElement("tr");var f=s.createElement("td");e.appendChild(f);f.className=
+c;f.colSpan=D(d);if(typeof b==="string")f.innerHTML=b;else i(f).html(b);b=i("tr",d.nTBody);i.inArray(a,b)!=-1&&i(e).insertAfter(a);d.aoOpenRows.push({nTr:e,nParent:a});return e}};this.fnPageChange=function(a,b){var c=C(this[l.ext.iApiIndex]);Ga(c,a);I(c);if(b===p||b)H(c)};this.fnSetColumnVis=function(a,b,c){var d=C(this[l.ext.iApiIndex]),e,f,g=d.aoColumns,j=d.aoData,k,m;if(g[a].bVisible!=b){if(b){for(e=f=0;e<a;e++)g[e].bVisible&&f++;m=f>=D(d);if(!m)for(e=a;e<g.length;e++)if(g[e].bVisible){k=e;break}e=
+0;for(f=j.length;e<f;e++)if(j[e].nTr!==null)m?j[e].nTr.appendChild(j[e]._anHidden[a]):j[e].nTr.insertBefore(j[e]._anHidden[a],W(d,e)[k])}else{e=0;for(f=j.length;e<f;e++)if(j[e].nTr!==null){k=W(d,e)[a];j[e]._anHidden[a]=k;k.parentNode.removeChild(k)}}g[a].bVisible=b;ia(d,d.aoHeader);d.nTFoot&&ia(d,d.aoFooter);e=0;for(f=d.aoOpenRows.length;e<f;e++)d.aoOpenRows[e].nTr.colSpan=D(d);if(c===p||c){o(d);H(d)}Ha(d)}};this.fnSettings=function(){return C(this[l.ext.iApiIndex])};this.fnSort=function(a){var b=
+C(this[l.ext.iApiIndex]);b.aaSorting=a;$(b)};this.fnSortListener=function(a,b,c){ya(C(this[l.ext.iApiIndex]),a,b,c)};this.fnUpdate=function(a,b,c,d,e){var f=C(this[l.ext.iApiIndex]);b=typeof b==="object"?V(f,b):b;if(i.isArray(a)&&c===p){f.aoData[b]._aData=a.slice();for(c=0;c<f.aoColumns.length;c++)this.fnUpdate(F(f,b,c),b,c,false,false)}else if(i.isPlainObject(a)&&c===p){f.aoData[b]._aData=i.extend(true,{},a);for(c=0;c<f.aoColumns.length;c++)this.fnUpdate(F(f,b,c),b,c,false,false)}else{S(f,b,c,a);
+a=F(f,b,c,"display");var g=f.aoColumns[c];if(g.fnRender!==null){a=da(f,b,c);g.bUseRendered&&S(f,b,c,a)}if(f.aoData[b].nTr!==null)W(f,b)[c].innerHTML=a}c=i.inArray(b,f.aiDisplay);f.asDataSearch[c]=Da(f,na(f,b,"filter",A(f,"bSearchable")));if(e===p||e)o(f);if(d===p||d)qa(f);return 0};this.fnVersionCheck=l.ext.fnVersionCheck;this.oApi={_fnExternApiFunc:jb,_fnInitialise:ra,_fnInitComplete:pa,_fnLanguageCompat:Fa,_fnAddColumn:n,_fnColumnOptions:q,_fnAddData:R,_fnCreateTr:ua,_fnGatherData:ea,_fnBuildHead:Ka,
+_fnDrawHead:ia,_fnDraw:H,_fnReDraw:qa,_fnAjaxUpdate:La,_fnAjaxParameters:Ta,_fnAjaxUpdateDraw:Ua,_fnServerParams:Aa,_fnAddOptionsHtml:Ma,_fnFeatureHtmlTable:Qa,_fnScrollDraw:$a,_fnAdjustColumnSizing:o,_fnFeatureHtmlFilter:Oa,_fnFilterComplete:X,_fnFilterCustom:Xa,_fnFilterColumn:Wa,_fnFilter:Va,_fnBuildSearchArray:Ba,_fnBuildSearchRow:Da,_fnFilterCreateSearch:Ca,_fnDataToSearch:Ya,_fnSort:$,_fnSortAttachListener:ya,_fnSortingClasses:ba,_fnFeatureHtmlPaginate:Sa,_fnPageChange:Ga,_fnFeatureHtmlInfo:Ra,
+_fnUpdateInfo:Za,_fnFeatureHtmlLength:Na,_fnFeatureHtmlProcessing:Pa,_fnProcessingDisplay:P,_fnVisibleToColumnIndex:v,_fnColumnIndexToVisible:w,_fnNodeToDataIndex:V,_fnVisbleColumns:D,_fnCalculateEnd:I,_fnConvertToWidth:ab,_fnCalculateColumnWidths:ta,_fnScrollingWidthAdjust:cb,_fnGetWidestNode:bb,_fnGetMaxLenString:db,_fnStringToCss:t,_fnDetectType:G,_fnSettingsFromNode:C,_fnGetDataMaster:oa,_fnGetTrNodes:fa,_fnGetTdNodes:W,_fnEscapeRegex:Ea,_fnDeleteIndex:xa,_fnReOrderIndex:E,_fnColumnOrdering:Y,
+_fnLog:O,_fnClearTable:wa,_fnSaveState:Ha,_fnLoadState:gb,_fnCreateCookie:lb,_fnReadCookie:mb,_fnDetectHeader:ha,_fnGetUniqueThs:Z,_fnScrollBarWidth:eb,_fnApplyToChildren:N,_fnMap:r,_fnGetRowData:na,_fnGetCellData:F,_fnSetCellData:S,_fnGetObjectDataFn:ca,_fnSetObjectDataFn:Ja,_fnApplyColumnDefs:ma,_fnBindAction:fb,_fnExtend:hb,_fnCallbackReg:J,_fnCallbackFire:K,_fnJsonString:kb,_fnRender:da,_fnNodeToColumnIndex:va,_fnInfoMacros:za,_fnBrowserDetect:ib,_fnGetColumns:A};i.extend(l.ext.oApi,this.oApi);
+for(var Ia in l.ext.oApi)if(Ia)this[Ia]=jb(Ia);var sa=this;this.each(function(){var a=0,b,c,d;c=this.getAttribute("id");var e=false,f=false;if(this.nodeName.toLowerCase()!="table")O(null,0,"Attempted to initialise DataTables on a node which is not a table: "+this.nodeName);else{a=0;for(b=l.settings.length;a<b;a++){if(l.settings[a].nTable==this)if(h===p||h.bRetrieve)return l.settings[a].oInstance;else if(h.bDestroy){l.settings[a].oInstance.fnDestroy();break}else{O(l.settings[a],0,"Cannot reinitialise DataTable.\n\nTo retrieve the DataTables object for this table, pass no arguments or see the docs for bRetrieve and bDestroy");
+return}if(l.settings[a].sTableId==this.id){l.settings.splice(a,1);break}}if(c===null||c==="")this.id=c="DataTables_Table_"+l.ext._oExternConfig.iNextUnique++;var g=i.extend(true,{},l.models.oSettings,{nTable:this,oApi:sa.oApi,oInit:h,sDestroyWidth:i(this).width(),sInstance:c,sTableId:c});l.settings.push(g);g.oInstance=sa.length===1?sa:i(this).dataTable();h||(h={});h.oLanguage&&Fa(h.oLanguage);h=hb(i.extend(true,{},l.defaults),h);r(g.oFeatures,h,"bPaginate");r(g.oFeatures,h,"bLengthChange");r(g.oFeatures,
+h,"bFilter");r(g.oFeatures,h,"bSort");r(g.oFeatures,h,"bInfo");r(g.oFeatures,h,"bProcessing");r(g.oFeatures,h,"bAutoWidth");r(g.oFeatures,h,"bSortClasses");r(g.oFeatures,h,"bServerSide");r(g.oFeatures,h,"bDeferRender");r(g.oScroll,h,"sScrollX","sX");r(g.oScroll,h,"sScrollXInner","sXInner");r(g.oScroll,h,"sScrollY","sY");r(g.oScroll,h,"bScrollCollapse","bCollapse");r(g.oScroll,h,"bScrollInfinite","bInfinite");r(g.oScroll,h,"iScrollLoadGap","iLoadGap");r(g.oScroll,h,"bScrollAutoCss","bAutoCss");r(g,
+h,"asStripeClasses");r(g,h,"asStripClasses","asStripeClasses");r(g,h,"fnServerData");r(g,h,"fnFormatNumber");r(g,h,"sServerMethod");r(g,h,"aaSorting");r(g,h,"aaSortingFixed");r(g,h,"aLengthMenu");r(g,h,"sPaginationType");r(g,h,"sAjaxSource");r(g,h,"sAjaxDataProp");r(g,h,"iCookieDuration");r(g,h,"sCookiePrefix");r(g,h,"sDom");r(g,h,"bSortCellsTop");r(g,h,"iTabIndex");r(g,h,"oSearch","oPreviousSearch");r(g,h,"aoSearchCols","aoPreSearchCols");r(g,h,"iDisplayLength","_iDisplayLength");r(g,h,"bJQueryUI",
+"bJUI");r(g,h,"fnCookieCallback");r(g,h,"fnStateLoad");r(g,h,"fnStateSave");r(g.oLanguage,h,"fnInfoCallback");J(g,"aoDrawCallback",h.fnDrawCallback,"user");J(g,"aoServerParams",h.fnServerParams,"user");J(g,"aoStateSaveParams",h.fnStateSaveParams,"user");J(g,"aoStateLoadParams",h.fnStateLoadParams,"user");J(g,"aoStateLoaded",h.fnStateLoaded,"user");J(g,"aoRowCallback",h.fnRowCallback,"user");J(g,"aoRowCreatedCallback",h.fnCreatedRow,"user");J(g,"aoHeaderCallback",h.fnHeaderCallback,"user");J(g,"aoFooterCallback",
+h.fnFooterCallback,"user");J(g,"aoInitComplete",h.fnInitComplete,"user");J(g,"aoPreDrawCallback",h.fnPreDrawCallback,"user");if(g.oFeatures.bServerSide&&g.oFeatures.bSort&&g.oFeatures.bSortClasses)J(g,"aoDrawCallback",ba,"server_side_sort_classes");else g.oFeatures.bDeferRender&&J(g,"aoDrawCallback",ba,"defer_sort_classes");if(h.bJQueryUI){i.extend(g.oClasses,l.ext.oJUIClasses);if(h.sDom===l.defaults.sDom&&l.defaults.sDom==="lfrtip")g.sDom='<"H"lfr>t<"F"ip>'}else i.extend(g.oClasses,l.ext.oStdClasses);
+i(this).addClass(g.oClasses.sTable);if(g.oScroll.sX!==""||g.oScroll.sY!=="")g.oScroll.iBarWidth=eb();if(g.iInitDisplayStart===p){g.iInitDisplayStart=h.iDisplayStart;g._iDisplayStart=h.iDisplayStart}if(h.bStateSave){g.oFeatures.bStateSave=true;gb(g,h);J(g,"aoDrawCallback",Ha,"state_save")}if(h.iDeferLoading!==null){g.bDeferLoading=true;a=i.isArray(h.iDeferLoading);g._iRecordsDisplay=a?h.iDeferLoading[0]:h.iDeferLoading;g._iRecordsTotal=a?h.iDeferLoading[1]:h.iDeferLoading}if(h.aaData!==null)f=true;
+if(h.oLanguage.sUrl!==""){g.oLanguage.sUrl=h.oLanguage.sUrl;i.getJSON(g.oLanguage.sUrl,null,function(k){Fa(k);i.extend(true,g.oLanguage,h.oLanguage,k);ra(g)});e=true}else i.extend(true,g.oLanguage,h.oLanguage);if(h.asStripeClasses===null)g.asStripeClasses=[g.oClasses.sStripeOdd,g.oClasses.sStripeEven];b=g.asStripeClasses.length;g.asDestroyStripes=[];if(b){c=false;d=i(this).children("tbody").children("tr:lt("+b+")");for(a=0;a<b;a++)if(d.hasClass(g.asStripeClasses[a])){c=true;g.asDestroyStripes.push(g.asStripeClasses[a])}c&&
+d.removeClass(g.asStripeClasses.join(" "))}c=[];a=this.getElementsByTagName("thead");if(a.length!==0){ha(g.aoHeader,a[0]);c=Z(g)}if(h.aoColumns===null){d=[];a=0;for(b=c.length;a<b;a++)d.push(null)}else d=h.aoColumns;a=0;for(b=d.length;a<b;a++){if(h.saved_aoColumns!==p&&h.saved_aoColumns.length==b){if(d[a]===null)d[a]={};d[a].bVisible=h.saved_aoColumns[a].bVisible}n(g,c?c[a]:null)}ma(g,h.aoColumnDefs,d,function(k,m){q(g,k,m)});a=0;for(b=g.aaSorting.length;a<b;a++){if(g.aaSorting[a][0]>=g.aoColumns.length)g.aaSorting[a][0]=
+0;var j=g.aoColumns[g.aaSorting[a][0]];if(g.aaSorting[a][2]===p)g.aaSorting[a][2]=0;if(h.aaSorting===p&&g.saved_aaSorting===p)g.aaSorting[a][1]=j.asSorting[0];c=0;for(d=j.asSorting.length;c<d;c++)if(g.aaSorting[a][1]==j.asSorting[c]){g.aaSorting[a][2]=c;break}}ba(g);ib(g);a=i(this).children("caption").each(function(){this._captionSide=i(this).css("caption-side")});b=i(this).children("thead");if(b.length===0){b=[s.createElement("thead")];this.appendChild(b[0])}g.nTHead=b[0];b=i(this).children("tbody");
+if(b.length===0){b=[s.createElement("tbody")];this.appendChild(b[0])}g.nTBody=b[0];g.nTBody.setAttribute("role","alert");g.nTBody.setAttribute("aria-live","polite");g.nTBody.setAttribute("aria-relevant","all");b=i(this).children("tfoot");if(b.length===0&&a.length>0&&(g.oScroll.sX!==""||g.oScroll.sY!=="")){b=[s.createElement("tfoot")];this.appendChild(b[0])}if(b.length>0){g.nTFoot=b[0];ha(g.aoFooter,g.nTFoot)}if(f)for(a=0;a<h.aaData.length;a++)R(g,h.aaData[a]);else ea(g);g.aiDisplay=g.aiDisplayMaster.slice();
+g.bInitialised=true;e===false&&ra(g)}});sa=null;return this};l.fnVersionCheck=function(h){var n=function(A,G){for(;A.length<G;)A+="0";return A},q=l.ext.sVersion.split(".");h=h.split(".");for(var o="",v="",w=0,D=h.length;w<D;w++){o+=n(q[w],3);v+=n(h[w],3)}return parseInt(o,10)>=parseInt(v,10)};l.fnIsDataTable=function(h){for(var n=l.settings,q=0;q<n.length;q++)if(n[q].nTable===h||n[q].nScrollHead===h||n[q].nScrollFoot===h)return true;return false};l.fnTables=function(h){var n=[];jQuery.each(l.settings,
+function(q,o){if(!h||h===true&&i(o.nTable).is(":visible"))n.push(o.nTable)});return n};l.version="1.9.4";l.settings=[];l.models={};l.models.ext={afnFiltering:[],afnSortData:[],aoFeatures:[],aTypes:[],fnVersionCheck:l.fnVersionCheck,iApiIndex:0,ofnSearch:{},oApi:{},oStdClasses:{},oJUIClasses:{},oPagination:{},oSort:{},sVersion:l.version,sErrMode:"alert",_oExternConfig:{iNextUnique:0}};l.models.oSearch={bCaseInsensitive:true,sSearch:"",bRegex:false,bSmart:true};l.models.oRow={nTr:null,_aData:[],_aSortData:[],
+_anHidden:[],_sRowStripe:""};l.models.oColumn={aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bUseRendered:null,bVisible:null,_bAutoType:true,fnCreatedCell:null,fnGetData:null,fnRender:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};l.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:null,
+aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:true,bDeferRender:false,bDestroy:false,bFilter:true,bInfo:true,bJQueryUI:false,bLengthChange:true,bPaginate:true,bProcessing:false,bRetrieve:false,bScrollAutoCss:true,bScrollCollapse:false,bScrollInfinite:false,bServerSide:false,bSort:true,bSortCellsTop:false,bSortClasses:true,bStateSave:false,fnCookieCallback:null,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(h){if(h<
+1E3)return h;var n=h+"";h=n.split("");var q="";n=n.length;for(var o=0;o<n;o++){if(o%3===0&&o!==0)q=this.oLanguage.sInfoThousands+q;q=h[n-o-1]+q}return q},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:function(h,n,q,o){o.jqXHR=i.ajax({url:h,data:n,success:function(v){v.sError&&o.oApi._fnLog(o,0,v.sError);i(o.oInstance).trigger("xhr",[o,v]);q(v)},dataType:"json",cache:false,type:o.sServerMethod,error:function(v,w){w=="parsererror"&&
+o.oApi._fnLog(o,0,"DataTables warning: JSON data from server could not be parsed. This is caused by a JSON formatting error.")}})},fnServerParams:null,fnStateLoad:function(h){h=this.oApi._fnReadCookie(h.sCookiePrefix+h.sInstance);var n;try{n=typeof i.parseJSON==="function"?i.parseJSON(h):eval("("+h+")")}catch(q){n=null}return n},fnStateLoadParams:null,fnStateLoaded:null,fnStateSave:function(h,n){this.oApi._fnCreateCookie(h.sCookiePrefix+h.sInstance,this.oApi._fnJsonString(n),h.iCookieDuration,h.sCookiePrefix,
+h.fnCookieCallback)},fnStateSaveParams:null,iCookieDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iScrollLoadGap:100,iTabIndex:0,oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",
+sInfoPostFix:"",sInfoThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sUrl:"",sZeroRecords:"No matching records found"},oSearch:i.extend({},l.models.oSearch),sAjaxDataProp:"aaData",sAjaxSource:null,sCookiePrefix:"SpryMedia_DataTables_",sDom:"lfrtip",sPaginationType:"two_button",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET"};l.defaults.columns={aDataSort:null,asSorting:["asc","desc"],bSearchable:true,bSortable:true,
+bUseRendered:true,bVisible:true,fnCreatedCell:null,fnRender:null,iDataSort:-1,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};l.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortClasses:null,bStateSave:null},oScroll:{bAutoCss:null,bCollapse:null,bInfinite:null,iBarWidth:0,iLoadGap:null,
+sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:false},aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aoColumns:[],aoHeader:[],aoFooter:[],asDataSearch:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:null,asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],
+aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:false,bInitialised:false,aoOpenRows:[],sDom:null,sPaginationType:"two_button",iCookieDuration:0,sCookiePrefix:"",fnCookieCallback:null,aoStateSave:[],aoStateLoad:[],oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:true,jqXHR:null,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:false,iDrawError:-1,_iDisplayLength:10,
+_iDisplayStart:0,_iDisplayEnd:10,_iRecordsTotal:0,_iRecordsDisplay:0,bJUI:null,oClasses:{},bFiltered:false,bSorted:false,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return this.oFeatures.bServerSide?parseInt(this._iRecordsTotal,10):this.aiDisplayMaster.length},fnRecordsDisplay:function(){return this.oFeatures.bServerSide?parseInt(this._iRecordsDisplay,10):this.aiDisplay.length},fnDisplayEnd:function(){return this.oFeatures.bServerSide?this.oFeatures.bPaginate===false||
+this._iDisplayLength==-1?this._iDisplayStart+this.aiDisplay.length:Math.min(this._iDisplayStart+this._iDisplayLength,this._iRecordsDisplay):this._iDisplayEnd},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null};l.ext=i.extend(true,{},l.models.ext);i.extend(l.ext.oStdClasses,{sTable:"dataTable",sPagePrevEnabled:"paginate_enabled_previous",sPagePrevDisabled:"paginate_disabled_previous",sPageNextEnabled:"paginate_enabled_next",sPageNextDisabled:"paginate_disabled_next",sPageJUINext:"",
+sPageJUIPrev:"",sPageButton:"paginate_button",sPageButtonActive:"paginate_active",sPageButtonStaticDisabled:"paginate_button paginate_button_disabled",sPageFirst:"first",sPagePrevious:"previous",sPageNext:"next",sPageLast:"last",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",
+sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",
+sFooterTH:"",sJUIHeader:"",sJUIFooter:""});i.extend(l.ext.oJUIClasses,l.ext.oStdClasses,{sPagePrevEnabled:"fg-button ui-button ui-state-default ui-corner-left",sPagePrevDisabled:"fg-button ui-button ui-state-default ui-corner-left ui-state-disabled",sPageNextEnabled:"fg-button ui-button ui-state-default ui-corner-right",sPageNextDisabled:"fg-button ui-button ui-state-default ui-corner-right ui-state-disabled",sPageJUINext:"ui-icon ui-icon-circle-arrow-e",sPageJUIPrev:"ui-icon ui-icon-circle-arrow-w",
+sPageButton:"fg-button ui-button ui-state-default",sPageButtonActive:"fg-button ui-button ui-state-default ui-state-disabled",sPageButtonStaticDisabled:"fg-button ui-button ui-state-default ui-state-disabled",sPageFirst:"first ui-corner-tl ui-corner-bl",sPageLast:"last ui-corner-tr ui-corner-br",sPaging:"dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",sSortAsc:"ui-state-default",sSortDesc:"ui-state-default",sSortable:"ui-state-default",sSortableAsc:"ui-state-default",
+sSortableDesc:"ui-state-default",sSortableNone:"ui-state-default",sSortJUIAsc:"css_right ui-icon ui-icon-triangle-1-n",sSortJUIDesc:"css_right ui-icon ui-icon-triangle-1-s",sSortJUI:"css_right ui-icon ui-icon-carat-2-n-s",sSortJUIAscAllowed:"css_right ui-icon ui-icon-carat-1-n",sSortJUIDescAllowed:"css_right ui-icon ui-icon-carat-1-s",sSortJUIWrapper:"DataTables_sort_wrapper",sSortIcon:"DataTables_sort_icon",sScrollHead:"dataTables_scrollHead ui-state-default",sScrollFoot:"dataTables_scrollFoot ui-state-default",
+sFooterTH:"ui-state-default",sJUIHeader:"fg-toolbar ui-toolbar ui-widget-header ui-corner-tl ui-corner-tr ui-helper-clearfix",sJUIFooter:"fg-toolbar ui-toolbar ui-widget-header ui-corner-bl ui-corner-br ui-helper-clearfix"});i.extend(l.ext.oPagination,{two_button:{fnInit:function(h,n,q){var o=h.oLanguage.oPaginate,v=function(D){h.oApi._fnPageChange(h,D.data.action)&&q(h)};o=!h.bJUI?'<a class="'+h.oClasses.sPagePrevDisabled+'" tabindex="'+h.iTabIndex+'" role="button">'+o.sPrevious+'</a><a class="'+
+h.oClasses.sPageNextDisabled+'" tabindex="'+h.iTabIndex+'" role="button">'+o.sNext+"</a>":'<a class="'+h.oClasses.sPagePrevDisabled+'" tabindex="'+h.iTabIndex+'" role="button"><span class="'+h.oClasses.sPageJUIPrev+'"></span></a><a class="'+h.oClasses.sPageNextDisabled+'" tabindex="'+h.iTabIndex+'" role="button"><span class="'+h.oClasses.sPageJUINext+'"></span></a>';i(n).append(o);var w=i("a",n);o=w[0];w=w[1];h.oApi._fnBindAction(o,{action:"previous"},v);h.oApi._fnBindAction(w,{action:"next"},v);
+if(!h.aanFeatures.p){n.id=h.sTableId+"_paginate";o.id=h.sTableId+"_previous";w.id=h.sTableId+"_next";o.setAttribute("aria-controls",h.sTableId);w.setAttribute("aria-controls",h.sTableId)}},fnUpdate:function(h){if(h.aanFeatures.p)for(var n=h.oClasses,q=h.aanFeatures.p,o,v=0,w=q.length;v<w;v++)if(o=q[v].firstChild){o.className=h._iDisplayStart===0?n.sPagePrevDisabled:n.sPagePrevEnabled;o=o.nextSibling;o.className=h.fnDisplayEnd()==h.fnRecordsDisplay()?n.sPageNextDisabled:n.sPageNextEnabled}}},iFullNumbersShowPages:5,
+full_numbers:{fnInit:function(h,n,q){var o=h.oLanguage.oPaginate,v=h.oClasses,w=function(G){h.oApi._fnPageChange(h,G.data.action)&&q(h)};i(n).append('<a tabindex="'+h.iTabIndex+'" class="'+v.sPageButton+" "+v.sPageFirst+'">'+o.sFirst+'</a><a tabindex="'+h.iTabIndex+'" class="'+v.sPageButton+" "+v.sPagePrevious+'">'+o.sPrevious+'</a><span></span><a tabindex="'+h.iTabIndex+'" class="'+v.sPageButton+" "+v.sPageNext+'">'+o.sNext+'</a><a tabindex="'+h.iTabIndex+'" class="'+v.sPageButton+" "+v.sPageLast+
+'">'+o.sLast+"</a>");var D=i("a",n);o=D[0];v=D[1];var A=D[2];D=D[3];h.oApi._fnBindAction(o,{action:"first"},w);h.oApi._fnBindAction(v,{action:"previous"},w);h.oApi._fnBindAction(A,{action:"next"},w);h.oApi._fnBindAction(D,{action:"last"},w);if(!h.aanFeatures.p){n.id=h.sTableId+"_paginate";o.id=h.sTableId+"_first";v.id=h.sTableId+"_previous";A.id=h.sTableId+"_next";D.id=h.sTableId+"_last"}},fnUpdate:function(h,n){if(h.aanFeatures.p){var q=l.ext.oPagination.iFullNumbersShowPages,o=Math.floor(q/2),v=
+Math.ceil(h.fnRecordsDisplay()/h._iDisplayLength),w=Math.ceil(h._iDisplayStart/h._iDisplayLength)+1,D="",A,G=h.oClasses,E,Y=h.aanFeatures.p,ma=function(R){h.oApi._fnBindAction(this,{page:R+A-1},function(ea){h.oApi._fnPageChange(h,ea.data.page);n(h);ea.preventDefault()})};if(h._iDisplayLength===-1)w=o=A=1;else if(v<q){A=1;o=v}else if(w<=o){A=1;o=q}else if(w>=v-o){A=v-q+1;o=v}else{A=w-Math.ceil(q/2)+1;o=A+q-1}for(q=A;q<=o;q++)D+=w!==q?'<a tabindex="'+h.iTabIndex+'" class="'+G.sPageButton+'">'+h.fnFormatNumber(q)+
+"</a>":'<a tabindex="'+h.iTabIndex+'" class="'+G.sPageButtonActive+'">'+h.fnFormatNumber(q)+"</a>";q=0;for(o=Y.length;q<o;q++){E=Y[q];if(E.hasChildNodes()){i("span:eq(0)",E).html(D).children("a").each(ma);E=E.getElementsByTagName("a");E=[E[0],E[1],E[E.length-2],E[E.length-1]];i(E).removeClass(G.sPageButton+" "+G.sPageButtonActive+" "+G.sPageButtonStaticDisabled);i([E[0],E[1]]).addClass(w==1?G.sPageButtonStaticDisabled:G.sPageButton);i([E[2],E[3]]).addClass(v===0||w===v||h._iDisplayLength===-1?G.sPageButtonStaticDisabled:
+G.sPageButton)}}}}}});i.extend(l.ext.oSort,{"string-pre":function(h){if(typeof h!="string")h=h!==null&&h.toString?h.toString():"";return h.toLowerCase()},"string-asc":function(h,n){return h<n?-1:h>n?1:0},"string-desc":function(h,n){return h<n?1:h>n?-1:0},"html-pre":function(h){return h.replace(/<.*?>/g,"").toLowerCase()},"html-asc":function(h,n){return h<n?-1:h>n?1:0},"html-desc":function(h,n){return h<n?1:h>n?-1:0},"date-pre":function(h){h=Date.parse(h);if(isNaN(h)||h==="")h=Date.parse("01/01/1970 00:00:00");
+return h},"date-asc":function(h,n){return h-n},"date-desc":function(h,n){return n-h},"numeric-pre":function(h){return h=="-"||h===""?0:h*1},"numeric-asc":function(h,n){return h-n},"numeric-desc":function(h,n){return n-h}});i.extend(l.ext.aTypes,[function(h){if(typeof h==="number")return"numeric";else if(typeof h!=="string")return null;var n,q=false;n=h.charAt(0);if("0123456789-".indexOf(n)==-1)return null;for(var o=1;o<h.length;o++){n=h.charAt(o);if("0123456789.".indexOf(n)==-1)return null;if(n==
+"."){if(q)return null;q=true}}return"numeric"},function(h){var n=Date.parse(h);if(n!==null&&!isNaN(n)||typeof h==="string"&&h.length===0)return"date";return null},function(h){if(typeof h==="string"&&h.indexOf("<")!=-1&&h.indexOf(">")!=-1)return"html";return null}]);i.fn.DataTable=l;i.fn.dataTable=l;i.fn.dataTableSettings=l.settings;i.fn.dataTableExt=l.ext})})(window,document);
diff --git a/slider-core/src/main/resources/webapps/static/hadoop-st.png b/slider-core/src/main/resources/webapps/static/hadoop-st.png
new file mode 100644
index 0000000..b481c04
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/hadoop-st.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js b/slider-core/src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js
new file mode 100644
index 0000000..bc3fbc8
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js
@@ -0,0 +1,2 @@
+/*! jQuery v1.8.2 jquery.com | jquery.org/license */
+(function(a,b){function G(a){var b=F[a]={};return p.each(a.split(s),function(a,c){b[c]=!0}),b}function J(a,c,d){if(d===b&&a.nodeType===1){var e="data-"+c.replace(I,"-$1").toLowerCase();d=a.getAttribute(e);if(typeof d=="string"){try{d=d==="true"?!0:d==="false"?!1:d==="null"?null:+d+""===d?+d:H.test(d)?p.parseJSON(d):d}catch(f){}p.data(a,c,d)}else d=b}return d}function K(a){var b;for(b in a){if(b==="data"&&p.isEmptyObject(a[b]))continue;if(b!=="toJSON")return!1}return!0}function ba(){return!1}function bb(){return!0}function bh(a){return!a||!a.parentNode||a.parentNode.nodeType===11}function bi(a,b){do a=a[b];while(a&&a.nodeType!==1);return a}function bj(a,b,c){b=b||0;if(p.isFunction(b))return p.grep(a,function(a,d){var e=!!b.call(a,d,a);return e===c});if(b.nodeType)return p.grep(a,function(a,d){return a===b===c});if(typeof b=="string"){var d=p.grep(a,function(a){return a.nodeType===1});if(be.test(b))return p.filter(b,d,!c);b=p.filter(b,d)}return p.grep(a,function(a,d){return p.inArray(a,b)>=0===c})}function bk(a){var b=bl.split("|"),c=a.createDocumentFragment();if(c.createElement)while(b.length)c.createElement(b.pop());return c}function bC(a,b){return a.getElementsByTagName(b)[0]||a.appendChild(a.ownerDocument.createElement(b))}function bD(a,b){if(b.nodeType!==1||!p.hasData(a))return;var c,d,e,f=p._data(a),g=p._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;d<e;d++)p.event.add(b,c,h[c][d])}g.data&&(g.data=p.extend({},g.data))}function bE(a,b){var c;if(b.nodeType!==1)return;b.clearAttributes&&b.clearAttributes(),b.mergeAttributes&&b.mergeAttributes(a),c=b.nodeName.toLowerCase(),c==="object"?(b.parentNode&&(b.outerHTML=a.outerHTML),p.support.html5Clone&&a.innerHTML&&!p.trim(b.innerHTML)&&(b.innerHTML=a.innerHTML)):c==="input"&&bv.test(a.type)?(b.defaultChecked=b.checked=a.checked,b.value!==a.value&&(b.value=a.value)):c==="option"?b.selected=a.defaultSelected:c==="input"||c==="textarea"?b.defaultValue=a.defaultValue:c==="script"&&b.text!==a.text&&(b.text=a.text),b.removeAttribute(p.expando)}function bF(a){return typeof a.getElementsByTagName!="undefined"?a.getElementsByTagName("*"):typeof a.querySelectorAll!="undefined"?a.querySelectorAll("*"):[]}function bG(a){bv.test(a.type)&&(a.defaultChecked=a.checked)}function bY(a,b){if(b in a)return b;var c=b.charAt(0).toUpperCase()+b.slice(1),d=b,e=bW.length;while(e--){b=bW[e]+c;if(b in a)return b}return d}function bZ(a,b){return a=b||a,p.css(a,"display")==="none"||!p.contains(a.ownerDocument,a)}function b$(a,b){var c,d,e=[],f=0,g=a.length;for(;f<g;f++){c=a[f];if(!c.style)continue;e[f]=p._data(c,"olddisplay"),b?(!e[f]&&c.style.display==="none"&&(c.style.display=""),c.style.display===""&&bZ(c)&&(e[f]=p._data(c,"olddisplay",cc(c.nodeName)))):(d=bH(c,"display"),!e[f]&&d!=="none"&&p._data(c,"olddisplay",d))}for(f=0;f<g;f++){c=a[f];if(!c.style)continue;if(!b||c.style.display==="none"||c.style.display==="")c.style.display=b?e[f]||"":"none"}return a}function b_(a,b,c){var d=bP.exec(b);return d?Math.max(0,d[1]-(c||0))+(d[2]||"px"):b}function ca(a,b,c,d){var e=c===(d?"border":"content")?4:b==="width"?1:0,f=0;for(;e<4;e+=2)c==="margin"&&(f+=p.css(a,c+bV[e],!0)),d?(c==="content"&&(f-=parseFloat(bH(a,"padding"+bV[e]))||0),c!=="margin"&&(f-=parseFloat(bH(a,"border"+bV[e]+"Width"))||0)):(f+=parseFloat(bH(a,"padding"+bV[e]))||0,c!=="padding"&&(f+=parseFloat(bH(a,"border"+bV[e]+"Width"))||0));return f}function cb(a,b,c){var d=b==="width"?a.offsetWidth:a.offsetHeight,e=!0,f=p.support.boxSizing&&p.css(a,"boxSizing")==="border-box";if(d<=0||d==null){d=bH(a,b);if(d<0||d==null)d=a.style[b];if(bQ.test(d))return d;e=f&&(p.support.boxSizingReliable||d===a.style[b]),d=parseFloat(d)||0}return d+ca(a,b,c||(f?"border":"content"),e)+"px"}function cc(a){if(bS[a])return bS[a];var b=p("<"+a+">").appendTo(e.body),c=b.css("display");b.remove();if(c==="none"||c===""){bI=e.body.appendChild(bI||p.extend(e.createElement("iframe"),{frameBorder:0,width:0,height:0}));if(!bJ||!bI.createElement)bJ=(bI.contentWindow||bI.contentDocument).document,bJ.write("<!doctype html><html><body>"),bJ.close();b=bJ.body.appendChild(bJ.createElement(a)),c=bH(b,"display"),e.body.removeChild(bI)}return bS[a]=c,c}function ci(a,b,c,d){var e;if(p.isArray(b))p.each(b,function(b,e){c||ce.test(a)?d(a,e):ci(a+"["+(typeof e=="object"?b:"")+"]",e,c,d)});else if(!c&&p.type(b)==="object")for(e in b)ci(a+"["+e+"]",b[e],c,d);else d(a,b)}function cz(a){return function(b,c){typeof b!="string"&&(c=b,b="*");var d,e,f,g=b.toLowerCase().split(s),h=0,i=g.length;if(p.isFunction(c))for(;h<i;h++)d=g[h],f=/^\+/.test(d),f&&(d=d.substr(1)||"*"),e=a[d]=a[d]||[],e[f?"unshift":"push"](c)}}function cA(a,c,d,e,f,g){f=f||c.dataTypes[0],g=g||{},g[f]=!0;var h,i=a[f],j=0,k=i?i.length:0,l=a===cv;for(;j<k&&(l||!h);j++)h=i[j](c,d,e),typeof h=="string"&&(!l||g[h]?h=b:(c.dataTypes.unshift(h),h=cA(a,c,d,e,h,g)));return(l||!h)&&!g["*"]&&(h=cA(a,c,d,e,"*",g)),h}function cB(a,c){var d,e,f=p.ajaxSettings.flatOptions||{};for(d in c)c[d]!==b&&((f[d]?a:e||(e={}))[d]=c[d]);e&&p.extend(!0,a,e)}function cC(a,c,d){var e,f,g,h,i=a.contents,j=a.dataTypes,k=a.responseFields;for(f in k)f in d&&(c[k[f]]=d[f]);while(j[0]==="*")j.shift(),e===b&&(e=a.mimeType||c.getResponseHeader("content-type"));if(e)for(f in i)if(i[f]&&i[f].test(e)){j.unshift(f);break}if(j[0]in d)g=j[0];else{for(f in d){if(!j[0]||a.converters[f+" "+j[0]]){g=f;break}h||(h=f)}g=g||h}if(g)return g!==j[0]&&j.unshift(g),d[g]}function cD(a,b){var c,d,e,f,g=a.dataTypes.slice(),h=g[0],i={},j=0;a.dataFilter&&(b=a.dataFilter(b,a.dataType));if(g[1])for(c in a.converters)i[c.toLowerCase()]=a.converters[c];for(;e=g[++j];)if(e!=="*"){if(h!=="*"&&h!==e){c=i[h+" "+e]||i["* "+e];if(!c)for(d in i){f=d.split(" ");if(f[1]===e){c=i[h+" "+f[0]]||i["* "+f[0]];if(c){c===!0?c=i[d]:i[d]!==!0&&(e=f[0],g.splice(j--,0,e));break}}}if(c!==!0)if(c&&a["throws"])b=c(b);else try{b=c(b)}catch(k){return{state:"parsererror",error:c?k:"No conversion from "+h+" to "+e}}}h=e}return{state:"success",data:b}}function cL(){try{return new a.XMLHttpRequest}catch(b){}}function cM(){try{return new a.ActiveXObject("Microsoft.XMLHTTP")}catch(b){}}function cU(){return setTimeout(function(){cN=b},0),cN=p.now()}function cV(a,b){p.each(b,function(b,c){var d=(cT[b]||[]).concat(cT["*"]),e=0,f=d.length;for(;e<f;e++)if(d[e].call(a,b,c))return})}function cW(a,b,c){var d,e=0,f=0,g=cS.length,h=p.Deferred().always(function(){delete i.elem}),i=function(){var b=cN||cU(),c=Math.max(0,j.startTime+j.duration-b),d=1-(c/j.duration||0),e=0,f=j.tweens.length;for(;e<f;e++)j.tweens[e].run(d);return h.notifyWith(a,[j,d,c]),d<1&&f?c:(h.resolveWith(a,[j]),!1)},j=h.promise({elem:a,props:p.extend({},b),opts:p.extend(!0,{specialEasing:{}},c),originalProperties:b,originalOptions:c,startTime:cN||cU(),duration:c.duration,tweens:[],createTween:function(b,c,d){var e=p.Tween(a,j.opts,b,c,j.opts.specialEasing[b]||j.opts.easing);return j.tweens.push(e),e},stop:function(b){var c=0,d=b?j.tweens.length:0;for(;c<d;c++)j.tweens[c].run(1);return b?h.resolveWith(a,[j,b]):h.rejectWith(a,[j,b]),this}}),k=j.props;cX(k,j.opts.specialEasing);for(;e<g;e++){d=cS[e].call(j,a,k,j.opts);if(d)return d}return cV(j,k),p.isFunction(j.opts.start)&&j.opts.start.call(a,j),p.fx.timer(p.extend(i,{anim:j,queue:j.opts.queue,elem:a})),j.progress(j.opts.progress).done(j.opts.done,j.opts.complete).fail(j.opts.fail).always(j.opts.always)}function cX(a,b){var c,d,e,f,g;for(c in a){d=p.camelCase(c),e=b[d],f=a[c],p.isArray(f)&&(e=f[1],f=a[c]=f[0]),c!==d&&(a[d]=f,delete a[c]),g=p.cssHooks[d];if(g&&"expand"in g){f=g.expand(f),delete a[d];for(c in f)c in a||(a[c]=f[c],b[c]=e)}else b[d]=e}}function cY(a,b,c){var d,e,f,g,h,i,j,k,l=this,m=a.style,n={},o=[],q=a.nodeType&&bZ(a);c.queue||(j=p._queueHooks(a,"fx"),j.unqueued==null&&(j.unqueued=0,k=j.empty.fire,j.empty.fire=function(){j.unqueued||k()}),j.unqueued++,l.always(function(){l.always(function(){j.unqueued--,p.queue(a,"fx").length||j.empty.fire()})})),a.nodeType===1&&("height"in b||"width"in b)&&(c.overflow=[m.overflow,m.overflowX,m.overflowY],p.css(a,"display")==="inline"&&p.css(a,"float")==="none"&&(!p.support.inlineBlockNeedsLayout||cc(a.nodeName)==="inline"?m.display="inline-block":m.zoom=1)),c.overflow&&(m.overflow="hidden",p.support.shrinkWrapBlocks||l.done(function(){m.overflow=c.overflow[0],m.overflowX=c.overflow[1],m.overflowY=c.overflow[2]}));for(d in b){f=b[d];if(cP.exec(f)){delete b[d];if(f===(q?"hide":"show"))continue;o.push(d)}}g=o.length;if(g){h=p._data(a,"fxshow")||p._data(a,"fxshow",{}),q?p(a).show():l.done(function(){p(a).hide()}),l.done(function(){var b;p.removeData(a,"fxshow",!0);for(b in n)p.style(a,b,n[b])});for(d=0;d<g;d++)e=o[d],i=l.createTween(e,q?h[e]:0),n[e]=h[e]||p.style(a,e),e in h||(h[e]=i.start,q&&(i.end=i.start,i.start=e==="width"||e==="height"?1:0))}}function cZ(a,b,c,d,e){return new cZ.prototype.init(a,b,c,d,e)}function c$(a,b){var c,d={height:a},e=0;b=b?1:0;for(;e<4;e+=2-b)c=bV[e],d["margin"+c]=d["padding"+c]=a;return b&&(d.opacity=d.width=a),d}function da(a){return p.isWindow(a)?a:a.nodeType===9?a.defaultView||a.parentWindow:!1}var c,d,e=a.document,f=a.location,g=a.navigator,h=a.jQuery,i=a.$,j=Array.prototype.push,k=Array.prototype.slice,l=Array.prototype.indexOf,m=Object.prototype.toString,n=Object.prototype.hasOwnProperty,o=String.prototype.trim,p=function(a,b){return new p.fn.init(a,b,c)},q=/[\-+]?(?:\d*\.|)\d+(?:[eE][\-+]?\d+|)/.source,r=/\S/,s=/\s+/,t=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,u=/^(?:[^#<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/,v=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,w=/^[\],:{}\s]*$/,x=/(?:^|:|,)(?:\s*\[)+/g,y=/\\(?:["\\\/bfnrt]|u[\da-fA-F]{4})/g,z=/"[^"\\\r\n]*"|true|false|null|-?(?:\d\d*\.|)\d+(?:[eE][\-+]?\d+|)/g,A=/^-ms-/,B=/-([\da-z])/gi,C=function(a,b){return(b+"").toUpperCase()},D=function(){e.addEventListener?(e.removeEventListener("DOMContentLoaded",D,!1),p.ready()):e.readyState==="complete"&&(e.detachEvent("onreadystatechange",D),p.ready())},E={};p.fn=p.prototype={constructor:p,init:function(a,c,d){var f,g,h,i;if(!a)return this;if(a.nodeType)return this.context=this[0]=a,this.length=1,this;if(typeof a=="string"){a.charAt(0)==="<"&&a.charAt(a.length-1)===">"&&a.length>=3?f=[null,a,null]:f=u.exec(a);if(f&&(f[1]||!c)){if(f[1])return c=c instanceof p?c[0]:c,i=c&&c.nodeType?c.ownerDocument||c:e,a=p.parseHTML(f[1],i,!0),v.test(f[1])&&p.isPlainObject(c)&&this.attr.call(a,c,!0),p.merge(this,a);g=e.getElementById(f[2]);if(g&&g.parentNode){if(g.id!==f[2])return d.find(a);this.length=1,this[0]=g}return this.context=e,this.selector=a,this}return!c||c.jquery?(c||d).find(a):this.constructor(c).find(a)}return p.isFunction(a)?d.ready(a):(a.selector!==b&&(this.selector=a.selector,this.context=a.context),p.makeArray(a,this))},selector:"",jquery:"1.8.2",length:0,size:function(){return this.length},toArray:function(){return k.call(this)},get:function(a){return a==null?this.toArray():a<0?this[this.length+a]:this[a]},pushStack:function(a,b,c){var d=p.merge(this.constructor(),a);return d.prevObject=this,d.context=this.context,b==="find"?d.selector=this.selector+(this.selector?" ":"")+c:b&&(d.selector=this.selector+"."+b+"("+c+")"),d},each:function(a,b){return p.each(this,a,b)},ready:function(a){return p.ready.promise().done(a),this},eq:function(a){return a=+a,a===-1?this.slice(a):this.slice(a,a+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(k.apply(this,arguments),"slice",k.call(arguments).join(","))},map:function(a){return this.pushStack(p.map(this,function(b,c){return a.call(b,c,b)}))},end:function(){return this.prevObject||this.constructor(null)},push:j,sort:[].sort,splice:[].splice},p.fn.init.prototype=p.fn,p.extend=p.fn.extend=function(){var a,c,d,e,f,g,h=arguments[0]||{},i=1,j=arguments.length,k=!1;typeof h=="boolean"&&(k=h,h=arguments[1]||{},i=2),typeof h!="object"&&!p.isFunction(h)&&(h={}),j===i&&(h=this,--i);for(;i<j;i++)if((a=arguments[i])!=null)for(c in a){d=h[c],e=a[c];if(h===e)continue;k&&e&&(p.isPlainObject(e)||(f=p.isArray(e)))?(f?(f=!1,g=d&&p.isArray(d)?d:[]):g=d&&p.isPlainObject(d)?d:{},h[c]=p.extend(k,g,e)):e!==b&&(h[c]=e)}return h},p.extend({noConflict:function(b){return a.$===p&&(a.$=i),b&&a.jQuery===p&&(a.jQuery=h),p},isReady:!1,readyWait:1,holdReady:function(a){a?p.readyWait++:p.ready(!0)},ready:function(a){if(a===!0?--p.readyWait:p.isReady)return;if(!e.body)return setTimeout(p.ready,1);p.isReady=!0;if(a!==!0&&--p.readyWait>0)return;d.resolveWith(e,[p]),p.fn.trigger&&p(e).trigger("ready").off("ready")},isFunction:function(a){return p.type(a)==="function"},isArray:Array.isArray||function(a){return p.type(a)==="array"},isWindow:function(a){return a!=null&&a==a.window},isNumeric:function(a){return!isNaN(parseFloat(a))&&isFinite(a)},type:function(a){return a==null?String(a):E[m.call(a)]||"object"},isPlainObject:function(a){if(!a||p.type(a)!=="object"||a.nodeType||p.isWindow(a))return!1;try{if(a.constructor&&!n.call(a,"constructor")&&!n.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}var d;for(d in a);return d===b||n.call(a,d)},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},error:function(a){throw new Error(a)},parseHTML:function(a,b,c){var d;return!a||typeof a!="string"?null:(typeof b=="boolean"&&(c=b,b=0),b=b||e,(d=v.exec(a))?[b.createElement(d[1])]:(d=p.buildFragment([a],b,c?null:[]),p.merge([],(d.cacheable?p.clone(d.fragment):d.fragment).childNodes)))},parseJSON:function(b){if(!b||typeof b!="string")return null;b=p.trim(b);if(a.JSON&&a.JSON.parse)return a.JSON.parse(b);if(w.test(b.replace(y,"@").replace(z,"]").replace(x,"")))return(new Function("return "+b))();p.error("Invalid JSON: "+b)},parseXML:function(c){var d,e;if(!c||typeof c!="string")return null;try{a.DOMParser?(e=new DOMParser,d=e.parseFromString(c,"text/xml")):(d=new ActiveXObject("Microsoft.XMLDOM"),d.async="false",d.loadXML(c))}catch(f){d=b}return(!d||!d.documentElement||d.getElementsByTagName("parsererror").length)&&p.error("Invalid XML: "+c),d},noop:function(){},globalEval:function(b){b&&r.test(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(A,"ms-").replace(B,C)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,c,d){var e,f=0,g=a.length,h=g===b||p.isFunction(a);if(d){if(h){for(e in a)if(c.apply(a[e],d)===!1)break}else for(;f<g;)if(c.apply(a[f++],d)===!1)break}else if(h){for(e in a)if(c.call(a[e],e,a[e])===!1)break}else for(;f<g;)if(c.call(a[f],f,a[f++])===!1)break;return a},trim:o&&!o.call(" ")?function(a){return a==null?"":o.call(a)}:function(a){return a==null?"":(a+"").replace(t,"")},makeArray:function(a,b){var c,d=b||[];return a!=null&&(c=p.type(a),a.length==null||c==="string"||c==="function"||c==="regexp"||p.isWindow(a)?j.call(d,a):p.merge(d,a)),d},inArray:function(a,b,c){var d;if(b){if(l)return l.call(b,a,c);d=b.length,c=c?c<0?Math.max(0,d+c):c:0;for(;c<d;c++)if(c in b&&b[c]===a)return c}return-1},merge:function(a,c){var d=c.length,e=a.length,f=0;if(typeof d=="number")for(;f<d;f++)a[e++]=c[f];else while(c[f]!==b)a[e++]=c[f++];return a.length=e,a},grep:function(a,b,c){var d,e=[],f=0,g=a.length;c=!!c;for(;f<g;f++)d=!!b(a[f],f),c!==d&&e.push(a[f]);return e},map:function(a,c,d){var e,f,g=[],h=0,i=a.length,j=a instanceof p||i!==b&&typeof i=="number"&&(i>0&&a[0]&&a[i-1]||i===0||p.isArray(a));if(j)for(;h<i;h++)e=c(a[h],h,d),e!=null&&(g[g.length]=e);else for(f in a)e=c(a[f],f,d),e!=null&&(g[g.length]=e);return g.concat.apply([],g)},guid:1,proxy:function(a,c){var d,e,f;return typeof c=="string"&&(d=a[c],c=a,a=d),p.isFunction(a)?(e=k.call(arguments,2),f=function(){return a.apply(c,e.concat(k.call(arguments)))},f.guid=a.guid=a.guid||p.guid++,f):b},access:function(a,c,d,e,f,g,h){var i,j=d==null,k=0,l=a.length;if(d&&typeof d=="object"){for(k in d)p.access(a,c,k,d[k],1,g,e);f=1}else if(e!==b){i=h===b&&p.isFunction(e),j&&(i?(i=c,c=function(a,b,c){return i.call(p(a),c)}):(c.call(a,e),c=null));if(c)for(;k<l;k++)c(a[k],d,i?e.call(a[k],k,c(a[k],d)):e,h);f=1}return f?a:j?c.call(a):l?c(a[0],d):g},now:function(){return(new Date).getTime()}}),p.ready.promise=function(b){if(!d){d=p.Deferred();if(e.readyState==="complete")setTimeout(p.ready,1);else if(e.addEventListener)e.addEventListener("DOMContentLoaded",D,!1),a.addEventListener("load",p.ready,!1);else{e.attachEvent("onreadystatechange",D),a.attachEvent("onload",p.ready);var c=!1;try{c=a.frameElement==null&&e.documentElement}catch(f){}c&&c.doScroll&&function g(){if(!p.isReady){try{c.doScroll("left")}catch(a){return setTimeout(g,50)}p.ready()}}()}}return d.promise(b)},p.each("Boolean Number String Function Array Date RegExp Object".split(" "),function(a,b){E["[object "+b+"]"]=b.toLowerCase()}),c=p(e);var F={};p.Callbacks=function(a){a=typeof a=="string"?F[a]||G(a):p.extend({},a);var c,d,e,f,g,h,i=[],j=!a.once&&[],k=function(b){c=a.memory&&b,d=!0,h=f||0,f=0,g=i.length,e=!0;for(;i&&h<g;h++)if(i[h].apply(b[0],b[1])===!1&&a.stopOnFalse){c=!1;break}e=!1,i&&(j?j.length&&k(j.shift()):c?i=[]:l.disable())},l={add:function(){if(i){var b=i.length;(function d(b){p.each(b,function(b,c){var e=p.type(c);e==="function"&&(!a.unique||!l.has(c))?i.push(c):c&&c.length&&e!=="string"&&d(c)})})(arguments),e?g=i.length:c&&(f=b,k(c))}return this},remove:function(){return i&&p.each(arguments,function(a,b){var c;while((c=p.inArray(b,i,c))>-1)i.splice(c,1),e&&(c<=g&&g--,c<=h&&h--)}),this},has:function(a){return p.inArray(a,i)>-1},empty:function(){return i=[],this},disable:function(){return i=j=c=b,this},disabled:function(){return!i},lock:function(){return j=b,c||l.disable(),this},locked:function(){return!j},fireWith:function(a,b){return b=b||[],b=[a,b.slice?b.slice():b],i&&(!d||j)&&(e?j.push(b):k(b)),this},fire:function(){return l.fireWith(this,arguments),this},fired:function(){return!!d}};return l},p.extend({Deferred:function(a){var b=[["resolve","done",p.Callbacks("once memory"),"resolved"],["reject","fail",p.Callbacks("once memory"),"rejected"],["notify","progress",p.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return p.Deferred(function(c){p.each(b,function(b,d){var f=d[0],g=a[b];e[d[1]](p.isFunction(g)?function(){var a=g.apply(this,arguments);a&&p.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f+"With"](this===e?c:this,[a])}:c[f])}),a=null}).promise()},promise:function(a){return a!=null?p.extend(a,d):d}},e={};return d.pipe=d.then,p.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[a^1][2].disable,b[2][2].lock),e[f[0]]=g.fire,e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=k.call(arguments),d=c.length,e=d!==1||a&&p.isFunction(a.promise)?d:0,f=e===1?a:p.Deferred(),g=function(a,b,c){return function(d){b[a]=this,c[a]=arguments.length>1?k.call(arguments):d,c===h?f.notifyWith(b,c):--e||f.resolveWith(b,c)}},h,i,j;if(d>1){h=new Array(d),i=new Array(d),j=new Array(d);for(;b<d;b++)c[b]&&p.isFunction(c[b].promise)?c[b].promise().done(g(b,j,c)).fail(f.reject).progress(g(b,i,h)):--e}return e||f.resolveWith(j,c),f.promise()}}),p.support=function(){var b,c,d,f,g,h,i,j,k,l,m,n=e.createElement("div");n.setAttribute("className","t"),n.innerHTML=" <link/><table></table><a href='/a'>a</a><input type='checkbox'/>",c=n.getElementsByTagName("*"),d=n.getElementsByTagName("a")[0],d.style.cssText="top:1px;float:left;opacity:.5";if(!c||!c.length)return{};f=e.createElement("select"),g=f.appendChild(e.createElement("option")),h=n.getElementsByTagName("input")[0],b={leadingWhitespace:n.firstChild.nodeType===3,tbody:!n.getElementsByTagName("tbody").length,htmlSerialize:!!n.getElementsByTagName("link").length,style:/top/.test(d.getAttribute("style")),hrefNormalized:d.getAttribute("href")==="/a",opacity:/^0.5/.test(d.style.opacity),cssFloat:!!d.style.cssFloat,checkOn:h.value==="on",optSelected:g.selected,getSetAttribute:n.className!=="t",enctype:!!e.createElement("form").enctype,html5Clone:e.createElement("nav").cloneNode(!0).outerHTML!=="<:nav></:nav>",boxModel:e.compatMode==="CSS1Compat",submitBubbles:!0,changeBubbles:!0,focusinBubbles:!1,deleteExpando:!0,noCloneEvent:!0,inlineBlockNeedsLayout:!1,shrinkWrapBlocks:!1,reliableMarginRight:!0,boxSizingReliable:!0,pixelPosition:!1},h.checked=!0,b.noCloneChecked=h.cloneNode(!0).checked,f.disabled=!0,b.optDisabled=!g.disabled;try{delete n.test}catch(o){b.deleteExpando=!1}!n.addEventListener&&n.attachEvent&&n.fireEvent&&(n.attachEvent("onclick",m=function(){b.noCloneEvent=!1}),n.cloneNode(!0).fireEvent("onclick"),n.detachEvent("onclick",m)),h=e.createElement("input"),h.value="t",h.setAttribute("type","radio"),b.radioValue=h.value==="t",h.setAttribute("checked","checked"),h.setAttribute("name","t"),n.appendChild(h),i=e.createDocumentFragment(),i.appendChild(n.lastChild),b.checkClone=i.cloneNode(!0).cloneNode(!0).lastChild.checked,b.appendChecked=h.checked,i.removeChild(h),i.appendChild(n);if(n.attachEvent)for(k in{submit:!0,change:!0,focusin:!0})j="on"+k,l=j in n,l||(n.setAttribute(j,"return;"),l=typeof n[j]=="function"),b[k+"Bubbles"]=l;return p(function(){var c,d,f,g,h="padding:0;margin:0;border:0;display:block;overflow:hidden;",i=e.getElementsByTagName("body")[0];if(!i)return;c=e.createElement("div"),c.style.cssText="visibility:hidden;border:0;width:0;height:0;position:static;top:0;margin-top:1px",i.insertBefore(c,i.firstChild),d=e.createElement("div"),c.appendChild(d),d.innerHTML="<table><tr><td></td><td>t</td></tr></table>",f=d.getElementsByTagName("td"),f[0].style.cssText="padding:0;margin:0;border:0;display:none",l=f[0].offsetHeight===0,f[0].style.display="",f[1].style.display="none",b.reliableHiddenOffsets=l&&f[0].offsetHeight===0,d.innerHTML="",d.style.cssText="box-sizing:border-box;-moz-box-sizing:border-box;-webkit-box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%;",b.boxSizing=d.offsetWidth===4,b.doesNotIncludeMarginInBodyOffset=i.offsetTop!==1,a.getComputedStyle&&(b.pixelPosition=(a.getComputedStyle(d,null)||{}).top!=="1%",b.boxSizingReliable=(a.getComputedStyle(d,null)||{width:"4px"}).width==="4px",g=e.createElement("div"),g.style.cssText=d.style.cssText=h,g.style.marginRight=g.style.width="0",d.style.width="1px",d.appendChild(g),b.reliableMarginRight=!parseFloat((a.getComputedStyle(g,null)||{}).marginRight)),typeof d.style.zoom!="undefined"&&(d.innerHTML="",d.style.cssText=h+"width:1px;padding:1px;display:inline;zoom:1",b.inlineBlockNeedsLayout=d.offsetWidth===3,d.style.display="block",d.style.overflow="visible",d.innerHTML="<div></div>",d.firstChild.style.width="5px",b.shrinkWrapBlocks=d.offsetWidth!==3,c.style.zoom=1),i.removeChild(c),c=d=f=g=null}),i.removeChild(n),c=d=f=g=h=i=n=null,b}();var H=/(?:\{[\s\S]*\}|\[[\s\S]*\])$/,I=/([A-Z])/g;p.extend({cache:{},deletedIds:[],uuid:0,expando:"jQuery"+(p.fn.jquery+Math.random()).replace(/\D/g,""),noData:{embed:!0,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",applet:!0},hasData:function(a){return a=a.nodeType?p.cache[a[p.expando]]:a[p.expando],!!a&&!K(a)},data:function(a,c,d,e){if(!p.acceptData(a))return;var f,g,h=p.expando,i=typeof c=="string",j=a.nodeType,k=j?p.cache:a,l=j?a[h]:a[h]&&h;if((!l||!k[l]||!e&&!k[l].data)&&i&&d===b)return;l||(j?a[h]=l=p.deletedIds.pop()||p.guid++:l=h),k[l]||(k[l]={},j||(k[l].toJSON=p.noop));if(typeof c=="object"||typeof c=="function")e?k[l]=p.extend(k[l],c):k[l].data=p.extend(k[l].data,c);return f=k[l],e||(f.data||(f.data={}),f=f.data),d!==b&&(f[p.camelCase(c)]=d),i?(g=f[c],g==null&&(g=f[p.camelCase(c)])):g=f,g},removeData:function(a,b,c){if(!p.acceptData(a))return;var d,e,f,g=a.nodeType,h=g?p.cache:a,i=g?a[p.expando]:p.expando;if(!h[i])return;if(b){d=c?h[i]:h[i].data;if(d){p.isArray(b)||(b in d?b=[b]:(b=p.camelCase(b),b in d?b=[b]:b=b.split(" ")));for(e=0,f=b.length;e<f;e++)delete d[b[e]];if(!(c?K:p.isEmptyObject)(d))return}}if(!c){delete h[i].data;if(!K(h[i]))return}g?p.cleanData([a],!0):p.support.deleteExpando||h!=h.window?delete h[i]:h[i]=null},_data:function(a,b,c){return p.data(a,b,c,!0)},acceptData:function(a){var b=a.nodeName&&p.noData[a.nodeName.toLowerCase()];return!b||b!==!0&&a.getAttribute("classid")===b}}),p.fn.extend({data:function(a,c){var d,e,f,g,h,i=this[0],j=0,k=null;if(a===b){if(this.length){k=p.data(i);if(i.nodeType===1&&!p._data(i,"parsedAttrs")){f=i.attributes;for(h=f.length;j<h;j++)g=f[j].name,g.indexOf("data-")||(g=p.camelCase(g.substring(5)),J(i,g,k[g]));p._data(i,"parsedAttrs",!0)}}return k}return typeof a=="object"?this.each(function(){p.data(this,a)}):(d=a.split(".",2),d[1]=d[1]?"."+d[1]:"",e=d[1]+"!",p.access(this,function(c){if(c===b)return k=this.triggerHandler("getData"+e,[d[0]]),k===b&&i&&(k=p.data(i,a),k=J(i,a,k)),k===b&&d[1]?this.data(d[0]):k;d[1]=c,this.each(function(){var b=p(this);b.triggerHandler("setData"+e,d),p.data(this,a,c),b.triggerHandler("changeData"+e,d)})},null,c,arguments.length>1,null,!1))},removeData:function(a){return this.each(function(){p.removeData(this,a)})}}),p.extend({queue:function(a,b,c){var d;if(a)return b=(b||"fx")+"queue",d=p._data(a,b),c&&(!d||p.isArray(c)?d=p._data(a,b,p.makeArray(c)):d.push(c)),d||[]},dequeue:function(a,b){b=b||"fx";var c=p.queue(a,b),d=c.length,e=c.shift(),f=p._queueHooks(a,b),g=function(){p.dequeue(a,b)};e==="inprogress"&&(e=c.shift(),d--),e&&(b==="fx"&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return p._data(a,c)||p._data(a,c,{empty:p.Callbacks("once memory").add(function(){p.removeData(a,b+"queue",!0),p.removeData(a,c,!0)})})}}),p.fn.extend({queue:function(a,c){var d=2;return typeof a!="string"&&(c=a,a="fx",d--),arguments.length<d?p.queue(this[0],a):c===b?this:this.each(function(){var b=p.queue(this,a,c);p._queueHooks(this,a),a==="fx"&&b[0]!=="inprogress"&&p.dequeue(this,a)})},dequeue:function(a){return this.each(function(){p.dequeue(this,a)})},delay:function(a,b){return a=p.fx?p.fx.speeds[a]||a:a,b=b||"fx",this.queue(b,function(b,c){var d=setTimeout(b,a);c.stop=function(){clearTimeout(d)}})},clearQueue:function(a){return this.queue(a||"fx",[])},promise:function(a,c){var d,e=1,f=p.Deferred(),g=this,h=this.length,i=function(){--e||f.resolveWith(g,[g])};typeof a!="string"&&(c=a,a=b),a=a||"fx";while(h--)d=p._data(g[h],a+"queueHooks"),d&&d.empty&&(e++,d.empty.add(i));return i(),f.promise(c)}});var L,M,N,O=/[\t\r\n]/g,P=/\r/g,Q=/^(?:button|input)$/i,R=/^(?:button|input|object|select|textarea)$/i,S=/^a(?:rea|)$/i,T=/^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i,U=p.support.getSetAttribute;p.fn.extend({attr:function(a,b){return p.access(this,p.attr,a,b,arguments.length>1)},removeAttr:function(a){return this.each(function(){p.removeAttr(this,a)})},prop:function(a,b){return p.access(this,p.prop,a,b,arguments.length>1)},removeProp:function(a){return a=p.propFix[a]||a,this.each(function(){try{this[a]=b,delete this[a]}catch(c){}})},addClass:function(a){var b,c,d,e,f,g,h;if(p.isFunction(a))return this.each(function(b){p(this).addClass(a.call(this,b,this.className))});if(a&&typeof a=="string"){b=a.split(s);for(c=0,d=this.length;c<d;c++){e=this[c];if(e.nodeType===1)if(!e.className&&b.length===1)e.className=a;else{f=" "+e.className+" ";for(g=0,h=b.length;g<h;g++)f.indexOf(" "+b[g]+" ")<0&&(f+=b[g]+" ");e.className=p.trim(f)}}}return this},removeClass:function(a){var c,d,e,f,g,h,i;if(p.isFunction(a))return this.each(function(b){p(this).removeClass(a.call(this,b,this.className))});if(a&&typeof a=="string"||a===b){c=(a||"").split(s);for(h=0,i=this.length;h<i;h++){e=this[h];if(e.nodeType===1&&e.className){d=(" "+e.className+" ").replace(O," ");for(f=0,g=c.length;f<g;f++)while(d.indexOf(" "+c[f]+" ")>=0)d=d.replace(" "+c[f]+" "," ");e.className=a?p.trim(d):""}}}return this},toggleClass:function(a,b){var c=typeof a,d=typeof b=="boolean";return p.isFunction(a)?this.each(function(c){p(this).toggleClass(a.call(this,c,this.className,b),b)}):this.each(function(){if(c==="string"){var e,f=0,g=p(this),h=b,i=a.split(s);while(e=i[f++])h=d?h:!g.hasClass(e),g[h?"addClass":"removeClass"](e)}else if(c==="undefined"||c==="boolean")this.className&&p._data(this,"__className__",this.className),this.className=this.className||a===!1?"":p._data(this,"__className__")||""})},hasClass:function(a){var b=" "+a+" ",c=0,d=this.length;for(;c<d;c++)if(this[c].nodeType===1&&(" "+this[c].className+" ").replace(O," ").indexOf(b)>=0)return!0;return!1},val:function(a){var c,d,e,f=this[0];if(!arguments.length){if(f)return c=p.valHooks[f.type]||p.valHooks[f.nodeName.toLowerCase()],c&&"get"in c&&(d=c.get(f,"value"))!==b?d:(d=f.value,typeof d=="string"?d.replace(P,""):d==null?"":d);return}return e=p.isFunction(a),this.each(function(d){var f,g=p(this);if(this.nodeType!==1)return;e?f=a.call(this,d,g.val()):f=a,f==null?f="":typeof f=="number"?f+="":p.isArray(f)&&(f=p.map(f,function(a){return a==null?"":a+""})),c=p.valHooks[this.type]||p.valHooks[this.nodeName.toLowerCase()];if(!c||!("set"in c)||c.set(this,f,"value")===b)this.value=f})}}),p.extend({valHooks:{option:{get:function(a){var b=a.attributes.value;return!b||b.specified?a.value:a.text}},select:{get:function(a){var b,c,d,e,f=a.selectedIndex,g=[],h=a.options,i=a.type==="select-one";if(f<0)return null;c=i?f:0,d=i?f+1:h.length;for(;c<d;c++){e=h[c];if(e.selected&&(p.support.optDisabled?!e.disabled:e.getAttribute("disabled")===null)&&(!e.parentNode.disabled||!p.nodeName(e.parentNode,"optgroup"))){b=p(e).val();if(i)return b;g.push(b)}}return i&&!g.length&&h.length?p(h[f]).val():g},set:function(a,b){var c=p.makeArray(b);return p(a).find("option").each(function(){this.selected=p.inArray(p(this).val(),c)>=0}),c.length||(a.selectedIndex=-1),c}}},attrFn:{},attr:function(a,c,d,e){var f,g,h,i=a.nodeType;if(!a||i===3||i===8||i===2)return;if(e&&p.isFunction(p.fn[c]))return p(a)[c](d);if(typeof a.getAttribute=="undefined")return p.prop(a,c,d);h=i!==1||!p.isXMLDoc(a),h&&(c=c.toLowerCase(),g=p.attrHooks[c]||(T.test(c)?M:L));if(d!==b){if(d===null){p.removeAttr(a,c);return}return g&&"set"in g&&h&&(f=g.set(a,d,c))!==b?f:(a.setAttribute(c,d+""),d)}return g&&"get"in g&&h&&(f=g.get(a,c))!==null?f:(f=a.getAttribute(c),f===null?b:f)},removeAttr:function(a,b){var c,d,e,f,g=0;if(b&&a.nodeType===1){d=b.split(s);for(;g<d.length;g++)e=d[g],e&&(c=p.propFix[e]||e,f=T.test(e),f||p.attr(a,e,""),a.removeAttribute(U?e:c),f&&c in a&&(a[c]=!1))}},attrHooks:{type:{set:function(a,b){if(Q.test(a.nodeName)&&a.parentNode)p.error("type property can't be changed");else if(!p.support.radioValue&&b==="radio"&&p.nodeName(a,"input")){var c=a.value;return a.setAttribute("type",b),c&&(a.value=c),b}}},value:{get:function(a,b){return L&&p.nodeName(a,"button")?L.get(a,b):b in a?a.value:null},set:function(a,b,c){if(L&&p.nodeName(a,"button"))return L.set(a,b,c);a.value=b}}},propFix:{tabindex:"tabIndex",readonly:"readOnly","for":"htmlFor","class":"className",maxlength:"maxLength",cellspacing:"cellSpacing",cellpadding:"cellPadding",rowspan:"rowSpan",colspan:"colSpan",usemap:"useMap",frameborder:"frameBorder",contenteditable:"contentEditable"},prop:function(a,c,d){var e,f,g,h=a.nodeType;if(!a||h===3||h===8||h===2)return;return g=h!==1||!p.isXMLDoc(a),g&&(c=p.propFix[c]||c,f=p.propHooks[c]),d!==b?f&&"set"in f&&(e=f.set(a,d,c))!==b?e:a[c]=d:f&&"get"in f&&(e=f.get(a,c))!==null?e:a[c]},propHooks:{tabIndex:{get:function(a){var c=a.getAttributeNode("tabindex");return c&&c.specified?parseInt(c.value,10):R.test(a.nodeName)||S.test(a.nodeName)&&a.href?0:b}}}}),M={get:function(a,c){var d,e=p.prop(a,c);return e===!0||typeof e!="boolean"&&(d=a.getAttributeNode(c))&&d.nodeValue!==!1?c.toLowerCase():b},set:function(a,b,c){var d;return b===!1?p.removeAttr(a,c):(d=p.propFix[c]||c,d in a&&(a[d]=!0),a.setAttribute(c,c.toLowerCase())),c}},U||(N={name:!0,id:!0,coords:!0},L=p.valHooks.button={get:function(a,c){var d;return d=a.getAttributeNode(c),d&&(N[c]?d.value!=="":d.specified)?d.value:b},set:function(a,b,c){var d=a.getAttributeNode(c);return d||(d=e.createAttribute(c),a.setAttributeNode(d)),d.value=b+""}},p.each(["width","height"],function(a,b){p.attrHooks[b]=p.extend(p.attrHooks[b],{set:function(a,c){if(c==="")return a.setAttribute(b,"auto"),c}})}),p.attrHooks.contenteditable={get:L.get,set:function(a,b,c){b===""&&(b="false"),L.set(a,b,c)}}),p.support.hrefNormalized||p.each(["href","src","width","height"],function(a,c){p.attrHooks[c]=p.extend(p.attrHooks[c],{get:function(a){var d=a.getAttribute(c,2);return d===null?b:d}})}),p.support.style||(p.attrHooks.style={get:function(a){return a.style.cssText.toLowerCase()||b},set:function(a,b){return a.style.cssText=b+""}}),p.support.optSelected||(p.propHooks.selected=p.extend(p.propHooks.selected,{get:function(a){var b=a.parentNode;return b&&(b.selectedIndex,b.parentNode&&b.parentNode.selectedIndex),null}})),p.support.enctype||(p.propFix.enctype="encoding"),p.support.checkOn||p.each(["radio","checkbox"],function(){p.valHooks[this]={get:function(a){return a.getAttribute("value")===null?"on":a.value}}}),p.each(["radio","checkbox"],function(){p.valHooks[this]=p.extend(p.valHooks[this],{set:function(a,b){if(p.isArray(b))return a.checked=p.inArray(p(a).val(),b)>=0}})});var V=/^(?:textarea|input|select)$/i,W=/^([^\.]*|)(?:\.(.+)|)$/,X=/(?:^|\s)hover(\.\S+|)\b/,Y=/^key/,Z=/^(?:mouse|contextmenu)|click/,$=/^(?:focusinfocus|focusoutblur)$/,_=function(a){return p.event.special.hover?a:a.replace(X,"mouseenter$1 mouseleave$1")};p.event={add:function(a,c,d,e,f){var g,h,i,j,k,l,m,n,o,q,r;if(a.nodeType===3||a.nodeType===8||!c||!d||!(g=p._data(a)))return;d.handler&&(o=d,d=o.handler,f=o.selector),d.guid||(d.guid=p.guid++),i=g.events,i||(g.events=i={}),h=g.handle,h||(g.handle=h=function(a){return typeof p!="undefined"&&(!a||p.event.triggered!==a.type)?p.event.dispatch.apply(h.elem,arguments):b},h.elem=a),c=p.trim(_(c)).split(" ");for(j=0;j<c.length;j++){k=W.exec(c[j])||[],l=k[1],m=(k[2]||"").split(".").sort(),r=p.event.special[l]||{},l=(f?r.delegateType:r.bindType)||l,r=p.event.special[l]||{},n=p.extend({type:l,origType:k[1],data:e,handler:d,guid:d.guid,selector:f,needsContext:f&&p.expr.match.needsContext.test(f),namespace:m.join(".")},o),q=i[l];if(!q){q=i[l]=[],q.delegateCount=0;if(!r.setup||r.setup.call(a,e,m,h)===!1)a.addEventListener?a.addEventListener(l,h,!1):a.attachEvent&&a.attachEvent("on"+l,h)}r.add&&(r.add.call(a,n),n.handler.guid||(n.handler.guid=d.guid)),f?q.splice(q.delegateCount++,0,n):q.push(n),p.event.global[l]=!0}a=null},global:{},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,n,o,q,r=p.hasData(a)&&p._data(a);if(!r||!(m=r.events))return;b=p.trim(_(b||"")).split(" ");for(f=0;f<b.length;f++){g=W.exec(b[f])||[],h=i=g[1],j=g[2];if(!h){for(h in m)p.event.remove(a,h+b[f],c,d,!0);continue}n=p.event.special[h]||{},h=(d?n.delegateType:n.bindType)||h,o=m[h]||[],k=o.length,j=j?new RegExp("(^|\\.)"+j.split(".").sort().join("\\.(?:.*\\.|)")+"(\\.|$)"):null;for(l=0;l<o.length;l++)q=o[l],(e||i===q.origType)&&(!c||c.guid===q.guid)&&(!j||j.test(q.namespace))&&(!d||d===q.selector||d==="**"&&q.selector)&&(o.splice(l--,1),q.selector&&o.delegateCount--,n.remove&&n.remove.call(a,q));o.length===0&&k!==o.length&&((!n.teardown||n.teardown.call(a,j,r.handle)===!1)&&p.removeEvent(a,h,r.handle),delete m[h])}p.isEmptyObject(m)&&(delete r.handle,p.removeData(a,"events",!0))},customEvent:{getData:!0,setData:!0,changeData:!0},trigger:function(c,d,f,g){if(!f||f.nodeType!==3&&f.nodeType!==8){var h,i,j,k,l,m,n,o,q,r,s=c.type||c,t=[];if($.test(s+p.event.triggered))return;s.indexOf("!")>=0&&(s=s.slice(0,-1),i=!0),s.indexOf(".")>=0&&(t=s.split("."),s=t.shift(),t.sort());if((!f||p.event.customEvent[s])&&!p.event.global[s])return;c=typeof c=="object"?c[p.expando]?c:new p.Event(s,c):new p.Event(s),c.type=s,c.isTrigger=!0,c.exclusive=i,c.namespace=t.join("."),c.namespace_re=c.namespace?new RegExp("(^|\\.)"+t.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,m=s.indexOf(":")<0?"on"+s:"";if(!f){h=p.cache;for(j in h)h[j].events&&h[j].events[s]&&p.event.trigger(c,d,h[j].handle.elem,!0);return}c.result=b,c.target||(c.target=f),d=d!=null?p.makeArray(d):[],d.unshift(c),n=p.event.special[s]||{};if(n.trigger&&n.trigger.apply(f,d)===!1)return;q=[[f,n.bindType||s]];if(!g&&!n.noBubble&&!p.isWindow(f)){r=n.delegateType||s,k=$.test(r+s)?f:f.parentNode;for(l=f;k;k=k.parentNode)q.push([k,r]),l=k;l===(f.ownerDocument||e)&&q.push([l.defaultView||l.parentWindow||a,r])}for(j=0;j<q.length&&!c.isPropagationStopped();j++)k=q[j][0],c.type=q[j][1],o=(p._data(k,"events")||{})[c.type]&&p._data(k,"handle"),o&&o.apply(k,d),o=m&&k[m],o&&p.acceptData(k)&&o.apply&&o.apply(k,d)===!1&&c.preventDefault();return c.type=s,!g&&!c.isDefaultPrevented()&&(!n._default||n._default.apply(f.ownerDocument,d)===!1)&&(s!=="click"||!p.nodeName(f,"a"))&&p.acceptData(f)&&m&&f[s]&&(s!=="focus"&&s!=="blur"||c.target.offsetWidth!==0)&&!p.isWindow(f)&&(l=f[m],l&&(f[m]=null),p.event.triggered=s,f[s](),p.event.triggered=b,l&&(f[m]=l)),c.result}return},dispatch:function(c){c=p.event.fix(c||a.event);var d,e,f,g,h,i,j,l,m,n,o=(p._data(this,"events")||{})[c.type]||[],q=o.delegateCount,r=k.call(arguments),s=!c.exclusive&&!c.namespace,t=p.event.special[c.type]||{},u=[];r[0]=c,c.delegateTarget=this;if(t.preDispatch&&t.preDispatch.call(this,c)===!1)return;if(q&&(!c.button||c.type!=="click"))for(f=c.target;f!=this;f=f.parentNode||this)if(f.disabled!==!0||c.type!=="click"){h={},j=[];for(d=0;d<q;d++)l=o[d],m=l.selector,h[m]===b&&(h[m]=l.needsContext?p(m,this).index(f)>=0:p.find(m,this,null,[f]).length),h[m]&&j.push(l);j.length&&u.push({elem:f,matches:j})}o.length>q&&u.push({elem:this,matches:o.slice(q)});for(d=0;d<u.length&&!c.isPropagationStopped();d++){i=u[d],c.currentTarget=i.elem;for(e=0;e<i.matches.length&&!c.isImmediatePropagationStopped();e++){l=i.matches[e];if(s||!c.namespace&&!l.namespace||c.namespace_re&&c.namespace_re.test(l.namespace))c.data=l.data,c.handleObj=l,g=((p.event.special[l.origType]||{}).handle||l.handler).apply(i.elem,r),g!==b&&(c.result=g,g===!1&&(c.preventDefault(),c.stopPropagation()))}}return t.postDispatch&&t.postDispatch.call(this,c),c.result},props:"attrChange attrName relatedNode srcElement altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "),fixHooks:{},keyHooks:{props:"char charCode key keyCode".split(" "),filter:function(a,b){return a.which==null&&(a.which=b.charCode!=null?b.charCode:b.keyCode),a}},mouseHooks:{props:"button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "),filter:function(a,c){var d,f,g,h=c.button,i=c.fromElement;return a.pageX==null&&c.clientX!=null&&(d=a.target.ownerDocument||e,f=d.documentElement,g=d.body,a.pageX=c.clientX+(f&&f.scrollLeft||g&&g.scrollLeft||0)-(f&&f.clientLeft||g&&g.clientLeft||0),a.pageY=c.clientY+(f&&f.scrollTop||g&&g.scrollTop||0)-(f&&f.clientTop||g&&g.clientTop||0)),!a.relatedTarget&&i&&(a.relatedTarget=i===a.target?c.toElement:i),!a.which&&h!==b&&(a.which=h&1?1:h&2?3:h&4?2:0),a}},fix:function(a){if(a[p.expando])return a;var b,c,d=a,f=p.event.fixHooks[a.type]||{},g=f.props?this.props.concat(f.props):this.props;a=p.Event(d);for(b=g.length;b;)c=g[--b],a[c]=d[c];return a.target||(a.target=d.srcElement||e),a.target.nodeType===3&&(a.target=a.target.parentNode),a.metaKey=!!a.metaKey,f.filter?f.filter(a,d):a},special:{load:{noBubble:!0},focus:{delegateType:"focusin"},blur:{delegateType:"focusout"},beforeunload:{setup:function(a,b,c){p.isWindow(this)&&(this.onbeforeunload=c)},teardown:function(a,b){this.onbeforeunload===b&&(this.onbeforeunload=null)}}},simulate:function(a,b,c,d){var e=p.extend(new p.Event,c,{type:a,isSimulated:!0,originalEvent:{}});d?p.event.trigger(e,null,b):p.event.dispatch.call(b,e),e.isDefaultPrevented()&&c.preventDefault()}},p.event.handle=p.event.dispatch,p.removeEvent=e.removeEventListener?function(a,b,c){a.removeEventListener&&a.removeEventListener(b,c,!1)}:function(a,b,c){var d="on"+b;a.detachEvent&&(typeof a[d]=="undefined"&&(a[d]=null),a.detachEvent(d,c))},p.Event=function(a,b){if(this instanceof p.Event)a&&a.type?(this.originalEvent=a,this.type=a.type,this.isDefaultPrevented=a.defaultPrevented||a.returnValue===!1||a.getPreventDefault&&a.getPreventDefault()?bb:ba):this.type=a,b&&p.extend(this,b),this.timeStamp=a&&a.timeStamp||p.now(),this[p.expando]=!0;else return new p.Event(a,b)},p.Event.prototype={preventDefault:function(){this.isDefaultPrevented=bb;var a=this.originalEvent;if(!a)return;a.preventDefault?a.preventDefault():a.returnValue=!1},stopPropagation:function(){this.isPropagationStopped=bb;var a=this.originalEvent;if(!a)return;a.stopPropagation&&a.stopPropagation(),a.cancelBubble=!0},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=bb,this.stopPropagation()},isDefaultPrevented:ba,isPropagationStopped:ba,isImmediatePropagationStopped:ba},p.each({mouseenter:"mouseover",mouseleave:"mouseout"},function(a,b){p.event.special[a]={delegateType:b,bindType:b,handle:function(a){var c,d=this,e=a.relatedTarget,f=a.handleObj,g=f.selector;if(!e||e!==d&&!p.contains(d,e))a.type=f.origType,c=f.handler.apply(this,arguments),a.type=b;return c}}}),p.support.submitBubbles||(p.event.special.submit={setup:function(){if(p.nodeName(this,"form"))return!1;p.event.add(this,"click._submit keypress._submit",function(a){var c=a.target,d=p.nodeName(c,"input")||p.nodeName(c,"button")?c.form:b;d&&!p._data(d,"_submit_attached")&&(p.event.add(d,"submit._submit",function(a){a._submit_bubble=!0}),p._data(d,"_submit_attached",!0))})},postDispatch:function(a){a._submit_bubble&&(delete a._submit_bubble,this.parentNode&&!a.isTrigger&&p.event.simulate("submit",this.parentNode,a,!0))},teardown:function(){if(p.nodeName(this,"form"))return!1;p.event.remove(this,"._submit")}}),p.support.changeBubbles||(p.event.special.change={setup:function(){if(V.test(this.nodeName)){if(this.type==="checkbox"||this.type==="radio")p.event.add(this,"propertychange._change",function(a){a.originalEvent.propertyName==="checked"&&(this._just_changed=!0)}),p.event.add(this,"click._change",function(a){this._just_changed&&!a.isTrigger&&(this._just_changed=!1),p.event.simulate("change",this,a,!0)});return!1}p.event.add(this,"beforeactivate._change",function(a){var b=a.target;V.test(b.nodeName)&&!p._data(b,"_change_attached")&&(p.event.add(b,"change._change",function(a){this.parentNode&&!a.isSimulated&&!a.isTrigger&&p.event.simulate("change",this.parentNode,a,!0)}),p._data(b,"_change_attached",!0))})},handle:function(a){var b=a.target;if(this!==b||a.isSimulated||a.isTrigger||b.type!=="radio"&&b.type!=="checkbox")return a.handleObj.handler.apply(this,arguments)},teardown:function(){return p.event.remove(this,"._change"),!V.test(this.nodeName)}}),p.support.focusinBubbles||p.each({focus:"focusin",blur:"focusout"},function(a,b){var c=0,d=function(a){p.event.simulate(b,a.target,p.event.fix(a),!0)};p.event.special[b]={setup:function(){c++===0&&e.addEventListener(a,d,!0)},teardown:function(){--c===0&&e.removeEventListener(a,d,!0)}}}),p.fn.extend({on:function(a,c,d,e,f){var g,h;if(typeof a=="object"){typeof c!="string"&&(d=d||c,c=b);for(h in a)this.on(h,c,d,a[h],f);return this}d==null&&e==null?(e=c,d=c=b):e==null&&(typeof c=="string"?(e=d,d=b):(e=d,d=c,c=b));if(e===!1)e=ba;else if(!e)return this;return f===1&&(g=e,e=function(a){return p().off(a),g.apply(this,arguments)},e.guid=g.guid||(g.guid=p.guid++)),this.each(function(){p.event.add(this,a,e,d,c)})},one:function(a,b,c,d){return this.on(a,b,c,d,1)},off:function(a,c,d){var e,f;if(a&&a.preventDefault&&a.handleObj)return e=a.handleObj,p(a.delegateTarget).off(e.namespace?e.origType+"."+e.namespace:e.origType,e.selector,e.handler),this;if(typeof a=="object"){for(f in a)this.off(f,c,a[f]);return this}if(c===!1||typeof c=="function")d=c,c=b;return d===!1&&(d=ba),this.each(function(){p.event.remove(this,a,d,c)})},bind:function(a,b,c){return this.on(a,null,b,c)},unbind:function(a,b){return this.off(a,null,b)},live:function(a,b,c){return p(this.context).on(a,this.selector,b,c),this},die:function(a,b){return p(this.context).off(a,this.selector||"**",b),this},delegate:function(a,b,c,d){return this.on(b,a,c,d)},undelegate:function(a,b,c){return arguments.length===1?this.off(a,"**"):this.off(b,a||"**",c)},trigger:function(a,b){return this.each(function(){p.event.trigger(a,b,this)})},triggerHandler:function(a,b){if(this[0])return p.event.trigger(a,b,this[0],!0)},toggle:function(a){var b=arguments,c=a.guid||p.guid++,d=0,e=function(c){var e=(p._data(this,"lastToggle"+a.guid)||0)%d;return p._data(this,"lastToggle"+a.guid,e+1),c.preventDefault(),b[e].apply(this,arguments)||!1};e.guid=c;while(d<b.length)b[d++].guid=c;return this.click(e)},hover:function(a,b){return this.mouseenter(a).mouseleave(b||a)}}),p.each("blur focus focusin focusout load resize scroll unload click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup error contextmenu".split(" "),function(a,b){p.fn[b]=function(a,c){return c==null&&(c=a,a=null),arguments.length>0?this.on(b,null,a,c):this.trigger(b)},Y.test(b)&&(p.event.fixHooks[b]=p.event.keyHooks),Z.test(b)&&(p.event.fixHooks[b]=p.event.mouseHooks)}),function(a,b){function bc(a,b,c,d){c=c||[],b=b||r;var e,f,i,j,k=b.nodeType;if(!a||typeof a!="string")return c;if(k!==1&&k!==9)return[];i=g(b);if(!i&&!d)if(e=P.exec(a))if(j=e[1]){if(k===9){f=b.getElementById(j);if(!f||!f.parentNode)return c;if(f.id===j)return c.push(f),c}else if(b.ownerDocument&&(f=b.ownerDocument.getElementById(j))&&h(b,f)&&f.id===j)return c.push(f),c}else{if(e[2])return w.apply(c,x.call(b.getElementsByTagName(a),0)),c;if((j=e[3])&&_&&b.getElementsByClassName)return w.apply(c,x.call(b.getElementsByClassName(j),0)),c}return bp(a.replace(L,"$1"),b,c,d,i)}function bd(a){return function(b){var c=b.nodeName.toLowerCase();return c==="input"&&b.type===a}}function be(a){return function(b){var c=b.nodeName.toLowerCase();return(c==="input"||c==="button")&&b.type===a}}function bf(a){return z(function(b){return b=+b,z(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function bg(a,b,c){if(a===b)return c;var d=a.nextSibling;while(d){if(d===b)return-1;d=d.nextSibling}return 1}function bh(a,b){var c,d,f,g,h,i,j,k=C[o][a];if(k)return b?0:k.slice(0);h=a,i=[],j=e.preFilter;while(h){if(!c||(d=M.exec(h)))d&&(h=h.slice(d[0].length)),i.push(f=[]);c=!1;if(d=N.exec(h))f.push(c=new q(d.shift())),h=h.slice(c.length),c.type=d[0].replace(L," ");for(g in e.filter)(d=W[g].exec(h))&&(!j[g]||(d=j[g](d,r,!0)))&&(f.push(c=new q(d.shift())),h=h.slice(c.length),c.type=g,c.matches=d);if(!c)break}return b?h.length:h?bc.error(a):C(a,i).slice(0)}function bi(a,b,d){var e=b.dir,f=d&&b.dir==="parentNode",g=u++;return b.first?function(b,c,d){while(b=b[e])if(f||b.nodeType===1)return a(b,c,d)}:function(b,d,h){if(!h){var i,j=t+" "+g+" ",k=j+c;while(b=b[e])if(f||b.nodeType===1){if((i=b[o])===k)return b.sizset;if(typeof i=="string"&&i.indexOf(j)===0){if(b.sizset)return b}else{b[o]=k;if(a(b,d,h))return b.sizset=!0,b;b.sizset=!1}}}else while(b=b[e])if(f||b.nodeType===1)if(a(b,d,h))return b}}function bj(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function bk(a,b,c,d,e){var f,g=[],h=0,i=a.length,j=b!=null;for(;h<i;h++)if(f=a[h])if(!c||c(f,d,e))g.push(f),j&&b.push(h);return g}function bl(a,b,c,d,e,f){return d&&!d[o]&&(d=bl(d)),e&&!e[o]&&(e=bl(e,f)),z(function(f,g,h,i){if(f&&e)return;var j,k,l,m=[],n=[],o=g.length,p=f||bo(b||"*",h.nodeType?[h]:h,[],f),q=a&&(f||!b)?bk(p,m,a,h,i):p,r=c?e||(f?a:o||d)?[]:g:q;c&&c(q,r,h,i);if(d){l=bk(r,n),d(l,[],h,i),j=l.length;while(j--)if(k=l[j])r[n[j]]=!(q[n[j]]=k)}if(f){j=a&&r.length;while(j--)if(k=r[j])f[m[j]]=!(g[m[j]]=k)}else r=bk(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):w.apply(g,r)})}function bm(a){var b,c,d,f=a.length,g=e.relative[a[0].type],h=g||e.relative[" "],i=g?1:0,j=bi(function(a){return a===b},h,!0),k=bi(function(a){return y.call(b,a)>-1},h,!0),m=[function(a,c,d){return!g&&(d||c!==l)||((b=c).nodeType?j(a,c,d):k(a,c,d))}];for(;i<f;i++)if(c=e.relative[a[i].type])m=[bi(bj(m),c)];else{c=e.filter[a[i].type].apply(null,a[i].matches);if(c[o]){d=++i;for(;d<f;d++)if(e.relative[a[d].type])break;return bl(i>1&&bj(m),i>1&&a.slice(0,i-1).join("").replace(L,"$1"),c,i<d&&bm(a.slice(i,d)),d<f&&bm(a=a.slice(d)),d<f&&a.join(""))}m.push(c)}return bj(m)}function bn(a,b){var d=b.length>0,f=a.length>0,g=function(h,i,j,k,m){var n,o,p,q=[],s=0,u="0",x=h&&[],y=m!=null,z=l,A=h||f&&e.find.TAG("*",m&&i.parentNode||i),B=t+=z==null?1:Math.E;y&&(l=i!==r&&i,c=g.el);for(;(n=A[u])!=null;u++){if(f&&n){for(o=0;p=a[o];o++)if(p(n,i,j)){k.push(n);break}y&&(t=B,c=++g.el)}d&&((n=!p&&n)&&s--,h&&x.push(n))}s+=u;if(d&&u!==s){for(o=0;p=b[o];o++)p(x,q,i,j);if(h){if(s>0)while(u--)!x[u]&&!q[u]&&(q[u]=v.call(k));q=bk(q)}w.apply(k,q),y&&!h&&q.length>0&&s+b.length>1&&bc.uniqueSort(k)}return y&&(t=B,l=z),x};return g.el=0,d?z(g):g}function bo(a,b,c,d){var e=0,f=b.length;for(;e<f;e++)bc(a,b[e],c,d);return c}function bp(a,b,c,d,f){var g,h,j,k,l,m=bh(a),n=m.length;if(!d&&m.length===1){h=m[0]=m[0].slice(0);if(h.length>2&&(j=h[0]).type==="ID"&&b.nodeType===9&&!f&&e.relative[h[1].type]){b=e.find.ID(j.matches[0].replace(V,""),b,f)[0];if(!b)return c;a=a.slice(h.shift().length)}for(g=W.POS.test(a)?-1:h.length-1;g>=0;g--){j=h[g];if(e.relative[k=j.type])break;if(l=e.find[k])if(d=l(j.matches[0].replace(V,""),R.test(h[0].type)&&b.parentNode||b,f)){h.splice(g,1),a=d.length&&h.join("");if(!a)return w.apply(c,x.call(d,0)),c;break}}}return i(a,m)(d,b,f,c,R.test(a)),c}function bq(){}var c,d,e,f,g,h,i,j,k,l,m=!0,n="undefined",o=("sizcache"+Math.random()).replace(".",""),q=String,r=a.document,s=r.documentElement,t=0,u=0,v=[].pop,w=[].push,x=[].slice,y=[].indexOf||function(a){var b=0,c=this.length;for(;b<c;b++)if(this[b]===a)return b;return-1},z=function(a,b){return a[o]=b==null||b,a},A=function(){var a={},b=[];return z(function(c,d){return b.push(c)>e.cacheLength&&delete a[b.shift()],a[c]=d},a)},B=A(),C=A(),D=A(),E="[\\x20\\t\\r\\n\\f]",F="(?:\\\\.|[-\\w]|[^\\x00-\\xa0])+",G=F.replace("w","w#"),H="([*^$|!~]?=)",I="\\["+E+"*("+F+")"+E+"*(?:"+H+E+"*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|("+G+")|)|)"+E+"*\\]",J=":("+F+")(?:\\((?:(['\"])((?:\\\\.|[^\\\\])*?)\\2|([^()[\\]]*|(?:(?:"+I+")|[^:]|\\\\.)*|.*))\\)|)",K=":(even|odd|eq|gt|lt|nth|first|last)(?:\\("+E+"*((?:-\\d)?\\d*)"+E+"*\\)|)(?=[^-]|$)",L=new RegExp("^"+E+"+|((?:^|[^\\\\])(?:\\\\.)*)"+E+"+$","g"),M=new RegExp("^"+E+"*,"+E+"*"),N=new RegExp("^"+E+"*([\\x20\\t\\r\\n\\f>+~])"+E+"*"),O=new RegExp(J),P=/^(?:#([\w\-]+)|(\w+)|\.([\w\-]+))$/,Q=/^:not/,R=/[\x20\t\r\n\f]*[+~]/,S=/:not\($/,T=/h\d/i,U=/input|select|textarea|button/i,V=/\\(?!\\)/g,W={ID:new RegExp("^#("+F+")"),CLASS:new RegExp("^\\.("+F+")"),NAME:new RegExp("^\\[name=['\"]?("+F+")['\"]?\\]"),TAG:new RegExp("^("+F.replace("w","w*")+")"),ATTR:new RegExp("^"+I),PSEUDO:new RegExp("^"+J),POS:new RegExp(K,"i"),CHILD:new RegExp("^:(only|nth|first|last)-child(?:\\("+E+"*(even|odd|(([+-]|)(\\d*)n|)"+E+"*(?:([+-]|)"+E+"*(\\d+)|))"+E+"*\\)|)","i"),needsContext:new RegExp("^"+E+"*[>+~]|"+K,"i")},X=function(a){var b=r.createElement("div");try{return a(b)}catch(c){return!1}finally{b=null}},Y=X(function(a){return a.appendChild(r.createComment("")),!a.getElementsByTagName("*").length}),Z=X(function(a){return a.innerHTML="<a href='#'></a>",a.firstChild&&typeof a.firstChild.getAttribute!==n&&a.firstChild.getAttribute("href")==="#"}),$=X(function(a){a.innerHTML="<select></select>";var b=typeof a.lastChild.getAttribute("multiple");return b!=="boolean"&&b!=="string"}),_=X(function(a){return a.innerHTML="<div class='hidden e'></div><div class='hidden'></div>",!a.getElementsByClassName||!a.getElementsByClassName("e").length?!1:(a.lastChild.className="e",a.getElementsByClassName("e").length===2)}),ba=X(function(a){a.id=o+0,a.innerHTML="<a name='"+o+"'></a><div name='"+o+"'></div>",s.insertBefore(a,s.firstChild);var b=r.getElementsByName&&r.getElementsByName(o).length===2+r.getElementsByName(o+0).length;return d=!r.getElementById(o),s.removeChild(a),b});try{x.call(s.childNodes,0)[0].nodeType}catch(bb){x=function(a){var b,c=[];for(;b=this[a];a++)c.push(b);return c}}bc.matches=function(a,b){return bc(a,null,null,b)},bc.matchesSelector=function(a,b){return bc(b,null,null,[a]).length>0},f=bc.getText=function(a){var b,c="",d=0,e=a.nodeType;if(e){if(e===1||e===9||e===11){if(typeof a.textContent=="string")return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=f(a)}else if(e===3||e===4)return a.nodeValue}else for(;b=a[d];d++)c+=f(b);return c},g=bc.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?b.nodeName!=="HTML":!1},h=bc.contains=s.contains?function(a,b){var c=a.nodeType===9?a.documentElement:a,d=b&&b.parentNode;return a===d||!!(d&&d.nodeType===1&&c.contains&&c.contains(d))}:s.compareDocumentPosition?function(a,b){return b&&!!(a.compareDocumentPosition(b)&16)}:function(a,b){while(b=b.parentNode)if(b===a)return!0;return!1},bc.attr=function(a,b){var c,d=g(a);return d||(b=b.toLowerCase()),(c=e.attrHandle[b])?c(a):d||$?a.getAttribute(b):(c=a.getAttributeNode(b),c?typeof a[b]=="boolean"?a[b]?b:null:c.specified?c.value:null:null)},e=bc.selectors={cacheLength:50,createPseudo:z,match:W,attrHandle:Z?{}:{href:function(a){return a.getAttribute("href",2)},type:function(a){return a.getAttribute("type")}},find:{ID:d?function(a,b,c){if(typeof b.getElementById!==n&&!c){var d=b.getElementById(a);return d&&d.parentNode?[d]:[]}}:function(a,c,d){if(typeof c.getElementById!==n&&!d){var e=c.getElementById(a);return e?e.id===a||typeof e.getAttributeNode!==n&&e.getAttributeNode("id").value===a?[e]:b:[]}},TAG:Y?function(a,b){if(typeof b.getElementsByTagName!==n)return b.getElementsByTagName(a)}:function(a,b){var c=b.getElementsByTagName(a);if(a==="*"){var d,e=[],f=0;for(;d=c[f];f++)d.nodeType===1&&e.push(d);return e}return c},NAME:ba&&function(a,b){if(typeof b.getElementsByName!==n)return b.getElementsByName(name)},CLASS:_&&function(a,b,c){if(typeof b.getElementsByClassName!==n&&!c)return b.getElementsByClassName(a)}},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(V,""),a[3]=(a[4]||a[5]||"").replace(V,""),a[2]==="~="&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),a[1]==="nth"?(a[2]||bc.error(a[0]),a[3]=+(a[3]?a[4]+(a[5]||1):2*(a[2]==="even"||a[2]==="odd")),a[4]=+(a[6]+a[7]||a[2]==="odd")):a[2]&&bc.error(a[0]),a},PSEUDO:function(a){var b,c;if(W.CHILD.test(a[0]))return null;if(a[3])a[2]=a[3];else if(b=a[4])O.test(b)&&(c=bh(b,!0))&&(c=b.indexOf(")",b.length-c)-b.length)&&(b=b.slice(0,c),a[0]=a[0].slice(0,c)),a[2]=b;return a.slice(0,3)}},filter:{ID:d?function(a){return a=a.replace(V,""),function(b){return b.getAttribute("id")===a}}:function(a){return a=a.replace(V,""),function(b){var c=typeof b.getAttributeNode!==n&&b.getAttributeNode("id");return c&&c.value===a}},TAG:function(a){return a==="*"?function(){return!0}:(a=a.replace(V,"").toLowerCase(),function(b){return b.nodeName&&b.nodeName.toLowerCase()===a})},CLASS:function(a){var b=B[o][a];return b||(b=B(a,new RegExp("(^|"+E+")"+a+"("+E+"|$)"))),function(a){return b.test(a.className||typeof a.getAttribute!==n&&a.getAttribute("class")||"")}},ATTR:function(a,b,c){return function(d,e){var f=bc.attr(d,a);return f==null?b==="!=":b?(f+="",b==="="?f===c:b==="!="?f!==c:b==="^="?c&&f.indexOf(c)===0:b==="*="?c&&f.indexOf(c)>-1:b==="$="?c&&f.substr(f.length-c.length)===c:b==="~="?(" "+f+" ").indexOf(c)>-1:b==="|="?f===c||f.substr(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d){return a==="nth"?function(a){var b,e,f=a.parentNode;if(c===1&&d===0)return!0;if(f){e=0;for(b=f.firstChild;b;b=b.nextSibling)if(b.nodeType===1){e++;if(a===b)break}}return e-=d,e===c||e%c===0&&e/c>=0}:function(b){var c=b;switch(a){case"only":case"first":while(c=c.previousSibling)if(c.nodeType===1)return!1;if(a==="first")return!0;c=b;case"last":while(c=c.nextSibling)if(c.nodeType===1)return!1;return!0}}},PSEUDO:function(a,b){var c,d=e.pseudos[a]||e.setFilters[a.toLowerCase()]||bc.error("unsupported pseudo: "+a);return d[o]?d(b):d.length>1?(c=[a,a,"",b],e.setFilters.hasOwnProperty(a.toLowerCase())?z(function(a,c){var e,f=d(a,b),g=f.length;while(g--)e=y.call(a,f[g]),a[e]=!(c[e]=f[g])}):function(a){return d(a,0,c)}):d}},pseudos:{not:z(function(a){var b=[],c=[],d=i(a.replace(L,"$1"));return d[o]?z(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)if(f=g[h])a[h]=!(b[h]=f)}):function(a,e,f){return b[0]=a,d(b,null,f,c),!c.pop()}}),has:z(function(a){return function(b){return bc(a,b).length>0}}),contains:z(function(a){return function(b){return(b.textContent||b.innerText||f(b)).indexOf(a)>-1}}),enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return b==="input"&&!!a.checked||b==="option"&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},parent:function(a){return!e.pseudos.empty(a)},empty:function(a){var b;a=a.firstChild;while(a){if(a.nodeName>"@"||(b=a.nodeType)===3||b===4)return!1;a=a.nextSibling}return!0},header:function(a){return T.test(a.nodeName)},text:function(a){var b,c;return a.nodeName.toLowerCase()==="input"&&(b=a.type)==="text"&&((c=a.getAttribute("type"))==null||c.toLowerCase()===b)},radio:bd("radio"),checkbox:bd("checkbox"),file:bd("file"),password:bd("password"),image:bd("image"),submit:be("submit"),reset:be("reset"),button:function(a){var b=a.nodeName.toLowerCase();return b==="input"&&a.type==="button"||b==="button"},input:function(a){return U.test(a.nodeName)},focus:function(a){var b=a.ownerDocument;return a===b.activeElement&&(!b.hasFocus||b.hasFocus())&&(!!a.type||!!a.href)},active:function(a){return a===a.ownerDocument.activeElement},first:bf(function(a,b,c){return[0]}),last:bf(function(a,b,c){return[b-1]}),eq:bf(function(a,b,c){return[c<0?c+b:c]}),even:bf(function(a,b,c){for(var d=0;d<b;d+=2)a.push(d);return a}),odd:bf(function(a,b,c){for(var d=1;d<b;d+=2)a.push(d);return a}),lt:bf(function(a,b,c){for(var d=c<0?c+b:c;--d>=0;)a.push(d);return a}),gt:bf(function(a,b,c){for(var d=c<0?c+b:c;++d<b;)a.push(d);return a})}},j=s.compareDocumentPosition?function(a,b){return a===b?(k=!0,0):(!a.compareDocumentPosition||!b.compareDocumentPosition?a.compareDocumentPosition:a.compareDocumentPosition(b)&4)?-1:1}:function(a,b){if(a===b)return k=!0,0;if(a.sourceIndex&&b.sourceIndex)return a.sourceIndex-b.sourceIndex;var c,d,e=[],f=[],g=a.parentNode,h=b.parentNode,i=g;if(g===h)return bg(a,b);if(!g)return-1;if(!h)return 1;while(i)e.unshift(i),i=i.parentNode;i=h;while(i)f.unshift(i),i=i.parentNode;c=e.length,d=f.length;for(var j=0;j<c&&j<d;j++)if(e[j]!==f[j])return bg(e[j],f[j]);return j===c?bg(a,f[j],-1):bg(e[j],b,1)},[0,0].sort(j),m=!k,bc.uniqueSort=function(a){var b,c=1;k=m,a.sort(j);if(k)for(;b=a[c];c++)b===a[c-1]&&a.splice(c--,1);return a},bc.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},i=bc.compile=function(a,b){var c,d=[],e=[],f=D[o][a];if(!f){b||(b=bh(a)),c=b.length;while(c--)f=bm(b[c]),f[o]?d.push(f):e.push(f);f=D(a,bn(e,d))}return f},r.querySelectorAll&&function(){var a,b=bp,c=/'|\\/g,d=/\=[\x20\t\r\n\f]*([^'"\]]*)[\x20\t\r\n\f]*\]/g,e=[":focus"],f=[":active",":focus"],h=s.matchesSelector||s.mozMatchesSelector||s.webkitMatchesSelector||s.oMatchesSelector||s.msMatchesSelector;X(function(a){a.innerHTML="<select><option selected=''></option></select>",a.querySelectorAll("[selected]").length||e.push("\\["+E+"*(?:checked|disabled|ismap|multiple|readonly|selected|value)"),a.querySelectorAll(":checked").length||e.push(":checked")}),X(function(a){a.innerHTML="<p test=''></p>",a.querySelectorAll("[test^='']").length&&e.push("[*^$]="+E+"*(?:\"\"|'')"),a.innerHTML="<input type='hidden'/>",a.querySelectorAll(":enabled").length||e.push(":enabled",":disabled")}),e=new RegExp(e.join("|")),bp=function(a,d,f,g,h){if(!g&&!h&&(!e||!e.test(a))){var i,j,k=!0,l=o,m=d,n=d.nodeType===9&&a;if(d.nodeType===1&&d.nodeName.toLowerCase()!=="object"){i=bh(a),(k=d.getAttribute("id"))?l=k.replace(c,"\\$&"):d.setAttribute("id",l),l="[id='"+l+"'] ",j=i.length;while(j--)i[j]=l+i[j].join("");m=R.test(a)&&d.parentNode||d,n=i.join(",")}if(n)try{return w.apply(f,x.call(m.querySelectorAll(n),0)),f}catch(p){}finally{k||d.removeAttribute("id")}}return b(a,d,f,g,h)},h&&(X(function(b){a=h.call(b,"div");try{h.call(b,"[test!='']:sizzle"),f.push("!=",J)}catch(c){}}),f=new RegExp(f.join("|")),bc.matchesSelector=function(b,c){c=c.replace(d,"='$1']");if(!g(b)&&!f.test(c)&&(!e||!e.test(c)))try{var i=h.call(b,c);if(i||a||b.document&&b.document.nodeType!==11)return i}catch(j){}return bc(c,null,null,[b]).length>0})}(),e.pseudos.nth=e.pseudos.eq,e.filters=bq.prototype=e.pseudos,e.setFilters=new bq,bc.attr=p.attr,p.find=bc,p.expr=bc.selectors,p.expr[":"]=p.expr.pseudos,p.unique=bc.uniqueSort,p.text=bc.getText,p.isXMLDoc=bc.isXML,p.contains=bc.contains}(a);var bc=/Until$/,bd=/^(?:parents|prev(?:Until|All))/,be=/^.[^:#\[\.,]*$/,bf=p.expr.match.needsContext,bg={children:!0,contents:!0,next:!0,prev:!0};p.fn.extend({find:function(a){var b,c,d,e,f,g,h=this;if(typeof a!="string")return p(a).filter(function(){for(b=0,c=h.length;b<c;b++)if(p.contains(h[b],this))return!0});g=this.pushStack("","find",a);for(b=0,c=this.length;b<c;b++){d=g.length,p.find(a,this[b],g);if(b>0)for(e=d;e<g.length;e++)for(f=0;f<d;f++)if(g[f]===g[e]){g.splice(e--,1);break}}return g},has:function(a){var b,c=p(a,this),d=c.length;return this.filter(function(){for(b=0;b<d;b++)if(p.contains(this,c[b]))return!0})},not:function(a){return this.pushStack(bj(this,a,!1),"not",a)},filter:function(a){return this.pushStack(bj(this,a,!0),"filter",a)},is:function(a){return!!a&&(typeof a=="string"?bf.test(a)?p(a,this.context).index(this[0])>=0:p.filter(a,this).length>0:this.filter(a).length>0)},closest:function(a,b){var c,d=0,e=this.length,f=[],g=bf.test(a)||typeof a!="string"?p(a,b||this.context):0;for(;d<e;d++){c=this[d];while(c&&c.ownerDocument&&c!==b&&c.nodeType!==11){if(g?g.index(c)>-1:p.find.matchesSelector(c,a)){f.push(c);break}c=c.parentNode}}return f=f.length>1?p.unique(f):f,this.pushStack(f,"closest",a)},index:function(a){return a?typeof a=="string"?p.inArray(this[0],p(a)):p.inArray(a.jquery?a[0]:a,this):this[0]&&this[0].parentNode?this.prevAll().length:-1},add:function(a,b){var c=typeof a=="string"?p(a,b):p.makeArray(a&&a.nodeType?[a]:a),d=p.merge(this.get(),c);return this.pushStack(bh(c[0])||bh(d[0])?d:p.unique(d))},addBack:function(a){return this.add(a==null?this.prevObject:this.prevObject.filter(a))}}),p.fn.andSelf=p.fn.addBack,p.each({parent:function(a){var b=a.parentNode;return b&&b.nodeType!==11?b:null},parents:function(a){return p.dir(a,"parentNode")},parentsUntil:function(a,b,c){return p.dir(a,"parentNode",c)},next:function(a){return bi(a,"nextSibling")},prev:function(a){return bi(a,"previousSibling")},nextAll:function(a){return p.dir(a,"nextSibling")},prevAll:function(a){return p.dir(a,"previousSibling")},nextUntil:function(a,b,c){return p.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return p.dir(a,"previousSibling",c)},siblings:function(a){return p.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return p.sibling(a.firstChild)},contents:function(a){return p.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:p.merge([],a.childNodes)}},function(a,b){p.fn[a]=function(c,d){var e=p.map(this,b,c);return bc.test(a)||(d=c),d&&typeof d=="string"&&(e=p.filter(d,e)),e=this.length>1&&!bg[a]?p.unique(e):e,this.length>1&&bd.test(a)&&(e=e.reverse()),this.pushStack(e,a,k.call(arguments).join(","))}}),p.extend({filter:function(a,b,c){return c&&(a=":not("+a+")"),b.length===1?p.find.matchesSelector(b[0],a)?[b[0]]:[]:p.find.matches(a,b)},dir:function(a,c,d){var e=[],f=a[c];while(f&&f.nodeType!==9&&(d===b||f.nodeType!==1||!p(f).is(d)))f.nodeType===1&&e.push(f),f=f[c];return e},sibling:function(a,b){var c=[];for(;a;a=a.nextSibling)a.nodeType===1&&a!==b&&c.push(a);return c}});var bl="abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",bm=/ jQuery\d+="(?:null|\d+)"/g,bn=/^\s+/,bo=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,bp=/<([\w:]+)/,bq=/<tbody/i,br=/<|&#?\w+;/,bs=/<(?:script|style|link)/i,bt=/<(?:script|object|embed|option|style)/i,bu=new RegExp("<(?:"+bl+")[\\s/>]","i"),bv=/^(?:checkbox|radio)$/,bw=/checked\s*(?:[^=]|=\s*.checked.)/i,bx=/\/(java|ecma)script/i,by=/^\s*<!(?:\[CDATA\[|\-\-)|[\]\-]{2}>\s*$/g,bz={option:[1,"<select multiple='multiple'>","</select>"],legend:[1,"<fieldset>","</fieldset>"],thead:[1,"<table>","</table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],col:[2,"<table><tbody></tbody><colgroup>","</colgroup></table>"],area:[1,"<map>","</map>"],_default:[0,"",""]},bA=bk(e),bB=bA.appendChild(e.createElement("div"));bz.optgroup=bz.option,bz.tbody=bz.tfoot=bz.colgroup=bz.caption=bz.thead,bz.th=bz.td,p.support.htmlSerialize||(bz._default=[1,"X<div>","</div>"]),p.fn.extend({text:function(a){return p.access(this,function(a){return a===b?p.text(this):this.empty().append((this[0]&&this[0].ownerDocument||e).createTextNode(a))},null,a,arguments.length)},wrapAll:function(a){if(p.isFunction(a))return this.each(function(b){p(this).wrapAll(a.call(this,b))});if(this[0]){var b=p(a,this[0].ownerDocument).eq(0).clone(!0);this[0].parentNode&&b.insertBefore(this[0]),b.map(function(){var a=this;while(a.firstChild&&a.firstChild.nodeType===1)a=a.firstChild;return a}).append(this)}return this},wrapInner:function(a){return p.isFunction(a)?this.each(function(b){p(this).wrapInner(a.call(this,b))}):this.each(function(){var b=p(this),c=b.contents();c.length?c.wrapAll(a):b.append(a)})},wrap:function(a){var b=p.isFunction(a);return this.each(function(c){p(this).wrapAll(b?a.call(this,c):a)})},unwrap:function(){return this.parent().each(function(){p.nodeName(this,"body")||p(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,!0,function(a){(this.nodeType===1||this.nodeType===11)&&this.appendChild(a)})},prepend:function(){return this.domManip(arguments,!0,function(a){(this.nodeType===1||this.nodeType===11)&&this.insertBefore(a,this.firstChild)})},before:function(){if(!bh(this[0]))return this.domManip(arguments,!1,function(a){this.parentNode.insertBefore(a,this)});if(arguments.length){var a=p.clean(arguments);return this.pushStack(p.merge(a,this),"before",this.selector)}},after:function(){if(!bh(this[0]))return this.domManip(arguments,!1,function(a){this.parentNode.insertBefore(a,this.nextSibling)});if(arguments.length){var a=p.clean(arguments);return this.pushStack(p.merge(this,a),"after",this.selector)}},remove:function(a,b){var c,d=0;for(;(c=this[d])!=null;d++)if(!a||p.filter(a,[c]).length)!b&&c.nodeType===1&&(p.cleanData(c.getElementsByTagName("*")),p.cleanData([c])),c.parentNode&&c.parentNode.removeChild(c);return this},empty:function(){var a,b=0;for(;(a=this[b])!=null;b++){a.nodeType===1&&p.cleanData(a.getElementsByTagName("*"));while(a.firstChild)a.removeChild(a.firstChild)}return this},clone:function(a,b){return a=a==null?!1:a,b=b==null?a:b,this.map(function(){return p.clone(this,a,b)})},html:function(a){return p.access(this,function(a){var c=this[0]||{},d=0,e=this.length;if(a===b)return c.nodeType===1?c.innerHTML.replace(bm,""):b;if(typeof a=="string"&&!bs.test(a)&&(p.support.htmlSerialize||!bu.test(a))&&(p.support.leadingWhitespace||!bn.test(a))&&!bz[(bp.exec(a)||["",""])[1].toLowerCase()]){a=a.replace(bo,"<$1></$2>");try{for(;d<e;d++)c=this[d]||{},c.nodeType===1&&(p.cleanData(c.getElementsByTagName("*")),c.innerHTML=a);c=0}catch(f){}}c&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(a){return bh(this[0])?this.length?this.pushStack(p(p.isFunction(a)?a():a),"replaceWith",a):this:p.isFunction(a)?this.each(function(b){var c=p(this),d=c.html();c.replaceWith(a.call(this,b,d))}):(typeof a!="string"&&(a=p(a).detach()),this.each(function(){var b=this.nextSibling,c=this.parentNode;p(this).remove(),b?p(b).before(a):p(c).append(a)}))},detach:function(a){return this.remove(a,!0)},domManip:function(a,c,d){a=[].concat.apply([],a);var e,f,g,h,i=0,j=a[0],k=[],l=this.length;if(!p.support.checkClone&&l>1&&typeof j=="string"&&bw.test(j))return this.each(function(){p(this).domManip(a,c,d)});if(p.isFunction(j))return this.each(function(e){var f=p(this);a[0]=j.call(this,e,c?f.html():b),f.domManip(a,c,d)});if(this[0]){e=p.buildFragment(a,this,k),g=e.fragment,f=g.firstChild,g.childNodes.length===1&&(g=f);if(f){c=c&&p.nodeName(f,"tr");for(h=e.cacheable||l-1;i<l;i++)d.call(c&&p.nodeName(this[i],"table")?bC(this[i],"tbody"):this[i],i===h?g:p.clone(g,!0,!0))}g=f=null,k.length&&p.each(k,function(a,b){b.src?p.ajax?p.ajax({url:b.src,type:"GET",dataType:"script",async:!1,global:!1,"throws":!0}):p.error("no ajax"):p.globalEval((b.text||b.textContent||b.innerHTML||"").replace(by,"")),b.parentNode&&b.parentNode.removeChild(b)})}return this}}),p.buildFragment=function(a,c,d){var f,g,h,i=a[0];return c=c||e,c=!c.nodeType&&c[0]||c,c=c.ownerDocument||c,a.length===1&&typeof i=="string"&&i.length<512&&c===e&&i.charAt(0)==="<"&&!bt.test(i)&&(p.support.checkClone||!bw.test(i))&&(p.support.html5Clone||!bu.test(i))&&(g=!0,f=p.fragments[i],h=f!==b),f||(f=c.createDocumentFragment(),p.clean(a,c,f,d),g&&(p.fragments[i]=h&&f)),{fragment:f,cacheable:g}},p.fragments={},p.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){p.fn[a]=function(c){var d,e=0,f=[],g=p(c),h=g.length,i=this.length===1&&this[0].parentNode;if((i==null||i&&i.nodeType===11&&i.childNodes.length===1)&&h===1)return g[b](this[0]),this;for(;e<h;e++)d=(e>0?this.clone(!0):this).get(),p(g[e])[b](d),f=f.concat(d);return this.pushStack(f,a,g.selector)}}),p.extend({clone:function(a,b,c){var d,e,f,g;p.support.html5Clone||p.isXMLDoc(a)||!bu.test("<"+a.nodeName+">")?g=a.cloneNode(!0):(bB.innerHTML=a.outerHTML,bB.removeChild(g=bB.firstChild));if((!p.support.noCloneEvent||!p.support.noCloneChecked)&&(a.nodeType===1||a.nodeType===11)&&!p.isXMLDoc(a)){bE(a,g),d=bF(a),e=bF(g);for(f=0;d[f];++f)e[f]&&bE(d[f],e[f])}if(b){bD(a,g);if(c){d=bF(a),e=bF(g);for(f=0;d[f];++f)bD(d[f],e[f])}}return d=e=null,g},clean:function(a,b,c,d){var f,g,h,i,j,k,l,m,n,o,q,r,s=b===e&&bA,t=[];if(!b||typeof b.createDocumentFragment=="undefined")b=e;for(f=0;(h=a[f])!=null;f++){typeof h=="number"&&(h+="");if(!h)continue;if(typeof h=="string")if(!br.test(h))h=b.createTextNode(h);else{s=s||bk(b),l=b.createElement("div"),s.appendChild(l),h=h.replace(bo,"<$1></$2>"),i=(bp.exec(h)||["",""])[1].toLowerCase(),j=bz[i]||bz._default,k=j[0],l.innerHTML=j[1]+h+j[2];while(k--)l=l.lastChild;if(!p.support.tbody){m=bq.test(h),n=i==="table"&&!m?l.firstChild&&l.firstChild.childNodes:j[1]==="<table>"&&!m?l.childNodes:[];for(g=n.length-1;g>=0;--g)p.nodeName(n[g],"tbody")&&!n[g].childNodes.length&&n[g].parentNode.removeChild(n[g])}!p.support.leadingWhitespace&&bn.test(h)&&l.insertBefore(b.createTextNode(bn.exec(h)[0]),l.firstChild),h=l.childNodes,l.parentNode.removeChild(l)}h.nodeType?t.push(h):p.merge(t,h)}l&&(h=l=s=null);if(!p.support.appendChecked)for(f=0;(h=t[f])!=null;f++)p.nodeName(h,"input")?bG(h):typeof h.getElementsByTagName!="undefined"&&p.grep(h.getElementsByTagName("input"),bG);if(c){q=function(a){if(!a.type||bx.test(a.type))return d?d.push(a.parentNode?a.parentNode.removeChild(a):a):c.appendChild(a)};for(f=0;(h=t[f])!=null;f++)if(!p.nodeName(h,"script")||!q(h))c.appendChild(h),typeof h.getElementsByTagName!="undefined"&&(r=p.grep(p.merge([],h.getElementsByTagName("script")),q),t.splice.apply(t,[f+1,0].concat(r)),f+=r.length)}return t},cleanData:function(a,b){var c,d,e,f,g=0,h=p.expando,i=p.cache,j=p.support.deleteExpando,k=p.event.special;for(;(e=a[g])!=null;g++)if(b||p.acceptData(e)){d=e[h],c=d&&i[d];if(c){if(c.events)for(f in c.events)k[f]?p.event.remove(e,f):p.removeEvent(e,f,c.handle);i[d]&&(delete i[d],j?delete e[h]:e.removeAttribute?e.removeAttribute(h):e[h]=null,p.deletedIds.push(d))}}}}),function(){var a,b;p.uaMatch=function(a){a=a.toLowerCase();var b=/(chrome)[ \/]([\w.]+)/.exec(a)||/(webkit)[ \/]([\w.]+)/.exec(a)||/(opera)(?:.*version|)[ \/]([\w.]+)/.exec(a)||/(msie) ([\w.]+)/.exec(a)||a.indexOf("compatible")<0&&/(mozilla)(?:.*? rv:([\w.]+)|)/.exec(a)||[];return{browser:b[1]||"",version:b[2]||"0"}},a=p.uaMatch(g.userAgent),b={},a.browser&&(b[a.browser]=!0,b.version=a.version),b.chrome?b.webkit=!0:b.webkit&&(b.safari=!0),p.browser=b,p.sub=function(){function a(b,c){return new a.fn.init(b,c)}p.extend(!0,a,this),a.superclass=this,a.fn=a.prototype=this(),a.fn.constructor=a,a.sub=this.sub,a.fn.init=function c(c,d){return d&&d instanceof p&&!(d instanceof a)&&(d=a(d)),p.fn.init.call(this,c,d,b)},a.fn.init.prototype=a.fn;var b=a(e);return a}}();var bH,bI,bJ,bK=/alpha\([^)]*\)/i,bL=/opacity=([^)]*)/,bM=/^(top|right|bottom|left)$/,bN=/^(none|table(?!-c[ea]).+)/,bO=/^margin/,bP=new RegExp("^("+q+")(.*)$","i"),bQ=new RegExp("^("+q+")(?!px)[a-z%]+$","i"),bR=new RegExp("^([-+])=("+q+")","i"),bS={},bT={position:"absolute",visibility:"hidden",display:"block"},bU={letterSpacing:0,fontWeight:400},bV=["Top","Right","Bottom","Left"],bW=["Webkit","O","Moz","ms"],bX=p.fn.toggle;p.fn.extend({css:function(a,c){return p.access(this,function(a,c,d){return d!==b?p.style(a,c,d):p.css(a,c)},a,c,arguments.length>1)},show:function(){return b$(this,!0)},hide:function(){return b$(this)},toggle:function(a,b){var c=typeof a=="boolean";return p.isFunction(a)&&p.isFunction(b)?bX.apply(this,arguments):this.each(function(){(c?a:bZ(this))?p(this).show():p(this).hide()})}}),p.extend({cssHooks:{opacity:{get:function(a,b){if(b){var c=bH(a,"opacity");return c===""?"1":c}}}},cssNumber:{fillOpacity:!0,fontWeight:!0,lineHeight:!0,opacity:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":p.support.cssFloat?"cssFloat":"styleFloat"},style:function(a,c,d,e){if(!a||a.nodeType===3||a.nodeType===8||!a.style)return;var f,g,h,i=p.camelCase(c),j=a.style;c=p.cssProps[i]||(p.cssProps[i]=bY(j,i)),h=p.cssHooks[c]||p.cssHooks[i];if(d===b)return h&&"get"in h&&(f=h.get(a,!1,e))!==b?f:j[c];g=typeof d,g==="string"&&(f=bR.exec(d))&&(d=(f[1]+1)*f[2]+parseFloat(p.css(a,c)),g="number");if(d==null||g==="number"&&isNaN(d))return;g==="number"&&!p.cssNumber[i]&&(d+="px");if(!h||!("set"in h)||(d=h.set(a,d,e))!==b)try{j[c]=d}catch(k){}},css:function(a,c,d,e){var f,g,h,i=p.camelCase(c);return c=p.cssProps[i]||(p.cssProps[i]=bY(a.style,i)),h=p.cssHooks[c]||p.cssHooks[i],h&&"get"in h&&(f=h.get(a,!0,e)),f===b&&(f=bH(a,c)),f==="normal"&&c in bU&&(f=bU[c]),d||e!==b?(g=parseFloat(f),d||p.isNumeric(g)?g||0:f):f},swap:function(a,b,c){var d,e,f={};for(e in b)f[e]=a.style[e],a.style[e]=b[e];d=c.call(a);for(e in b)a.style[e]=f[e];return d}}),a.getComputedStyle?bH=function(b,c){var d,e,f,g,h=a.getComputedStyle(b,null),i=b.style;return h&&(d=h[c],d===""&&!p.contains(b.ownerDocument,b)&&(d=p.style(b,c)),bQ.test(d)&&bO.test(c)&&(e=i.width,f=i.minWidth,g=i.maxWidth,i.minWidth=i.maxWidth=i.width=d,d=h.width,i.width=e,i.minWidth=f,i.maxWidth=g)),d}:e.documentElement.currentStyle&&(bH=function(a,b){var c,d,e=a.currentStyle&&a.currentStyle[b],f=a.style;return e==null&&f&&f[b]&&(e=f[b]),bQ.test(e)&&!bM.test(b)&&(c=f.left,d=a.runtimeStyle&&a.runtimeStyle.left,d&&(a.runtimeStyle.left=a.currentStyle.left),f.left=b==="fontSize"?"1em":e,e=f.pixelLeft+"px",f.left=c,d&&(a.runtimeStyle.left=d)),e===""?"auto":e}),p.each(["height","width"],function(a,b){p.cssHooks[b]={get:function(a,c,d){if(c)return a.offsetWidth===0&&bN.test(bH(a,"display"))?p.swap(a,bT,function(){return cb(a,b,d)}):cb(a,b,d)},set:function(a,c,d){return b_(a,c,d?ca(a,b,d,p.support.boxSizing&&p.css(a,"boxSizing")==="border-box"):0)}}}),p.support.opacity||(p.cssHooks.opacity={get:function(a,b){return bL.test((b&&a.currentStyle?a.currentStyle.filter:a.style.filter)||"")?.01*parseFloat(RegExp.$1)+"":b?"1":""},set:function(a,b){var c=a.style,d=a.currentStyle,e=p.isNumeric(b)?"alpha(opacity="+b*100+")":"",f=d&&d.filter||c.filter||"";c.zoom=1;if(b>=1&&p.trim(f.replace(bK,""))===""&&c.removeAttribute){c.removeAttribute("filter");if(d&&!d.filter)return}c.filter=bK.test(f)?f.replace(bK,e):f+" "+e}}),p(function(){p.support.reliableMarginRight||(p.cssHooks.marginRight={get:function(a,b){return p.swap(a,{display:"inline-block"},function(){if(b)return bH(a,"marginRight")})}}),!p.support.pixelPosition&&p.fn.position&&p.each(["top","left"],function(a,b){p.cssHooks[b]={get:function(a,c){if(c){var d=bH(a,b);return bQ.test(d)?p(a).position()[b]+"px":d}}}})}),p.expr&&p.expr.filters&&(p.expr.filters.hidden=function(a){return a.offsetWidth===0&&a.offsetHeight===0||!p.support.reliableHiddenOffsets&&(a.style&&a.style.display||bH(a,"display"))==="none"},p.expr.filters.visible=function(a){return!p.expr.filters.hidden(a)}),p.each({margin:"",padding:"",border:"Width"},function(a,b){p.cssHooks[a+b]={expand:function(c){var d,e=typeof c=="string"?c.split(" "):[c],f={};for(d=0;d<4;d++)f[a+bV[d]+b]=e[d]||e[d-2]||e[0];return f}},bO.test(a)||(p.cssHooks[a+b].set=b_)});var cd=/%20/g,ce=/\[\]$/,cf=/\r?\n/g,cg=/^(?:color|date|datetime|datetime-local|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,ch=/^(?:select|textarea)/i;p.fn.extend({serialize:function(){return p.param(this.serializeArray())},serializeArray:function(){return this.map(function(){return this.elements?p.makeArray(this.elements):this}).filter(function(){return this.name&&!this.disabled&&(this.checked||ch.test(this.nodeName)||cg.test(this.type))}).map(function(a,b){var c=p(this).val();return c==null?null:p.isArray(c)?p.map(c,function(a,c){return{name:b.name,value:a.replace(cf,"\r\n")}}):{name:b.name,value:c.replace(cf,"\r\n")}}).get()}}),p.param=function(a,c){var d,e=[],f=function(a,b){b=p.isFunction(b)?b():b==null?"":b,e[e.length]=encodeURIComponent(a)+"="+encodeURIComponent(b)};c===b&&(c=p.ajaxSettings&&p.ajaxSettings.traditional);if(p.isArray(a)||a.jquery&&!p.isPlainObject(a))p.each(a,function(){f(this.name,this.value)});else for(d in a)ci(d,a[d],c,f);return e.join("&").replace(cd,"+")};var cj,ck,cl=/#.*$/,cm=/^(.*?):[ \t]*([^\r\n]*)\r?$/mg,cn=/^(?:about|app|app\-storage|.+\-extension|file|res|widget):$/,co=/^(?:GET|HEAD)$/,cp=/^\/\//,cq=/\?/,cr=/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,cs=/([?&])_=[^&]*/,ct=/^([\w\+\.\-]+:)(?:\/\/([^\/?#:]*)(?::(\d+)|)|)/,cu=p.fn.load,cv={},cw={},cx=["*/"]+["*"];try{ck=f.href}catch(cy){ck=e.createElement("a"),ck.href="",ck=ck.href}cj=ct.exec(ck.toLowerCase())||[],p.fn.load=function(a,c,d){if(typeof a!="string"&&cu)return cu.apply(this,arguments);if(!this.length)return this;var e,f,g,h=this,i=a.indexOf(" ");return i>=0&&(e=a.slice(i,a.length),a=a.slice(0,i)),p.isFunction(c)?(d=c,c=b):c&&typeof c=="object"&&(f="POST"),p.ajax({url:a,type:f,dataType:"html",data:c,complete:function(a,b){d&&h.each(d,g||[a.responseText,b,a])}}).done(function(a){g=arguments,h.html(e?p("<div>").append(a.replace(cr,"")).find(e):a)}),this},p.each("ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split(" "),function(a,b){p.fn[b]=function(a){return this.on(b,a)}}),p.each(["get","post"],function(a,c){p[c]=function(a,d,e,f){return p.isFunction(d)&&(f=f||e,e=d,d=b),p.ajax({type:c,url:a,data:d,success:e,dataType:f})}}),p.extend({getScript:function(a,c){return p.get(a,b,c,"script")},getJSON:function(a,b,c){return p.get(a,b,c,"json")},ajaxSetup:function(a,b){return b?cB(a,p.ajaxSettings):(b=a,a=p.ajaxSettings),cB(a,b),a},ajaxSettings:{url:ck,isLocal:cn.test(cj[1]),global:!0,type:"GET",contentType:"application/x-www-form-urlencoded; charset=UTF-8",processData:!0,async:!0,accepts:{xml:"application/xml, text/xml",html:"text/html",text:"text/plain",json:"application/json, text/javascript","*":cx},contents:{xml:/xml/,html:/html/,json:/json/},responseFields:{xml:"responseXML",text:"responseText"},converters:{"* text":a.String,"text html":!0,"text json":p.parseJSON,"text xml":p.parseXML},flatOptions:{context:!0,url:!0}},ajaxPrefilter:cz(cv),ajaxTransport:cz(cw),ajax:function(a,c){function y(a,c,f,i){var k,s,t,u,w,y=c;if(v===2)return;v=2,h&&clearTimeout(h),g=b,e=i||"",x.readyState=a>0?4:0,f&&(u=cC(l,x,f));if(a>=200&&a<300||a===304)l.ifModified&&(w=x.getResponseHeader("Last-Modified"),w&&(p.lastModified[d]=w),w=x.getResponseHeader("Etag"),w&&(p.etag[d]=w)),a===304?(y="notmodified",k=!0):(k=cD(l,u),y=k.state,s=k.data,t=k.error,k=!t);else{t=y;if(!y||a)y="error",a<0&&(a=0)}x.status=a,x.statusText=(c||y)+"",k?o.resolveWith(m,[s,y,x]):o.rejectWith(m,[x,y,t]),x.statusCode(r),r=b,j&&n.trigger("ajax"+(k?"Success":"Error"),[x,l,k?s:t]),q.fireWith(m,[x,y]),j&&(n.trigger("ajaxComplete",[x,l]),--p.active||p.event.trigger("ajaxStop"))}typeof a=="object"&&(c=a,a=b),c=c||{};var d,e,f,g,h,i,j,k,l=p.ajaxSetup({},c),m=l.context||l,n=m!==l&&(m.nodeType||m instanceof p)?p(m):p.event,o=p.Deferred(),q=p.Callbacks("once memory"),r=l.statusCode||{},t={},u={},v=0,w="canceled",x={readyState:0,setRequestHeader:function(a,b){if(!v){var c=a.toLowerCase();a=u[c]=u[c]||a,t[a]=b}return this},getAllResponseHeaders:function(){return v===2?e:null},getResponseHeader:function(a){var c;if(v===2){if(!f){f={};while(c=cm.exec(e))f[c[1].toLowerCase()]=c[2]}c=f[a.toLowerCase()]}return c===b?null:c},overrideMimeType:function(a){return v||(l.mimeType=a),this},abort:function(a){return a=a||w,g&&g.abort(a),y(0,a),this}};o.promise(x),x.success=x.done,x.error=x.fail,x.complete=q.add,x.statusCode=function(a){if(a){var b;if(v<2)for(b in a)r[b]=[r[b],a[b]];else b=a[x.status],x.always(b)}return this},l.url=((a||l.url)+"").replace(cl,"").replace(cp,cj[1]+"//"),l.dataTypes=p.trim(l.dataType||"*").toLowerCase().split(s),l.crossDomain==null&&(i=ct.exec(l.url.toLowerCase())||!1,l.crossDomain=i&&i.join(":")+(i[3]?"":i[1]==="http:"?80:443)!==cj.join(":")+(cj[3]?"":cj[1]==="http:"?80:443)),l.data&&l.processData&&typeof l.data!="string"&&(l.data=p.param(l.data,l.traditional)),cA(cv,l,c,x);if(v===2)return x;j=l.global,l.type=l.type.toUpperCase(),l.hasContent=!co.test(l.type),j&&p.active++===0&&p.event.trigger("ajaxStart");if(!l.hasContent){l.data&&(l.url+=(cq.test(l.url)?"&":"?")+l.data,delete l.data),d=l.url;if(l.cache===!1){var z=p.now(),A=l.url.replace(cs,"$1_="+z);l.url=A+(A===l.url?(cq.test(l.url)?"&":"?")+"_="+z:"")}}(l.data&&l.hasContent&&l.contentType!==!1||c.contentType)&&x.setRequestHeader("Content-Type",l.contentType),l.ifModified&&(d=d||l.url,p.lastModified[d]&&x.setRequestHeader("If-Modified-Since",p.lastModified[d]),p.etag[d]&&x.setRequestHeader("If-None-Match",p.etag[d])),x.setRequestHeader("Accept",l.dataTypes[0]&&l.accepts[l.dataTypes[0]]?l.accepts[l.dataTypes[0]]+(l.dataTypes[0]!=="*"?", "+cx+"; q=0.01":""):l.accepts["*"]);for(k in l.headers)x.setRequestHeader(k,l.headers[k]);if(!l.beforeSend||l.beforeSend.call(m,x,l)!==!1&&v!==2){w="abort";for(k in{success:1,error:1,complete:1})x[k](l[k]);g=cA(cw,l,c,x);if(!g)y(-1,"No Transport");else{x.readyState=1,j&&n.trigger("ajaxSend",[x,l]),l.async&&l.timeout>0&&(h=setTimeout(function(){x.abort("timeout")},l.timeout));try{v=1,g.send(t,y)}catch(B){if(v<2)y(-1,B);else throw B}}return x}return x.abort()},active:0,lastModified:{},etag:{}});var cE=[],cF=/\?/,cG=/(=)\?(?=&|$)|\?\?/,cH=p.now();p.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var a=cE.pop()||p.expando+"_"+cH++;return this[a]=!0,a}}),p.ajaxPrefilter("json jsonp",function(c,d,e){var f,g,h,i=c.data,j=c.url,k=c.jsonp!==!1,l=k&&cG.test(j),m=k&&!l&&typeof i=="string"&&!(c.contentType||"").indexOf("application/x-www-form-urlencoded")&&cG.test(i);if(c.dataTypes[0]==="jsonp"||l||m)return f=c.jsonpCallback=p.isFunction(c.jsonpCallback)?c.jsonpCallback():c.jsonpCallback,g=a[f],l?c.url=j.replace(cG,"$1"+f):m?c.data=i.replace(cG,"$1"+f):k&&(c.url+=(cF.test(j)?"&":"?")+c.jsonp+"="+f),c.converters["script json"]=function(){return h||p.error(f+" was not called"),h[0]},c.dataTypes[0]="json",a[f]=function(){h=arguments},e.always(function(){a[f]=g,c[f]&&(c.jsonpCallback=d.jsonpCallback,cE.push(f)),h&&p.isFunction(g)&&g(h[0]),h=g=b}),"script"}),p.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/javascript|ecmascript/},converters:{"text script":function(a){return p.globalEval(a),a}}}),p.ajaxPrefilter("script",function(a){a.cache===b&&(a.cache=!1),a.crossDomain&&(a.type="GET",a.global=!1)}),p.ajaxTransport("script",function(a){if(a.crossDomain){var c,d=e.head||e.getElementsByTagName("head")[0]||e.documentElement;return{send:function(f,g){c=e.createElement("script"),c.async="async",a.scriptCharset&&(c.charset=a.scriptCharset),c.src=a.url,c.onload=c.onreadystatechange=function(a,e){if(e||!c.readyState||/loaded|complete/.test(c.readyState))c.onload=c.onreadystatechange=null,d&&c.parentNode&&d.removeChild(c),c=b,e||g(200,"success")},d.insertBefore(c,d.firstChild)},abort:function(){c&&c.onload(0,1)}}}});var cI,cJ=a.ActiveXObject?function(){for(var a in cI)cI[a](0,1)}:!1,cK=0;p.ajaxSettings.xhr=a.ActiveXObject?function(){return!this.isLocal&&cL()||cM()}:cL,function(a){p.extend(p.support,{ajax:!!a,cors:!!a&&"withCredentials"in a})}(p.ajaxSettings.xhr()),p.support.ajax&&p.ajaxTransport(function(c){if(!c.crossDomain||p.support.cors){var d;return{send:function(e,f){var g,h,i=c.xhr();c.username?i.open(c.type,c.url,c.async,c.username,c.password):i.open(c.type,c.url,c.async);if(c.xhrFields)for(h in c.xhrFields)i[h]=c.xhrFields[h];c.mimeType&&i.overrideMimeType&&i.overrideMimeType(c.mimeType),!c.crossDomain&&!e["X-Requested-With"]&&(e["X-Requested-With"]="XMLHttpRequest");try{for(h in e)i.setRequestHeader(h,e[h])}catch(j){}i.send(c.hasContent&&c.data||null),d=function(a,e){var h,j,k,l,m;try{if(d&&(e||i.readyState===4)){d=b,g&&(i.onreadystatechange=p.noop,cJ&&delete cI[g]);if(e)i.readyState!==4&&i.abort();else{h=i.status,k=i.getAllResponseHeaders(),l={},m=i.responseXML,m&&m.documentElement&&(l.xml=m);try{l.text=i.responseText}catch(a){}try{j=i.statusText}catch(n){j=""}!h&&c.isLocal&&!c.crossDomain?h=l.text?200:404:h===1223&&(h=204)}}}catch(o){e||f(-1,o)}l&&f(h,j,l,k)},c.async?i.readyState===4?setTimeout(d,0):(g=++cK,cJ&&(cI||(cI={},p(a).unload(cJ)),cI[g]=d),i.onreadystatechange=d):d()},abort:function(){d&&d(0,1)}}}});var cN,cO,cP=/^(?:toggle|show|hide)$/,cQ=new RegExp("^(?:([-+])=|)("+q+")([a-z%]*)$","i"),cR=/queueHooks$/,cS=[cY],cT={"*":[function(a,b){var c,d,e=this.createTween(a,b),f=cQ.exec(b),g=e.cur(),h=+g||0,i=1,j=20;if(f){c=+f[2],d=f[3]||(p.cssNumber[a]?"":"px");if(d!=="px"&&h){h=p.css(e.elem,a,!0)||c||1;do i=i||".5",h=h/i,p.style(e.elem,a,h+d);while(i!==(i=e.cur()/g)&&i!==1&&--j)}e.unit=d,e.start=h,e.end=f[1]?h+(f[1]+1)*c:c}return e}]};p.Animation=p.extend(cW,{tweener:function(a,b){p.isFunction(a)?(b=a,a=["*"]):a=a.split(" ");var c,d=0,e=a.length;for(;d<e;d++)c=a[d],cT[c]=cT[c]||[],cT[c].unshift(b)},prefilter:function(a,b){b?cS.unshift(a):cS.push(a)}}),p.Tween=cZ,cZ.prototype={constructor:cZ,init:function(a,b,c,d,e,f){this.elem=a,this.prop=c,this.easing=e||"swing",this.options=b,this.start=this.now=this.cur(),this.end=d,this.unit=f||(p.cssNumber[c]?"":"px")},cur:function(){var a=cZ.propHooks[this.prop];return a&&a.get?a.get(this):cZ.propHooks._default.get(this)},run:function(a){var b,c=cZ.propHooks[this.prop];return this.options.duration?this.pos=b=p.easing[this.easing](a,this.options.duration*a,0,1,this.options.duration):this.pos=b=a,this.now=(this.end-this.start)*b+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),c&&c.set?c.set(this):cZ.propHooks._default.set(this),this}},cZ.prototype.init.prototype=cZ.prototype,cZ.propHooks={_default:{get:function(a){var b;return a.elem[a.prop]==null||!!a.elem.style&&a.elem.style[a.prop]!=null?(b=p.css(a.elem,a.prop,!1,""),!b||b==="auto"?0:b):a.elem[a.prop]},set:function(a){p.fx.step[a.prop]?p.fx.step[a.prop](a):a.elem.style&&(a.elem.style[p.cssProps[a.prop]]!=null||p.cssHooks[a.prop])?p.style(a.elem,a.prop,a.now+a.unit):a.elem[a.prop]=a.now}}},cZ.propHooks.scrollTop=cZ.propHooks.scrollLeft={set:function(a){a.elem.nodeType&&a.elem.parentNode&&(a.elem[a.prop]=a.now)}},p.each(["toggle","show","hide"],function(a,b){var c=p.fn[b];p.fn[b]=function(d,e,f){return d==null||typeof d=="boolean"||!a&&p.isFunction(d)&&p.isFunction(e)?c.apply(this,arguments):this.animate(c$(b,!0),d,e,f)}}),p.fn.extend({fadeTo:function(a,b,c,d){return this.filter(bZ).css("opacity",0).show().end().animate({opacity:b},a,c,d)},animate:function(a,b,c,d){var e=p.isEmptyObject(a),f=p.speed(b,c,d),g=function(){var b=cW(this,p.extend({},a),f);e&&b.stop(!0)};return e||f.queue===!1?this.each(g):this.queue(f.queue,g)},stop:function(a,c,d){var e=function(a){var b=a.stop;delete a.stop,b(d)};return typeof a!="string"&&(d=c,c=a,a=b),c&&a!==!1&&this.queue(a||"fx",[]),this.each(function(){var b=!0,c=a!=null&&a+"queueHooks",f=p.timers,g=p._data(this);if(c)g[c]&&g[c].stop&&e(g[c]);else for(c in g)g[c]&&g[c].stop&&cR.test(c)&&e(g[c]);for(c=f.length;c--;)f[c].elem===this&&(a==null||f[c].queue===a)&&(f[c].anim.stop(d),b=!1,f.splice(c,1));(b||!d)&&p.dequeue(this,a)})}}),p.each({slideDown:c$("show"),slideUp:c$("hide"),slideToggle:c$("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(a,b){p.fn[a]=function(a,c,d){return this.animate(b,a,c,d)}}),p.speed=function(a,b,c){var d=a&&typeof a=="object"?p.extend({},a):{complete:c||!c&&b||p.isFunction(a)&&a,duration:a,easing:c&&b||b&&!p.isFunction(b)&&b};d.duration=p.fx.off?0:typeof d.duration=="number"?d.duration:d.duration in p.fx.speeds?p.fx.speeds[d.duration]:p.fx.speeds._default;if(d.queue==null||d.queue===!0)d.queue="fx";return d.old=d.complete,d.complete=function(){p.isFunction(d.old)&&d.old.call(this),d.queue&&p.dequeue(this,d.queue)},d},p.easing={linear:function(a){return a},swing:function(a){return.5-Math.cos(a*Math.PI)/2}},p.timers=[],p.fx=cZ.prototype.init,p.fx.tick=function(){var a,b=p.timers,c=0;for(;c<b.length;c++)a=b[c],!a()&&b[c]===a&&b.splice(c--,1);b.length||p.fx.stop()},p.fx.timer=function(a){a()&&p.timers.push(a)&&!cO&&(cO=setInterval(p.fx.tick,p.fx.interval))},p.fx.interval=13,p.fx.stop=function(){clearInterval(cO),cO=null},p.fx.speeds={slow:600,fast:200,_default:400},p.fx.step={},p.expr&&p.expr.filters&&(p.expr.filters.animated=function(a){return p.grep(p.timers,function(b){return a===b.elem}).length});var c_=/^(?:body|html)$/i;p.fn.offset=function(a){if(arguments.length)return a===b?this:this.each(function(b){p.offset.setOffset(this,a,b)});var c,d,e,f,g,h,i,j={top:0,left:0},k=this[0],l=k&&k.ownerDocument;if(!l)return;return(d=l.body)===k?p.offset.bodyOffset(k):(c=l.documentElement,p.contains(c,k)?(typeof k.getBoundingClientRect!="undefined"&&(j=k.getBoundingClientRect()),e=da(l),f=c.clientTop||d.clientTop||0,g=c.clientLeft||d.clientLeft||0,h=e.pageYOffset||c.scrollTop,i=e.pageXOffset||c.scrollLeft,{top:j.top+h-f,left:j.left+i-g}):j)},p.offset={bodyOffset:function(a){var b=a.offsetTop,c=a.offsetLeft;return p.support.doesNotIncludeMarginInBodyOffset&&(b+=parseFloat(p.css(a,"marginTop"))||0,c+=parseFloat(p.css(a,"marginLeft"))||0),{top:b,left:c}},setOffset:function(a,b,c){var d=p.css(a,"position");d==="static"&&(a.style.position="relative");var e=p(a),f=e.offset(),g=p.css(a,"top"),h=p.css(a,"left"),i=(d==="absolute"||d==="fixed")&&p.inArray("auto",[g,h])>-1,j={},k={},l,m;i?(k=e.position(),l=k.top,m=k.left):(l=parseFloat(g)||0,m=parseFloat(h)||0),p.isFunction(b)&&(b=b.call(a,c,f)),b.top!=null&&(j.top=b.top-f.top+l),b.left!=null&&(j.left=b.left-f.left+m),"using"in b?b.using.call(a,j):e.css(j)}},p.fn.extend({position:function(){if(!this[0])return;var a=this[0],b=this.offsetParent(),c=this.offset(),d=c_.test(b[0].nodeName)?{top:0,left:0}:b.offset();return c.top-=parseFloat(p.css(a,"marginTop"))||0,c.left-=parseFloat(p.css(a,"marginLeft"))||0,d.top+=parseFloat(p.css(b[0],"borderTopWidth"))||0,d.left+=parseFloat(p.css(b[0],"borderLeftWidth"))||0,{top:c.top-d.top,left:c.left-d.left}},offsetParent:function(){return this.map(function(){var a=this.offsetParent||e.body;while(a&&!c_.test(a.nodeName)&&p.css(a,"position")==="static")a=a.offsetParent;return a||e.body})}}),p.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(a,c){var d=/Y/.test(c);p.fn[a]=function(e){return p.access(this,function(a,e,f){var g=da(a);if(f===b)return g?c in g?g[c]:g.document.documentElement[e]:a[e];g?g.scrollTo(d?p(g).scrollLeft():f,d?f:p(g).scrollTop()):a[e]=f},a,e,arguments.length,null)}}),p.each({Height:"height",Width:"width"},function(a,c){p.each({padding:"inner"+a,content:c,"":"outer"+a},function(d,e){p.fn[e]=function(e,f){var g=arguments.length&&(d||typeof e!="boolean"),h=d||(e===!0||f===!0?"margin":"border");return p.access(this,function(c,d,e){var f;return p.isWindow(c)?c.document.documentElement["client"+a]:c.nodeType===9?(f=c.documentElement,Math.max(c.body["scroll"+a],f["scroll"+a],c.body["offset"+a],f["offset"+a],f["client"+a])):e===b?p.css(c,d,e,h):p.style(c,d,e,h)},c,g?e:b,g,null)}})}),a.jQuery=a.$=p,typeof define=="function"&&define.amd&&define.amd.jQuery&&define("jquery",[],function(){return p})})(window);
\ No newline at end of file
diff --git a/slider-core/src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js b/slider-core/src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js
new file mode 100644
index 0000000..aa7a923
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js
@@ -0,0 +1,6 @@
+/*! jQuery UI - v1.9.1 - 2012-10-25
+* http://jqueryui.com
+* Includes: jquery.ui.core.js, jquery.ui.widget.js, jquery.ui.mouse.js, jquery.ui.position.js, jquery.ui.accordion.js, jquery.ui.autocomplete.js, jquery.ui.button.js, jquery.ui.datepicker.js, jquery.ui.dialog.js, jquery.ui.draggable.js, jquery.ui.droppable.js, jquery.ui.effect.js, jquery.ui.effect-blind.js, jquery.ui.effect-bounce.js, jquery.ui.effect-clip.js, jquery.ui.effect-drop.js, jquery.ui.effect-explode.js, jquery.ui.effect-fade.js, jquery.ui.effect-fold.js, jquery.ui.effect-highlight.js, jquery.ui.effect-pulsate.js, jquery.ui.effect-scale.js, jquery.ui.effect-shake.js, jquery.ui.effect-slide.js, jquery.ui.effect-transfer.js, jquery.ui.menu.js, jquery.ui.progressbar.js, jquery.ui.resizable.js, jquery.ui.selectable.js, jquery.ui.slider.js, jquery.ui.sortable.js, jquery.ui.spinner.js, jquery.ui.tabs.js, jquery.ui.tooltip.js
+* Copyright (c) 2012 jQuery Foundation and other contributors Licensed MIT */
+
+(function(e,t){function i(t,n){var r,i,o,u=t.nodeName.toLowerCase();return"area"===u?(r=t.parentNode,i=r.name,!t.href||!i||r.nodeName.toLowerCase()!=="map"?!1:(o=e("img[usemap=#"+i+"]")[0],!!o&&s(o))):(/input|select|textarea|button|object/.test(u)?!t.disabled:"a"===u?t.href||n:n)&&s(t)}function s(t){return e.expr.filters.visible(t)&&!e(t).parents().andSelf().filter(function(){return e.css(this,"visibility")==="hidden"}).length}var n=0,r=/^ui-id-\d+$/;e.ui=e.ui||{};if(e.ui.version)return;e.extend(e.ui,{version:"1.9.1",keyCode:{BACKSPACE:8,COMMA:188,DELETE:46,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,LEFT:37,NUMPAD_ADD:107,NUMPAD_DECIMAL:110,NUMPAD_DIVIDE:111,NUMPAD_ENTER:108,NUMPAD_MULTIPLY:106,NUMPAD_SUBTRACT:109,PAGE_DOWN:34,PAGE_UP:33,PERIOD:190,RIGHT:39,SPACE:32,TAB:9,UP:38}}),e.fn.extend({_focus:e.fn.focus,focus:function(t,n){return typeof t=="number"?this.each(function(){var r=this;setTimeout(function(){e(r).focus(),n&&n.call(r)},t)}):this._focus.apply(this,arguments)},scrollParent:function(){var t;return e.ui.ie&&/(static|relative)/.test(this.css("position"))||/absolute/.test(this.css("position"))?t=this.parents().filter(function(){return/(relative|absolute|fixed)/.test(e.css(this,"position"))&&/(auto|scroll)/.test(e.css(this,"overflow")+e.css(this,"overflow-y")+e.css(this,"overflow-x"))}).eq(0):t=this.parents().filter(function(){return/(auto|scroll)/.test(e.css(this,"overflow")+e.css(this,"overflow-y")+e.css(this,"overflow-x"))}).eq(0),/fixed/.test(this.css("position"))||!t.length?e(document):t},zIndex:function(n){if(n!==t)return this.css("zIndex",n);if(this.length){var r=e(this[0]),i,s;while(r.length&&r[0]!==document){i=r.css("position");if(i==="absolute"||i==="relative"||i==="fixed"){s=parseInt(r.css("zIndex"),10);if(!isNaN(s)&&s!==0)return s}r=r.parent()}}return 0},uniqueId:function(){return this.each(function(){this.id||(this.id="ui-id-"+ ++n)})},removeUniqueId:function(){return this.each(function(){r.test(this.id)&&e(this).removeAttr("id")})}}),e("<a>").outerWidth(1).jquery||e.each(["Width","Height"],function(n,r){function u(t,n,r,s){return e.each(i,function(){n-=parseFloat(e.css(t,"padding"+this))||0,r&&(n-=parseFloat(e.css(t,"border"+this+"Width"))||0),s&&(n-=parseFloat(e.css(t,"margin"+this))||0)}),n}var i=r==="Width"?["Left","Right"]:["Top","Bottom"],s=r.toLowerCase(),o={innerWidth:e.fn.innerWidth,innerHeight:e.fn.innerHeight,outerWidth:e.fn.outerWidth,outerHeight:e.fn.outerHeight};e.fn["inner"+r]=function(n){return n===t?o["inner"+r].call(this):this.each(function(){e(this).css(s,u(this,n)+"px")})},e.fn["outer"+r]=function(t,n){return typeof t!="number"?o["outer"+r].call(this,t):this.each(function(){e(this).css(s,u(this,t,!0,n)+"px")})}}),e.extend(e.expr[":"],{data:e.expr.createPseudo?e.expr.createPseudo(function(t){return function(n){return!!e.data(n,t)}}):function(t,n,r){return!!e.data(t,r[3])},focusable:function(t){return i(t,!isNaN(e.attr(t,"tabindex")))},tabbable:function(t){var n=e.attr(t,"tabindex"),r=isNaN(n);return(r||n>=0)&&i(t,!r)}}),e(function(){var t=document.body,n=t.appendChild(n=document.createElement("div"));n.offsetHeight,e.extend(n.style,{minHeight:"100px",height:"auto",padding:0,borderWidth:0}),e.support.minHeight=n.offsetHeight===100,e.support.selectstart="onselectstart"in n,t.removeChild(n).style.display="none"}),function(){var t=/msie ([\w.]+)/.exec(navigator.userAgent.toLowerCase())||[];e.ui.ie=t.length?!0:!1,e.ui.ie6=parseFloat(t[1],10)===6}(),e.fn.extend({disableSelection:function(){return this.bind((e.support.selectstart?"selectstart":"mousedown")+".ui-disableSelection",function(e){e.preventDefault()})},enableSelection:function(){return this.unbind(".ui-disableSelection")}}),e.extend(e.ui,{plugin:{add:function(t,n,r){var i,s=e.ui[t].prototype;for(i in r)s.plugins[i]=s.plugins[i]||[],s.plugins[i].push([n,r[i]])},call:function(e,t,n){var r,i=e.plugins[t];if(!i||!e.element[0].parentNode||e.element[0].parentNode.nodeType===11)return;for(r=0;r<i.length;r++)e.options[i[r][0]]&&i[r][1].apply(e.element,n)}},contains:e.contains,hasScroll:function(t,n){if(e(t).css("overflow")==="hidden")return!1;var r=n&&n==="left"?"scrollLeft":"scrollTop",i=!1;return t[r]>0?!0:(t[r]=1,i=t[r]>0,t[r]=0,i)},isOverAxis:function(e,t,n){return e>t&&e<t+n},isOver:function(t,n,r,i,s,o){return e.ui.isOverAxis(t,r,s)&&e.ui.isOverAxis(n,i,o)}})})(jQuery);(function(e,t){var n=0,r=Array.prototype.slice,i=e.cleanData;e.cleanData=function(t){for(var n=0,r;(r=t[n])!=null;n++)try{e(r).triggerHandler("remove")}catch(s){}i(t)},e.widget=function(t,n,r){var i,s,o,u,a=t.split(".")[0];t=t.split(".")[1],i=a+"-"+t,r||(r=n,n=e.Widget),e.expr[":"][i.toLowerCase()]=function(t){return!!e.data(t,i)},e[a]=e[a]||{},s=e[a][t],o=e[a][t]=function(e,t){if(!this._createWidget)return new o(e,t);arguments.length&&this._createWidget(e,t)},e.extend(o,s,{version:r.version,_proto:e.extend({},r),_childConstructors:[]}),u=new n,u.options=e.widget.extend({},u.options),e.each(r,function(t,i){e.isFunction(i)&&(r[t]=function(){var e=function(){return n.prototype[t].apply(this,arguments)},r=function(e){return n.prototype[t].apply(this,e)};return function(){var t=this._super,n=this._superApply,s;return this._super=e,this._superApply=r,s=i.apply(this,arguments),this._super=t,this._superApply=n,s}}())}),o.prototype=e.widget.extend(u,{widgetEventPrefix:u.widgetEventPrefix||t},r,{constructor:o,namespace:a,widgetName:t,widgetBaseClass:i,widgetFullName:i}),s?(e.each(s._childConstructors,function(t,n){var r=n.prototype;e.widget(r.namespace+"."+r.widgetName,o,n._proto)}),delete s._childConstructors):n._childConstructors.push(o),e.widget.bridge(t,o)},e.widget.extend=function(n){var i=r.call(arguments,1),s=0,o=i.length,u,a;for(;s<o;s++)for(u in i[s])a=i[s][u],i[s].hasOwnProperty(u)&&a!==t&&(e.isPlainObject(a)?n[u]=e.isPlainObject(n[u])?e.widget.extend({},n[u],a):e.widget.extend({},a):n[u]=a);return n},e.widget.bridge=function(n,i){var s=i.prototype.widgetFullName;e.fn[n]=function(o){var u=typeof o=="string",a=r.call(arguments,1),f=this;return o=!u&&a.length?e.widget.extend.apply(null,[o].concat(a)):o,u?this.each(function(){var r,i=e.data(this,s);if(!i)return e.error("cannot call methods on "+n+" prior to initialization; "+"attempted to call method '"+o+"'");if(!e.isFunction(i[o])||o.charAt(0)==="_")return e.error("no such method '"+o+"' for "+n+" widget instance");r=i[o].apply(i,a);if(r!==i&&r!==t)return f=r&&r.jquery?f.pushStack(r.get()):r,!1}):this.each(function(){var t=e.data(this,s);t?t.option(o||{})._init():new i(o,this)}),f}},e.Widget=function(){},e.Widget._childConstructors=[],e.Widget.prototype={widgetName:"widget",widgetEventPrefix:"",defaultElement:"<div>",options:{disabled:!1,create:null},_createWidget:function(t,r){r=e(r||this.defaultElement||this)[0],this.element=e(r),this.uuid=n++,this.eventNamespace="."+this.widgetName+this.uuid,this.options=e.widget.extend({},this.options,this._getCreateOptions(),t),this.bindings=e(),this.hoverable=e(),this.focusable=e(),r!==this&&(e.data(r,this.widgetName,this),e.data(r,this.widgetFullName,this),this._on(this.element,{remove:function(e){e.target===r&&this.destroy()}}),this.document=e(r.style?r.ownerDocument:r.document||r),this.window=e(this.document[0].defaultView||this.document[0].parentWindow)),this._create(),this._trigger("create",null,this._getCreateEventData()),this._init()},_getCreateOptions:e.noop,_getCreateEventData:e.noop,_create:e.noop,_init:e.noop,destroy:function(){this._destroy(),this.element.unbind(this.eventNamespace).removeData(this.widgetName).removeData(this.widgetFullName).removeData(e.camelCase(this.widgetFullName)),this.widget().unbind(this.eventNamespace).removeAttr("aria-disabled").removeClass(this.widgetFullName+"-disabled "+"ui-state-disabled"),this.bindings.unbind(this.eventNamespace),this.hoverable.removeClass("ui-state-hover"),this.focusable.removeClass("ui-state-focus")},_destroy:e.noop,widget:function(){return this.element},option:function(n,r){var i=n,s,o,u;if(arguments.length===0)return e.widget.extend({},this.options);if(typeof n=="string"){i={},s=n.split("."),n=s.shift();if(s.length){o=i[n]=e.widget.extend({},this.options[n]);for(u=0;u<s.length-1;u++)o[s[u]]=o[s[u]]||{},o=o[s[u]];n=s.pop();if(r===t)return o[n]===t?null:o[n];o[n]=r}else{if(r===t)return this.options[n]===t?null:this.options[n];i[n]=r}}return this._setOptions(i),this},_setOptions:function(e){var t;for(t in e)this._setOption(t,e[t]);return this},_setOption:function(e,t){return this.options[e]=t,e==="disabled"&&(this.widget().toggleClass(this.widgetFullName+"-disabled ui-state-disabled",!!t).attr("aria-disabled",t),this.hoverable.removeClass("ui-state-hover"),this.focusable.removeClass("ui-state-focus")),this},enable:function(){return this._setOption("disabled",!1)},disable:function(){return this._setOption("disabled",!0)},_on:function(t,n){var r,i=this;n?(t=r=e(t),this.bindings=this.bindings.add(t)):(n=t,t=this.element,r=this.widget()),e.each(n,function(n,s){function o(){if(i.options.disabled===!0||e(this).hasClass("ui-state-disabled"))return;return(typeof s=="string"?i[s]:s).apply(i,arguments)}typeof s!="string"&&(o.guid=s.guid=s.guid||o.guid||e.guid++);var u=n.match(/^(\w+)\s*(.*)$/),a=u[1]+i.eventNamespace,f=u[2];f?r.delegate(f,a,o):t.bind(a,o)})},_off:function(e,t){t=(t||"").split(" ").join(this.eventNamespace+" ")+this.eventNamespace,e.unbind(t).undelegate(t)},_delay:function(e,t){function n(){return(typeof e=="string"?r[e]:e).apply(r,arguments)}var r=this;return setTimeout(n,t||0)},_hoverable:function(t){this.hoverable=this.hoverable.add(t),this._on(t,{mouseenter:function(t){e(t.currentTarget).addClass("ui-state-hover")},mouseleave:function(t){e(t.currentTarget).removeClass("ui-state-hover")}})},_focusable:function(t){this.focusable=this.focusable.add(t),this._on(t,{focusin:function(t){e(t.currentTarget).addClass("ui-state-focus")},focusout:function(t){e(t.currentTarget).removeClass("ui-state-focus")}})},_trigger:function(t,n,r){var i,s,o=this.options[t];r=r||{},n=e.Event(n),n.type=(t===this.widgetEventPrefix?t:this.widgetEventPrefix+t).toLowerCase(),n.target=this.element[0],s=n.originalEvent;if(s)for(i in s)i in n||(n[i]=s[i]);return this.element.trigger(n,r),!(e.isFunction(o)&&o.apply(this.element[0],[n].concat(r))===!1||n.isDefaultPrevented())}},e.each({show:"fadeIn",hide:"fadeOut"},function(t,n){e.Widget.prototype["_"+t]=function(r,i,s){typeof i=="string"&&(i={effect:i});var o,u=i?i===!0||typeof i=="number"?n:i.effect||n:t;i=i||{},typeof i=="number"&&(i={duration:i}),o=!e.isEmptyObject(i),i.complete=s,i.delay&&r.delay(i.delay),o&&e.effects&&(e.effects.effect[u]||e.uiBackCompat!==!1&&e.effects[u])?r[t](i):u!==t&&r[u]?r[u](i.duration,i.easing,s):r.queue(function(n){e(this)[t](),s&&s.call(r[0]),n()})}}),e.uiBackCompat!==!1&&(e.Widget.prototype._getCreateOptions=function(){return e.metadata&&e.metadata.get(this.element[0])[this.widgetName]})})(jQuery);(function(e,t){var n=!1;e(document).mouseup(function(e){n=!1}),e.widget("ui.mouse",{version:"1.9.1",options:{cancel:"input,textarea,button,select,option",distance:1,delay:0},_mouseInit:function(){var t=this;this.element.bind("mousedown."+this.widgetName,function(e){return t._mouseDown(e)}).bind("click."+this.widgetName,function(n){if(!0===e.data(n.target,t.widgetName+".preventClickEvent"))return e.removeData(n.target,t.widgetName+".preventClickEvent"),n.stopImmediatePropagation(),!1}),this.started=!1},_mouseDestroy:function(){this.element.unbind("."+this.widgetName),this._mouseMoveDelegate&&e(document).unbind("mousemove."+this.widgetName,this._mouseMoveDelegate).unbind("mouseup."+this.widgetName,this._mouseUpDelegate)},_mouseDown:function(t){if(n)return;this._mouseStarted&&this._mouseUp(t),this._mouseDownEvent=t;var r=this,i=t.which===1,s=typeof this.options.cancel=="string"&&t.target.nodeName?e(t.target).closest(this.options.cancel).length:!1;if(!i||s||!this._mouseCapture(t))return!0;this.mouseDelayMet=!this.options.delay,this.mouseDelayMet||(this._mouseDelayTimer=setTimeout(function(){r.mouseDelayMet=!0},this.options.delay));if(this._mouseDistanceMet(t)&&this._mouseDelayMet(t)){this._mouseStarted=this._mouseStart(t)!==!1;if(!this._mouseStarted)return t.preventDefault(),!0}return!0===e.data(t.target,this.widgetName+".preventClickEvent")&&e.removeData(t.target,this.widgetName+".preventClickEvent"),this._mouseMoveDelegate=function(e){return r._mouseMove(e)},this._mouseUpDelegate=function(e){return r._mouseUp(e)},e(document).bind("mousemove."+this.widgetName,this._mouseMoveDelegate).bind("mouseup."+this.widgetName,this._mouseUpDelegate),t.preventDefault(),n=!0,!0},_mouseMove:function(t){return!e.ui.ie||document.documentMode>=9||!!t.button?this._mouseStarted?(this._mouseDrag(t),t.preventDefault()):(this._mouseDistanceMet(t)&&this._mouseDelayMet(t)&&(this._mouseStarted=this._mouseStart(this._mouseDownEvent,t)!==!1,this._mouseStarted?this._mouseDrag(t):this._mouseUp(t)),!this._mouseStarted):this._mouseUp(t)},_mouseUp:function(t){return e(document).unbind("mousemove."+this.widgetName,this._mouseMoveDelegate).unbind("mouseup."+this.widgetName,this._mouseUpDelegate),this._mouseStarted&&(this._mouseStarted=!1,t.target===this._mouseDownEvent.target&&e.data(t.target,this.widgetName+".preventClickEvent",!0),this._mouseStop(t)),!1},_mouseDistanceMet:function(e){return Math.max(Math.abs(this._mouseDownEvent.pageX-e.pageX),Math.abs(this._mouseDownEvent.pageY-e.pageY))>=this.options.distance},_mouseDelayMet:function(e){return this.mouseDelayMet},_mouseStart:function(e){},_mouseDrag:function(e){},_mouseStop:function(e){},_mouseCapture:function(e){return!0}})})(jQuery);(function(e,t){function h(e,t,n){return[parseInt(e[0],10)*(l.test(e[0])?t/100:1),parseInt(e[1],10)*(l.test(e[1])?n/100:1)]}function p(t,n){return parseInt(e.css(t,n),10)||0}e.ui=e.ui||{};var n,r=Math.max,i=Math.abs,s=Math.round,o=/left|center|right/,u=/top|center|bottom/,a=/[\+\-]\d+%?/,f=/^\w+/,l=/%$/,c=e.fn.position;e.position={scrollbarWidth:function(){if(n!==t)return n;var r,i,s=e("<div style='display:block;width:50px;height:50px;overflow:hidden;'><div style='height:100px;width:auto;'></div></div>"),o=s.children()[0];return e("body").append(s),r=o.offsetWidth,s.css("overflow","scroll"),i=o.offsetWidth,r===i&&(i=s[0].clientWidth),s.remove(),n=r-i},getScrollInfo:function(t){var n=t.isWindow?"":t.element.css("overflow-x"),r=t.isWindow?"":t.element.css("overflow-y"),i=n==="scroll"||n==="auto"&&t.width<t.element[0].scrollWidth,s=r==="scroll"||r==="auto"&&t.height<t.element[0].scrollHeight;return{width:i?e.position.scrollbarWidth():0,height:s?e.position.scrollbarWidth():0}},getWithinInfo:function(t){var n=e(t||window),r=e.isWindow(n[0]);return{element:n,isWindow:r,offset:n.offset()||{left:0,top:0},scrollLeft:n.scrollLeft(),scrollTop:n.scrollTop(),width:r?n.width():n.outerWidth(),height:r?n.height():n.outerHeight()}}},e.fn.position=function(t){if(!t||!t.of)return c.apply(this,arguments);t=e.extend({},t);var n,l,d,v,m,g=e(t.of),y=e.position.getWithinInfo(t.within),b=e.position.getScrollInfo(y),w=g[0],E=(t.collision||"flip").split(" "),S={};return w.nodeType===9?(l=g.width(),d=g.height(),v={top:0,left:0}):e.isWindow(w)?(l=g.width(),d=g.height(),v={top:g.scrollTop(),left:g.scrollLeft()}):w.preventDefault?(t.at="left top",l=d=0,v={top:w.pageY,left:w.pageX}):(l=g.outerWidth(),d=g.outerHeight(),v=g.offset()),m=e.extend({},v),e.each(["my","at"],function(){var e=(t[this]||"").split(" "),n,r;e.length===1&&(e=o.test(e[0])?e.concat(["center"]):u.test(e[0])?["center"].concat(e):["center","center"]),e[0]=o.test(e[0])?e[0]:"center",e[1]=u.test(e[1])?e[1]:"center",n=a.exec(e[0]),r=a.exec(e[1]),S[this]=[n?n[0]:0,r?r[0]:0],t[this]=[f.exec(e[0])[0],f.exec(e[1])[0]]}),E.length===1&&(E[1]=E[0]),t.at[0]==="right"?m.left+=l:t.at[0]==="center"&&(m.left+=l/2),t.at[1]==="bottom"?m.top+=d:t.at[1]==="center"&&(m.top+=d/2),n=h(S.at,l,d),m.left+=n[0],m.top+=n[1],this.each(function(){var o,u,a=e(this),f=a.outerWidth(),c=a.outerHeight(),w=p(this,"marginLeft"),x=p(this,"marginTop"),T=f+w+p(this,"marginRight")+b.width,N=c+x+p(this,"marginBottom")+b.height,C=e.extend({},m),k=h(S.my,a.outerWidth(),a.outerHeight());t.my[0]==="right"?C.left-=f:t.my[0]==="center"&&(C.left-=f/2),t.my[1]==="bottom"?C.top-=c:t.my[1]==="center"&&(C.top-=c/2),C.left+=k[0],C.top+=k[1],e.support.offsetFractions||(C.left=s(C.left),C.top=s(C.top)),o={marginLeft:w,marginTop:x},e.each(["left","top"],function(r,i){e.ui.position[E[r]]&&e.ui.position[E[r]][i](C,{targetWidth:l,targetHeight:d,elemWidth:f,elemHeight:c,collisionPosition:o,collisionWidth:T,collisionHeight:N,offset:[n[0]+k[0],n[1]+k[1]],my:t.my,at:t.at,within:y,elem:a})}),e.fn.bgiframe&&a.bgiframe(),t.using&&(u=function(e){var n=v.left-C.left,s=n+l-f,o=v.top-C.top,u=o+d-c,h={target:{element:g,left:v.left,top:v.top,width:l,height:d},element:{element:a,left:C.left,top:C.top,width:f,height:c},horizontal:s<0?"left":n>0?"right":"center",vertical:u<0?"top":o>0?"bottom":"middle"};l<f&&i(n+s)<l&&(h.horizontal="center"),d<c&&i(o+u)<d&&(h.vertical="middle"),r(i(n),i(s))>r(i(o),i(u))?h.important="horizontal":h.important="vertical",t.using.call(this,e,h)}),a.offset(e.extend(C,{using:u}))})},e.ui.position={fit:{left:function(e,t){var n=t.within,i=n.isWindow?n.scrollLeft:n.offset.left,s=n.width,o=e.left-t.collisionPosition.marginLeft,u=i-o,a=o+t.collisionWidth-s-i,f;t.collisionWidth>s?u>0&&a<=0?(f=e.left+u+t.collisionWidth-s-i,e.left+=u-f):a>0&&u<=0?e.left=i:u>a?e.left=i+s-t.collisionWidth:e.left=i:u>0?e.left+=u:a>0?e.left-=a:e.left=r(e.left-o,e.left)},top:function(e,t){var n=t.within,i=n.isWindow?n.scrollTop:n.offset.top,s=t.within.height,o=e.top-t.collisionPosition.marginTop,u=i-o,a=o+t.collisionHeight-s-i,f;t.collisionHeight>s?u>0&&a<=0?(f=e.top+u+t.collisionHeight-s-i,e.top+=u-f):a>0&&u<=0?e.top=i:u>a?e.top=i+s-t.collisionHeight:e.top=i:u>0?e.top+=u:a>0?e.top-=a:e.top=r(e.top-o,e.top)}},flip:{left:function(e,t){var n=t.within,r=n.offset.left+n.scrollLeft,s=n.width,o=n.isWindow?n.scrollLeft:n.offset.left,u=e.left-t.collisionPosition.marginLeft,a=u-o,f=u+t.collisionWidth-s-o,l=t.my[0]==="left"?-t.elemWidth:t.my[0]==="right"?t.elemWidth:0,c=t.at[0]==="left"?t.targetWidth:t.at[0]==="right"?-t.targetWidth:0,h=-2*t.offset[0],p,d;if(a<0){p=e.left+l+c+h+t.collisionWidth-s-r;if(p<0||p<i(a))e.left+=l+c+h}else if(f>0){d=e.left-t.collisionPosition.marginLeft+l+c+h-o;if(d>0||i(d)<f)e.left+=l+c+h}},top:function(e,t){var n=t.within,r=n.offset.top+n.scrollTop,s=n.height,o=n.isWindow?n.scrollTop:n.offset.top,u=e.top-t.collisionPosition.marginTop,a=u-o,f=u+t.collisionHeight-s-o,l=t.my[1]==="top",c=l?-t.elemHeight:t.my[1]==="bottom"?t.elemHeight:0,h=t.at[1]==="top"?t.targetHeight:t.at[1]==="bottom"?-t.targetHeight:0,p=-2*t.offset[1],d,v;a<0?(v=e.top+c+h+p+t.collisionHeight-s-r,e.top+c+h+p>a&&(v<0||v<i(a))&&(e.top+=c+h+p)):f>0&&(d=e.top-t.collisionPosition.marginTop+c+h+p-o,e.top+c+h+p>f&&(d>0||i(d)<f)&&(e.top+=c+h+p))}},flipfit:{left:function(){e.ui.position.flip.left.apply(this,arguments),e.ui.position.fit.left.apply(this,arguments)},top:function(){e.ui.position.flip.top.apply(this,arguments),e.ui.position.fit.top.apply(this,arguments)}}},function(){var t,n,r,i,s,o=document.getElementsByTagName("body")[0],u=document.createElement("div");t=document.createElement(o?"div":"body"),r={visibility:"hidden",width:0,height:0,border:0,margin:0,background:"none"},o&&e.extend(r,{position:"absolute",left:"-1000px",top:"-1000px"});for(s in r)t.style[s]=r[s];t.appendChild(u),n=o||document.documentElement,n.insertBefore(t,n.firstChild),u.style.cssText="position: absolute; left: 10.7432222px;",i=e(u).offset().left,e.support.offsetFractions=i>10&&i<11,t.innerHTML="",n.removeChild(t)}(),e.uiBackCompat!==!1&&function(e){var n=e.fn.position;e.fn.position=function(r){if(!r||!r.offset)return n.call(this,r);var i=r.offset.split(" "),s=r.at.split(" ");return i.length===1&&(i[1]=i[0]),/^\d/.test(i[0])&&(i[0]="+"+i[0]),/^\d/.test(i[1])&&(i[1]="+"+i[1]),s.length===1&&(/left|center|right/.test(s[0])?s[1]="center":(s[1]=s[0],s[0]="center")),n.call(this,e.extend(r,{at:s[0]+i[0]+" "+s[1]+i[1],offset:t}))}}(jQuery)})(jQuery);(function(e,t){var n=0,r={},i={};r.height=r.paddingTop=r.paddingBottom=r.borderTopWidth=r.borderBottomWidth="hide",i.height=i.paddingTop=i.paddingBottom=i.borderTopWidth=i.borderBottomWidth="show",e.widget("ui.accordion",{version:"1.9.1",options:{active:0,animate:{},collapsible:!1,event:"click",header:"> li > :first-child,> :not(li):even",heightStyle:"auto",icons:{activeHeader:"ui-icon-triangle-1-s",header:"ui-icon-triangle-1-e"},activate:null,beforeActivate:null},_create:function(){var t=this.accordionId="ui-accordion-"+(this.element.attr("id")||++n),r=this.options;this.prevShow=this.prevHide=e(),this.element.addClass("ui-accordion ui-widget ui-helper-reset"),this.headers=this.element.find(r.header).addClass("ui-accordion-header ui-helper-reset ui-state-default ui-corner-all"),this._hoverable(this.headers),this._focusable(this.headers),this.headers.next().addClass("ui-accordion-content ui-helper-reset ui-widget-content ui-corner-bottom").hide(),!r.collapsible&&(r.active===!1||r.active==null)&&(r.active=0),r.active<0&&(r.active+=this.headers.length),this.active=this._findActive(r.active).addClass("ui-accordion-header-active ui-state-active").toggleClass("ui-corner-all ui-corner-top"),this.active.next().addClass("ui-accordion-content-active").show(),this._createIcons(),this.refresh(),this.element.attr("role","tablist"),this.headers.attr("role","tab").each(function(n){var r=e(this),i=r.attr("id"),s=r.next(),o=s.attr("id");i||(i=t+"-header-"+n,r.attr("id",i)),o||(o=t+"-panel-"+n,s.attr("id",o)),r.attr("aria-controls",o),s.attr("aria-labelledby",i)}).next().attr("role","tabpanel"),this.headers.not(this.active).attr({"aria-selected":"false",tabIndex:-1}).next().attr({"aria-expanded":"false","aria-hidden":"true"}).hide(),this.active.length?this.active.attr({"aria-selected":"true",tabIndex:0}).next().attr({"aria-expanded":"true","aria-hidden":"false"}):this.headers.eq(0).attr("tabIndex",0),this._on(this.headers,{keydown:"_keydown"}),this._on(this.headers.next(),{keydown:"_panelKeyDown"}),this._setupEvents(r.event)},_getCreateEventData:function(){return{header:this.active,content:this.active.length?this.active.next():e()}},_createIcons:function(){var t=this.options.icons;t&&(e("<span>").addClass("ui-accordion-header-icon ui-icon "+t.header).prependTo(this.headers),this.active.children(".ui-accordion-header-icon").removeClass(t.header).addClass(t.activeHeader),this.headers.addClass("ui-accordion-icons"))},_destroyIcons:function(){this.headers.removeClass("ui-accordion-icons").children(".ui-accordion-header-icon").remove()},_destroy:function(){var e;this.element.removeClass("ui-accordion ui-widget ui-helper-reset").removeAttr("role"),this.headers.removeClass("ui-accordion-header ui-accordion-header-active ui-helper-reset ui-state-default ui-corner-all ui-state-active ui-state-disabled ui-corner-top").removeAttr("role").removeAttr("aria-selected").removeAttr("aria-controls").removeAttr("tabIndex").each(function(){/^ui-accordion/.test(this.id)&&this.removeAttribute("id")}),this._destroyIcons(),e=this.headers.next().css("display","").removeAttr("role").removeAttr("aria-expanded").removeAttr("aria-hidden").removeAttr("aria-labelledby").removeClass("ui-helper-reset ui-widget-content ui-corner-bottom ui-accordion-content ui-accordion-content-active ui-state-disabled").each(function(){/^ui-accordion/.test(this.id)&&this.removeAttribute("id")}),this.options.heightStyle!=="content"&&e.css("height","")},_setOption:function(e,t){if(e==="active"){this._activate(t);return}e==="event"&&(this.options.event&&this._off(this.headers,this.options.event),this._setupEvents(t)),this._super(e,t),e==="collapsible"&&!t&&this.options.active===!1&&this._activate(0),e==="icons"&&(this._destroyIcons(),t&&this._createIcons()),e==="disabled"&&this.headers.add(this.headers.next()).toggleClass("ui-state-disabled",!!t)},_keydown:function(t){if(t.altKey||t.ctrlKey)return;var n=e.ui.keyCode,r=this.headers.length,i=this.headers.index(t.target),s=!1;switch(t.keyCode){case n.RIGHT:case n.DOWN:s=this.headers[(i+1)%r];break;case n.LEFT:case n.UP:s=this.headers[(i-1+r)%r];break;case n.SPACE:case n.ENTER:this._eventHandler(t);break;case n.HOME:s=this.headers[0];break;case n.END:s=this.headers[r-1]}s&&(e(t.target).attr("tabIndex",-1),e(s).attr("tabIndex",0),s.focus(),t.preventDefault())},_panelKeyDown:function(t){t.keyCode===e.ui.keyCode.UP&&t.ctrlKey&&e(t.currentTarget).prev().focus()},refresh:function(){var t,n,r=this.options.heightStyle,i=this.element.parent();r==="fill"?(e.support.minHeight||(n=i.css("overflow"),i.css("overflow","hidden")),t=i.height(),this.element.siblings(":visible").each(function(){var n=e(this),r=n.css("position");if(r==="absolute"||r==="fixed")return;t-=n.outerHeight(!0)}),n&&i.css("overflow",n),this.headers.each(function(){t-=e(this).outerHeight(!0)}),this.headers.next().each(function(){e(this).height(Math.max(0,t-e(this).innerHeight()+e(this).height()))}).css("overflow","auto")):r==="auto"&&(t=0,this.headers.next().each(function(){t=Math.max(t,e(this).height("").height())}).height(t))},_activate:function(t){var n=this._findActive(t)[0];if(n===this.active[0])return;n=n||this.active[0],this._eventHandler({target:n,currentTarget:n,preventDefault:e.noop})},_findActive:function(t){return typeof t=="number"?this.headers.eq(t):e()},_setupEvents:function(t){var n={};if(!t)return;e.each(t.split(" "),function(e,t){n[t]="_eventHandler"}),this._on(this.headers,n)},_eventHandler:function(t){var n=this.options,r=this.active,i=e(t.currentTarget),s=i[0]===r[0],o=s&&n.collapsible,u=o?e():i.next(),a=r.next(),f={oldHeader:r,oldPanel:a,newHeader:o?e():i,newPanel:u};t.preventDefault();if(s&&!n.collapsible||this._trigger("beforeActivate",t,f)===!1)return;n.active=o?!1:this.headers.index(i),this.active=s?e():i,this._toggle(f),r.removeClass("ui-accordion-header-active ui-state-active"),n.icons&&r.children(".ui-accordion-header-icon").removeClass(n.icons.activeHeader).addClass(n.icons.header),s||(i.removeClass("ui-corner-all").addClass("ui-accordion-header-active ui-state-active ui-corner-top"),n.icons&&i.children(".ui-accordion-header-icon").removeClass(n.icons.header).addClass(n.icons.activeHeader),i.next().addClass("ui-accordion-content-active"))},_toggle:function(t){var n=t.newPanel,r=this.prevShow.length?this.prevShow:t.oldPanel;this.prevShow.add(this.prevHide).stop(!0,!0),this.prevShow=n,this.prevHide=r,this.options.animate?this._animate(n,r,t):(r.hide(),n.show(),this._toggleComplete(t)),r.attr({"aria-expanded":"false","aria-hidden":"true"}),r.prev().attr("aria-selected","false"),n.length&&r.length?r.prev().attr("tabIndex",-1):n.length&&this.headers.filter(function(){return e(this).attr("tabIndex")===0}).attr("tabIndex",-1),n.attr({"aria-expanded":"true","aria-hidden":"false"}).prev().attr({"aria-selected":"true",tabIndex:0})},_animate:function(e,t,n){var s,o,u,a=this,f=0,l=e.length&&(!t.length||e.index()<t.index()),c=this.options.animate||{},h=l&&c.down||c,p=function(){a._toggleComplete(n)};typeof h=="number"&&(u=h),typeof h=="string"&&(o=h),o=o||h.easing||c.easing,u=u||h.duration||c.duration;if(!t.length)return e.animate(i,u,o,p);if(!e.length)return t.animate(r,u,o,p);s=e.show().outerHeight(),t.animate(r,{duration:u,easing:o,step:function(e,t){t.now=Math.round(e)}}),e.hide().animate(i,{duration:u,easing:o,complete:p,step:function(e,n){n.now=Math.round(e),n.prop!=="height"?f+=n.now:a.options.heightStyle!=="content"&&(n.now=Math.round(s-t.outerHeight()-f),f=0)}})},_toggleComplete:function(e){var t=e.oldPanel;t.removeClass("ui-accordion-content-active").prev().removeClass("ui-corner-top").addClass("ui-corner-all"),t.length&&(t.parent()[0].className=t.parent()[0].className),this._trigger("activate",null,e)}}),e.uiBackCompat!==!1&&(function(e,t){e.extend(t.options,{navigation:!1,navigationFilter:function(){return this.href.toLowerCase()===location.href.toLowerCase()}});var n=t._create;t._create=function(){if(this.options.navigation){var t=this,r=this.element.find(this.options.header),i=r.next(),s=r.add(i).find("a").filter(this.options.navigationFilter)[0];s&&r.add(i).each(function(n){if(e.contains(this,s))return t.options.active=Math.floor(n/2),!1})}n.call(this)}}(jQuery,jQuery.ui.accordion.prototype),function(e,t){e.extend(t.options,{heightStyle:null,autoHeight:!0,clearStyle:!1,fillSpace:!1});var n=t._create,r=t._setOption;e.extend(t,{_create:function(){this.options.heightStyle=this.options.heightStyle||this._mergeHeightStyle(),n.call(this)},_setOption:function(e){if(e==="autoHeight"||e==="clearStyle"||e==="fillSpace")this.options.heightStyle=this._mergeHeightStyle();r.apply(this,arguments)},_mergeHeightStyle:function(){var e=this.options;if(e.fillSpace)return"fill";if(e.clearStyle)return"content";if(e.autoHeight)return"auto"}})}(jQuery,jQuery.ui.accordion.prototype),function(e,t){e.extend(t.options.icons,{activeHeader:null,headerSelected:"ui-icon-triangle-1-s"});var n=t._createIcons;t._createIcons=function(){this.options.icons&&(this.options.icons.activeHeader=this.options.icons.activeHeader||this.options.icons.headerSelected),n.call(this)}}(jQuery,jQuery.ui.accordion.prototype),function(e,t){t.activate=t._activate;var n=t._findActive;t._findActive=function(e){return e===-1&&(e=!1),e&&typeof e!="number"&&(e=this.headers.index(this.headers.filter(e)),e===-1&&(e=!1)),n.call(this,e)}}(jQuery,jQuery.ui.accordion.prototype),jQuery.ui.accordion.prototype.resize=jQuery.ui.accordion.prototype.refresh,function(e,t){e.extend(t.options,{change:null,changestart:null});var n=t._trigger;t._trigger=function(e,t,r){var i=n.apply(this,arguments);return i?(e==="beforeActivate"?i=n.call(this,"changestart",t,{oldHeader:r.oldHeader,oldContent:r.oldPanel,newHeader:r.newHeader,newContent:r.newPanel}):e==="activate"&&(i=n.call(this,"change",t,{oldHeader:r.oldHeader,oldContent:r.oldPanel,newHeader:r.newHeader,newContent:r.newPanel})),i):!1}}(jQuery,jQuery.ui.accordion.prototype),function(e,t){e.extend(t.options,{animate:null,animated:"slide"});var n=t._create;t._create=function(){var e=this.options;e.animate===null&&(e.animated?e.animated==="slide"?e.animate=300:e.animated==="bounceslide"?e.animate={duration:200,down:{easing:"easeOutBounce",duration:1e3}}:e.animate=e.animated:e.animate=!1),n.call(this)}}(jQuery,jQuery.ui.accordion.prototype))})(jQuery);(function(e,t){var n=0;e.widget("ui.autocomplete",{version:"1.9.1",defaultElement:"<input>",options:{appendTo:"body",autoFocus:!1,delay:300,minLength:1,position:{my:"left top",at:"left bottom",collision:"none"},source:null,change:null,close:null,focus:null,open:null,response:null,search:null,select:null},pending:0,_create:function(){var t,n,r;this.isMultiLine=this._isMultiLine(),this.valueMethod=this.element[this.element.is("input,textarea")?"val":"text"],this.isNewMenu=!0,this.element.addClass("ui-autocomplete-input").attr("autocomplete","off"),this._on(this.element,{keydown:function(i){if(this.element.prop("readOnly")){t=!0,r=!0,n=!0;return}t=!1,r=!1,n=!1;var s=e.ui.keyCode;switch(i.keyCode){case s.PAGE_UP:t=!0,this._move("previousPage",i);break;case s.PAGE_DOWN:t=!0,this._move("nextPage",i);break;case s.UP:t=!0,this._keyEvent("previous",i);break;case s.DOWN:t=!0,this._keyEvent("next",i);break;case s.ENTER:case s.NUMPAD_ENTER:this.menu.active&&(t=!0,i.preventDefault(),this.menu.select(i));break;case s.TAB:this.menu.active&&this.menu.select(i);break;case s.ESCAPE:this.menu.element.is(":visible")&&(this._value(this.term),this.close(i),i.preventDefault());break;default:n=!0,this._searchTimeout(i)}},keypress:function(r){if(t){t=!1,r.preventDefault();return}if(n)return;var i=e.ui.keyCode;switch(r.keyCode){case i.PAGE_UP:this._move("previousPage",r);break;case i.PAGE_DOWN:this._move("nextPage",r);break;case i.UP:this._keyEvent("previous",r);break;case i.DOWN:this._keyEvent("next",r)}},input:function(e){if(r){r=!1,e.preventDefault();return}this._searchTimeout(e)},focus:function(){this.selectedItem=null,this.previous=this._value()},blur:function(e){if(this.cancelBlur){delete this.cancelBlur;return}clearTimeout(this.searching),this.close(e),this._change(e)}}),this._initSource(),this.menu=e("<ul>").addClass("ui-autocomplete").appendTo(this.document.find(this.options.appendTo||"body")[0]).menu({input:e(),role:null}).zIndex(this.element.zIndex()+1).hide().data("menu"),this._on(this.menu.element,{mousedown:function(t){t.preventDefault(),this.cancelBlur=!0,this._delay(function(){delete this.cancelBlur});var n=this.menu.element[0];e(t.target).closest(".ui-menu-item").length||this._delay(function(){var t=this;this.document.one("mousedown",function(r){r.target!==t.element[0]&&r.target!==n&&!e.contains(n,r.target)&&t.close()})})},menufocus:function(t,n){if(this.isNewMenu){this.isNewMenu=!1;if(t.originalEvent&&/^mouse/.test(t.originalEvent.type)){this.menu.blur(),this.document.one("mousemove",function(){e(t.target).trigger(t.originalEvent)});return}}var r=n.item.data("ui-autocomplete-item")||n.item.data("item.autocomplete");!1!==this._trigger("focus",t,{item:r})?t.originalEvent&&/^key/.test(t.originalEvent.type)&&this._value(r.value):this.liveRegion.text(r.value)},menuselect:function(e,t){var n=t.item.data("ui-autocomplete-item")||t.item.data("item.autocomplete"),r=this.previous;this.element[0]!==this.document[0].activeElement&&(this.element.focus(),this.previous=r,this._delay(function(){this.previous=r,this.selectedItem=n})),!1!==this._trigger("select",e,{item:n})&&this._value(n.value),this.term=this._value(),this.close(e),this.selectedItem=n}}),this.liveRegion=e("<span>",{role:"status","aria-live":"polite"}).addClass("ui-helper-hidden-accessible").insertAfter(this.element),e.fn.bgiframe&&this.menu.element.bgiframe(),this._on(this.window,{beforeunload:function(){this.element.removeAttr("autocomplete")}})},_destroy:function(){clearTimeout(this.searching),this.element.removeClass("ui-autocomplete-input").removeAttr("autocomplete"),this.menu.element.remove(),this.liveRegion.remove()},_setOption:function(e,t){this._super(e,t),e==="source"&&this._initSource(),e==="appendTo"&&this.menu.element.appendTo(this.document.find(t||"body")[0]),e==="disabled"&&t&&this.xhr&&this.xhr.abort()},_isMultiLine:function(){return this.element.is("textarea")?!0:this.element.is("input")?!1:this.element.prop("isContentEditable")},_initSource:function(){var t,n,r=this;e.isArray(this.options.source)?(t=this.options.source,this.source=function(n,r){r(e.ui.autocomplete.filter(t,n.term))}):typeof this.options.source=="string"?(n=this.options.source,this.source=function(t,i){r.xhr&&r.xhr.abort(),r.xhr=e.ajax({url:n,data:t,dataType:"json",success:function(e){i(e)},error:function(){i([])}})}):this.source=this.options.source},_searchTimeout:function(e){clearTimeout(this.searching),this.searching=this._delay(function(){this.term!==this._value()&&(this.selectedItem=null,this.search(null,e))},this.options.delay)},search:function(e,t){e=e!=null?e:this._value(),this.term=this._value();if(e.length<this.options.minLength)return this.close(t);if(this._trigger("search",t)===!1)return;return this._search(e)},_search:function(e){this.pending++,this.element.addClass("ui-autocomplete-loading"),this.cancelSearch=!1,this.source({term:e},this._response())},_response:function(){var e=this,t=++n;return function(r){t===n&&e.__response(r),e.pending--,e.pending||e.element.removeClass("ui-autocomplete-loading")}},__response:function(e){e&&(e=this._normalize(e)),this._trigger("response",null,{content:e}),!this.options.disabled&&e&&e.length&&!this.cancelSearch?(this._suggest(e),this._trigger("open")):this._close()},close:function(e){this.cancelSearch=!0,this._close(e)},_close:function(e){this.menu.element.is(":visible")&&(this.menu.element.hide(),this.menu.blur(),this.isNewMenu=!0,this._trigger("close",e))},_change:function(e){this.previous!==this._value()&&this._trigger("change",e,{item:this.selectedItem})},_normalize:function(t){return t.length&&t[0].label&&t[0].value?t:e.map(t,function(t){return typeof t=="string"?{label:t,value:t}:e.extend({label:t.label||t.value,value:t.value||t.label},t)})},_suggest:function(t){var n=this.menu.element.empty().zIndex(this.element.zIndex()+1);this._renderMenu(n,t),this.menu.refresh(),n.show(),this._resizeMenu(),n.position(e.extend({of:this.element},this.options.position)),this.options.autoFocus&&this.menu.next()},_resizeMenu:function(){var e=this.menu.element;e.outerWidth(Math.max(e.width("").outerWidth()+1,this.element.outerWidth()))},_renderMenu:function(t,n){var r=this;e.each(n,function(e,n){r._renderItemData(t,n)})},_renderItemData:function(e,t){return this._renderItem(e,t).data("ui-autocomplete-item",t)},_renderItem:function(t,n){return e("<li>").append(e("<a>").text(n.label)).appendTo(t)},_move:function(e,t){if(!this.menu.element.is(":visible")){this.search(null,t);return}if(this.menu.isFirstItem()&&/^previous/.test(e)||this.menu.isLastItem()&&/^next/.test(e)){this._value(this.term),this.menu.blur();return}this.menu[e](t)},widget:function(){return this.menu.element},_value:function(){return this.valueMethod.apply(this.element,arguments)},_keyEvent:function(e,t){if(!this.isMultiLine||this.menu.element.is(":visible"))this._move(e,t),t.preventDefault()}}),e.extend(e.ui.autocomplete,{escapeRegex:function(e){return e.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g,"\\$&")},filter:function(t,n){var r=new RegExp(e.ui.autocomplete.escapeRegex(n),"i");return e.grep(t,function(e){return r.test(e.label||e.value||e)})}}),e.widget("ui.autocomplete",e.ui.autocomplete,{options:{messages:{noResults:"No search results.",results:function(e){return e+(e>1?" results are":" result is")+" available, use up and down arrow keys to navigate."}}},__response:function(e){var t;this._superApply(arguments);if(this.options.disabled||this.cancelSearch)return;e&&e.length?t=this.options.messages.results(e.length):t=this.options.messages.noResults,this.liveRegion.text(t)}})})(jQuery);(function(e,t){var n,r,i,s,o="ui-button ui-widget ui-state-default ui-corner-all",u="ui-state-hover ui-state-active ",a="ui-button-icons-only ui-button-icon-only ui-button-text-icons ui-button-text-icon-primary ui-button-text-icon-secondary ui-button-text-only",f=function(){var t=e(this).find(":ui-button");setTimeout(function(){t.button("refresh")},1)},l=function(t){var n=t.name,r=t.form,i=e([]);return n&&(r?i=e(r).find("[name='"+n+"']"):i=e("[name='"+n+"']",t.ownerDocument).filter(function(){return!this.form})),i};e.widget("ui.button",{version:"1.9.1",defaultElement:"<button>",options:{disabled:null,text:!0,label:null,icons:{primary:null,secondary:null}},_create:function(){this.element.closest("form").unbind("reset"+this.eventNamespace).bind("reset"+this.eventNamespace,f),typeof this.options.disabled!="boolean"?this.options.disabled=!!this.element.prop("disabled"):this.element.prop("disabled",this.options.disabled),this._determineButtonType(),this.hasTitle=!!this.buttonElement.attr("title");var t=this,u=this.options,a=this.type==="checkbox"||this.type==="radio",c="ui-state-hover"+(a?"":" ui-state-active"),h="ui-state-focus";u.label===null&&(u.label=this.type==="input"?this.buttonElement.val():this.buttonElement.html()),this.buttonElement.addClass(o).attr("role","button").bind("mouseenter"+this.eventNamespace,function(){if(u.disabled)return;e(this).addClass("ui-state-hover"),this===n&&e(this).addClass("ui-state-active")}).bind("mouseleave"+this.eventNamespace,function(){if(u.disabled)return;e(this).removeClass(c)}).bind("click"+this.eventNamespace,function(e){u.disabled&&(e.preventDefault(),e.stopImmediatePropagation())}),this.element.bind("focus"+this.eventNamespace,function(){t.buttonElement.addClass(h)}).bind("blur"+this.eventNamespace,function(){t.buttonElement.removeClass(h)}),a&&(this.element.bind("change"+this.eventNamespace,function(){if(s)return;t.refresh()}),this.buttonElement.bind("mousedown"+this.eventNamespace,function(e){if(u.disabled)return;s=!1,r=e.pageX,i=e.pageY}).bind("mouseup"+this.eventNamespace,function(e){if(u.disabled)return;if(r!==e.pageX||i!==e.pageY)s=!0})),this.type==="checkbox"?this.buttonElement.bind("click"+this.eventNamespace,function(){if(u.disabled||s)return!1;e(this).toggleClass("ui-state-active"),t.buttonElement.attr("aria-pressed",t.element[0].checked)}):this.type==="radio"?this.buttonElement.bind("click"+this.eventNamespace,function(){if(u.disabled||s)return!1;e(this).addClass("ui-state-active"),t.buttonElement.attr("aria-pressed","true");var n=t.element[0];l(n).not(n).map(function(){return e(this).button("widget")[0]}).removeClass("ui-state-active").attr("aria-pressed","false")}):(this.buttonElement.bind("mousedown"+this.eventNamespace,function(){if(u.disabled)return!1;e(this).addClass("ui-state-active"),n=this,t.document.one("mouseup",function(){n=null})}).bind("mouseup"+this.eventNamespace,function(){if(u.disabled)return!1;e(this).removeClass("ui-state-active")}).bind("keydown"+this.eventNamespace,function(t){if(u.disabled)return!1;(t.keyCode===e.ui.keyCode.SPACE||t.keyCode===e.ui.keyCode.ENTER)&&e(this).addClass("ui-state-active")}).bind("keyup"+this.eventNamespace,function(){e(this).removeClass("ui-state-active")}),this.buttonElement.is("a")&&this.buttonElement.keyup(function(t){t.keyCode===e.ui.keyCode.SPACE&&e(this).click()})),this._setOption("disabled",u.disabled),this._resetButton()},_determineButtonType:function(){var e,t,n;this.element.is("[type=checkbox]")?this.type="checkbox":this.element.is("[type=radio]")?this.type="radio":this.element.is("input")?this.type="input":this.type="button",this.type==="checkbox"||this.type==="radio"?(e=this.element.parents().last(),t="label[for='"+this.element.attr("id")+"']",this.buttonElement=e.find(t),this.buttonElement.length||(e=e.length?e.siblings():this.element.siblings(),this.buttonElement=e.filter(t),this.buttonElement.length||(this.buttonElement=e.find(t))),this.element.addClass("ui-helper-hidden-accessible"),n=this.element.is(":checked"),n&&this.buttonElement.addClass("ui-state-active"),this.buttonElement.prop("aria-pressed",n)):this.buttonElement=this.element},widget:function(){return this.buttonElement},_destroy:function(){this.element.removeClass("ui-helper-hidden-accessible"),this.buttonElement.removeClass(o+" "+u+" "+a).removeAttr("role").removeAttr("aria-pressed").html(this.buttonElement.find(".ui-button-text").html()),this.hasTitle||this.buttonElement.removeAttr("title")},_setOption:function(e,t){this._super(e,t);if(e==="disabled"){t?this.element.prop("disabled",!0):this.element.prop("disabled",!1);return}this._resetButton()},refresh:function(){var t=this.element.is(":disabled")||this.element.hasClass("ui-button-disabled");t!==this.options.disabled&&this._setOption("disabled",t),this.type==="radio"?l(this.element[0]).each(function(){e(this).is(":checked")?e(this).button("widget").addClass("ui-state-active").attr("aria-pressed","true"):e(this).button("widget").removeClass("ui-state-active").attr("aria-pressed","false")}):this.type==="checkbox"&&(this.element.is(":checked")?this.buttonElement.addClass("ui-state-active").attr("aria-pressed","true"):this.buttonElement.removeClass("ui-state-active").attr("aria-pressed","false"))},_resetButton:function(){if(this.type==="input"){this.options.label&&this.element.val(this.options.label);return}var t=this.buttonElement.removeClass(a),n=e("<span></span>",this.document[0]).addClass("ui-button-text").html(this.options.label).appendTo(t.empty()).text(),r=this.options.icons,i=r.primary&&r.secondary,s=[];r.primary||r.secondary?(this.options.text&&s.push("ui-button-text-icon"+(i?"s":r.primary?"-primary":"-secondary")),r.primary&&t.prepend("<span class='ui-button-icon-primary ui-icon "+r.primary+"'></span>"),r.secondary&&t.append("<span class='ui-button-icon-secondary ui-icon "+r.secondary+"'></span>"),this.options.text||(s.push(i?"ui-button-icons-only":"ui-button-icon-only"),this.hasTitle||t.attr("title",e.trim(n)))):s.push("ui-button-text-only"),t.addClass(s.join(" "))}}),e.widget("ui.buttonset",{version:"1.9.1",options:{items:"button, input[type=button], input[type=submit], input[type=reset], input[type=checkbox], input[type=radio], a, :data(button)"},_create:function(){this.element.addClass("ui-buttonset")},_init:function(){this.refresh()},_setOption:function(e,t){e==="disabled"&&this.buttons.button("option",e,t),this._super(e,t)},refresh:function(){var t=this.element.css("direction")==="rtl";this.buttons=this.element.find(this.options.items).filter(":ui-button").button("refresh").end().not(":ui-button").button().end().map(function(){return e(this).button("widget")[0]}).removeClass("ui-corner-all ui-corner-left ui-corner-right").filter(":first").addClass(t?"ui-corner-right":"ui-corner-left").end().filter(":last").addClass(t?"ui-corner-left":"ui-corner-right").end().end()},_destroy:function(){this.element.removeClass("ui-buttonset"),this.buttons.map(function(){return e(this).button("widget")[0]}).removeClass("ui-corner-left ui-corner-right").end().button("destroy")}})})(jQuery);(function($,undefined){function Datepicker(){this.debug=!1,this._curInst=null,this._keyEvent=!1,this._disabledInputs=[],this._datepickerShowing=!1,this._inDialog=!1,this._mainDivId="ui-datepicker-div",this._inlineClass="ui-datepicker-inline",this._appendClass="ui-datepicker-append",this._triggerClass="ui-datepicker-trigger",this._dialogClass="ui-datepicker-dialog",this._disableClass="ui-datepicker-disabled",this._unselectableClass="ui-datepicker-unselectable",this._currentClass="ui-datepicker-current-day",this._dayOverClass="ui-datepicker-days-cell-over",this.regional=[],this.regional[""]={closeText:"Done",prevText:"Prev",nextText:"Next",currentText:"Today",monthNames:["January","February","March","April","May","June","July","August","September","October","November","December"],monthNamesShort:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"],dayNames:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],dayNamesShort:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],dayNamesMin:["Su","Mo","Tu","We","Th","Fr","Sa"],weekHeader:"Wk",dateFormat:"mm/dd/yy",firstDay:0,isRTL:!1,showMonthAfterYear:!1,yearSuffix:""},this._defaults={showOn:"focus",showAnim:"fadeIn",showOptions:{},defaultDate:null,appendText:"",buttonText:"...",buttonImage:"",buttonImageOnly:!1,hideIfNoPrevNext:!1,navigationAsDateFormat:!1,gotoCurrent:!1,changeMonth:!1,changeYear:!1,yearRange:"c-10:c+10",showOtherMonths:!1,selectOtherMonths:!1,showWeek:!1,calculateWeek:this.iso8601Week,shortYearCutoff:"+10",minDate:null,maxDate:null,duration:"fast",beforeShowDay:null,beforeShow:null,onSelect:null,onChangeMonthYear:null,onClose:null,numberOfMonths:1,showCurrentAtPos:0,stepMonths:1,stepBigMonths:12,altField:"",altFormat:"",constrainInput:!0,showButtonPanel:!1,autoSize:!1,disabled:!1},$.extend(this._defaults,this.regional[""]),this.dpDiv=bindHover($('<div id="'+this._mainDivId+'" class="ui-datepicker ui-widget ui-widget-content ui-helper-clearfix ui-corner-all"></div>'))}function bindHover(e){var t="button, .ui-datepicker-prev, .ui-datepicker-next, .ui-datepicker-calendar td a";return e.delegate(t,"mouseout",function(){$(this).removeClass("ui-state-hover"),this.className.indexOf("ui-datepicker-prev")!=-1&&$(this).removeClass("ui-datepicker-prev-hover"),this.className.indexOf("ui-datepicker-next")!=-1&&$(this).removeClass("ui-datepicker-next-hover")}).delegate(t,"mouseover",function(){$.datepicker._isDisabledDatepicker(instActive.inline?e.parent()[0]:instActive.input[0])||($(this).parents(".ui-datepicker-calendar").find("a").removeClass("ui-state-hover"),$(this).addClass("ui-state-hover"),this.className.indexOf("ui-datepicker-prev")!=-1&&$(this).addClass("ui-datepicker-prev-hover"),this.className.indexOf("ui-datepicker-next")!=-1&&$(this).addClass("ui-datepicker-next-hover"))})}function extendRemove(e,t){$.extend(e,t);for(var n in t)if(t[n]==null||t[n]==undefined)e[n]=t[n];return e}$.extend($.ui,{datepicker:{version:"1.9.1"}});var PROP_NAME="datepicker",dpuuid=(new Date).getTime(),instActive;$.extend(Datepicker.prototype,{markerClassName:"hasDatepicker",maxRows:4,log:function(){this.debug&&console.log.apply("",arguments)},_widgetDatepicker:function(){return this.dpDiv},setDefaults:function(e){return extendRemove(this._defaults,e||{}),this},_attachDatepicker:function(target,settings){var inlineSettings=null;for(var attrName in this._defaults){var attrValue=target.getAttribute("date:"+attrName);if(attrValue){inlineSettings=inlineSettings||{};try{inlineSettings[attrName]=eval(attrValue)}catch(err){inlineSettings[attrName]=attrValue}}}var nodeName=target.nodeName.toLowerCase(),inline=nodeName=="div"||nodeName=="span";target.id||(this.uuid+=1,target.id="dp"+this.uuid);var inst=this._newInst($(target),inline);inst.settings=$.extend({},settings||{},inlineSettings||{}),nodeName=="input"?this._connectDatepicker(target,inst):inline&&this._inlineDatepicker(target,inst)},_newInst:function(e,t){var n=e[0].id.replace(/([^A-Za-z0-9_-])/g,"\\\\$1");return{id:n,input:e,selectedDay:0,selectedMonth:0,selectedYear:0,drawMonth:0,drawYear:0,inline:t,dpDiv:t?bindHover($('<div class="'+this._inlineClass+' ui-datepicker ui-widget ui-widget-content ui-helper-clearfix ui-corner-all"></div>')):this.dpDiv}},_connectDatepicker:function(e,t){var n=$(e);t.append=$([]),t.trigger=$([]);if(n.hasClass(this.markerClassName))return;this._attachments(n,t),n.addClass(this.markerClassName).keydown(this._doKeyDown).keypress(this._doKeyPress).keyup(this._doKeyUp).bind("setData.datepicker",function(e,n,r){t.settings[n]=r}).bind("getData.datepicker",function(e,n){return this._get(t,n)}),this._autoSize(t),$.data(e,PROP_NAME,t),t.settings.disabled&&this._disableDatepicker(e)},_attachments:function(e,t){var n=this._get(t,"appendText"),r=this._get(t,"isRTL");t.append&&t.append.remove(),n&&(t.append=$('<span class="'+this._appendClass+'">'+n+"</span>"),e[r?"before":"after"](t.append)),e.unbind("focus",this._showDatepicker),t.trigger&&t.trigger.remove();var i=this._get(t,"showOn");(i=="focus"||i=="both")&&e.focus(this._showDatepicker);if(i=="button"||i=="both"){var s=this._get(t,"buttonText"),o=this._get(t,"buttonImage");t.trigger=$(this._get(t,"buttonImageOnly")?$("<img/>").addClass(this._triggerClass).attr({src:o,alt:s,title:s}):$('<button type="button"></button>').addClass(this._triggerClass).html(o==""?s:$("<img/>").attr({src:o,alt:s,title:s}))),e[r?"before":"after"](t.trigger),t.trigger.click(function(){return $.datepicker._datepickerShowing&&$.datepicker._lastInput==e[0]?$.datepicker._hideDatepicker():$.datepicker._datepickerShowing&&$.datepicker._lastInput!=e[0]?($.datepicker._hideDatepicker(),$.datepicker._showDatepicker(e[0])):$.datepicker._showDatepicker(e[0]),!1})}},_autoSize:function(e){if(this._get(e,"autoSize")&&!e.inline){var t=new Date(2009,11,20),n=this._get(e,"dateFormat");if(n.match(/[DM]/)){var r=function(e){var t=0,n=0;for(var r=0;r<e.length;r++)e[r].length>t&&(t=e[r].length,n=r);return n};t.setMonth(r(this._get(e,n.match(/MM/)?"monthNames":"monthNamesShort"))),t.setDate(r(this._get(e,n.match(/DD/)?"dayNames":"dayNamesShort"))+20-t.getDay())}e.input.attr("size",this._formatDate(e,t).length)}},_inlineDatepicker:function(e,t){var n=$(e);if(n.hasClass(this.markerClassName))return;n.addClass(this.markerClassName).append(t.dpDiv).bind("setData.datepicker",function(e,n,r){t.settings[n]=r}).bind("getData.datepicker",function(e,n){return this._get(t,n)}),$.data(e,PROP_NAME,t),this._setDate(t,this._getDefaultDate(t),!0),this._updateDatepicker(t),this._updateAlternate(t),t.settings.disabled&&this._disableDatepicker(e),t.dpDiv.css("display","block")},_dialogDatepicker:function(e,t,n,r,i){var s=this._dialogInst;if(!s){this.uuid+=1;var o="dp"+this.uuid;this._dialogInput=$('<input type="text" id="'+o+'" style="position: absolute; top: -100px; width: 0px;"/>'),this._dialogInput.keydown(this._doKeyDown),$("body").append(this._dialogInput),s=this._dialogInst=this._newInst(this._dialogInput,!1),s.settings={},$.data(this._dialogInput[0],PROP_NAME,s)}extendRemove(s.settings,r||{}),t=t&&t.constructor==Date?this._formatDate(s,t):t,this._dialogInput.val(t),this._pos=i?i.length?i:[i.pageX,i.pageY]:null;if(!this._pos){var u=document.documentElement.clientWidth,a=document.documentElement.clientHeight,f=document.documentElement.scrollLeft||document.body.scrollLeft,l=document.documentElement.scrollTop||document.body.scrollTop;this._pos=[u/2-100+f,a/2-150+l]}return this._dialogInput.css("left",this._pos[0]+20+"px").css("top",this._pos[1]+"px"),s.settings.onSelect=n,this._inDialog=!0,this.dpDiv.addClass(this._dialogClass),this._showDatepicker(this._dialogInput[0]),$.blockUI&&$.blockUI(this.dpDiv),$.data(this._dialogInput[0],PROP_NAME,s),this},_destroyDatepicker:function(e){var t=$(e),n=$.data(e,PROP_NAME);if(!t.hasClass(this.markerClassName))return;var r=e.nodeName.toLowerCase();$.removeData(e,PROP_NAME),r=="input"?(n.append.remove(),n.trigger.remove(),t.removeClass(this.markerClassName).unbind("focus",this._showDatepicker).unbind("keydown",this._doKeyDown).unbind("keypress",this._doKeyPress).unbind("keyup",this._doKeyUp)):(r=="div"||r=="span")&&t.removeClass(this.markerClassName).empty()},_enableDatepicker:function(e){var t=$(e),n=$.data(e,PROP_NAME);if(!t.hasClass(this.markerClassName))return;var r=e.nodeName.toLowerCase();if(r=="input")e.disabled=!1,n.trigger.filter("button").each(function(){this.disabled=!1}).end().filter("img").css({opacity:"1.0",cursor:""});else if(r=="div"||r=="span"){var i=t.children("."+this._inlineClass);i.children().removeClass("ui-state-disabled"),i.find("select.ui-datepicker-month, select.ui-datepicker-year").prop("disabled",!1)}this._disabledInputs=$.map(this._disabledInputs,function(t){return t==e?null:t})},_disableDatepicker:function(e){var t=$(e),n=$.data(e,PROP_NAME);if(!t.hasClass(this.markerClassName))return;var r=e.nodeName.toLowerCase();if(r=="input")e.disabled=!0,n.trigger.filter("button").each(function(){this.disabled=!0}).end().filter("img").css({opacity:"0.5",cursor:"default"});else if(r=="div"||r=="span"){var i=t.children("."+this._inlineClass);i.children().addClass("ui-state-disabled"),i.find("select.ui-datepicker-month, select.ui-datepicker-year").prop("disabled",!0)}this._disabledInputs=$.map(this._disabledInputs,function(t){return t==e?null:t}),this._disabledInputs[this._disabledInputs.length]=e},_isDisabledDatepicker:function(e){if(!e)return!1;for(var t=0;t<this._disabledInputs.length;t++)if(this._disabledInputs[t]==e)return!0;return!1},_getInst:function(e){try{return $.data(e,PROP_NAME)}catch(t){throw"Missing instance data for this datepicker"}},_optionDatepicker:function(e,t,n){var r=this._getInst(e);if(arguments.length==2&&typeof t=="string")return t=="defaults"?$.extend({},$.datepicker._defaults):r?t=="all"?$.extend({},r.settings):this._get(r,t):null;var i=t||{};typeof t=="string"&&(i={},i[t]=n);if(r){this._curInst==r&&this._hideDatepicker();var s=this._getDateDatepicker(e,!0),o=this._getMinMaxDate(r,"min"),u=this._getMinMaxDate(r,"max");extendRemove(r.settings,i),o!==null&&i.dateFormat!==undefined&&i.minDate===undefined&&(r.settings.minDate=this._formatDate(r,o)),u!==null&&i.dateFormat!==undefined&&i.maxDate===undefined&&(r.settings.maxDate=this._formatDate(r,u)),this._attachments($(e),r),this._autoSize(r),this._setDate(r,s),this._updateAlternate(r),this._updateDatepicker(r)}},_changeDatepicker:function(e,t,n){this._optionDatepicker(e,t,n)},_refreshDatepicker:function(e){var t=this._getInst(e);t&&this._updateDatepicker(t)},_setDateDatepicker:function(e,t){var n=this._getInst(e);n&&(this._setDate(n,t),this._updateDatepicker(n),this._updateAlternate(n))},_getDateDatepicker:function(e,t){var n=this._getInst(e);return n&&!n.inline&&this._setDateFromField(n,t),n?this._getDate(n):null},_doKeyDown:function(e){var t=$.datepicker._getInst(e.target),n=!0,r=t.dpDiv.is(".ui-datepicker-rtl");t._keyEvent=!0;if($.datepicker._datepickerShowing)switch(e.keyCode){case 9:$.datepicker._hideDatepicker(),n=!1;break;case 13:var i=$("td."+$.datepicker._dayOverClass+":not(."+$.datepicker._currentClass+")",t.dpDiv);i[0]&&$.datepicker._selectDay(e.target,t.selectedMonth,t.selectedYear,i[0]);var s=$.datepicker._get(t,"onSelect");if(s){var o=$.datepicker._formatDate(t);s.apply(t.input?t.input[0]:null,[o,t])}else $.datepicker._hideDatepicker();return!1;case 27:$.datepicker._hideDatepicker();break;case 33:$.datepicker._adjustDate(e.target,e.ctrlKey?-$.datepicker._get(t,"stepBigMonths"):-$.datepicker._get(t,"stepMonths"),"M");break;case 34:$.datepicker._adjustDate(e.target,e.ctrlKey?+$.datepicker._get(t,"stepBigMonths"):+$.datepicker._get(t,"stepMonths"),"M");break;case 35:(e.ctrlKey||e.metaKey)&&$.datepicker._clearDate(e.target),n=e.ctrlKey||e.metaKey;break;case 36:(e.ctrlKey||e.metaKey)&&$.datepicker._gotoToday(e.target),n=e.ctrlKey||e.metaKey;break;case 37:(e.ctrlKey||e.metaKey)&&$.datepicker._adjustDate(e.target,r?1:-1,"D"),n=e.ctrlKey||e.metaKey,e.originalEvent.altKey&&$.datepicker._adjustDate(e.target,e.ctrlKey?-$.datepicker._get(t,"stepBigMonths"):-$.datepicker._get(t,"stepMonths"),"M");break;case 38:(e.ctrlKey||e.metaKey)&&$.datepicker._adjustDate(e.target,-7,"D"),n=e.ctrlKey||e.metaKey;break;case 39:(e.ctrlKey||e.metaKey)&&$.datepicker._adjustDate(e.target,r?-1:1,"D"),n=e.ctrlKey||e.metaKey,e.originalEvent.altKey&&$.datepicker._adjustDate(e.target,e.ctrlKey?+$.datepicker._get(t,"stepBigMonths"):+$.datepicker._get(t,"stepMonths"),"M");break;case 40:(e.ctrlKey||e.metaKey)&&$.datepicker._adjustDate(e.target,7,"D"),n=e.ctrlKey||e.metaKey;break;default:n=!1}else e.keyCode==36&&e.ctrlKey?$.datepicker._showDatepicker(this):n=!1;n&&(e.preventDefault(),e.stopPropagation())},_doKeyPress:function(e){var t=$.datepicker._getInst(e.target);if($.datepicker._get(t,"constrainInput")){var n=$.datepicker._possibleChars($.datepicker._get(t,"dateFormat")),r=String.fromCharCode(e.charCode==undefined?e.keyCode:e.charCode);return e.ctrlKey||e.metaKey||r<" "||!n||n.indexOf(r)>-1}},_doKeyUp:function(e){var t=$.datepicker._getInst(e.target);if(t.input.val()!=t.lastVal)try{var n=$.datepicker.parseDate($.datepicker._get(t,"dateFormat"),t.input?t.input.val():null,$.datepicker._getFormatConfig(t));n&&($.datepicker._setDateFromField(t),$.datepicker._updateAlternate(t),$.datepicker._updateDatepicker(t))}catch(r){$.datepicker.log(r)}return!0},_showDatepicker:function(e){e=e.target||e,e.nodeName.toLowerCase()!="input"&&(e=$("input",e.parentNode)[0]);if($.datepicker._isDisabledDatepicker(e)||$.datepicker._lastInput==e)return;var t=$.datepicker._getInst(e);$.datepicker._curInst&&$.datepicker._curInst!=t&&($.datepicker._curInst.dpDiv.stop(!0,!0),t&&$.datepicker._datepickerShowing&&$.datepicker._hideDatepicker($.datepicker._curInst.input[0]));var n=$.datepicker._get(t,"beforeShow"),r=n?n.apply(e,[e,t]):{};if(r===!1)return;extendRemove(t.settings,r),t.lastVal=null,$.datepicker._lastInput=e,$.datepicker._setDateFromField(t),$.datepicker._inDialog&&(e.value=""),$.datepicker._pos||($.datepicker._pos=$.datepicker._findPos(e),$.datepicker._pos[1]+=e.offsetHeight);var i=!1;$(e).parents().each(function(){return i|=$(this).css("position")=="fixed",!i});var s={left:$.datepicker._pos[0],top:$.datepicker._pos[1]};$.datepicker._pos=null,t.dpDiv.empty(),t.dpDiv.css({position:"absolute",display:"block",top:"-1000px"}),$.datepicker._updateDatepicker(t),s=$.datepicker._checkOffset(t,s,i),t.dpDiv.css({position:$.datepicker._inDialog&&$.blockUI?"static":i?"fixed":"absolute",display:"none",left:s.left+"px",top:s.top+"px"});if(!t.inline){var o=$.datepicker._get(t,"showAnim"),u=$.datepicker._get(t,"duration"),a=function(){var e=t.dpDiv.find("iframe.ui-datepicker-cover");if(!!e.length){var n=$.datepicker._getBorders(t.dpDiv);e.css({left:-n[0],top:-n[1],width:t.dpDiv.outerWidth(),height:t.dpDiv.outerHeight()})}};t.dpDiv.zIndex($(e).zIndex()+1),$.datepicker._datepickerShowing=!0,$.effects&&($.effects.effect[o]||$.effects[o])?t.dpDiv.show(o,$.datepicker._get(t,"showOptions"),u,a):t.dpDiv[o||"show"](o?u:null,a),(!o||!u)&&a(),t.input.is(":visible")&&!t.input.is(":disabled")&&t.input.focus(),$.datepicker._curInst=t}},_updateDatepicker:function(e){this.maxRows=4;var t=$.datepicker._getBorders(e.dpDiv);instActive=e,e.dpDiv.empty().append(this._generateHTML(e)),this._attachHandlers(e);var n=e.dpDiv.find("iframe.ui-datepicker-cover");!n.length||n.css({left:-t[0],top:-t[1],width:e.dpDiv.outerWidth(),height:e.dpDiv.outerHeight()}),e.dpDiv.find("."+this._dayOverClass+" a").mouseover();var r=this._getNumberOfMonths(e),i=r[1],s=17;e.dpDiv.removeClass("ui-datepicker-multi-2 ui-datepicker-multi-3 ui-datepicker-multi-4").width(""),i>1&&e.dpDiv.addClass("ui-datepicker-multi-"+i).css("width",s*i+"em"),e.dpDiv[(r[0]!=1||r[1]!=1?"add":"remove")+"Class"]("ui-datepicker-multi"),e.dpDiv[(this._get(e,"isRTL")?"add":"remove")+"Class"]("ui-datepicker-rtl"),e==$.datepicker._curInst&&$.datepicker._datepickerShowing&&e.input&&e.input.is(":visible")&&!e.input.is(":disabled")&&e.input[0]!=document.activeElement&&e.input.focus();if(e.yearshtml){var o=e.yearshtml;setTimeout(function(){o===e.yearshtml&&e.yearshtml&&e.dpDiv.find("select.ui-datepicker-year:first").replaceWith(e.yearshtml),o=e.yearshtml=null},0)}},_getBorders:function(e){var t=function(e){return{thin:1,medium:2,thick:3}[e]||e};return[parseFloat(t(e.css("border-left-width"))),parseFloat(t(e.css("border-top-width")))]},_checkOffset:function(e,t,n){var r=e.dpDiv.outerWidth(),i=e.dpDiv.outerHeight(),s=e.input?e.input.outerWidth():0,o=e.input?e.input.outerHeight():0,u=document.documentElement.clientWidth+(n?0:$(document).scrollLeft()),a=document.documentElement.clientHeight+(n?0:$(document).scrollTop());return t.left-=this._get(e,"isRTL")?r-s:0,t.left-=n&&t.left==e.input.offset().left?$(document).scrollLeft():0,t.top-=n&&t.top==e.input.offset().top+o?$(document).scrollTop():0,t.left-=Math.min(t.left,t.left+r>u&&u>r?Math.abs(t.left+r-u):0),t.top-=Math.min(t.top,t.top+i>a&&a>i?Math.abs(i+o):0),t},_findPos:function(e){var t=this._getInst(e),n=this._get(t,"isRTL");while(e&&(e.type=="hidden"||e.nodeType!=1||$.expr.filters.hidden(e)))e=e[n?"previousSibling":"nextSibling"];var r=$(e).offset();return[r.left,r.top]},_hideDatepicker:function(e){var t=this._curInst;if(!t||e&&t!=$.data(e,PROP_NAME))return;if(this._datepickerShowing){var n=this._get(t,"showAnim"),r=this._get(t,"duration"),i=function(){$.datepicker._tidyDialog(t)};$.effects&&($.effects.effect[n]||$.effects[n])?t.dpDiv.hide(n,$.datepicker._get(t,"showOptions"),r,i):t.dpDiv[n=="slideDown"?"slideUp":n=="fadeIn"?"fadeOut":"hide"](n?r:null,i),n||i(),this._datepickerShowing=!1;var s=this._get(t,"onClose");s&&s.apply(t.input?t.input[0]:null,[t.input?t.input.val():"",t]),this._lastInput=null,this._inDialog&&(this._dialogInput.css({position:"absolute",left:"0",top:"-100px"}),$.blockUI&&($.unblockUI(),$("body").append(this.dpDiv))),this._inDialog=!1}},_tidyDialog:function(e){e.dpDiv.removeClass(this._dialogClass).unbind(".ui-datepicker-calendar")},_checkExternalClick:function(e){if(!$.datepicker._curInst)return;var t=$(e.target),n=$.datepicker._getInst(t[0]);(t[0].id!=$.datepicker._mainDivId&&t.parents("#"+$.datepicker._mainDivId).length==0&&!t.hasClass($.datepicker.markerClassName)&&!t.closest("."+$.datepicker._triggerClass).length&&$.datepicker._datepickerShowing&&(!$.datepicker._inDialog||!$.blockUI)||t.hasClass($.datepicker.markerClassName)&&$.datepicker._curInst!=n)&&$.datepicker._hideDatepicker()},_adjustDate:function(e,t,n){var r=$(e),i=this._getInst(r[0]);if(this._isDisabledDatepicker(r[0]))return;this._adjustInstDate(i,t+(n=="M"?this._get(i,"showCurrentAtPos"):0),n),this._updateDatepicker(i)},_gotoToday:function(e){var t=$(e),n=this._getInst(t[0]);if(this._get(n,"gotoCurrent")&&n.currentDay)n.selectedDay=n.currentDay,n.drawMonth=n.selectedMonth=n.currentMonth,n.drawYear=n.selectedYear=n.currentYear;else{var r=new Date;n.selectedDay=r.getDate(),n.drawMonth=n.selectedMonth=r.getMonth(),n.drawYear=n.selectedYear=r.getFullYear()}this._notifyChange(n),this._adjustDate(t)},_selectMonthYear:function(e,t,n){var r=$(e),i=this._getInst(r[0]);i["selected"+(n=="M"?"Month":"Year")]=i["draw"+(n=="M"?"Month":"Year")]=parseInt(t.options[t.selectedIndex].value,10),this._notifyChange(i),this._adjustDate(r)},_selectDay:function(e,t,n,r){var i=$(e);if($(r).hasClass(this._unselectableClass)||this._isDisabledDatepicker(i[0]))return;var s=this._getInst(i[0]);s.selectedDay=s.currentDay=$("a",r).html(),s.selectedMonth=s.currentMonth=t,s.selectedYear=s.currentYear=n,this._selectDate(e,this._formatDate(s,s.currentDay,s.currentMonth,s.currentYear))},_clearDate:function(e){var t=$(e),n=this._getInst(t[0]);this._selectDate(t,"")},_selectDate:function(e,t){var n=$(e),r=this._getInst(n[0]);t=t!=null?t:this._formatDate(r),r.input&&r.input.val(t),this._updateAlternate(r);var i=this._get(r,"onSelect");i?i.apply(r.input?r.input[0]:null,[t,r]):r.input&&r.input.trigger("change"),r.inline?this._updateDatepicker(r):(this._hideDatepicker(),this._lastInput=r.input[0],typeof r.input[0]!="object"&&r.input.focus(),this._lastInput=null)},_updateAlternate:function(e){var t=this._get(e,"altField");if(t){var n=this._get(e,"altFormat")||this._get(e,"dateFormat"),r=this._getDate(e),i=this.formatDate(n,r,this._getFormatConfig(e));$(t).each(function(){$(this).val(i)})}},noWeekends:function(e){var t=e.getDay();return[t>0&&t<6,""]},iso8601Week:function(e){var t=new Date(e.getTime());t.setDate(t.getDate()+4-(t.getDay()||7));var n=t.getTime();return t.setMonth(0),t.setDate(1),Math.floor(Math.round((n-t)/864e5)/7)+1},parseDate:function(e,t,n){if(e==null||t==null)throw"Invalid arguments";t=typeof t=="object"?t.toString():t+"";if(t=="")return null;var r=(n?n.shortYearCutoff:null)||this._defaults.shortYearCutoff;r=typeof r!="string"?r:(new Date).getFullYear()%100+parseInt(r,10);var i=(n?n.dayNamesShort:null)||this._defaults.dayNamesShort,s=(n?n.dayNames:null)||this._defaults.dayNames,o=(n?n.monthNamesShort:null)||this._defaults.monthNamesShort,u=(n?n.monthNames:null)||this._defaults.monthNames,a=-1,f=-1,l=-1,c=-1,h=!1,p=function(t){var n=y+1<e.length&&e.charAt(y+1)==t;return n&&y++,n},d=function(e){var n=p(e),r=e=="@"?14:e=="!"?20:e=="y"&&n?4:e=="o"?3:2,i=new RegExp("^\\d{1,"+r+"}"),s=t.substring(g).match(i);if(!s)throw"Missing number at position "+g;return g+=s[0].length,parseInt(s[0],10)},v=function(e,n,r){var i=$.map(p(e)?r:n,function(e,t){return[[t,e]]}).sort(function(e,t){return-(e[1].length-t[1].length)}),s=-1;$.each(i,function(e,n){var r=n[1];if(t.substr(g,r.length).toLowerCase()==r.toLowerCase())return s=n[0],g+=r.length,!1});if(s!=-1)return s+1;throw"Unknown name at position "+g},m=function(){if(t.charAt(g)!=e.charAt(y))throw"Unexpected literal at position "+g;g++},g=0;for(var y=0;y<e.length;y++)if(h)e.charAt(y)=="'"&&!p("'")?h=!1:m();else switch(e.charAt(y)){case"d":l=d("d");break;case"D":v("D",i,s);break;case"o":c=d("o");break;case"m":f=d("m");break;case"M":f=v("M",o,u);break;case"y":a=d("y");break;case"@":var b=new Date(d("@"));a=b.getFullYear(),f=b.getMonth()+1,l=b.getDate();break;case"!":var b=new Date((d("!")-this._ticksTo1970)/1e4);a=b.getFullYear(),f=b.getMonth()+1,l=b.getDate();break;case"'":p("'")?m():h=!0;break;default:m()}if(g<t.length){var w=t.substr(g);if(!/^\s+/.test(w))throw"Extra/unparsed characters found in date: "+w}a==-1?a=(new Date).getFullYear():a<100&&(a+=(new Date).getFullYear()-(new Date).getFullYear()%100+(a<=r?0:-100));if(c>-1){f=1,l=c;do{var E=this._getDaysInMonth(a,f-1);if(l<=E)break;f++,l-=E}while(!0)}var b=this._daylightSavingAdjust(new Date(a,f-1,l));if(b.getFullYear()!=a||b.getMonth()+1!=f||b.getDate()!=l)throw"Invalid date";return b},ATOM:"yy-mm-dd",COOKIE:"D, dd M yy",ISO_8601:"yy-mm-dd",RFC_822:"D, d M y",RFC_850:"DD, dd-M-y",RFC_1036:"D, d M y",RFC_1123:"D, d M yy",RFC_2822:"D, d M yy",RSS:"D, d M y",TICKS:"!",TIMESTAMP:"@",W3C:"yy-mm-dd",_ticksTo1970:(718685+Math.floor(492.5)-Math.floor(19.7)+Math.floor(4.925))*24*60*60*1e7,formatDate:function(e,t,n){if(!t)return"";var r=(n?n.dayNamesShort:null)||this._defaults.dayNamesShort,i=(n?n.dayNames:null)||this._defaults.dayNames,s=(n?n.monthNamesShort:null)||this._defaults.monthNamesShort,o=(n?n.monthNames:null)||this._defaults.monthNames,u=function(t){var n=h+1<e.length&&e.charAt(h+1)==t;return n&&h++,n},a=function(e,t,n){var r=""+t;if(u(e))while(r.length<n)r="0"+r;return r},f=function(e,t,n,r){return u(e)?r[t]:n[t]},l="",c=!1;if(t)for(var h=0;h<e.length;h++)if(c)e.charAt(h)=="'"&&!u("'")?c=!1:l+=e.charAt(h);else switch(e.charAt(h)){case"d":l+=a("d",t.getDate(),2);break;case"D":l+=f("D",t.getDay(),r,i);break;case"o":l+=a("o",Math.round(((new Date(t.getFullYear(),t.getMonth(),t.getDate())).getTime()-(new Date(t.getFullYear(),0,0)).getTime())/864e5),3);break;case"m":l+=a("m",t.getMonth()+1,2);break;case"M":l+=f("M",t.getMonth(),s,o);break;case"y":l+=u("y")?t.getFullYear():(t.getYear()%100<10?"0":"")+t.getYear()%100;break;case"@":l+=t.getTime();break;case"!":l+=t.getTime()*1e4+this._ticksTo1970;break;case"'":u("'")?l+="'":c=!0;break;default:l+=e.charAt(h)}return l},_possibleChars:function(e){var t="",n=!1,r=function(t){var n=i+1<e.length&&e.charAt(i+1)==t;return n&&i++,n};for(var i=0;i<e.length;i++)if(n)e.charAt(i)=="'"&&!r("'")?n=!1:t+=e.charAt(i);else switch(e.charAt(i)){case"d":case"m":case"y":case"@":t+="0123456789";break;case"D":case"M":return null;case"'":r("'")?t+="'":n=!0;break;default:t+=e.charAt(i)}return t},_get:function(e,t){return e.settings[t]!==undefined?e.settings[t]:this._defaults[t]},_setDateFromField:function(e,t){if(e.input.val()==e.lastVal)return;var n=this._get(e,"dateFormat"),r=e.lastVal=e.input?e.input.val():null,i,s;i=s=this._getDefaultDate(e);var o=this._getFormatConfig(e);try{i=this.parseDate(n,r,o)||s}catch(u){this.log(u),r=t?"":r}e.selectedDay=i.getDate(),e.drawMonth=e.selectedMonth=i.getMonth(),e.drawYear=e.selectedYear=i.getFullYear(),e.currentDay=r?i.getDate():0,e.currentMonth=r?i.getMonth():0,e.currentYear=r?i.getFullYear():0,this._adjustInstDate(e)},_getDefaultDate:function(e){return this._restrictMinMax(e,this._determineDate(e,this._get(e,"defaultDate"),new Date))},_determineDate:function(e,t,n){var r=function(e){var t=new Date;return t.setDate(t.getDate()+e),t},i=function(t){try{return $.datepicker.parseDate($.datepicker._get(e,"dateFormat"),t,$.datepicker._getFormatConfig(e))}catch(n){}var r=(t.toLowerCase().match(/^c/)?$.datepicker._getDate(e):null)||new Date,i=r.getFullYear(),s=r.getMonth(),o=r.getDate(),u=/([+-]?[0-9]+)\s*(d|D|w|W|m|M|y|Y)?/g,a=u.exec(t);while(a){switch(a[2]||"d"){case"d":case"D":o+=parseInt(a[1],10);break;case"w":case"W":o+=parseInt(a[1],10)*7;break;case"m":case"M":s+=parseInt(a[1],10),o=Math.min(o,$.datepicker._getDaysInMonth(i,s));break;case"y":case"Y":i+=parseInt(a[1],10),o=Math.min(o,$.datepicker._getDaysInMonth(i,s))}a=u.exec(t)}return new Date(i,s,o)},s=t==null||t===""?n:typeof t=="string"?i(t):typeof t=="number"?isNaN(t)?n:r(t):new Date(t.getTime());return s=s&&s.toString()=="Invalid Date"?n:s,s&&(s.setHours(0),s.setMinutes(0),s.setSeconds(0),s.setMilliseconds(0)),this._daylightSavingAdjust(s)},_daylightSavingAdjust:function(e){return e?(e.setHours(e.getHours()>12?e.getHours()+2:0),e):null},_setDate:function(e,t,n){var r=!t,i=e.selectedMonth,s=e.selectedYear,o=this._restrictMinMax(e,this._determineDate(e,t,new Date));e.selectedDay=e.currentDay=o.getDate(),e.drawMonth=e.selectedMonth=e.currentMonth=o.getMonth(),e.drawYear=e.selectedYear=e.currentYear=o.getFullYear(),(i!=e.selectedMonth||s!=e.selectedYear)&&!n&&this._notifyChange(e),this._adjustInstDate(e),e.input&&e.input.val(r?"":this._formatDate(e))},_getDate:function(e){var t=!e.currentYear||e.input&&e.input.val()==""?null:this._daylightSavingAdjust(new Date(e.currentYear,e.currentMonth,e.currentDay));return t},_attachHandlers:function(e){var t=this._get(e,"stepMonths"),n="#"+e.id.replace(/\\\\/g,"\\");e.dpDiv.find("[data-handler]").map(function(){var e={prev:function(){window["DP_jQuery_"+dpuuid].datepicker._adjustDate(n,-t,"M")},next:function(){window["DP_jQuery_"+dpuuid].datepicker._adjustDate(n,+t,"M")},hide:function(){window["DP_jQuery_"+dpuuid].datepicker._hideDatepicker()},today:function(){window["DP_jQuery_"+dpuuid].datepicker._gotoToday(n)},selectDay:function(){return window["DP_jQuery_"+dpuuid].datepicker._selectDay(n,+this.getAttribute("data-month"),+this.getAttribute("data-year"),this),!1},selectMonth:function(){return window["DP_jQuery_"+dpuuid].datepicker._selectMonthYear(n,this,"M"),!1},selectYear:function(){return window["DP_jQuery_"+dpuuid].datepicker._selectMonthYear(n,this,"Y"),!1}};$(this).bind(this.getAttribute("data-event"),e[this.getAttribute("data-handler")])})},_generateHTML:function(e){var t=new Date;t=this._daylightSavingAdjust(new Date(t.getFullYear(),t.getMonth(),t.getDate()));var n=this._get(e,"isRTL"),r=this._get(e,"showButtonPanel"),i=this._get(e,"hideIfNoPrevNext"),s=this._get(e,"navigationAsDateFormat"),o=this._getNumberOfMonths(e),u=this._get(e,"showCurrentAtPos"),a=this._get(e,"stepMonths"),f=o[0]!=1||o[1]!=1,l=this._daylightSavingAdjust(e.currentDay?new Date(e.currentYear,e.currentMonth,e.currentDay):new Date(9999,9,9)),c=this._getMinMaxDate(e,"min"),h=this._getMinMaxDate(e,"max"),p=e.drawMonth-u,d=e.drawYear;p<0&&(p+=12,d--);if(h){var v=this._daylightSavingAdjust(new Date(h.getFullYear(),h.getMonth()-o[0]*o[1]+1,h.getDate()));v=c&&v<c?c:v;while(this._daylightSavingAdjust(new Date(d,p,1))>v)p--,p<0&&(p=11,d--)}e.drawMonth=p,e.drawYear=d;var m=this._get(e,"prevText");m=s?this.formatDate(m,this._daylightSavingAdjust(new Date(d,p-a,1)),this._getFormatConfig(e)):m;var g=this._canAdjustMonth(e,-1,d,p)?'<a class="ui-datepicker-prev ui-corner-all" data-handler="prev" data-event="click" title="'+m+'"><span class="ui-icon ui-icon-circle-triangle-'+(n?"e":"w")+'">'+m+"</span></a>":i?"":'<a class="ui-datepicker-prev ui-corner-all ui-state-disabled" title="'+m+'"><span class="ui-icon ui-icon-circle-triangle-'+(n?"e":"w")+'">'+m+"</span></a>",y=this._get(e,"nextText");y=s?this.formatDate(y,this._daylightSavingAdjust(new Date(d,p+a,1)),this._getFormatConfig(e)):y;var b=this._canAdjustMonth(e,1,d,p)?'<a class="ui-datepicker-next ui-corner-all" data-handler="next" data-event="click" title="'+y+'"><span class="ui-icon ui-icon-circle-triangle-'+(n?"w":"e")+'">'+y+"</span></a>":i?"":'<a class="ui-datepicker-next ui-corner-all ui-state-disabled" title="'+y+'"><span class="ui-icon ui-icon-circle-triangle-'+(n?"w":"e")+'">'+y+"</span></a>",w=this._get(e,"currentText"),E=this._get(e,"gotoCurrent")&&e.currentDay?l:t;w=s?this.formatDate(w,E,this._getFormatConfig(e)):w;var S=e.inline?"":'<button type="button" class="ui-datepicker-close ui-state-default ui-priority-primary ui-corner-all" data-handler="hide" data-event="click">'+this._get(e,"closeText")+"</button>",x=r?'<div class="ui-datepicker-buttonpane ui-widget-content">'+(n?S:"")+(this._isInRange(e,E)?'<button type="button" class="ui-datepicker-current ui-state-default ui-priority-secondary ui-corner-all" data-handler="today" data-event="click">'+w+"</button>":"")+(n?"":S)+"</div>":"",T=parseInt(this._get(e,"firstDay"),10);T=isNaN(T)?0:T;var N=this._get(e,"showWeek"),C=this._get(e,"dayNames"),k=this._get(e,"dayNamesShort"),L=this._get(e,"dayNamesMin"),A=this._get(e,"monthNames"),O=this._get(e,"monthNamesShort"),M=this._get(e,"beforeShowDay"),_=this._get(e,"showOtherMonths"),D=this._get(e,"selectOtherMonths"),P=this._get(e,"calculateWeek")||this.iso8601Week,H=this._getDefaultDate(e),B="";for(var j=0;j<o[0];j++){var F="";this.maxRows=4;for(var I=0;I<o[1];I++){var q=this._daylightSavingAdjust(new Date(d,p,e.selectedDay)),R=" ui-corner-all",U="";if(f){U+='<div class="ui-datepicker-group';if(o[1]>1)switch(I){case 0:U+=" ui-datepicker-group-first",R=" ui-corner-"+(n?"right":"left");break;case o[1]-1:U+=" ui-datepicker-group-last",R=" ui-corner-"+(n?"left":"right");break;default:U+=" ui-datepicker-group-middle",R=""}U+='">'}U+='<div class="ui-datepicker-header ui-widget-header ui-helper-clearfix'+R+'">'+(/all|left/.test(R)&&j==0?n?b:g:"")+(/all|right/.test(R)&&j==0?n?g:b:"")+this._generateMonthYearHeader(e,p,d,c,h,j>0||I>0,A,O)+'</div><table class="ui-datepicker-calendar"><thead>'+"<tr>";var z=N?'<th class="ui-datepicker-week-col">'+this._get(e,"weekHeader")+"</th>":"";for(var W=0;W<7;W++){var X=(W+T)%7;z+="<th"+((W+T+6)%7>=5?' class="ui-datepicker-week-end"':"")+">"+'<span title="'+C[X]+'">'+L[X]+"</span></th>"}U+=z+"</tr></thead><tbody>";var V=this._getDaysInMonth(d,p);d==e.selectedYear&&p==e.selectedMonth&&(e.selectedDay=Math.min(e.selectedDay,V));var J=(this._getFirstDayOfMonth(d,p)-T+7)%7,K=Math.ceil((J+V)/7),Q=f?this.maxRows>K?this.maxRows:K:K;this.maxRows=Q;var G=this._daylightSavingAdjust(new Date(d,p,1-J));for(var Y=0;Y<Q;Y++){U+="<tr>";var Z=N?'<td class="ui-datepicker-week-col">'+this._get(e,"calculateWeek")(G)+"</td>":"";for(var W=0;W<7;W++){var et=M?M.apply(e.input?e.input[0]:null,[G]):[!0,""],tt=G.getMonth()!=p,nt=tt&&!D||!et[0]||c&&G<c||h&&G>h;Z+='<td class="'+((W+T+6)%7>=5?" ui-datepicker-week-end":"")+(tt?" ui-datepicker-other-month":"")+(G.getTime()==q.getTime()&&p==e.selectedMonth&&e._keyEvent||H.getTime()==G.getTime()&&H.getTime()==q.getTime()?" "+this._dayOverClass:"")+(nt?" "+this._unselectableClass+" ui-state-disabled":"")+(tt&&!_?"":" "+et[1]+(G.getTime()==l.getTime()?" "+this._currentClass:"")+(G.getTime()==t.getTime()?" ui-datepicker-today":""))+'"'+((!tt||_)&&et[2]?' title="'+et[2]+'"':"")+(nt?"":' data-handler="selectDay" data-event="click" data-month="'+G.getMonth()+'" data-year="'+G.getFullYear()+'"')+">"+(tt&&!_?" ":nt?'<span class="ui-state-default">'+G.getDate()+"</span>":'<a class="ui-state-default'+(G.getTime()==t.getTime()?" ui-state-highlight":"")+(G.getTime()==l.getTime()?" ui-state-active":"")+(tt?" ui-priority-secondary":"")+'" href="#">'+G.getDate()+"</a>")+"</td>",G.setDate(G.getDate()+1),G=this._daylightSavingAdjust(G)}U+=Z+"</tr>"}p++,p>11&&(p=0,d++),U+="</tbody></table>"+(f?"</div>"+(o[0]>0&&I==o[1]-1?'<div class="ui-datepicker-row-break"></div>':""):""),F+=U}B+=F}return B+=x+($.ui.ie6&&!e.inline?'<iframe src="javascript:false;" class="ui-datepicker-cover" frameborder="0"></iframe>':""),e._keyEvent=!1,B},_generateMonthYearHeader:function(e,t,n,r,i,s,o,u){var a=this._get(e,"changeMonth"),f=this._get(e,"changeYear"),l=this._get(e,"showMonthAfterYear"),c='<div class="ui-datepicker-title">',h="";if(s||!a)h+='<span class="ui-datepicker-month">'+o[t]+"</span>";else{var p=r&&r.getFullYear()==n,d=i&&i.getFullYear()==n;h+='<select class="ui-datepicker-month" data-handler="selectMonth" data-event="change">';for(var v=0;v<12;v++)(!p||v>=r.getMonth())&&(!d||v<=i.getMonth())&&(h+='<option value="'+v+'"'+(v==t?' selected="selected"':"")+">"+u[v]+"</option>");h+="</select>"}l||(c+=h+(s||!a||!f?" ":""));if(!e.yearshtml){e.yearshtml="";if(s||!f)c+='<span class="ui-datepicker-year">'+n+"</span>";else{var m=this._get(e,"yearRange").split(":"),g=(new Date).getFullYear(),y=function(e){var t=e.match(/c[+-].*/)?n+parseInt(e.substring(1),10):e.match(/[+-].*/)?g+parseInt(e,10):parseInt(e,10);return isNaN(t)?g:t},b=y(m[0]),w=Math.max(b,y(m[1]||""));b=r?Math.max(b,r.getFullYear()):b,w=i?Math.min(w,i.getFullYear()):w,e.yearshtml+='<select class="ui-datepicker-year" data-handler="selectYear" data-event="change">';for(;b<=w;b++)e.yearshtml+='<option value="'+b+'"'+(b==n?' selected="selected"':"")+">"+b+"</option>";e.yearshtml+="</select>",c+=e.yearshtml,e.yearshtml=null}}return c+=this._get(e,"yearSuffix"),l&&(c+=(s||!a||!f?" ":"")+h),c+="</div>",c},_adjustInstDate:function(e,t,n){var r=e.drawYear+(n=="Y"?t:0),i=e.drawMonth+(n=="M"?t:0),s=Math.min(e.selectedDay,this._getDaysInMonth(r,i))+(n=="D"?t:0),o=this._restrictMinMax(e,this._daylightSavingAdjust(new Date(r,i,s)));e.selectedDay=o.getDate(),e.drawMonth=e.selectedMonth=o.getMonth(),e.drawYear=e.selectedYear=o.getFullYear(),(n=="M"||n=="Y")&&this._notifyChange(e)},_restrictMinMax:function(e,t){var n=this._getMinMaxDate(e,"min"),r=this._getMinMaxDate(e,"max"),i=n&&t<n?n:t;return i=r&&i>r?r:i,i},_notifyChange:function(e){var t=this._get(e,"onChangeMonthYear");t&&t.apply(e.input?e.input[0]:null,[e.selectedYear,e.selectedMonth+1,e])},_getNumberOfMonths:function(e){var t=this._get(e,"numberOfMonths");return t==null?[1,1]:typeof t=="number"?[1,t]:t},_getMinMaxDate:function(e,t){return this._determineDate(e,this._get(e,t+"Date"),null)},_getDaysInMonth:function(e,t){return 32-this._daylightSavingAdjust(new Date(e,t,32)).getDate()},_getFirstDayOfMonth:function(e,t){return(new Date(e,t,1)).getDay()},_canAdjustMonth:function(e,t,n,r){var i=this._getNumberOfMonths(e),s=this._daylightSavingAdjust(new Date(n,r+(t<0?t:i[0]*i[1]),1));return t<0&&s.setDate(this._getDaysInMonth(s.getFullYear(),s.getMonth())),this._isInRange(e,s)},_isInRange:function(e,t){var n=this._getMinMaxDate(e,"min"),r=this._getMinMaxDate(e,"max");return(!n||t.getTime()>=n.getTime())&&(!r||t.getTime()<=r.getTime())},_getFormatConfig:function(e){var t=this._get(e,"shortYearCutoff");return t=typeof t!="string"?t:(new Date).getFullYear()%100+parseInt(t,10),{shortYearCutoff:t,dayNamesShort:this._get(e,"dayNamesShort"),dayNames:this._get(e,"dayNames"),monthNamesShort:this._get(e,"monthNamesShort"),monthNames:this._get(e,"monthNames")}},_formatDate:function(e,t,n,r){t||(e.currentDay=e.selectedDay,e.currentMonth=e.selectedMonth,e.currentYear=e.selectedYear);var i=t?typeof t=="object"?t:this._daylightSavingAdjust(new Date(r,n,t)):this._daylightSavingAdjust(new Date(e.currentYear,e.currentMonth,e.currentDay));return this.formatDate(this._get(e,"dateFormat"),i,this._getFormatConfig(e))}}),$.fn.datepicker=function(e){if(!this.length)return this;$.datepicker.initialized||($(document).mousedown($.datepicker._checkExternalClick).find(document.body).append($.datepicker.dpDiv),$.datepicker.initialized=!0);var t=Array.prototype.slice.call(arguments,1);return typeof e!="string"||e!="isDisabled"&&e!="getDate"&&e!="widget"?e=="option"&&arguments.length==2&&typeof arguments[1]=="string"?$.datepicker["_"+e+"Datepicker"].apply($.datepicker,[this[0]].concat(t)):this.each(function(){typeof e=="string"?$.datepicker["_"+e+"Datepicker"].apply($.datepicker,[this].concat(t)):$.datepicker._attachDatepicker(this,e)}):$.datepicker["_"+e+"Datepicker"].apply($.datepicker,[this[0]].concat(t))},$.datepicker=new Datepicker,$.datepicker.initialized=!1,$.datepicker.uuid=(new Date).getTime(),$.datepicker.version="1.9.1",window["DP_jQuery_"+dpuuid]=$})(jQuery);(function(e,t){var n="ui-dialog ui-widget ui-widget-content ui-corner-all ",r={buttons:!0,height:!0,maxHeight:!0,maxWidth:!0,minHeight:!0,minWidth:!0,width:!0},i={maxHeight:!0,maxWidth:!0,minHeight:!0,minWidth:!0};e.widget("ui.dialog",{version:"1.9.1",options:{autoOpen:!0,buttons:{},closeOnEscape:!0,closeText:"close",dialogClass:"",draggable:!0,hide:null,height:"auto",maxHeight:!1,maxWidth:!1,minHeight:150,minWidth:150,modal:!1,position:{my:"center",at:"center",of:window,collision:"fit",using:function(t){var n=e(this).css(t).offset().top;n<0&&e(this).css("top",t.top-n)}},resizable:!0,show:null,stack:!0,title:"",width:300,zIndex:1e3},_create:function(){this.originalTitle=this.element.attr("title"),typeof this.originalTitle!="string"&&(this.originalTitle=""),this.oldPosition={parent:this.element.parent(),index:this.element.parent().children().index(this.element)},this.options.title=this.options.title||this.originalTitle;var t=this,r=this.options,i=r.title||" ",s,o,u,a,f;s=(this.uiDialog=e("<div>")).addClass(n+r.dialogClass).css({display:"none",outline:0,zIndex:r.zIndex}).attr("tabIndex",-1).keydown(function(n){r.closeOnEscape&&!n.isDefaultPrevented()&&n.keyCode&&n.keyCode===e.ui.keyCode.ESCAPE&&(t.close(n),n.preventDefault())}).mousedown(function(e){t.moveToTop(!1,e)}).appendTo("body"),this.element.show().removeAttr("title").addClass("ui-dialog-content ui-widget-content").appendTo(s),o=(this.uiDialogTitlebar=e("<div>")).addClass("ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix").bind("mousedown",function(){s.focus()}).prependTo(s),u=e("<a href='#'></a>").addClass("ui-dialog-titlebar-close ui-corner-all").attr("role","button").click(function(e){e.preventDefault(),t.close(e)}).appendTo(o),(this.uiDialogTitlebarCloseText=e("<span>")).addClass("ui-icon ui-icon-closethick").text(r.closeText).appendTo(u),a=e("<span>").uniqueId().addClass("ui-dialog-title").html(i).prependTo(o),f=(this.uiDialogButtonPane=e("<div>")).addClass("ui-dialog-buttonpane ui-widget-content ui-helper-clearfix"),(this.uiButtonSet=e("<div>")).addClass("ui-dialog-buttonset").appendTo(f),s.attr({role:"dialog","aria-labelledby":a.attr("id")}),o.find("*").add(o).disableSelection(),this._hoverable(u),this._focusable(u),r.draggable&&e.fn.draggable&&this._makeDraggable(),r.resizable&&e.fn.resizable&&this._makeResizable(),this._createButtons(r.buttons),this._isOpen=!1,e.fn.bgiframe&&s.bgiframe(),this._on(s,{keydown:function(t){if(!r.modal||t.keyCode!==e.ui.keyCode.TAB)return;var n=e(":tabbable",s),i=n.filter(":first"),o=n.filter(":last");if(t.target===o[0]&&!t.shiftKey)return i.focus(1),!1;if(t.target===i[0]&&t.shiftKey)return o.focus(1),!1}})},_init:function(){this.options.autoOpen&&this.open()},_destroy:function(){var e,t=this.oldPosition;this.overlay&&this.overlay.destroy(),this.uiDialog.hide(),this.element.removeClass("ui-dialog-content ui-widget-content").hide().appendTo("body"),this.uiDialog.remove(),this.originalTitle&&this.element.attr("title",this.originalTitle),e=t.parent.children().eq(t.index),e.length&&e[0]!==this.element[0]?e.before(this.element):t.parent.append(this.element)},widget:function(){return this.uiDialog},close:function(t){var n=this,r,i;if(!this._isOpen)return;if(!1===this._trigger("beforeClose",t))return;return this._isOpen=!1,this.overlay&&this.overlay.destroy(),this.options.hide?this._hide(this.uiDialog,this.options.hide,function(){n._trigger("close",t)}):(this.uiDialog.hide(),this._trigger("close",t)),e.ui.dialog.overlay.resize(),this.options.modal&&(r=0,e(".ui-dialog").each(function(){this!==n.uiDialog[0]&&(i=e(this).css("z-index"),isNaN(i)||(r=Math.max(r,i)))}),e.ui.dialog.maxZ=r),this},isOpen:function(){return this._isOpen},moveToTop:function(t,n){var r=this.options,i;return r.modal&&!t||!r.stack&&!r.modal?this._trigger("focus",n):(r.zIndex>e.ui.dialog.maxZ&&(e.ui.dialog.maxZ=r.zIndex),this.overlay&&(e.ui.dialog.maxZ+=1,e.ui.dialog.overlay.maxZ=e.ui.dialog.maxZ,this.overlay.$el.css("z-index",e.ui.dialog.overlay.maxZ)),i={scrollTop:this.element.scrollTop(),scrollLeft:this.element.scrollLeft()},e.ui.dialog.maxZ+=1,this.uiDialog.css("z-index",e.ui.dialog.maxZ),this.element.attr(i),this._trigger("focus",n),this)},open:function(){if(this._isOpen)return;var t,n=this.options,r=this.uiDialog;return this._size(),this._position(n.position),r.show(n.show),this.overlay=n.modal?new e.ui.dialog.overlay(this):null,this.moveToTop(!0),t=this.element.find(":tabbable"),t.length||(t=this.uiDialogButtonPane.find(":tabbable"),t.length||(t=r)),t.eq(0).focus(),this._isOpen=!0,this._trigger("open"),this},_createButtons:function(t){var n=this,r=!1;this.uiDialogButtonPane.remove(),this.uiButtonSet.empty(),typeof t=="object"&&t!==null&&e.each(t,function(){return!(r=!0)}),r?(e.each(t,function(t,r){r=e.isFunction(r)?{click:r,text:t}:r;var i=e("<button type='button'></button>").attr(r,!0).unbind("click").click(function(){r.click.apply(n.element[0],arguments)}).appendTo(n.uiButtonSet);e.fn.button&&i.button()}),this.uiDialog.addClass("ui-dialog-buttons"),this.uiDialogButtonPane.appendTo(this.uiDialog)):this.uiDialog.removeClass("ui-dialog-buttons")},_makeDraggable:function(){function r(e){return{position:e.position,offset:e.offset}}var t=this,n=this.options;this.uiDialog.draggable({cancel:".ui-dialog-content, .ui-dialog-titlebar-close",handle:".ui-dialog-titlebar",containment:"document",start:function(n,i){e(this).addClass("ui-dialog-dragging"),t._trigger("dragStart",n,r(i))},drag:function(e,n){t._trigger("drag",e,r(n))},stop:function(i,s){n.position=[s.position.left-t.document.scrollLeft(),s.position.top-t.document.scrollTop()],e(this).removeClass("ui-dialog-dragging"),t._trigger("dragStop",i,r(s)),e.ui.dialog.overlay.resize()}})},_makeResizable:function(n){function u(e){return{originalPosition:e.originalPosition,originalSize:e.originalSize,position:e.position,size:e.size}}n=n===t?this.options.resizable:n;var r=this,i=this.options,s=this.uiDialog.css("position"),o=typeof n=="string"?n:"n,e,s,w,se,sw,ne,nw";this.uiDialog.resizable({cancel:".ui-dialog-content",containment:"document",alsoResize:this.element,maxWidth:i.maxWidth,maxHeight:i.maxHeight,minWidth:i.minWidth,minHeight:this._minHeight(),handles:o,start:function(t,n){e(this).addClass("ui-dialog-resizing"),r._trigger("resizeStart",t,u(n))},resize:function(e,t){r._trigger("resize",e,u(t))},stop:function(t,n){e(this).removeClass("ui-dialog-resizing"),i.height=e(this).height(),i.width=e(this).width(),r._trigger("resizeStop",t,u(n)),e.ui.dialog.overlay.resize()}}).css("position",s).find(".ui-resizable-se").addClass("ui-icon ui-icon-grip-diagonal-se")},_minHeight:function(){var e=this.options;return e.height==="auto"?e.minHeight:Math.min(e.minHeight,e.height)},_position:function(t){var n=[],r=[0,0],i;if(t){if(typeof t=="string"||typeof t=="object"&&"0"in t)n=t.split?t.split(" "):[t[0],t[1]],n.length===1&&(n[1]=n[0]),e.each(["left","top"],function(e,t){+n[e]===n[e]&&(r[e]=n[e],n[e]=t)}),t={my:n[0]+(r[0]<0?r[0]:"+"+r[0])+" "+n[1]+(r[1]<0?r[1]:"+"+r[1]),at:n.join(" ")};t=e.extend({},e.ui.dialog.prototype.options.position,t)}else t=e.ui.dialog.prototype.options.position;i=this.uiDialog.is(":visible"),i||this.uiDialog.show(),this.uiDialog.position(t),i||this.uiDialog.hide()},_setOptions:function(t){var n=this,s={},o=!1;e.each(t,function(e,t){n._setOption(e,t),e in r&&(o=!0),e in i&&(s[e]=t)}),o&&this._size(),this.uiDialog.is(":data(resizable)")&&this.uiDialog.resizable("option",s)},_setOption:function(t,r){var i,s,o=this.uiDialog;switch(t){case"buttons":this._createButtons(r);break;case"closeText":this.uiDialogTitlebarCloseText.text(""+r);break;case"dialogClass":o.removeClass(this.options.dialogClass).addClass(n+r);break;case"disabled":r?o.addClass("ui-dialog-disabled"):o.removeClass("ui-dialog-disabled");break;case"draggable":i=o.is(":data(draggable)"),i&&!r&&o.draggable("destroy"),!i&&r&&this._makeDraggable();break;case"position":this._position(r);break;case"resizable":s=o.is(":data(resizable)"),s&&!r&&o.resizable("destroy"),s&&typeof r=="string"&&o.resizable("option","handles",r),!s&&r!==!1&&this._makeResizable(r);break;case"title":e(".ui-dialog-title",this.uiDialogTitlebar).html(""+(r||" "))}this._super(t,r)},_size:function(){var t,n,r,i=this.options,s=this.uiDialog.is(":visible");this.element.show().css({width:"auto",minHeight:0,height:0}),i.minWidth>i.width&&(i.width=i.minWidth),t=this.uiDialog.css({height:"auto",width:i.width}).outerHeight(),n=Math.max(0,i.minHeight-t),i.height==="auto"?e.support.minHeight?this.element.css({minHeight:n,height:"auto"}):(this.uiDialog.show(),r=this.element.css("height","auto").height(),s||this.uiDialog.hide(),this.element.height(Math.max(r,n))):this.element.height(Math.max(i.height-t,0)),this.uiDialog.is(":data(resizable)")&&this.uiDialog.resizable("option","minHeight",this._minHeight())}}),e.extend(e.ui.dialog,{uuid:0,maxZ:0,getTitleId:function(e){var t=e.attr("id");return t||(this.uuid+=1,t=this.uuid),"ui-dialog-title-"+t},overlay:function(t){this.$el=e.ui.dialog.overlay.create(t)}}),e.extend(e.ui.dialog.overlay,{instances:[],oldInstances:[],maxZ:0,events:e.map("focus,mousedown,mouseup,keydown,keypress,click".split(","),function(e){return e+".dialog-overlay"}).join(" "),create:function(t){this.instances.length===0&&(setTimeout(function(){e.ui.dialog.overlay.instances.length&&e(document).bind(e.ui.dialog.overlay.events,function(t){if(e(t.target).zIndex()<e.ui.dialog.overlay.maxZ)return!1})},1),e(window).bind("resize.dialog-overlay",e.ui.dialog.overlay.resize));var n=this.oldInstances.pop()||e("<div>").addClass("ui-widget-overlay");return e(document).bind("keydown.dialog-overlay",function(r){var i=e.ui.dialog.overlay.instances;i.length!==0&&i[i.length-1]===n&&t.options.closeOnEscape&&!r.isDefaultPrevented()&&r.keyCode&&r.keyCode===e.ui.keyCode.ESCAPE&&(t.close(r),r.preventDefault())}),n.appendTo(document.body).css({width:this.width(),height:this.height()}),e.fn.bgiframe&&n.bgiframe(),this.instances.push(n),n},destroy:function(t){var n=e.inArray(t,this.instances),r=0;n!==-1&&this.oldInstances.push(this.instances.splice(n,1)[0]),this.instances.length===0&&e([document,window]).unbind(".dialog-overlay"),t.height(0).width(0).remove(),e.each(this.instances,function(){r=Math.max(r,this.css("z-index"))}),this.maxZ=r},height:function(){var t,n;return e.ui.ie?(t=Math.max(document.documentElement.scrollHeight,document.body.scrollHeight),n=Math.max(document.documentElement.offsetHeight,document.body.offsetHeight),t<n?e(window).height()+"px":t+"px"):e(document).height()+"px"},width:function(){var t,n;return e.ui.ie?(t=Math.max(document.documentElement.scrollWidth,document.body.scrollWidth),n=Math.max(document.documentElement.offsetWidth,document.body.offsetWidth),t<n?e(window).width()+"px":t+"px"):e(document).width()+"px"},resize:function(){var t=e([]);e.each(e.ui.dialog.overlay.instances,function(){t=t.add(this)}),t.css({width:0,height:0}).css({width:e.ui.dialog.overlay.width(),height:e.ui.dialog.overlay.height()})}}),e.extend(e.ui.dialog.overlay.prototype,{destroy:function(){e.ui.dialog.overlay.destroy(this.$el)}})})(jQuery);(function(e,t){e.widget("ui.draggable",e.ui.mouse,{version:"1.9.1",widgetEventPrefix:"drag",options:{addClasses:!0,appendTo:"parent",axis:!1,connectToSortable:!1,containment:!1,cursor:"auto",cursorAt:!1,grid:!1,handle:!1,helper:"original",iframeFix:!1,opacity:!1,refreshPositions:!1,revert:!1,revertDuration:500,scope:"default",scroll:!0,scrollSensitivity:20,scrollSpeed:20,snap:!1,snapMode:"both",snapTolerance:20,stack:!1,zIndex:!1},_create:function(){this.options.helper=="original"&&!/^(?:r|a|f)/.test(this.element.css("position"))&&(this.element[0].style.position="relative"),this.options.addClasses&&this.element.addClass("ui-draggable"),this.options.disabled&&this.element.addClass("ui-draggable-disabled"),this._mouseInit()},_destroy:function(){this.element.removeClass("ui-draggable ui-draggable-dragging ui-draggable-disabled"),this._mouseDestroy()},_mouseCapture:function(t){var n=this.options;return this.helper||n.disabled||e(t.target).is(".ui-resizable-handle")?!1:(this.handle=this._getHandle(t),this.handle?(e(n.iframeFix===!0?"iframe":n.iframeFix).each(function(){e('<div class="ui-draggable-iframeFix" style="background: #fff;"></div>').css({width:this.offsetWidth+"px",height:this.offsetHeight+"px",position:"absolute",opacity:"0.001",zIndex:1e3}).css(e(this).offset()).appendTo("body")}),!0):!1)},_mouseStart:function(t){var n=this.options;return this.helper=this._createHelper(t),this.helper.addClass("ui-draggable-dragging"),this._cacheHelperProportions(),e.ui.ddmanager&&(e.ui.ddmanager.current=this),this._cacheMargins(),this.cssPosition=this.helper.css("position"),this.scrollParent=this.helper.scrollParent(),this.offset=this.positionAbs=this.element.offset(),this.offset={top:this.offset.top-this.margins.top,left:this.offset.left-this.margins.left},e.extend(this.offset,{click:{left:t.pageX-this.offset.left,top:t.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()}),this.originalPosition=this.position=this._generatePosition(t),this.originalPageX=t.pageX,this.originalPageY=t.pageY,n.cursorAt&&this._adjustOffsetFromHelper(n.cursorAt),n.containment&&this._setContainment(),this._trigger("start",t)===!1?(this._clear(),!1):(this._cacheHelperProportions(),e.ui.ddmanager&&!n.dropBehaviour&&e.ui.ddmanager.prepareOffsets(this,t),this._mouseDrag(t,!0),e.ui.ddmanager&&e.ui.ddmanager.dragStart(this,t),!0)},_mouseDrag:function(t,n){this.position=this._generatePosition(t),this.positionAbs=this._convertPositionTo("absolute");if(!n){var r=this._uiHash();if(this._trigger("drag",t,r)===!1)return this._mouseUp({}),!1;this.position=r.position}if(!this.options.axis||this.options.axis!="y")this.helper[0].style.left=this.position.left+"px";if(!this.options.axis||this.options.axis!="x")this.helper[0].style.top=this.position.top+"px";return e.ui.ddmanager&&e.ui.ddmanager.drag(this,t),!1},_mouseStop:function(t){var n=!1;e.ui.ddmanager&&!this.options.dropBehaviour&&(n=e.ui.ddmanager.drop(this,t)),this.dropped&&(n=this.dropped,this.dropped=!1);var r=this.element[0],i=!1;while(r&&(r=r.parentNode))r==document&&(i=!0);if(!i&&this.options.helper==="original")return!1;if(this.options.revert=="invalid"&&!n||this.options.revert=="valid"&&n||this.options.revert===!0||e.isFunction(this.options.revert)&&this.options.revert.call(this.element,n)){var s=this;e(this.helper).animate(this.originalPosition,parseInt(this.options.revertDuration,10),function(){s._trigger("stop",t)!==!1&&s._clear()})}else this._trigger("stop",t)!==!1&&this._clear();return!1},_mouseUp:function(t){return e("div.ui-draggable-iframeFix").each(function(){this.parentNode.removeChild(this)}),e.ui.ddmanager&&e.ui.ddmanager.dragStop(this,t),e.ui.mouse.prototype._mouseUp.call(this,t)},cancel:function(){return this.helper.is(".ui-draggable-dragging")?this._mouseUp({}):this._clear(),this},_getHandle:function(t){var n=!this.options.handle||!e(this.options.handle,this.element).length?!0:!1;return e(this.options.handle,this.element).find("*").andSelf().each(function(){this==t.target&&(n=!0)}),n},_createHelper:function(t){var n=this.options,r=e.isFunction(n.helper)?e(n.helper.apply(this.element[0],[t])):n.helper=="clone"?this.element.clone().removeAttr("id"):this.element;return r.parents("body").length||r.appendTo(n.appendTo=="parent"?this.element[0].parentNode:n.appendTo),r[0]!=this.element[0]&&!/(fixed|absolute)/.test(r.css("position"))&&r.css("position","absolute"),r},_adjustOffsetFromHelper:function(t){typeof t=="string"&&(t=t.split(" ")),e.isArray(t)&&(t={left:+t[0],top:+t[1]||0}),"left"in t&&(this.offset.click.left=t.left+this.margins.left),"right"in t&&(this.offset.click.left=this.helperProportions.width-t.right+this.margins.left),"top"in t&&(this.offset.click.top=t.top+this.margins.top),"bottom"in t&&(this.offset.click.top=this.helperProportions.height-t.bottom+this.margins.top)},_getParentOffset:function(){this.offsetParent=this.helper.offsetParent();var t=this.offsetParent.offset();this.cssPosition=="absolute"&&this.scrollParent[0]!=document&&e.contains(this.scrollParent[0],this.offsetParent[0])&&(t.left+=this.scrollParent.scrollLeft(),t.top+=this.scrollParent.scrollTop());if(this.offsetParent[0]==document.body||this.offsetParent[0].tagName&&this.offsetParent[0].tagName.toLowerCase()=="html"&&e.ui.ie)t={top:0,left:0};return{top:t.top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:t.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if(this.cssPosition=="relative"){var e=this.element.position();return{top:e.top-(parseInt(this.helper.css("top"),10)||0)+this.scrollParent.scrollTop(),left:e.left-(parseInt(this.helper.css("left"),10)||0)+this.scrollParent.scrollLeft()}}return{top:0,left:0}},_cacheMargins:function(){this.margins={left:parseInt(this.element.css("marginLeft"),10)||0,top:parseInt(this.element.css("marginTop"),10)||0,right:parseInt(this.element.css("marginRight"),10)||0,bottom:parseInt(this.element.css("marginBottom"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(),height:this.helper.outerHeight()}},_setContainment:function(){var t=this.options;t.containment=="parent"&&(t.containment=this.helper[0].parentNode);if(t.containment=="document"||t.containment=="window")this.containment=[t.containment=="document"?0:e(window).scrollLeft()-this.offset.relative.left-this.offset.parent.left,t.containment=="document"?0:e(window).scrollTop()-this.offset.relative.top-this.offset.parent.top,(t.containment=="document"?0:e(window).scrollLeft())+e(t.containment=="document"?document:window).width()-this.helperProportions.width-this.margins.left,(t.containment=="document"?0:e(window).scrollTop())+(e(t.containment=="document"?document:window).height()||document.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top];if(!/^(document|window|parent)$/.test(t.containment)&&t.containment.constructor!=Array){var n=e(t.containment),r=n[0];if(!r)return;var i=n.offset(),s=e(r).css("overflow")!="hidden";this.containment=[(parseInt(e(r).css("borderLeftWidth"),10)||0)+(parseInt(e(r).css("paddingLeft"),10)||0),(parseInt(e(r).css("borderTopWidth"),10)||0)+(parseInt(e(r).css("paddingTop"),10)||0),(s?Math.max(r.scrollWidth,r.offsetWidth):r.offsetWidth)-(parseInt(e(r).css("borderLeftWidth"),10)||0)-(parseInt(e(r).css("paddingRight"),10)||0)-this.helperProportions.width-this.margins.left-this.margins.right,(s?Math.max(r.scrollHeight,r.offsetHeight):r.offsetHeight)-(parseInt(e(r).css("borderTopWidth"),10)||0)-(parseInt(e(r).css("paddingBottom"),10)||0)-this.helperProportions.height-this.margins.top-this.margins.bottom],this.relative_container=n}else t.containment.constructor==Array&&(this.containment=t.containment)},_convertPositionTo:function(t,n){n||(n=this.position);var r=t=="absolute"?1:-1,i=this.options,s=this.cssPosition!="absolute"||this.scrollParent[0]!=document&&!!e.contains(this.scrollParent[0],this.offsetParent[0])?this.scrollParent:this.offsetParent,o=/(html|body)/i.test(s[0].tagName);return{top:n.top+this.offset.relative.top*r+this.offset.parent.top*r-(this.cssPosition=="fixed"?-this.scrollParent.scrollTop():o?0:s.scrollTop())*r,left:n.left+this.offset.relative.left*r+this.offset.parent.left*r-(this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():o?0:s.scrollLeft())*r}},_generatePosition:function(t){var n=this.options,r=this.cssPosition!="absolute"||this.scrollParent[0]!=document&&!!e.contains(this.scrollParent[0],this.offsetParent[0])?this.scrollParent:this.offsetParent,i=/(html|body)/i.test(r[0].tagName),s=t.pageX,o=t.pageY;if(this.originalPosition){var u;if(this.containment){if(this.relative_container){var a=this.relative_container.offset();u=[this.containment[0]+a.left,this.containment[1]+a.top,this.containment[2]+a.left,this.containment[3]+a.top]}else u=this.containment;t.pageX-this.offset.click.left<u[0]&&(s=u[0]+this.offset.click.left),t.pageY-this.offset.click.top<u[1]&&(o=u[1]+this.offset.click.top),t.pageX-this.offset.click.left>u[2]&&(s=u[2]+this.offset.click.left),t.pageY-this.offset.click.top>u[3]&&(o=u[3]+this.offset.click.top)}if(n.grid){var f=n.grid[1]?this.originalPageY+Math.round((o-this.originalPageY)/n.grid[1])*n.grid[1]:this.originalPageY;o=u?f-this.offset.click.top<u[1]||f-this.offset.click.top>u[3]?f-this.offset.click.top<u[1]?f+n.grid[1]:f-n.grid[1]:f:f;var l=n.grid[0]?this.originalPageX+Math.round((s-this.originalPageX)/n.grid[0])*n.grid[0]:this.originalPageX;s=u?l-this.offset.click.left<u[0]||l-this.offset.click.left>u[2]?l-this.offset.click.left<u[0]?l+n.grid[0]:l-n.grid[0]:l:l}}return{top:o-this.offset.click.top-this.offset.relative.top-this.offset.parent.top+(this.cssPosition=="fixed"?-this.scrollParent.scrollTop():i?0:r.scrollTop()),left:s-this.offset.click.left-this.offset.relative.left-this.offset.parent.left+(this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():i?0:r.scrollLeft())}},_clear:function(){this.helper.removeClass("ui-draggable-dragging"),this.helper[0]!=this.element[0]&&!this.cancelHelperRemoval&&this.helper.remove(),this.helper=null,this.cancelHelperRemoval=!1},_trigger:function(t,n,r){return r=r||this._uiHash(),e.ui.plugin.call(this,t,[n,r]),t=="drag"&&(this.positionAbs=this._convertPositionTo("absolute")),e.Widget.prototype._trigger.call(this,t,n,r)},plugins:{},_uiHash:function(e){return{helper:this.helper,position:this.position,originalPosition:this.originalPosition,offset:this.positionAbs}}}),e.ui.plugin.add("draggable","connectToSortable",{start:function(t,n){var r=e(this).data("draggable"),i=r.options,s=e.extend({},n,{item:r.element});r.sortables=[],e(i.connectToSortable).each(function(){var n=e.data(this,"sortable");n&&!n.options.disabled&&(r.sortables.push({instance:n,shouldRevert:n.options.revert}),n.refreshPositions(),n._trigger("activate",t,s))})},stop:function(t,n){var r=e(this).data("draggable"),i=e.extend({},n,{item:r.element});e.each(r.sortables,function(){this.instance.isOver?(this.instance.isOver=0,r.cancelHelperRemoval=!0,this.instance.cancelHelperRemoval=!1,this.shouldRevert&&(this.instance.options.revert=!0),this.instance._mouseStop(t),this.instance.options.helper=this.instance.options._helper,r.options.helper=="original"&&this.instance.currentItem.css({top:"auto",left:"auto"})):(this.instance.cancelHelperRemoval=!1,this.instance._trigger("deactivate",t,i))})},drag:function(t,n){var r=e(this).data("draggable"),i=this,s=function(t){var n=this.offset.click.top,r=this.offset.click.left,i=this.positionAbs.top,s=this.positionAbs.left,o=t.height,u=t.width,a=t.top,f=t.left;return e.ui.isOver(i+n,s+r,a,f,o,u)};e.each(r.sortables,function(s){var o=!1,u=this;this.instance.positionAbs=r.positionAbs,this.instance.helperProportions=r.helperProportions,this.instance.offset.click=r.offset.click,this.instance._intersectsWith(this.instance.containerCache)&&(o=!0,e.each(r.sortables,function(){return this.instance.positionAbs=r.positionAbs,this.instance.helperProportions=r.helperProportions,this.instance.offset.click=r.offset.click,this!=u&&this.instance._intersectsWith(this.instance.containerCache)&&e.ui.contains(u.instance.element[0],this.instance.element[0])&&(o=!1),o})),o?(this.instance.isOver||(this.instance.isOver=1,this.instance.currentItem=e(i).clone().removeAttr("id").appendTo(this.instance.element).data("sortable-item",!0),this.instance.options._helper=this.instance.options.helper,this.instance.options.helper=function(){return n.helper[0]},t.target=this.instance.currentItem[0],this.instance._mouseCapture(t,!0),this.instance._mouseStart(t,!0,!0),this.instance.offset.click.top=r.offset.click.top,this.instance.offset.click.left=r.offset.click.left,this.instance.offset.parent.left-=r.offset.parent.left-this.instance.offset.parent.left,this.instance.offset.parent.top-=r.offset.parent.top-this.instance.offset.parent.top,r._trigger("toSortable",t),r.dropped=this.instance.element,r.currentItem=r.element,this.instance.fromOutside=r),this.instance.currentItem&&this.instance._mouseDrag(t)):this.instance.isOver&&(this.instance.isOver=0,this.instance.cancelHelperRemoval=!0,this.instance.options.revert=!1,this.instance._trigger("out",t,this.instance._uiHash(this.instance)),this.instance._mouseStop(t,!0),this.instance.options.helper=this.instance.options._helper,this.instance.currentItem.remove(),this.instance.placeholder&&this.instance.placeholder.remove(),r._trigger("fromSortable",t),r.dropped=!1)})}}),e.ui.plugin.add("draggable","cursor",{start:function(t,n){var r=e("body"),i=e(this).data("draggable").options;r.css("cursor")&&(i._cursor=r.css("cursor")),r.css("cursor",i.cursor)},stop:function(t,n){var r=e(this).data("draggable").options;r._cursor&&e("body").css("cursor",r._cursor)}}),e.ui.plugin.add("draggable","opacity",{start:function(t,n){var r=e(n.helper),i=e(this).data("draggable").options;r.css("opacity")&&(i._opacity=r.css("opacity")),r.css("opacity",i.opacity)},stop:function(t,n){var r=e(this).data("draggable").options;r._opacity&&e(n.helper).css("opacity",r._opacity)}}),e.ui.plugin.add("draggable","scroll",{start:function(t,n){var r=e(this).data("draggable");r.scrollParent[0]!=document&&r.scrollParent[0].tagName!="HTML"&&(r.overflowOffset=r.scrollParent.offset())},drag:function(t,n){var r=e(this).data("draggable"),i=r.options,s=!1;if(r.scrollParent[0]!=document&&r.scrollParent[0].tagName!="HTML"){if(!i.axis||i.axis!="x")r.overflowOffset.top+r.scrollParent[0].offsetHeight-t.pageY<i.scrollSensitivity?r.scrollParent[0].scrollTop=s=r.scrollParent[0].scrollTop+i.scrollSpeed:t.pageY-r.overflowOffset.top<i.scrollSensitivity&&(r.scrollParent[0].scrollTop=s=r.scrollParent[0].scrollTop-i.scrollSpeed);if(!i.axis||i.axis!="y")r.overflowOffset.left+r.scrollParent[0].offsetWidth-t.pageX<i.scrollSensitivity?r.scrollParent[0].scrollLeft=s=r.scrollParent[0].scrollLeft+i.scrollSpeed:t.pageX-r.overflowOffset.left<i.scrollSensitivity&&(r.scrollParent[0].scrollLeft=s=r.scrollParent[0].scrollLeft-i.scrollSpeed)}else{if(!i.axis||i.axis!="x")t.pageY-e(document).scrollTop()<i.scrollSensitivity?s=e(document).scrollTop(e(document).scrollTop()-i.scrollSpeed):e(window).height()-(t.pageY-e(document).scrollTop())<i.scrollSensitivity&&(s=e(document).scrollTop(e(document).scrollTop()+i.scrollSpeed));if(!i.axis||i.axis!="y")t.pageX-e(document).scrollLeft()<i.scrollSensitivity?s=e(document).scrollLeft(e(document).scrollLeft()-i.scrollSpeed):e(window).width()-(t.pageX-e(document).scrollLeft())<i.scrollSensitivity&&(s=e(document).scrollLeft(e(document).scrollLeft()+i.scrollSpeed))}s!==!1&&e.ui.ddmanager&&!i.dropBehaviour&&e.ui.ddmanager.prepareOffsets(r,t)}}),e.ui.plugin.add("draggable","snap",{start:function(t,n){var r=e(this).data("draggable"),i=r.options;r.snapElements=[],e(i.snap.constructor!=String?i.snap.items||":data(draggable)":i.snap).each(function(){var t=e(this),n=t.offset();this!=r.element[0]&&r.snapElements.push({item:this,width:t.outerWidth(),height:t.outerHeight(),top:n.top,left:n.left})})},drag:function(t,n){var r=e(this).data("draggable"),i=r.options,s=i.snapTolerance,o=n.offset.left,u=o+r.helperProportions.width,a=n.offset.top,f=a+r.helperProportions.height;for(var l=r.snapElements.length-1;l>=0;l--){var c=r.snapElements[l].left,h=c+r.snapElements[l].width,p=r.snapElements[l].top,d=p+r.snapElements[l].height;if(!(c-s<o&&o<h+s&&p-s<a&&a<d+s||c-s<o&&o<h+s&&p-s<f&&f<d+s||c-s<u&&u<h+s&&p-s<a&&a<d+s||c-s<u&&u<h+s&&p-s<f&&f<d+s)){r.snapElements[l].snapping&&r.options.snap.release&&r.options.snap.release.call(r.element,t,e.extend(r._uiHash(),{snapItem:r.snapElements[l].item})),r.snapElements[l].snapping=!1;continue}if(i.snapMode!="inner"){var v=Math.abs(p-f)<=s,m=Math.abs(d-a)<=s,g=Math.abs(c-u)<=s,y=Math.abs(h-o)<=s;v&&(n.position.top=r._convertPositionTo("relative",{top:p-r.helperProportions.height,left:0}).top-r.margins.top),m&&(n.position.top=r._convertPositionTo("relative",{top:d,left:0}).top-r.margins.top),g&&(n.position.left=r._convertPositionTo("relative",{top:0,left:c-r.helperProportions.width}).left-r.margins.left),y&&(n.position.left=r._convertPositionTo("relative",{top:0,left:h}).left-r.margins.left)}var b=v||m||g||y;if(i.snapMode!="outer"){var v=Math.abs(p-a)<=s,m=Math.abs(d-f)<=s,g=Math.abs(c-o)<=s,y=Math.abs(h-u)<=s;v&&(n.position.top=r._convertPositionTo("relative",{top:p,left:0}).top-r.margins.top),m&&(n.position.top=r._convertPositionTo("relative",{top:d-r.helperProportions.height,left:0}).top-r.margins.top),g&&(n.position.left=r._convertPositionTo("relative",{top:0,left:c}).left-r.margins.left),y&&(n.position.left=r._convertPositionTo("relative",{top:0,left:h-r.helperProportions.width}).left-r.margins.left)}!r.snapElements[l].snapping&&(v||m||g||y||b)&&r.options.snap.snap&&r.options.snap.snap.call(r.element,t,e.extend(r._uiHash(),{snapItem:r.snapElements[l].item})),r.snapElements[l].snapping=v||m||g||y||b}}}),e.ui.plugin.add("draggable","stack",{start:function(t,n){var r=e(this).data("draggable").options,i=e.makeArray(e(r.stack)).sort(function(t,n){return(parseInt(e(t).css("zIndex"),10)||0)-(parseInt(e(n).css("zIndex"),10)||0)});if(!i.length)return;var s=parseInt(i[0].style.zIndex)||0;e(i).each(function(e){this.style.zIndex=s+e}),this[0].style.zIndex=s+i.length}}),e.ui.plugin.add("draggable","zIndex",{start:function(t,n){var r=e(n.helper),i=e(this).data("draggable").options;r.css("zIndex")&&(i._zIndex=r.css("zIndex")),r.css("zIndex",i.zIndex)},stop:function(t,n){var r=e(this).data("draggable").options;r._zIndex&&e(n.helper).css("zIndex",r._zIndex)}})})(jQuery);(function(e,t){e.widget("ui.droppable",{version:"1.9.1",widgetEventPrefix:"drop",options:{accept:"*",activeClass:!1,addClasses:!0,greedy:!1,hoverClass:!1,scope:"default",tolerance:"intersect"},_create:function(){var t=this.options,n=t.accept;this.isover=0,this.isout=1,this.accept=e.isFunction(n)?n:function(e){return e.is(n)},this.proportions={width:this.element[0].offsetWidth,height:this.element[0].offsetHeight},e.ui.ddmanager.droppables[t.scope]=e.ui.ddmanager.droppables[t.scope]||[],e.ui.ddmanager.droppables[t.scope].push(this),t.addClasses&&this.element.addClass("ui-droppable")},_destroy:function(){var t=e.ui.ddmanager.droppables[this.options.scope];for(var n=0;n<t.length;n++)t[n]==this&&t.splice(n,1);this.element.removeClass("ui-droppable ui-droppable-disabled")},_setOption:function(t,n){t=="accept"&&(this.accept=e.isFunction(n)?n:function(e){return e.is(n)}),e.Widget.prototype._setOption.apply(this,arguments)},_activate:function(t){var n=e.ui.ddmanager.current;this.options.activeClass&&this.element.addClass(this.options.activeClass),n&&this._trigger("activate",t,this.ui(n))},_deactivate:function(t){var n=e.ui.ddmanager.current;this.options.activeClass&&this.element.removeClass(this.options.activeClass),n&&this._trigger("deactivate",t,this.ui(n))},_over:function(t){var n=e.ui.ddmanager.current;if(!n||(n.currentItem||n.element)[0]==this.element[0])return;this.accept.call(this.element[0],n.currentItem||n.element)&&(this.options.hoverClass&&this.element.addClass(this.options.hoverClass),this._trigger("over",t,this.ui(n)))},_out:function(t){var n=e.ui.ddmanager.current;if(!n||(n.currentItem||n.element)[0]==this.element[0])return;this.accept.call(this.element[0],n.currentItem||n.element)&&(this.options.hoverClass&&this.element.removeClass(this.options.hoverClass),this._trigger("out",t,this.ui(n)))},_drop:function(t,n){var r=n||e.ui.ddmanager.current;if(!r||(r.currentItem||r.element)[0]==this.element[0])return!1;var i=!1;return this.element.find(":data(droppable)").not(".ui-draggable-dragging").each(function(){var t=e.data(this,"droppable");if(t.options.greedy&&!t.options.disabled&&t.options.scope==r.options.scope&&t.accept.call(t.element[0],r.currentItem||r.element)&&e.ui.intersect(r,e.extend(t,{offset:t.element.offset()}),t.options.tolerance))return i=!0,!1}),i?!1:this.accept.call(this.element[0],r.currentItem||r.element)?(this.options.activeClass&&this.element.removeClass(this.options.activeClass),this.options.hoverClass&&this.element.removeClass(this.options.hoverClass),this._trigger("drop",t,this.ui(r)),this.element):!1},ui:function(e){return{draggable:e.currentItem||e.element,helper:e.helper,position:e.position,offset:e.positionAbs}}}),e.ui.intersect=function(t,n,r){if(!n.offset)return!1;var i=(t.positionAbs||t.position.absolute).left,s=i+t.helperProportions.width,o=(t.positionAbs||t.position.absolute).top,u=o+t.helperProportions.height,a=n.offset.left,f=a+n.proportions.width,l=n.offset.top,c=l+n.proportions.height;switch(r){case"fit":return a<=i&&s<=f&&l<=o&&u<=c;case"intersect":return a<i+t.helperProportions.width/2&&s-t.helperProportions.width/2<f&&l<o+t.helperProportions.height/2&&u-t.helperProportions.height/2<c;case"pointer":var h=(t.positionAbs||t.position.absolute).left+(t.clickOffset||t.offset.click).left,p=(t.positionAbs||t.position.absolute).top+(t.clickOffset||t.offset.click).top,d=e.ui.isOver(p,h,l,a,n.proportions.height,n.proportions.width);return d;case"touch":return(o>=l&&o<=c||u>=l&&u<=c||o<l&&u>c)&&(i>=a&&i<=f||s>=a&&s<=f||i<a&&s>f);default:return!1}},e.ui.ddmanager={current:null,droppables:{"default":[]},prepareOffsets:function(t,n){var r=e.ui.ddmanager.droppables[t.options.scope]||[],i=n?n.type:null,s=(t.currentItem||t.element).find(":data(droppable)").andSelf();e:for(var o=0;o<r.length;o++){if(r[o].options.disabled||t&&!r[o].accept.call(r[o].element[0],t.currentItem||t.element))continue;for(var u=0;u<s.length;u++)if(s[u]==r[o].element[0]){r[o].proportions.height=0;continue e}r[o].visible=r[o].element.css("display")!="none";if(!r[o].visible)continue;i=="mousedown"&&r[o]._activate.call(r[o],n),r[o].offset=r[o].element.offset(),r[o].proportions={width:r[o].element[0].offsetWidth,height:r[o].element[0].offsetHeight}}},drop:function(t,n){var r=!1;return e.each(e.ui.ddmanager.droppables[t.options.scope]||[],function(){if(!this.options)return;!this.options.disabled&&this.visible&&e.ui.intersect(t,this,this.options.tolerance)&&(r=this._drop.call(this,n)||r),!this.options.disabled&&this.visible&&this.accept.call(this.element[0],t.currentItem||t.element)&&(this.isout=1,this.isover=0,this._deactivate.call(this,n))}),r},dragStart:function(t,n){t.element.parentsUntil("body").bind("scroll.droppable",function(){t.options.refreshPositions||e.ui.ddmanager.prepareOffsets(t,n)})},drag:function(t,n){t.options.refreshPositions&&e.ui.ddmanager.prepareOffsets(t,n),e.each(e.ui.ddmanager.droppables[t.options.scope]||[],function(){if(this.options.disabled||this.greedyChild||!this.visible)return;var r=e.ui.intersect(t,this,this.options.tolerance),i=!r&&this.isover==1?"isout":r&&this.isover==0?"isover":null;if(!i)return;var s;if(this.options.greedy){var o=this.options.scope,u=this.element.parents(":data(droppable)").filter(function(){return e.data(this,"droppable").options.scope===o});u.length&&(s=e.data(u[0],"droppable"),s.greedyChild=i=="isover"?1:0)}s&&i=="isover"&&(s.isover=0,s.isout=1,s._out.call(s,n)),this[i]=1,this[i=="isout"?"isover":"isout"]=0,this[i=="isover"?"_over":"_out"].call(this,n),s&&i=="isout"&&(s.isout=0,s.isover=1,s._over.call(s,n))})},dragStop:function(t,n){t.element.parentsUntil("body").unbind("scroll.droppable"),t.options.refreshPositions||e.ui.ddmanager.prepareOffsets(t,n)}}})(jQuery);jQuery.effects||function(e,t){var n=e.uiBackCompat!==!1,r="ui-effects-";e.effects={effect:{}},function(t,n){function p(e,t,n){var r=a[t.type]||{};return e==null?n||!t.def?null:t.def:(e=r.floor?~~e:parseFloat(e),isNaN(e)?t.def:r.mod?(e+r.mod)%r.mod:0>e?0:r.max<e?r.max:e)}function d(e){var n=o(),r=n._rgba=[];return e=e.toLowerCase(),h(s,function(t,i){var s,o=i.re.exec(e),a=o&&i.parse(o),f=i.space||"rgba";if(a)return s=n[f](a),n[u[f].cache]=s[u[f].cache],r=n._rgba=s._rgba,!1}),r.length?(r.join()==="0,0,0,0"&&t.extend(r,c.transparent),n):c[e]}function v(e,t,n){return n=(n+1)%1,n*6<1?e+(t-e)*n*6:n*2<1?t:n*3<2?e+(t-e)*(2/3-n)*6:e}var r="backgroundColor borderBottomColor borderLeftColor borderRightColor borderTopColor color columnRuleColor outlineColor textDecorationColor textEmphasisColor".split(" "),i=/^([\-+])=\s*(\d+\.?\d*)/,s=[{re:/rgba?\(\s*(\d{1,3})\s*,\s*(\d{1,3})\s*,\s*(\d{1,3})\s*(?:,\s*(\d+(?:\.\d+)?)\s*)?\)/,parse:function(e){return[e[1],e[2],e[3],e[4]]}},{re:/rgba?\(\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*(?:,\s*(\d+(?:\.\d+)?)\s*)?\)/,parse:function(e){return[e[1]*2.55,e[2]*2.55,e[3]*2.55,e[4]]}},{re:/#([a-f0-9]{2})([a-f0-9]{2})([a-f0-9]{2})/,parse:function(e){return[parseInt(e[1],16),parseInt(e[2],16),parseInt(e[3],16)]}},{re:/#([a-f0-9])([a-f0-9])([a-f0-9])/,parse:function(e){return[parseInt(e[1]+e[1],16),parseInt(e[2]+e[2],16),parseInt(e[3]+e[3],16)]}},{re:/hsla?\(\s*(\d+(?:\.\d+)?)\s*,\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*(?:,\s*(\d+(?:\.\d+)?)\s*)?\)/,space:"hsla",parse:function(e){return[e[1],e[2]/100,e[3]/100,e[4]]}}],o=t.Color=function(e,n,r,i){return new t.Color.fn.parse(e,n,r,i)},u={rgba:{props:{red:{idx:0,type:"byte"},green:{idx:1,type:"byte"},blue:{idx:2,type:"byte"}}},hsla:{props:{hue:{idx:0,type:"degrees"},saturation:{idx:1,type:"percent"},lightness:{idx:2,type:"percent"}}}},a={"byte":{floor:!0,max:255},percent:{max:1},degrees:{mod:360,floor:!0}},f=o.support={},l=t("<p>")[0],c,h=t.each;l.style.cssText="background-color:rgba(1,1,1,.5)",f.rgba=l.style.backgroundColor.indexOf("rgba")>-1,h(u,function(e,t){t.cache="_"+e,t.props.alpha={idx:3,type:"percent",def:1}}),o.fn=t.extend(o.prototype,{parse:function(r,i,s,a){if(r===n)return this._rgba=[null,null,null,null],this;if(r.jquery||r.nodeType)r=t(r).css(i),i=n;var f=this,l=t.type(r),v=this._rgba=[];i!==n&&(r=[r,i,s,a],l="array");if(l==="string")return this.parse(d(r)||c._default);if(l==="array")return h(u.rgba.props,function(e,t){v[t.idx]=p(r[t.idx],t)}),this;if(l==="object")return r instanceof o?h(u,function(e,t){r[t.cache]&&(f[t.cache]=r[t.cache].slice())}):h(u,function(t,n){var i=n.cache;h(n.props,function(e,t){if(!f[i]&&n.to){if(e==="alpha"||r[e]==null)return;f[i]=n.to(f._rgba)}f[i][t.idx]=p(r[e],t,!0)}),f[i]&&e.inArray(null,f[i].slice(0,3))<0&&(f[i][3]=1,n.from&&(f._rgba=n.from(f[i])))}),this},is:function(e){var t=o(e),n=!0,r=this;return h(u,function(e,i){var s,o=t[i.cache];return o&&(s=r[i.cache]||i.to&&i.to(r._rgba)||[],h(i.props,function(e,t){if(o[t.idx]!=null)return n=o[t.idx]===s[t.idx],n})),n}),n},_space:function(){var e=[],t=this;return h(u,function(n,r){t[r.cache]&&e.push(n)}),e.pop()},transition:function(e,t){var n=o(e),r=n._space(),i=u[r],s=this.alpha()===0?o("transparent"):this,f=s[i.cache]||i.to(s._rgba),l=f.slice();return n=n[i.cache],h(i.props,function(e,r){var i=r.idx,s=f[i],o=n[i],u=a[r.type]||{};if(o===null)return;s===null?l[i]=o:(u.mod&&(o-s>u.mod/2?s+=u.mod:s-o>u.mod/2&&(s-=u.mod)),l[i]=p((o-s)*t+s,r))}),this[r](l)},blend:function(e){if(this._rgba[3]===1)return this;var n=this._rgba.slice(),r=n.pop(),i=o(e)._rgba;return o(t.map(n,function(e,t){return(1-r)*i[t]+r*e}))},toRgbaString:function(){var e="rgba(",n=t.map(this._rgba,function(e,t){return e==null?t>2?1:0:e});return n[3]===1&&(n.pop(),e="rgb("),e+n.join()+")"},toHslaString:function(){var e="hsla(",n=t.map(this.hsla(),function(e,t){return e==null&&(e=t>2?1:0),t&&t<3&&(e=Math.round(e*100)+"%"),e});return n[3]===1&&(n.pop(),e="hsl("),e+n.join()+")"},toHexString:function(e){var n=this._rgba.slice(),r=n.pop();return e&&n.push(~~(r*255)),"#"+t.map(n,function(e){return e=(e||0).toString(16),e.length===1?"0"+e:e}).join("")},toString:function(){return this._rgba[3]===0?"transparent":this.toRgbaString()}}),o.fn.parse.prototype=o.fn,u.hsla.to=function(e){if(e[0]==null||e[1]==null||e[2]==null)return[null,null,null,e[3]];var t=e[0]/255,n=e[1]/255,r=e[2]/255,i=e[3],s=Math.max(t,n,r),o=Math.min(t,n,r),u=s-o,a=s+o,f=a*.5,l,c;return o===s?l=0:t===s?l=60*(n-r)/u+360:n===s?l=60*(r-t)/u+120:l=60*(t-n)/u+240,f===0||f===1?c=f:f<=.5?c=u/a:c=u/(2-a),[Math.round(l)%360,c,f,i==null?1:i]},u.hsla.from=function(e){if(e[0]==null||e[1]==null||e[2]==null)return[null,null,null,e[3]];var t=e[0]/360,n=e[1],r=e[2],i=e[3],s=r<=.5?r*(1+n):r+n-r*n,o=2*r-s;return[Math.round(v(o,s,t+1/3)*255),Math.round(v(o,s,t)*255),Math.round(v(o,s,t-1/3)*255),i]},h(u,function(e,r){var s=r.props,u=r.cache,a=r.to,f=r.from;o.fn[e]=function(e){a&&!this[u]&&(this[u]=a(this._rgba));if(e===n)return this[u].slice();var r,i=t.type(e),l=i==="array"||i==="object"?e:arguments,c=this[u].slice();return h(s,function(e,t){var n=l[i==="object"?e:t.idx];n==null&&(n=c[t.idx]),c[t.idx]=p(n,t)}),f?(r=o(f(c)),r[u]=c,r):o(c)},h(s,function(n,r){if(o.fn[n])return;o.fn[n]=function(s){var o=t.type(s),u=n==="alpha"?this._hsla?"hsla":"rgba":e,a=this[u](),f=a[r.idx],l;return o==="undefined"?f:(o==="function"&&(s=s.call(this,f),o=t.type(s)),s==null&&r.empty?this:(o==="string"&&(l=i.exec(s),l&&(s=f+parseFloat(l[2])*(l[1]==="+"?1:-1))),a[r.idx]=s,this[u](a)))}})}),h(r,function(e,n){t.cssHooks[n]={set:function(e,r){var i,s,u="";if(t.type(r)!=="string"||(i=d(r))){r=o(i||r);if(!f.rgba&&r._rgba[3]!==1){s=n==="backgroundColor"?e.parentNode:e;while((u===""||u==="transparent")&&s&&s.style)try{u=t.css(s,"backgroundColor"),s=s.parentNode}catch(a){}r=r.blend(u&&u!=="transparent"?u:"_default")}r=r.toRgbaString()}try{e.style[n]=r}catch(l){}}},t.fx.step[n]=function(e){e.colorInit||(e.start=o(e.elem,n),e.end=o(e.end),e.colorInit=!0),t.cssHooks[n].set(e.elem,e.start.transition(e.end,e.pos))}}),t.cssHooks.borderColor={expand:function(e){var t={};return h(["Top","Right","Bottom","Left"],function(n,r){t["border"+r+"Color"]=e}),t}},c=t.Color.names={aqua:"#00ffff",black:"#000000",blue:"#0000ff",fuchsia:"#ff00ff",gray:"#808080",green:"#008000",lime:"#00ff00",maroon:"#800000",navy:"#000080",olive:"#808000",purple:"#800080",red:"#ff0000",silver:"#c0c0c0",teal:"#008080",white:"#ffffff",yellow:"#ffff00",transparent:[null,null,null,0],_default:"#ffffff"}}(jQuery),function(){function i(){var t=this.ownerDocument.defaultView?this.ownerDocument.defaultView.getComputedStyle(this,null):this.currentStyle,n={},r,i;if(t&&t.length&&t[0]&&t[t[0]]){i=t.length;while(i--)r=t[i],typeof t[r]=="string"&&(n[e.camelCase(r)]=t[r])}else for(r in t)typeof t[r]=="string"&&(n[r]=t[r]);return n}function s(t,n){var i={},s,o;for(s in n)o=n[s],t[s]!==o&&!r[s]&&(e.fx.step[s]||!isNaN(parseFloat(o)))&&(i[s]=o);return i}var n=["add","remove","toggle"],r={border:1,borderBottom:1,borderColor:1,borderLeft:1,borderRight:1,borderTop:1,borderWidth:1,margin:1,padding:1};e.each(["borderLeftStyle","borderRightStyle","borderBottomStyle","borderTopStyle"],function(t,n){e.fx.step[n]=function(e){if(e.end!=="none"&&!e.setAttr||e.pos===1&&!e.setAttr)jQuery.style(e.elem,n,e.end),e.setAttr=!0}}),e.effects.animateClass=function(t,r,o,u){var a=e.speed(r,o,u);return this.queue(function(){var r=e(this),o=r.attr("class")||"",u,f=a.children?r.find("*").andSelf():r;f=f.map(function(){var t=e(this);return{el:t,start:i.call(this)}}),u=function(){e.each(n,function(e,n){t[n]&&r[n+"Class"](t[n])})},u(),f=f.map(function(){return this.end=i.call(this.el[0]),this.diff=s(this.start,this.end),this}),r.attr("class",o),f=f.map(function(){var t=this,n=e.Deferred(),r=jQuery.extend({},a,{queue:!1,complete:function(){n.resolve(t)}});return this.el.animate(this.diff,r),n.promise()}),e.when.apply(e,f.get()).done(function(){u(),e.each(arguments,function(){var t=this.el;e.each(this.diff,function(e){t.css(e,"")})}),a.complete.call(r[0])})})},e.fn.extend({_addClass:e.fn.addClass,addClass:function(t,n,r,i){return n?e.effects.animateClass.call(this,{add:t},n,r,i):this._addClass(t)},_removeClass:e.fn.removeClass,removeClass:function(t,n,r,i){return n?e.effects.animateClass.call(this,{remove:t},n,r,i):this._removeClass(t)},_toggleClass:e.fn.toggleClass,toggleClass:function(n,r,i,s,o){return typeof r=="boolean"||r===t?i?e.effects.animateClass.call(this,r?{add:n}:{remove:n},i,s,o):this._toggleClass(n,r):e.effects.animateClass.call(this,{toggle:n},r,i,s)},switchClass:function(t,n,r,i,s){return e.effects.animateClass.call(this,{add:n,remove:t},r,i,s)}})}(),function(){function i(t,n,r,i){e.isPlainObject(t)&&(n=t,t=t.effect),t={effect:t},n==null&&(n={}),e.isFunction(n)&&(i=n,r=null,n={});if(typeof n=="number"||e.fx.speeds[n])i=r,r=n,n={};return e.isFunction(r)&&(i=r,r=null),n&&e.extend(t,n),r=r||n.duration,t.duration=e.fx.off?0:typeof r=="number"?r:r in e.fx.speeds?e.fx.speeds[r]:e.fx.speeds._default,t.complete=i||n.complete,t}function s(t){return!t||typeof t=="number"||e.fx.speeds[t]?!0:typeof t=="string"&&!e.effects.effect[t]?n&&e.effects[t]?!1:!0:!1}e.extend(e.effects,{version:"1.9.1",save:function(e,t){for(var n=0;n<t.length;n++)t[n]!==null&&e.data(r+t[n],e[0].style[t[n]])},restore:function(e,n){var i,s;for(s=0;s<n.length;s++)n[s]!==null&&(i=e.data(r+n[s]),i===t&&(i=""),e.css(n[s],i))},setMode:function(e,t){return t==="toggle"&&(t=e.is(":hidden")?"show":"hide"),t},getBaseline:function(e,t){var n,r;switch(e[0]){case"top":n=0;break;case"middle":n=.5;break;case"bottom":n=1;break;default:n=e[0]/t.height}switch(e[1]){case"left":r=0;break;case"center":r=.5;break;case"right":r=1;break;default:r=e[1]/t.width}return{x:r,y:n}},createWrapper:function(t){if(t.parent().is(".ui-effects-wrapper"))return t.parent();var n={width:t.outerWidth(!0),height:t.outerHeight(!0),"float":t.css("float")},r=e("<div></div>").addClass("ui-effects-wrapper").css({fontSize:"100%",background:"transparent",border:"none",margin:0,padding:0}),i={width:t.width(),height:t.height()},s=document.activeElement;try{s.id}catch(o){s=document.body}return t.wrap(r),(t[0]===s||e.contains(t[0],s))&&e(s).focus(),r=t.parent(),t.css("position")==="static"?(r.css({position:"relative"}),t.css({position:"relative"})):(e.extend(n,{position:t.css("position"),zIndex:t.css("z-index")}),e.each(["top","left","bottom","right"],function(e,r){n[r]=t.css(r),isNaN(parseInt(n[r],10))&&(n[r]="auto")}),t.css({position:"relative",top:0,left:0,right:"auto",bottom:"auto"})),t.css(i),r.css(n).show()},removeWrapper:function(t){var n=document.activeElement;return t.parent().is(".ui-effects-wrapper")&&(t.parent().replaceWith(t),(t[0]===n||e.contains(t[0],n))&&e(n).focus()),t},setTransition:function(t,n,r,i){return i=i||{},e.each(n,function(e,n){var s=t.cssUnit(n);s[0]>0&&(i[n]=s[0]*r+s[1])}),i}}),e.fn.extend({effect:function(){function a(n){function u(){e.isFunction(i)&&i.call(r[0]),e.isFunction(n)&&n()}var r=e(this),i=t.complete,s=t.mode;(r.is(":hidden")?s==="hide":s==="show")?u():o.call(r[0],t,u)}var t=i.apply(this,arguments),r=t.mode,s=t.queue,o=e.effects.effect[t.effect],u=!o&&n&&e.effects[t.effect];return e.fx.off||!o&&!u?r?this[r](t.duration,t.complete):this.each(function(){t.complete&&t.complete.call(this)}):o?s===!1?this.each(a):this.queue(s||"fx",a):u.call(this,{options:t,duration:t.duration,callback:t.complete,mode:t.mode})},_show:e.fn.show,show:function(e){if(s(e))return this._show.apply(this,arguments);var t=i.apply(this,arguments);return t.mode="show",this.effect.call(this,t)},_hide:e.fn.hide,hide:function(e){if(s(e))return this._hide.apply(this,arguments);var t=i.apply(this,arguments);return t.mode="hide",this.effect.call(this,t)},__toggle:e.fn.toggle,toggle:function(t){if(s(t)||typeof t=="boolean"||e.isFunction(t))return this.__toggle.apply(this,arguments);var n=i.apply(this,arguments);return n.mode="toggle",this.effect.call(this,n)},cssUnit:function(t){var n=this.css(t),r=[];return e.each(["em","px","%","pt"],function(e,t){n.indexOf(t)>0&&(r=[parseFloat(n),t])}),r}})}(),function(){var t={};e.each(["Quad","Cubic","Quart","Quint","Expo"],function(e,n){t[n]=function(t){return Math.pow(t,e+2)}}),e.extend(t,{Sine:function(e){return 1-Math.cos(e*Math.PI/2)},Circ:function(e){return 1-Math.sqrt(1-e*e)},Elastic:function(e){return e===0||e===1?e:-Math.pow(2,8*(e-1))*Math.sin(((e-1)*80-7.5)*Math.PI/15)},Back:function(e){return e*e*(3*e-2)},Bounce:function(e){var t,n=4;while(e<((t=Math.pow(2,--n))-1)/11);return 1/Math.pow(4,3-n)-7.5625*Math.pow((t*3-2)/22-e,2)}}),e.each(t,function(t,n){e.easing["easeIn"+t]=n,e.easing["easeOut"+t]=function(e){return 1-n(1-e)},e.easing["easeInOut"+t]=function(e){return e<.5?n(e*2)/2:1-n(e*-2+2)/2}})}()}(jQuery);(function(e,t){var n=/up|down|vertical/,r=/up|left|vertical|horizontal/;e.effects.effect.blind=function(t,i){var s=e(this),o=["position","top","bottom","left","right","height","width"],u=e.effects.setMode(s,t.mode||"hide"),a=t.direction||"up",f=n.test(a),l=f?"height":"width",c=f?"top":"left",h=r.test(a),p={},d=u==="show",v,m,g;s.parent().is(".ui-effects-wrapper")?e.effects.save(s.parent(),o):e.effects.save(s,o),s.show(),v=e.effects.createWrapper(s).css({overflow:"hidden"}),m=v[l](),g=parseFloat(v.css(c))||0,p[l]=d?m:0,h||(s.css(f?"bottom":"right",0).css(f?"top":"left","auto").css({position:"absolute"}),p[c]=d?g:m+g),d&&(v.css(l,0),h||v.css(c,g+m)),v.animate(p,{duration:t.duration,easing:t.easing,queue:!1,complete:function(){u==="hide"&&s.hide(),e.effects.restore(s,o),e.effects.removeWrapper(s),i()}})}})(jQuery);(function(e,t){e.effects.effect.bounce=function(t,n){var r=e(this),i=["position","top","bottom","left","right","height","width"],s=e.effects.setMode(r,t.mode||"effect"),o=s==="hide",u=s==="show",a=t.direction||"up",f=t.distance,l=t.times||5,c=l*2+(u||o?1:0),h=t.duration/c,p=t.easing,d=a==="up"||a==="down"?"top":"left",v=a==="up"||a==="left",m,g,y,b=r.queue(),w=b.length;(u||o)&&i.push("opacity"),e.effects.save(r,i),r.show(),e.effects.createWrapper(r),f||(f=r[d==="top"?"outerHeight":"outerWidth"]()/3),u&&(y={opacity:1},y[d]=0,r.css("opacity",0).css(d,v?-f*2:f*2).animate(y,h,p)),o&&(f/=Math.pow(2,l-1)),y={},y[d]=0;for(m=0;m<l;m++)g={},g[d]=(v?"-=":"+=")+f,r.animate(g,h,p).animate(y,h,p),f=o?f*2:f/2;o&&(g={opacity:0},g[d]=(v?"-=":"+=")+f,r.animate(g,h,p)),r.queue(function(){o&&r.hide(),e.effects.restore(r,i),e.effects.removeWrapper(r),n()}),w>1&&b.splice.apply(b,[1,0].concat(b.splice(w,c+1))),r.dequeue()}})(jQuery);(function(e,t){e.effects.effect.clip=function(t,n){var r=e(this),i=["position","top","bottom","left","right","height","width"],s=e.effects.setMode(r,t.mode||"hide"),o=s==="show",u=t.direction||"vertical",a=u==="vertical",f=a?"height":"width",l=a?"top":"left",c={},h,p,d;e.effects.save(r,i),r.show(),h=e.effects.createWrapper(r).css({overflow:"hidden"}),p=r[0].tagName==="IMG"?h:r,d=p[f](),o&&(p.css(f,0),p.css(l,d/2)),c[f]=o?d:0,c[l]=o?0:d/2,p.animate(c,{queue:!1,duration:t.duration,easing:t.easing,complete:function(){o||r.hide(),e.effects.restore(r,i),e.effects.removeWrapper(r),n()}})}})(jQuery);(function(e,t){e.effects.effect.drop=function(t,n){var r=e(this),i=["position","top","bottom","left","right","opacity","height","width"],s=e.effects.setMode(r,t.mode||"hide"),o=s==="show",u=t.direction||"left",a=u==="up"||u==="down"?"top":"left",f=u==="up"||u==="left"?"pos":"neg",l={opacity:o?1:0},c;e.effects.save(r,i),r.show(),e.effects.createWrapper(r),c=t.distance||r[a==="top"?"outerHeight":"outerWidth"](!0)/2,o&&r.css("opacity",0).css(a,f==="pos"?-c:c),l[a]=(o?f==="pos"?"+=":"-=":f==="pos"?"-=":"+=")+c,r.animate(l,{queue:!1,duration:t.duration,easing:t.easing,complete:function(){s==="hide"&&r.hide(),e.effects.restore(r,i),e.effects.removeWrapper(r),n()}})}})(jQuery);(function(e,t){e.effects.effect.explode=function(t,n){function y(){c.push(this),c.length===r*i&&b()}function b(){s.css({visibility:"visible"}),e(c).remove(),u||s.hide(),n()}var r=t.pieces?Math.round(Math.sqrt(t.pieces)):3,i=r,s=e(this),o=e.effects.setMode(s,t.mode||"hide"),u=o==="show",a=s.show().css("visibility","hidden").offset(),f=Math.ceil(s.outerWidth()/i),l=Math.ceil(s.outerHeight()/r),c=[],h,p,d,v,m,g;for(h=0;h<r;h++){v=a.top+h*l,g=h-(r-1)/2;for(p=0;p<i;p++)d=a.left+p*f,m=p-(i-1)/2,s.clone().appendTo("body").wrap("<div></div>").css({position:"absolute",visibility:"visible",left:-p*f,top:-h*l}).parent().addClass("ui-effects-explode").css({position:"absolute",overflow:"hidden",width:f,height:l,left:d+(u?m*f:0),top:v+(u?g*l:0),opacity:u?0:1}).animate({left:d+(u?0:m*f),top:v+(u?0:g*l),opacity:u?1:0},t.duration||500,t.easing,y)}}})(jQuery);(function(e,t){e.effects.effect.fade=function(t,n){var r=e(this),i=e.effects.setMode(r,t.mode||"toggle");r.animate({opacity:i},{queue:!1,duration:t.duration,easing:t.easing,complete:n})}})(jQuery);(function(e,t){e.effects.effect.fold=function(t,n){var r=e(this),i=["position","top","bottom","left","right","height","width"],s=e.effects.setMode(r,t.mode||"hide"),o=s==="show",u=s==="hide",a=t.size||15,f=/([0-9]+)%/.exec(a),l=!!t.horizFirst,c=o!==l,h=c?["width","height"]:["height","width"],p=t.duration/2,d,v,m={},g={};e.effects.save(r,i),r.show(),d=e.effects.createWrapper(r).css({overflow:"hidden"}),v=c?[d.width(),d.height()]:[d.height(),d.width()],f&&(a=parseInt(f[1],10)/100*v[u?0:1]),o&&d.css(l?{height:0,width:a}:{height:a,width:0}),m[h[0]]=o?v[0]:a,g[h[1]]=o?v[1]:0,d.animate(m,p,t.easing).animate(g,p,t.easing,function(){u&&r.hide(),e.effects.restore(r,i),e.effects.removeWrapper(r),n()})}})(jQuery);(function(e,t){e.effects.effect.highlight=function(t,n){var r=e(this),i=["backgroundImage","backgroundColor","opacity"],s=e.effects.setMode(r,t.mode||"show"),o={backgroundColor:r.css("backgroundColor")};s==="hide"&&(o.opacity=0),e.effects.save(r,i),r.show().css({backgroundImage:"none",backgroundColor:t.color||"#ffff99"}).animate(o,{queue:!1,duration:t.duration,easing:t.easing,complete:function(){s==="hide"&&r.hide(),e.effects.restore(r,i),n()}})}})(jQuery);(function(e,t){e.effects.effect.pulsate=function(t,n){var r=e(this),i=e.effects.setMode(r,t.mode||"show"),s=i==="show",o=i==="hide",u=s||i==="hide",a=(t.times||5)*2+(u?1:0),f=t.duration/a,l=0,c=r.queue(),h=c.length,p;if(s||!r.is(":visible"))r.css("opacity",0).show(),l=1;for(p=1;p<a;p++)r.animate({opacity:l},f,t.easing),l=1-l;r.animate({opacity:l},f,t.easing),r.queue(function(){o&&r.hide(),n()}),h>1&&c.splice.apply(c,[1,0].concat(c.splice(h,a+1))),r.dequeue()}})(jQuery);(function(e,t){e.effects.effect.puff=function(t,n){var r=e(this),i=e.effects.setMode(r,t.mode||"hide"),s=i==="hide",o=parseInt(t.percent,10)||150,u=o/100,a={height:r.height(),width:r.width()};e.extend(t,{effect:"scale",queue:!1,fade:!0,mode:i,complete:n,percent:s?o:100,from:s?a:{height:a.height*u,width:a.width*u}}),r.effect(t)},e.effects.effect.scale=function(t,n){var r=e(this),i=e.extend(!0,{},t),s=e.effects.setMode(r,t.mode||"effect"),o=parseInt(t.percent,10)||(parseInt(t.percent,10)===0?0:s==="hide"?0:100),u=t.direction||"both",a=t.origin,f={height:r.height(),width:r.width(),outerHeight:r.outerHeight(),outerWidth:r.outerWidth()},l={y:u!=="horizontal"?o/100:1,x:u!=="vertical"?o/100:1};i.effect="size",i.queue=!1,i.complete=n,s!=="effect"&&(i.origin=a||["middle","center"],i.restore=!0),i.from=t.from||(s==="show"?{height:0,width:0}:f),i.to={height:f.height*l.y,width:f.width*l.x,outerHeight:f.outerHeight*l.y,outerWidth:f.outerWidth*l.x},i.fade&&(s==="show"&&(i.from.opacity=0,i.to.opacity=1),s==="hide"&&(i.from.opacity=1,i.to.opacity=0)),r.effect(i)},e.effects.effect.size=function(t,n){var r,i,s,o=e(this),u=["position","top","bottom","left","right","width","height","overflow","opacity"],a=["position","top","bottom","left","right","overflow","opacity"],f=["width","height","overflow"],l=["fontSize"],c=["borderTopWidth","borderBottomWidth","paddingTop","paddingBottom"],h=["borderLeftWidth","borderRightWidth","paddingLeft","paddingRight"],p=e.effects.setMode(o,t.mode||"effect"),d=t.restore||p!=="effect",v=t.scale||"both",m=t.origin||["middle","center"],g=o.css("position"),y=d?u:a,b={height:0,width:0};p==="show"&&o.show(),r={height:o.height(),width:o.width(),outerHeight:o.outerHeight(),outerWidth:o.outerWidth()},t.mode==="toggle"&&p==="show"?(o.from=t.to||b,o.to=t.from||r):(o.from=t.from||(p==="show"?b:r),o.to=t.to||(p==="hide"?b:r)),s={from:{y:o.from.height/r.height,x:o.from.width/r.width},to:{y:o.to.height/r.height,x:o.to.width/r.width}};if(v==="box"||v==="both")s.from.y!==s.to.y&&(y=y.concat(c),o.from=e.effects.setTransition(o,c,s.from.y,o.from),o.to=e.effects.setTransition(o,c,s.to.y,o.to)),s.from.x!==s.to.x&&(y=y.concat(h),o.from=e.effects.setTransition(o,h,s.from.x,o.from),o.to=e.effects.setTransition(o,h,s.to.x,o.to));(v==="content"||v==="both")&&s.from.y!==s.to.y&&(y=y.concat(l).concat(f),o.from=e.effects.setTransition(o,l,s.from.y,o.from),o.to=e.effects.setTransition(o,l,s.to.y,o.to)),e.effects.save(o,y),o.show(),e.effects.createWrapper(o),o.css("overflow","hidden").css(o.from),m&&(i=e.effects.getBaseline(m,r),o.from.top=(r.outerHeight-o.outerHeight())*i.y,o.from.left=(r.outerWidth-o.outerWidth())*i.x,o.to.top=(r.outerHeight-o.to.outerHeight)*i.y,o.to.left=(r.outerWidth-o.to.outerWidth)*i.x),o.css(o.from);if(v==="content"||v==="both")c=c.concat(["marginTop","marginBottom"]).concat(l),h=h.concat(["marginLeft","marginRight"]),f=u.concat(c).concat(h),o.find("*[width]").each(function(){var n=e(this),r={height:n.height(),width:n.width()};d&&e.effects.save(n,f),n.from={height:r.height*s.from.y,width:r.width*s.from.x},n.to={height:r.height*s.to.y,width:r.width*s.to.x},s.from.y!==s.to.y&&(n.from=e.effects.setTransition(n,c,s.from.y,n.from),n.to=e.effects.setTransition(n,c,s.to.y,n.to)),s.from.x!==s.to.x&&(n.from=e.effects.setTransition(n,h,s.from.x,n.from),n.to=e.effects.setTransition(n,h,s.to.x,n.to)),n.css(n.from),n.animate(n.to,t.duration,t.easing,function(){d&&e.effects.restore(n,f)})});o.animate(o.to,{queue:!1,duration:t.duration,easing:t.easing,complete:function(){o.to.opacity===0&&o.css("opacity",o.from.opacity),p==="hide"&&o.hide(),e.effects.restore(o,y),d||(g==="static"?o.css({position:"relative",top:o.to.top,left:o.to.left}):e.each(["top","left"],function(e,t){o.css(t,function(t,n){var r=parseInt(n,10),i=e?o.to.left:o.to.top;return n==="auto"?i+"px":r+i+"px"})})),e.effects.removeWrapper(o),n()}})}})(jQuery);(function(e,t){e.effects.effect.shake=function(t,n){var r=e(this),i=["position","top","bottom","left","right","height","width"],s=e.effects.setMode(r,t.mode||"effect"),o=t.direction||"left",u=t.distance||20,a=t.times||3,f=a*2+1,l=Math.round(t.duration/f),c=o==="up"||o==="down"?"top":"left",h=o==="up"||o==="left",p={},d={},v={},m,g=r.queue(),y=g.length;e.effects.save(r,i),r.show(),e.effects.createWrapper(r),p[c]=(h?"-=":"+=")+u,d[c]=(h?"+=":"-=")+u*2,v[c]=(h?"-=":"+=")+u*2,r.animate(p,l,t.easing);for(m=1;m<a;m++)r.animate(d,l,t.easing).animate(v,l,t.easing);r.animate(d,l,t.easing).animate(p,l/2,t.easing).queue(function(){s==="hide"&&r.hide(),e.effects.restore(r,i),e.effects.removeWrapper(r),n()}),y>1&&g.splice.apply(g,[1,0].concat(g.splice(y,f+1))),r.dequeue()}})(jQuery);(function(e,t){e.effects.effect.slide=function(t,n){var r=e(this),i=["position","top","bottom","left","right","width","height"],s=e.effects.setMode(r,t.mode||"show"),o=s==="show",u=t.direction||"left",a=u==="up"||u==="down"?"top":"left",f=u==="up"||u==="left",l,c={};e.effects.save(r,i),r.show(),l=t.distance||r[a==="top"?"outerHeight":"outerWidth"](!0),e.effects.createWrapper(r).css({overflow:"hidden"}),o&&r.css(a,f?isNaN(l)?"-"+l:-l:l),c[a]=(o?f?"+=":"-=":f?"-=":"+=")+l,r.animate(c,{queue:!1,duration:t.duration,easing:t.easing,complete:function(){s==="hide"&&r.hide(),e.effects.restore(r,i),e.effects.removeWrapper(r),n()}})}})(jQuery);(function(e,t){e.effects.effect.transfer=function(t,n){var r=e(this),i=e(t.to),s=i.css("position")==="fixed",o=e("body"),u=s?o.scrollTop():0,a=s?o.scrollLeft():0,f=i.offset(),l={top:f.top-u,left:f.left-a,height:i.innerHeight(),width:i.innerWidth()},c=r.offset(),h=e('<div class="ui-effects-transfer"></div>').appendTo(document.body).addClass(t.className).css({top:c.top-u,left:c.left-a,height:r.innerHeight(),width:r.innerWidth(),position:s?"fixed":"absolute"}).animate(l,t.duration,t.easing,function(){h.remove(),n()})}})(jQuery);(function(e,t){var n=!1;e.widget("ui.menu",{version:"1.9.1",defaultElement:"<ul>",delay:300,options:{icons:{submenu:"ui-icon-carat-1-e"},menus:"ul",position:{my:"left top",at:"right top"},role:"menu",blur:null,focus:null,select:null},_create:function(){this.activeMenu=this.element,this.element.uniqueId().addClass("ui-menu ui-widget ui-widget-content ui-corner-all").toggleClass("ui-menu-icons",!!this.element.find(".ui-icon").length).attr({role:this.options.role,tabIndex:0}).bind("click"+this.eventNamespace,e.proxy(function(e){this.options.disabled&&e.preventDefault()},this)),this.options.disabled&&this.element.addClass("ui-state-disabled").attr("aria-disabled","true"),this._on({"mousedown .ui-menu-item > a":function(e){e.preventDefault()},"click .ui-state-disabled > a":function(e){e.preventDefault()},"click .ui-menu-item:has(a)":function(t){var r=e(t.target).closest(".ui-menu-item");!n&&r.not(".ui-state-disabled").length&&(n=!0,this.select(t),r.has(".ui-menu").length?this.expand(t):this.element.is(":focus")||(this.element.trigger("focus",[!0]),this.active&&this.active.parents(".ui-menu").length===1&&clearTimeout(this.timer)))},"mouseenter .ui-menu-item":function(t){var n=e(t.currentTarget);n.siblings().children(".ui-state-active").removeClass("ui-state-active"),this.focus(t,n)},mouseleave:"collapseAll","mouseleave .ui-menu":"collapseAll",focus:function(e,t){var n=this.active||this.element.children(".ui-menu-item").eq(0);t||this.focus(e,n)},blur:function(t){this._delay(function(){e.contains(this.element[0],this.document[0].activeElement)||this.collapseAll(t)})},keydown:"_keydown"}),this.refresh(),this._on(this.document,{click:function(t){e(t.target).closest(".ui-menu").length||this.collapseAll(t),n=!1}})},_destroy:function(){this.element.removeAttr("aria-activedescendant").find(".ui-menu").andSelf().removeClass("ui-menu ui-widget ui-widget-content ui-corner-all ui-menu-icons").removeAttr("role").removeAttr("tabIndex").removeAttr("aria-labelledby").removeAttr("aria-expanded").removeAttr("aria-hidden").removeAttr("aria-disabled").removeUniqueId().show(),this.element.find(".ui-menu-item").removeClass("ui-menu-item").removeAttr("role").removeAttr("aria-disabled").children("a").removeUniqueId().removeClass("ui-corner-all ui-state-hover").removeAttr("tabIndex").removeAttr("role").removeAttr("aria-haspopup").children().each(function(){var t=e(this);t.data("ui-menu-submenu-carat")&&t.remove()}),this.element.find(".ui-menu-divider").removeClass("ui-menu-divider ui-widget-content")},_keydown:function(t){function a(e){return e.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g,"\\$&")}var n,r,i,s,o,u=!0;switch(t.keyCode){case e.ui.keyCode.PAGE_UP:this.previousPage(t);break;case e.ui.keyCode.PAGE_DOWN:this.nextPage(t);break;case e.ui.keyCode.HOME:this._move("first","first",t);break;case e.ui.keyCode.END:this._move("last","last",t);break;case e.ui.keyCode.UP:this.previous(t);break;case e.ui.keyCode.DOWN:this.next(t);break;case e.ui.keyCode.LEFT:this.collapse(t);break;case e.ui.keyCode.RIGHT:this.active&&!this.active.is(".ui-state-disabled")&&this.expand(t);break;case e.ui.keyCode.ENTER:case e.ui.keyCode.SPACE:this._activate(t);break;case e.ui.keyCode.ESCAPE:this.collapse(t);break;default:u=!1,r=this.previousFilter||"",i=String.fromCharCode(t.keyCode),s=!1,clearTimeout(this.filterTimer),i===r?s=!0:i=r+i,o=new RegExp("^"+a(i),"i"),n=this.activeMenu.children(".ui-menu-item").filter(function(){return o.test(e(this).children("a").text())}),n=s&&n.index(this.active.next())!==-1?this.active.nextAll(".ui-menu-item"):n,n.length||(i=String.fromCharCode(t.keyCode),o=new RegExp("^"+a(i),"i"),n=this.activeMenu.children(".ui-menu-item").filter(function(){return o.test(e(this).children("a").text())})),n.length?(this.focus(t,n),n.length>1?(this.previousFilter=i,this.filterTimer=this._delay(function(){delete this.previousFilter},1e3)):delete this.previousFilter):delete this.previousFilter}u&&t.preventDefault()},_activate:function(e){this.active.is(".ui-state-disabled")||(this.active.children("a[aria-haspopup='true']").length?this.expand(e):this.select(e))},refresh:function(){var t,n=this.options.icons.submenu,r=this.element.find(this.options.menus+":not(.ui-menu)").addClass("ui-menu ui-widget ui-widget-content ui-corner-all").hide().attr({role:this.options.role,"aria-hidden":"true","aria-expanded":"false"});t=r.add(this.element),t.children(":not(.ui-menu-item):has(a)").addClass("ui-menu-item").attr("role","presentation").children("a").uniqueId().addClass("ui-corner-all").attr({tabIndex:-1,role:this._itemRole()}),t.children(":not(.ui-menu-item)").each(function(){var t=e(this);/[^\-—–\s]/.test(t.text())||t.addClass("ui-widget-content ui-menu-divider")}),t.children(".ui-state-disabled").attr("aria-disabled","true"),r.each(function(){var t=e(this),r=t.prev("a"),i=e("<span>").addClass("ui-menu-icon ui-icon "+n).data("ui-menu-submenu-carat",!0);r.attr("aria-haspopup","true").prepend(i),t.attr("aria-labelledby",r.attr("id"))}),this.active&&!e.contains(this.element[0],this.active[0])&&this.blur()},_itemRole:function(){return{menu:"menuitem",listbox:"option"}[this.options.role]},focus:function(e,t){var n,r;this.blur(e,e&&e.type==="focus"),this._scrollIntoView(t),this.active=t.first(),r=this.active.children("a").addClass("ui-state-focus"),this.options.role&&this.element.attr("aria-activedescendant",r.attr("id")),this.active.parent().closest(".ui-menu-item").children("a:first").addClass("ui-state-active"),e&&e.type==="keydown"?this._close():this.timer=this._delay(function(){this._close()},this.delay),n=t.children(".ui-menu"),n.length&&/^mouse/.test(e.type)&&this._startOpening(n),this.activeMenu=t.parent(),this._trigger("focus",e,{item:t})},_scrollIntoView:function(t){var n,r,i,s,o,u;this._hasScroll()&&(n=parseFloat(e.css(this.activeMenu[0],"borderTopWidth"))||0,r=parseFloat(e.css(this.activeMenu[0],"paddingTop"))||0,i=t.offset().top-this.activeMenu.offset().top-n-r,s=this.activeMenu.scrollTop(),o=this.activeMenu.height(),u=t.height(),i<0?this.activeMenu.scrollTop(s+i):i+u>o&&this.activeMenu.scrollTop(s+i-o+u))},blur:function(e,t){t||clearTimeout(this.timer);if(!this.active)return;this.active.children("a").removeClass("ui-state-focus"),this.active=null,this._trigger("blur",e,{item:this.active})},_startOpening:function(e){clearTimeout(this.timer);if(e.attr("aria-hidden")!=="true")return;this.timer=this._delay(function(){this._close(),this._open(e)},this.delay)},_open:function(t){var n=e.extend({of:this.active},this.options.position);clearTimeout(this.timer),this.element.find(".ui-menu").not(t.parents(".ui-menu")).hide().attr("aria-hidden","true"),t.show().removeAttr("aria-hidden").attr("aria-expanded","true").position(n)},collapseAll:function(t,n){clearTimeout(this.timer),this.timer=this._delay(function(){var r=n?this.element:e(t&&t.target).closest(this.element.find(".ui-menu"));r.length||(r=this.element),this._close(r),this.blur(t),this.activeMenu=r},this.delay)},_close:function(e){e||(e=this.active?this.active.parent():this.element),e.find(".ui-menu").hide().attr("aria-hidden","true").attr("aria-expanded","false").end().find("a.ui-state-active").removeClass("ui-state-active")},collapse:function(e){var t=this.active&&this.active.parent().closest(".ui-menu-item",this.element);t&&t.length&&(this._close(),this.focus(e,t))},expand:function(e){var t=this.active&&this.active.children(".ui-menu ").children(".ui-menu-item").first();t&&t.length&&(this._open(t.parent()),this._delay(function(){this.focus(e,t)}))},next:function(e){this._move("next","first",e)},previous:function(e){this._move("prev","last",e)},isFirstItem:function(){return this.active&&!this.active.prevAll(".ui-menu-item").length},isLastItem:function(){return this.active&&!this.active.nextAll(".ui-menu-item").length},_move:function(e,t,n){var r;this.active&&(e==="first"||e==="last"?r=this.active[e==="first"?"prevAll":"nextAll"](".ui-menu-item").eq(-1):r=this.active[e+"All"](".ui-menu-item").eq(0));if(!r||!r.length||!this.active)r=this.activeMenu.children(".ui-menu-item")[t]();this.focus(n,r)},nextPage:function(t){var n,r,i;if(!this.active){this.next(t);return}if(this.isLastItem())return;this._hasScroll()?(r=this.active.offset().top,i=this.element.height(),this.active.nextAll(".ui-menu-item").each(function(){return n=e(this),n.offset().top-r-i<0}),this.focus(t,n)):this.focus(t,this.activeMenu.children(".ui-menu-item")[this.active?"last":"first"]())},previousPage:function(t){var n,r,i;if(!this.active){this.next(t);return}if(this.isFirstItem())return;this._hasScroll()?(r=this.active.offset().top,i=this.element.height(),this.active.prevAll(".ui-menu-item").each(function(){return n=e(this),n.offset().top-r+i>0}),this.focus(t,n)):this.focus(t,this.activeMenu.children(".ui-menu-item").first())},_hasScroll:function(){return this.element.outerHeight()<this.element.prop("scrollHeight")},select:function(t){this.active=this.active||e(t.target).closest(".ui-menu-item");var n={item:this.active};this.active.has(".ui-menu").length||this.collapseAll(t,!0),this._trigger("select",t,n)}})})(jQuery);(function(e,t){e.widget("ui.progressbar",{version:"1.9.1",options:{value:0,max:100},min:0,_create:function(){this.element.addClass("ui-progressbar ui-widget ui-widget-content ui-corner-all").attr({role:"progressbar","aria-valuemin":this.min,"aria-valuemax":this.options.max,"aria-valuenow":this._value()}),this.valueDiv=e("<div class='ui-progressbar-value ui-widget-header ui-corner-left'></div>").appendTo(this.element),this.oldValue=this._value(),this._refreshValue()},_destroy:function(){this.element.removeClass("ui-progressbar ui-widget ui-widget-content ui-corner-all").removeAttr("role").removeAttr("aria-valuemin").removeAttr("aria-valuemax").removeAttr("aria-valuenow"),this.valueDiv.remove()},value:function(e){return e===t?this._value():(this._setOption("value",e),this)},_setOption:function(e,t){e==="value"&&(this.options.value=t,this._refreshValue(),this._value()===this.options.max&&this._trigger("complete")),this._super(e,t)},_value:function(){var e=this.options.value;return typeof e!="number"&&(e=0),Math.min(this.options.max,Math.max(this.min,e))},_percentage:function(){return 100*this._value()/this.options.max},_refreshValue:function(){var e=this.value(),t=this._percentage();this.oldValue!==e&&(this.oldValue=e,this._trigger("change")),this.valueDiv.toggle(e>this.min).toggleClass("ui-corner-right",e===this.options.max).width(t.toFixed(0)+"%"),this.element.attr("aria-valuenow",e)}})})(jQuery);(function(e,t){e.widget("ui.resizable",e.ui.mouse,{version:"1.9.1",widgetEventPrefix:"resize",options:{alsoResize:!1,animate:!1,animateDuration:"slow",animateEasing:"swing",aspectRatio:!1,autoHide:!1,containment:!1,ghost:!1,grid:!1,handles:"e,s,se",helper:!1,maxHeight:null,maxWidth:null,minHeight:10,minWidth:10,zIndex:1e3},_create:function(){var t=this,n=this.options;this.element.addClass("ui-resizable"),e.extend(this,{_aspectRatio:!!n.aspectRatio,aspectRatio:n.aspectRatio,originalElement:this.element,_proportionallyResizeElements:[],_helper:n.helper||n.ghost||n.animate?n.helper||"ui-resizable-helper":null}),this.element[0].nodeName.match(/canvas|textarea|input|select|button|img/i)&&(this.element.wrap(e('<div class="ui-wrapper" style="overflow: hidden;"></div>').css({position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(),top:this.element.css("top"),left:this.element.css("left")})),this.element=this.element.parent().data("resizable",this.element.data("resizable")),this.elementIsWrapper=!0,this.element.css({marginLeft:this.originalElement.css("marginLeft"),marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom")}),this.originalElement.css({marginLeft:0,marginTop:0,marginRight:0,marginBottom:0}),this.originalResizeStyle=this.originalElement.css("resize"),this.originalElement.css("resize","none"),this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"})),this.originalElement.css({margin:this.originalElement.css("margin")}),this._proportionallyResize()),this.handles=n.handles||(e(".ui-resizable-handle",this.element).length?{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne",nw:".ui-resizable-nw"}:"e,s,se");if(this.handles.constructor==String){this.handles=="all"&&(this.handles="n,e,s,w,se,sw,ne,nw");var r=this.handles.split(",");this.handles={};for(var i=0;i<r.length;i++){var s=e.trim(r[i]),o="ui-resizable-"+s,u=e('<div class="ui-resizable-handle '+o+'"></div>');u.css({zIndex:n.zIndex}),"se"==s&&u.addClass("ui-icon ui-icon-gripsmall-diagonal-se"),this.handles[s]=".ui-resizable-"+s,this.element.append(u)}}this._renderAxis=function(t){t=t||this.element;for(var n in this.handles){this.handles[n].constructor==String&&(this.handles[n]=e(this.handles[n],this.element).show());if(this.elementIsWrapper&&this.originalElement[0].nodeName.match(/textarea|input|select|button/i)){var r=e(this.handles[n],this.element),i=0;i=/sw|ne|nw|se|n|s/.test(n)?r.outerHeight():r.outerWidth();var s=["padding",/ne|nw|n/.test(n)?"Top":/se|sw|s/.test(n)?"Bottom":/^e$/.test(n)?"Right":"Left"].join("");t.css(s,i),this._proportionallyResize()}if(!e(this.handles[n]).length)continue}},this._renderAxis(this.element),this._handles=e(".ui-resizable-handle",this.element).disableSelection(),this._handles.mouseover(function(){if(!t.resizing){if(this.className)var e=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i);t.axis=e&&e[1]?e[1]:"se"}}),n.autoHide&&(this._handles.hide(),e(this.element).addClass("ui-resizable-autohide").mouseenter(function(){if(n.disabled)return;e(this).removeClass("ui-resizable-autohide"),t._handles.show()}).mouseleave(function(){if(n.disabled)return;t.resizing||(e(this).addClass("ui-resizable-autohide"),t._handles.hide())})),this._mouseInit()},_destroy:function(){this._mouseDestroy();var t=function(t){e(t).removeClass("ui-resizable ui-resizable-disabled ui-resizable-resizing").removeData("resizable").removeData("ui-resizable").unbind(".resizable").find(".ui-resizable-handle").remove()};if(this.elementIsWrapper){t(this.element);var n=this.element;this.originalElement.css({position:n.css("position"),width:n.outerWidth(),height:n.outerHeight(),top:n.css("top"),left:n.css("left")}).insertAfter(n),n.remove()}return this.originalElement.css("resize",this.originalResizeStyle),t(this.originalElement),this},_mouseCapture:function(t){var n=!1;for(var r in this.handles)e(this.handles[r])[0]==t.target&&(n=!0);return!this.options.disabled&&n},_mouseStart:function(t){var r=this.options,i=this.element.position(),s=this.element;this.resizing=!0,this.documentScroll={top:e(document).scrollTop(),left:e(document).scrollLeft()},(s.is(".ui-draggable")||/absolute/.test(s.css("position")))&&s.css({position:"absolute",top:i.top,left:i.left}),this._renderProxy();var o=n(this.helper.css("left")),u=n(this.helper.css("top"));r.containment&&(o+=e(r.containment).scrollLeft()||0,u+=e(r.containment).scrollTop()||0),this.offset=this.helper.offset(),this.position={left:o,top:u},this.size=this._helper?{width:s.outerWidth(),height:s.outerHeight()}:{width:s.width(),height:s.height()},this.originalSize=this._helper?{width:s.outerWidth(),height:s.outerHeight()}:{width:s.width(),height:s.height()},this.originalPosition={left:o,top:u},this.sizeDiff={width:s.outerWidth()-s.width(),height:s.outerHeight()-s.height()},this.originalMousePosition={left:t.pageX,top:t.pageY},this.aspectRatio=typeof r.aspectRatio=="number"?r.aspectRatio:this.originalSize.width/this.originalSize.height||1;var a=e(".ui-resizable-"+this.axis).css("cursor");return e("body").css("cursor",a=="auto"?this.axis+"-resize":a),s.addClass("ui-resizable-resizing"),this._propagate("start",t),!0},_mouseDrag:function(e){var t=this.helper,n=this.options,r={},i=this,s=this.originalMousePosition,o=this.axis,u=e.pageX-s.left||0,a=e.pageY-s.top||0,f=this._change[o];if(!f)return!1;var l=f.apply(this,[e,u,a]);this._updateVirtualBoundaries(e.shiftKey);if(this._aspectRatio||e.shiftKey)l=this._updateRatio(l,e);return l=this._respectSize(l,e),this._propagate("resize",e),t.css({top:this.position.top+"px",left:this.position.left+"px",width:this.size.width+"px",height:this.size.height+"px"}),!this._helper&&this._proportionallyResizeElements.length&&this._proportionallyResize(),this._updateCache(l),this._trigger("resize",e,this.ui()),!1},_mouseStop:function(t){this.resizing=!1;var n=this.options,r=this;if(this._helper){var i=this._proportionallyResizeElements,s=i.length&&/textarea/i.test(i[0].nodeName),o=s&&e.ui.hasScroll(i[0],"left")?0:r.sizeDiff.height,u=s?0:r.sizeDiff.width,a={width:r.helper.width()-u,height:r.helper.height()-o},f=parseInt(r.element.css("left"),10)+(r.position.left-r.originalPosition.left)||null,l=parseInt(r.element.css("top"),10)+(r.position.top-r.originalPosition.top)||null;n.animate||this.element.css(e.extend(a,{top:l,left:f})),r.helper.height(r.size.height),r.helper.width(r.size.width),this._helper&&!n.animate&&this._proportionallyResize()}return e("body").css("cursor","auto"),this.element.removeClass("ui-resizable-resizing"),this._propagate("stop",t),this._helper&&this.helper.remove(),!1},_updateVirtualBoundaries:function(e){var t=this.options,n,i,s,o,u;u={minWidth:r(t.minWidth)?t.minWidth:0,maxWidth:r(t.maxWidth)?t.maxWidth:Infinity,minHeight:r(t.minHeight)?t.minHeight:0,maxHeight:r(t.maxHeight)?t.maxHeight:Infinity};if(this._aspectRatio||e)n=u.minHeight*this.aspectRatio,s=u.minWidth/this.aspectRatio,i=u.maxHeight*this.aspectRatio,o=u.maxWidth/this.aspectRatio,n>u.minWidth&&(u.minWidth=n),s>u.minHeight&&(u.minHeight=s),i<u.maxWidth&&(u.maxWidth=i),o<u.maxHeight&&(u.maxHeight=o);this._vBoundaries=u},_updateCache:function(e){var t=this.options;this.offset=this.helper.offset(),r(e.left)&&(this.position.left=e.left),r(e.top)&&(this.position.top=e.top),r(e.height)&&(this.size.height=e.height),r(e.width)&&(this.size.width=e.width)},_updateRatio:function(e,t){var n=this.options,i=this.position,s=this.size,o=this.axis;return r(e.height)?e.width=e.height*this.aspectRatio:r(e.width)&&(e.height=e.width/this.aspectRatio),o=="sw"&&(e.left=i.left+(s.width-e.width),e.top=null),o=="nw"&&(e.top=i.top+(s.height-e.height),e.left=i.left+(s.width-e.width)),e},_respectSize:function(e,t){var n=this.helper,i=this._vBoundaries,s=this._aspectRatio||t.shiftKey,o=this.axis,u=r(e.width)&&i.maxWidth&&i.maxWidth<e.width,a=r(e.height)&&i.maxHeight&&i.maxHeight<e.height,f=r(e.width)&&i.minWidth&&i.minWidth>e.width,l=r(e.height)&&i.minHeight&&i.minHeight>e.height;f&&(e.width=i.minWidth),l&&(e.height=i.minHeight),u&&(e.width=i.maxWidth),a&&(e.height=i.maxHeight);var c=this.originalPosition.left+this.originalSize.width,h=this.position.top+this.size.height,p=/sw|nw|w/.test(o),d=/nw|ne|n/.test(o);f&&p&&(e.left=c-i.minWidth),u&&p&&(e.left=c-i.maxWidth),l&&d&&(e.top=h-i.minHeight),a&&d&&(e.top=h-i.maxHeight);var v=!e.width&&!e.height;return v&&!e.left&&e.top?e.top=null:v&&!e.top&&e.left&&(e.left=null),e},_proportionallyResize:function(){var t=this.options;if(!this._proportionallyResizeElements.length)return;var n=this.helper||this.element;for(var r=0;r<this._proportionallyResizeElements.length;r++){var i=this._proportionallyResizeElements[r];if(!this.borderDif){var s=[i.css("borderTopWidth"),i.css("borderRightWidth"),i.css("borderBottomWidth"),i.css("borderLeftWidth")],o=[i.css("paddingTop"),i.css("paddingRight"),i.css("paddingBottom"),i.css("paddingLeft")];this.borderDif=e.map(s,function(e,t){var n=parseInt(e,10)||0,r=parseInt(o[t],10)||0;return n+r})}i.css({height:n.height()-this.borderDif[0]-this.borderDif[2]||0,width:n.width()-this.borderDif[1]-this.borderDif[3]||0})}},_renderProxy:function(){var t=this.element,n=this.options;this.elementOffset=t.offset();if(this._helper){this.helper=this.helper||e('<div style="overflow:hidden;"></div>');var r=e.ui.ie6?1:0,i=e.ui.ie6?2:-1;this.helper.addClass(this._helper).css({width:this.element.outerWidth()+i,height:this.element.outerHeight()+i,position:"absolute",left:this.elementOffset.left-r+"px",top:this.elementOffset.top-r+"px",zIndex:++n.zIndex}),this.helper.appendTo("body").disableSelection()}else this.helper=this.element},_change:{e:function(e,t,n){return{width:this.originalSize.width+t}},w:function(e,t,n){var r=this.options,i=this.originalSize,s=this.originalPosition;return{left:s.left+t,width:i.width-t}},n:function(e,t,n){var r=this.options,i=this.originalSize,s=this.originalPosition;return{top:s.top+n,height:i.height-n}},s:function(e,t,n){return{height:this.originalSize.height+n}},se:function(t,n,r){return e.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[t,n,r]))},sw:function(t,n,r){return e.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[t,n,r]))},ne:function(t,n,r){return e.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[t,n,r]))},nw:function(t,n,r){return e.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[t,n,r]))}},_propagate:function(t,n){e.ui.plugin.call(this,t,[n,this.ui()]),t!="resize"&&this._trigger(t,n,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}}),e.ui.plugin.add("resizable","alsoResize",{start:function(t,n){var r=e(this).data("resizable"),i=r.options,s=function(t){e(t).each(function(){var t=e(this);t.data("resizable-alsoresize",{width:parseInt(t.width(),10),height:parseInt(t.height(),10),left:parseInt(t.css("left"),10),top:parseInt(t.css("top"),10)})})};typeof i.alsoResize=="object"&&!i.alsoResize.parentNode?i.alsoResize.length?(i.alsoResize=i.alsoResize[0],s(i.alsoResize)):e.each(i.alsoResize,function(e){s(e)}):s(i.alsoResize)},resize:function(t,n){var r=e(this).data("resizable"),i=r.options,s=r.originalSize,o=r.originalPosition,u={height:r.size.height-s.height||0,width:r.size.width-s.width||0,top:r.position.top-o.top||0,left:r.position.left-o.left||0},a=function(t,r){e(t).each(function(){var t=e(this),i=e(this).data("resizable-alsoresize"),s={},o=r&&r.length?r:t.parents(n.originalElement[0]).length?["width","height"]:["width","height","top","left"];e.each(o,function(e,t){var n=(i[t]||0)+(u[t]||0);n&&n>=0&&(s[t]=n||null)}),t.css(s)})};typeof i.alsoResize=="object"&&!i.alsoResize.nodeType?e.each(i.alsoResize,function(e,t){a(e,t)}):a(i.alsoResize)},stop:function(t,n){e(this).removeData("resizable-alsoresize")}}),e.ui.plugin.add("resizable","animate",{stop:function(t,n){var r=e(this).data("resizable"),i=r.options,s=r._proportionallyResizeElements,o=s.length&&/textarea/i.test(s[0].nodeName),u=o&&e.ui.hasScroll(s[0],"left")?0:r.sizeDiff.height,a=o?0:r.sizeDiff.width,f={width:r.size.width-a,height:r.size.height-u},l=parseInt(r.element.css("left"),10)+(r.position.left-r.originalPosition.left)||null,c=parseInt(r.element.css("top"),10)+(r.position.top-r.originalPosition.top)||null;r.element.animate(e.extend(f,c&&l?{top:c,left:l}:{}),{duration:i.animateDuration,easing:i.animateEasing,step:function(){var n={width:parseInt(r.element.css("width"),10),height:parseInt(r.element.css("height"),10),top:parseInt(r.element.css("top"),10),left:parseInt(r.element.css("left"),10)};s&&s.length&&e(s[0]).css({width:n.width,height:n.height}),r._updateCache(n),r._propagate("resize",t)}})}}),e.ui.plugin.add("resizable","containment",{start:function(t,r){var i=e(this).data("resizable"),s=i.options,o=i.element,u=s.containment,a=u instanceof e?u.get(0):/parent/.test(u)?o.parent().get(0):u;if(!a)return;i.containerElement=e(a);if(/document/.test(u)||u==document)i.containerOffset={left:0,top:0},i.containerPosition={left:0,top:0},i.parentData={element:e(document),left:0,top:0,width:e(document).width(),height:e(document).height()||document.body.parentNode.scrollHeight};else{var f=e(a),l=[];e(["Top","Right","Left","Bottom"]).each(function(e,t){l[e]=n(f.css("padding"+t))}),i.containerOffset=f.offset(),i.containerPosition=f.position(),i.containerSize={height:f.innerHeight()-l[3],width:f.innerWidth()-l[1]};var c=i.containerOffset,h=i.containerSize.height,p=i.containerSize.width,d=e.ui.hasScroll(a,"left")?a.scrollWidth:p,v=e.ui.hasScroll(a)?a.scrollHeight:h;i.parentData={element:a,left:c.left,top:c.top,width:d,height:v}}},resize:function(t,n){var r=e(this).data("resizable"),i=r.options,s=r.containerSize,o=r.containerOffset,u=r.size,a=r.position,f=r._aspectRatio||t.shiftKey,l={top:0,left:0},c=r.containerElement;c[0]!=document&&/static/.test(c.css("position"))&&(l=o),a.left<(r._helper?o.left:0)&&(r.size.width=r.size.width+(r._helper?r.position.left-o.left:r.position.left-l.left),f&&(r.size.height=r.size.width/r.aspectRatio),r.position.left=i.helper?o.left:0),a.top<(r._helper?o.top:0)&&(r.size.height=r.size.height+(r._helper?r.position.top-o.top:r.position.top),f&&(r.size.width=r.size.height*r.aspectRatio),r.position.top=r._helper?o.top:0),r.offset.left=r.parentData.left+r.position.left,r.offset.top=r.parentData.top+r.position.top;var h=Math.abs((r._helper?r.offset.left-l.left:r.offset.left-l.left)+r.sizeDiff.width),p=Math.abs((r._helper?r.offset.top-l.top:r.offset.top-o.top)+r.sizeDiff.height),d=r.containerElement.get(0)==r.element.parent().get(0),v=/relative|absolute/.test(r.containerElement.css("position"));d&&v&&(h-=r.parentData.left),h+r.size.width>=r.parentData.width&&(r.size.width=r.parentData.width-h,f&&(r.size.height=r.size.width/r.aspectRatio)),p+r.size.height>=r.parentData.height&&(r.size.height=r.parentData.height-p,f&&(r.size.width=r.size.height*r.aspectRatio))},stop:function(t,n){var r=e(this).data("resizable"),i=r.options,s=r.position,o=r.containerOffset,u=r.containerPosition,a=r.containerElement,f=e(r.helper),l=f.offset(),c=f.outerWidth()-r.sizeDiff.width,h=f.outerHeight()-r.sizeDiff.height;r._helper&&!i.animate&&/relative/.test(a.css("position"))&&e(this).css({left:l.left-u.left-o.left,width:c,height:h}),r._helper&&!i.animate&&/static/.test(a.css("position"))&&e(this).css({left:l.left-u.left-o.left,width:c,height:h})}}),e.ui.plugin.add("resizable","ghost",{start:function(t,n){var r=e(this).data("resizable"),i=r.options,s=r.size;r.ghost=r.originalElement.clone(),r.ghost.css({opacity:.25,display:"block",position:"relative",height:s.height,width:s.width,margin:0,left:0,top:0}).addClass("ui-resizable-ghost").addClass(typeof i.ghost=="string"?i.ghost:""),r.ghost.appendTo(r.helper)},resize:function(t,n){var r=e(this).data("resizable"),i=r.options;r.ghost&&r.ghost.css({position:"relative",height:r.size.height,width:r.size.width})},stop:function(t,n){var r=e(this).data("resizable"),i=r.options;r.ghost&&r.helper&&r.helper.get(0).removeChild(r.ghost.get(0))}}),e.ui.plugin.add("resizable","grid",{resize:function(t,n){var r=e(this).data("resizable"),i=r.options,s=r.size,o=r.originalSize,u=r.originalPosition,a=r.axis,f=i._aspectRatio||t.shiftKey;i.grid=typeof i.grid=="number"?[i.grid,i.grid]:i.grid;var l=Math.round((s.width-o.width)/(i.grid[0]||1))*(i.grid[0]||1),c=Math.round((s.height-o.height)/(i.grid[1]||1))*(i.grid[1]||1);/^(se|s|e)$/.test(a)?(r.size.width=o.width+l,r.size.height=o.height+c):/^(ne)$/.test(a)?(r.size.width=o.width+l,r.size.height=o.height+c,r.position.top=u.top-c):/^(sw)$/.test(a)?(r.size.width=o.width+l,r.size.height=o.height+c,r.position.left=u.left-l):(r.size.width=o.width+l,r.size.height=o.height+c,r.position.top=u.top-c,r.position.left=u.left-l)}});var n=function(e){return parseInt(e,10)||0},r=function(e){return!isNaN(parseInt(e,10))}})(jQuery);(function(e,t){e.widget("ui.selectable",e.ui.mouse,{version:"1.9.1",options:{appendTo:"body",autoRefresh:!0,distance:0,filter:"*",tolerance:"touch"},_create:function(){var t=this;this.element.addClass("ui-selectable"),this.dragged=!1;var n;this.refresh=function(){n=e(t.options.filter,t.element[0]),n.addClass("ui-selectee"),n.each(function(){var t=e(this),n=t.offset();e.data(this,"selectable-item",{element:this,$element:t,left:n.left,top:n.top,right:n.left+t.outerWidth(),bottom:n.top+t.outerHeight(),startselected:!1,selected:t.hasClass("ui-selected"),selecting:t.hasClass("ui-selecting"),unselecting:t.hasClass("ui-unselecting")})})},this.refresh(),this.selectees=n.addClass("ui-selectee"),this._mouseInit(),this.helper=e("<div class='ui-selectable-helper'></div>")},_destroy:function(){this.selectees.removeClass("ui-selectee").removeData("selectable-item"),this.element.removeClass("ui-selectable ui-selectable-disabled"),this._mouseDestroy()},_mouseStart:function(t){var n=this;this.opos=[t.pageX,t.pageY];if(this.options.disabled)return;var r=this.options;this.selectees=e(r.filter,this.element[0]),this._trigger("start",t),e(r.appendTo).append(this.helper),this.helper.css({left:t.clientX,top:t.clientY,width:0,height:0}),r.autoRefresh&&this.refresh(),this.selectees.filter(".ui-selected").each(function(){var r=e.data(this,"selectable-item");r.startselected=!0,!t.metaKey&&!t.ctrlKey&&(r.$element.removeClass("ui-selected"),r.selected=!1,r.$element.addClass("ui-unselecting"),r.unselecting=!0,n._trigger("unselecting",t,{unselecting:r.element}))}),e(t.target).parents().andSelf().each(function(){var r=e.data(this,"selectable-item");if(r){var i=!t.metaKey&&!t.ctrlKey||!r.$element.hasClass("ui-selected");return r.$element.removeClass(i?"ui-unselecting":"ui-selected").addClass(i?"ui-selecting":"ui-unselecting"),r.unselecting=!i,r.selecting=i,r.selected=i,i?n._trigger("selecting",t,{selecting:r.element}):n._trigger("unselecting",t,{unselecting:r.element}),!1}})},_mouseDrag:function(t){var n=this;this.dragged=!0;if(this.options.disabled)return;var r=this.options,i=this.opos[0],s=this.opos[1],o=t.pageX,u=t.pageY;if(i>o){var a=o;o=i,i=a}if(s>u){var a=u;u=s,s=a}return this.helper.css({left:i,top:s,width:o-i,height:u-s}),this.selectees.each(function(){var a=e.data(this,"selectable-item");if(!a||a.element==n.element[0])return;var f=!1;r.tolerance=="touch"?f=!(a.left>o||a.right<i||a.top>u||a.bottom<s):r.tolerance=="fit"&&(f=a.left>i&&a.right<o&&a.top>s&&a.bottom<u),f?(a.selected&&(a.$element.removeClass("ui-selected"),a.selected=!1),a.unselecting&&(a.$element.removeClass("ui-unselecting"),a.unselecting=!1),a.selecting||(a.$element.addClass("ui-selecting"),a.selecting=!0,n._trigger("selecting",t,{selecting:a.element}))):(a.selecting&&((t.metaKey||t.ctrlKey)&&a.startselected?(a.$element.removeClass("ui-selecting"),a.selecting=!1,a.$element.addClass("ui-selected"),a.selected=!0):(a.$element.removeClass("ui-selecting"),a.selecting=!1,a.startselected&&(a.$element.addClass("ui-unselecting"),a.unselecting=!0),n._trigger("unselecting",t,{unselecting:a.element}))),a.selected&&!t.metaKey&&!t.ctrlKey&&!a.startselected&&(a.$element.removeClass("ui-selected"),a.selected=!1,a.$element.addClass("ui-unselecting"),a.unselecting=!0,n._trigger("unselecting",t,{unselecting:a.element})))}),!1},_mouseStop:function(t){var n=this;this.dragged=!1;var r=this.options;return e(".ui-unselecting",this.element[0]).each(function(){var r=e.data(this,"selectable-item");r.$element.removeClass("ui-unselecting"),r.unselecting=!1,r.startselected=!1,n._trigger("unselected",t,{unselected:r.element})}),e(".ui-selecting",this.element[0]).each(function(){var r=e.data(this,"selectable-item");r.$element.removeClass("ui-selecting").addClass("ui-selected"),r.selecting=!1,r.selected=!0,r.startselected=!0,n._trigger("selected",t,{selected:r.element})}),this._trigger("stop",t),this.helper.remove(),!1}})})(jQuery);(function(e,t){var n=5;e.widget("ui.slider",e.ui.mouse,{version:"1.9.1",widgetEventPrefix:"slide",options:{animate:!1,distance:0,max:100,min:0,orientation:"horizontal",range:!1,step:1,value:0,values:null},_create:function(){var t,r,i=this.options,s=this.element.find(".ui-slider-handle").addClass("ui-state-default ui-corner-all"),o="<a class='ui-slider-handle ui-state-default ui-corner-all' href='#'></a>",u=[];this._keySliding=!1,this._mouseSliding=!1,this._animateOff=!0,this._handleIndex=null,this._detectOrientation(),this._mouseInit(),this.element.addClass("ui-slider ui-slider-"+this.orientation+" ui-widget"+" ui-widget-content"+" ui-corner-all"+(i.disabled?" ui-slider-disabled ui-disabled":"")),this.range=e([]),i.range&&(i.range===!0&&(i.values||(i.values=[this._valueMin(),this._valueMin()]),i.values.length&&i.values.length!==2&&(i.values=[i.values[0],i.values[0]])),this.range=e("<div></div>").appendTo(this.element).addClass("ui-slider-range ui-widget-header"+(i.range==="min"||i.range==="max"?" ui-slider-range-"+i.range:""))),r=i.values&&i.values.length||1;for(t=s.length;t<r;t++)u.push(o);this.handles=s.add(e(u.join("")).appendTo(this.element)),this.handle=this.handles.eq(0),this.handles.add(this.range).filter("a").click(function(e){e.preventDefault()}).mouseenter(function(){i.disabled||e(this).addClass("ui-state-hover")}).mouseleave(function(){e(this).removeClass("ui-state-hover")}).focus(function(){i.disabled?e(this).blur():(e(".ui-slider .ui-state-focus").removeClass("ui-state-focus"),e(this).addClass("ui-state-focus"))}).blur(function(){e(this).removeClass("ui-state-focus")}),this.handles.each(function(t){e(this).data("ui-slider-handle-index",t)}),this._on(this.handles,{keydown:function(t){var r,i,s,o,u=e(t.target).data("ui-slider-handle-index");switch(t.keyCode){case e.ui.keyCode.HOME:case e.ui.keyCode.END:case e.ui.keyCode.PAGE_UP:case e.ui.keyCode.PAGE_DOWN:case e.ui.keyCode.UP:case e.ui.keyCode.RIGHT:case e.ui.keyCode.DOWN:case e.ui.keyCode.LEFT:t.preventDefault();if(!this._keySliding){this._keySliding=!0,e(t.target).addClass("ui-state-active"),r=this._start(t,u);if(r===!1)return}}o=this.options.step,this.options.values&&this.options.values.length?i=s=this.values(u):i=s=this.value();switch(t.keyCode){case e.ui.keyCode.HOME:s=this._valueMin();break;case e.ui.keyCode.END:s=this._valueMax();break;case e.ui.keyCode.PAGE_UP:s=this._trimAlignValue(i+(this._valueMax()-this._valueMin())/n);break;case e.ui.keyCode.PAGE_DOWN:s=this._trimAlignValue(i-(this._valueMax()-this._valueMin())/n);break;case e.ui.keyCode.UP:case e.ui.keyCode.RIGHT:if(i===this._valueMax())return;s=this._trimAlignValue(i+o);break;case e.ui.keyCode.DOWN:case e.ui.keyCode.LEFT:if(i===this._valueMin())return;s=this._trimAlignValue(i-o)}this._slide(t,u,s)},keyup:function(t){var n=e(t.target).data("ui-slider-handle-index");this._keySliding&&(this._keySliding=!1,this._stop(t,n),this._change(t,n),e(t.target).removeClass("ui-state-active"))}}),this._refreshValue(),this._animateOff=!1},_destroy:function(){this.handles.remove(),this.range.remove(),this.element.removeClass("ui-slider ui-slider-horizontal ui-slider-vertical ui-slider-disabled ui-widget ui-widget-content ui-corner-all"),this._mouseDestroy()},_mouseCapture:function(t){var n,r,i,s,o,u,a,f,l=this,c=this.options;return c.disabled?!1:(this.elementSize={width:this.element.outerWidth(),height:this.element.outerHeight()},this.elementOffset=this.element.offset(),n={x:t.pageX,y:t.pageY},r=this._normValueFromMouse(n),i=this._valueMax()-this._valueMin()+1,this.handles.each(function(t){var n=Math.abs(r-l.values(t));i>n&&(i=n,s=e(this),o=t)}),c.range===!0&&this.values(1)===c.min&&(o+=1,s=e(this.handles[o])),u=this._start(t,o),u===!1?!1:(this._mouseSliding=!0,this._handleIndex=o,s.addClass("ui-state-active").focus(),a=s.offset(),f=!e(t.target).parents().andSelf().is(".ui-slider-handle"),this._clickOffset=f?{left:0,top:0}:{left:t.pageX-a.left-s.width()/2,top:t.pageY-a.top-s.height()/2-(parseInt(s.css("borderTopWidth"),10)||0)-(parseInt(s.css("borderBottomWidth"),10)||0)+(parseInt(s.css("marginTop"),10)||0)},this.handles.hasClass("ui-state-hover")||this._slide(t,o,r),this._animateOff=!0,!0))},_mouseStart:function(){return!0},_mouseDrag:function(e){var t={x:e.pageX,y:e.pageY},n=this._normValueFromMouse(t);return this._slide(e,this._handleIndex,n),!1},_mouseStop:function(e){return this.handles.removeClass("ui-state-active"),this._mouseSliding=!1,this._stop(e,this._handleIndex),this._change(e,this._handleIndex),this._handleIndex=null,this._clickOffset=null,this._animateOff=!1,!1},_detectOrientation:function(){this.orientation=this.options.orientation==="vertical"?"vertical":"horizontal"},_normValueFromMouse:function(e){var t,n,r,i,s;return this.orientation==="horizontal"?(t=this.elementSize.width,n=e.x-this.elementOffset.left-(this._clickOffset?this._clickOffset.left:0)):(t=this.elementSize.height,n=e.y-this.elementOffset.top-(this._clickOffset?this._clickOffset.top:0)),r=n/t,r>1&&(r=1),r<0&&(r=0),this.orientation==="vertical"&&(r=1-r),i=this._valueMax()-this._valueMin(),s=this._valueMin()+r*i,this._trimAlignValue(s)},_start:function(e,t){var n={handle:this.handles[t],value:this.value()};return this.options.values&&this.options.values.length&&(n.value=this.values(t),n.values=this.values()),this._trigger("start",e,n)},_slide:function(e,t,n){var r,i,s;this.options.values&&this.options.values.length?(r=this.values(t?0:1),this.options.values.length===2&&this.options.range===!0&&(t===0&&n>r||t===1&&n<r)&&(n=r),n!==this.values(t)&&(i=this.values(),i[t]=n,s=this._trigger("slide",e,{handle:this.handles[t],value:n,values:i}),r=this.values(t?0:1),s!==!1&&this.values(t,n,!0))):n!==this.value()&&(s=this._trigger("slide",e,{handle:this.handles[t],value:n}),s!==!1&&this.value(n))},_stop:function(e,t){var n={handle:this.handles[t],value:this.value()};this.options.values&&this.options.values.length&&(n.value=this.values(t),n.values=this.values()),this._trigger("stop",e,n)},_change:function(e,t){if(!this._keySliding&&!this._mouseSliding){var n={handle:this.handles[t],value:this.value()};this.options.values&&this.options.values.length&&(n.value=this.values(t),n.values=this.values()),this._trigger("change",e,n)}},value:function(e){if(arguments.length){this.options.value=this._trimAlignValue(e),this._refreshValue(),this._change(null,0);return}return this._value()},values:function(t,n){var r,i,s;if(arguments.length>1){this.options.values[t]=this._trimAlignValue(n),this._refreshValue(),this._change(null,t);return}if(!arguments.length)return this._values();if(!e.isArray(arguments[0]))return this.options.values&&this.options.values.length?this._values(t):this.value();r=this.options.values,i=arguments[0];for(s=0;s<r.length;s+=1)r[s]=this._trimAlignValue(i[s]),this._change(null,s);this._refreshValue()},_setOption:function(t,n){var r,i=0;e.isArray(this.options.values)&&(i=this.options.values.length),e.Widget.prototype._setOption.apply(this,arguments);switch(t){case"disabled":n?(this.handles.filter(".ui-state-focus").blur(),this.handles.removeClass("ui-state-hover"),this.handles.prop("disabled",!0),this.element.addClass("ui-disabled")):(this.handles.prop("disabled",!1),this.element.removeClass("ui-disabled"));break;case"orientation":this._detectOrientation(),this.element.removeClass("ui-slider-horizontal ui-slider-vertical").addClass("ui-slider-"+this.orientation),this._refreshValue();break;case"value":this._animateOff=!0,this._refreshValue(),this._change(null,0),this._animateOff=!1;break;case"values":this._animateOff=!0,this._refreshValue();for(r=0;r<i;r+=1)this._change(null,r);this._animateOff=!1;break;case"min":case"max":this._animateOff=!0,this._refreshValue(),this._animateOff=!1}},_value:function(){var e=this.options.value;return e=this._trimAlignValue(e),e},_values:function(e){var t,n,r;if(arguments.length)return t=this.options.values[e],t=this._trimAlignValue(t),t;n=this.options.values.slice();for(r=0;r<n.length;r+=1)n[r]=this._trimAlignValue(n[r]);return n},_trimAlignValue:function(e){if(e<=this._valueMin())return this._valueMin();if(e>=this._valueMax())return this._valueMax();var t=this.options.step>0?this.options.step:1,n=(e-this._valueMin())%t,r=e-n;return Math.abs(n)*2>=t&&(r+=n>0?t:-t),parseFloat(r.toFixed(5))},_valueMin:function(){return this.options.min},_valueMax:function(){return this.options.max},_refreshValue:function(){var t,n,r,i,s,o=this.options.range,u=this.options,a=this,f=this._animateOff?!1:u.animate,l={};this.options.values&&this.options.values.length?this.handles.each(function(r){n=(a.values(r)-a._valueMin())/(a._valueMax()-a._valueMin())*100,l[a.orientation==="horizontal"?"left":"bottom"]=n+"%",e(this).stop(1,1)[f?"animate":"css"](l,u.animate),a.options.range===!0&&(a.orientation==="horizontal"?(r===0&&a.range.stop(1,1)[f?"animate":"css"]({left:n+"%"},u.animate),r===1&&a.range[f?"animate":"css"]({width:n-t+"%"},{queue:!1,duration:u.animate})):(r===0&&a.range.stop(1,1)[f?"animate":"css"]({bottom:n+"%"},u.animate),r===1&&a.range[f?"animate":"css"]({height:n-t+"%"},{queue:!1,duration:u.animate}))),t=n}):(r=this.value(),i=this._valueMin(),s=this._valueMax(),n=s!==i?(r-i)/(s-i)*100:0,l[this.orientation==="horizontal"?"left":"bottom"]=n+"%",this.handle.stop(1,1)[f?"animate":"css"](l,u.animate),o==="min"&&this.orientation==="horizontal"&&this.range.stop(1,1)[f?"animate":"css"]({width:n+"%"},u.animate),o==="max"&&this.orientation==="horizontal"&&this.range[f?"animate":"css"]({width:100-n+"%"},{queue:!1,duration:u.animate}),o==="min"&&this.orientation==="vertical"&&this.range.stop(1,1)[f?"animate":"css"]({height:n+"%"},u.animate),o==="max"&&this.orientation==="vertical"&&this.range[f?"animate":"css"]({height:100-n+"%"},{queue:!1,duration:u.animate}))}})})(jQuery);(function(e,t){e.widget("ui.sortable",e.ui.mouse,{version:"1.9.1",widgetEventPrefix:"sort",ready:!1,options:{appendTo:"parent",axis:!1,connectWith:!1,containment:!1,cursor:"auto",cursorAt:!1,dropOnEmpty:!0,forcePlaceholderSize:!1,forceHelperSize:!1,grid:!1,handle:!1,helper:"original",items:"> *",opacity:!1,placeholder:!1,revert:!1,scroll:!0,scrollSensitivity:20,scrollSpeed:20,scope:"default",tolerance:"intersect",zIndex:1e3},_create:function(){var e=this.options;this.containerCache={},this.element.addClass("ui-sortable"),this.refresh(),this.floating=this.items.length?e.axis==="x"||/left|right/.test(this.items[0].item.css("float"))||/inline|table-cell/.test(this.items[0].item.css("display")):!1,this.offset=this.element.offset(),this._mouseInit(),this.ready=!0},_destroy:function(){this.element.removeClass("ui-sortable ui-sortable-disabled"),this._mouseDestroy();for(var e=this.items.length-1;e>=0;e--)this.items[e].item.removeData(this.widgetName+"-item");return this},_setOption:function(t,n){t==="disabled"?(this.options[t]=n,this.widget().toggleClass("ui-sortable-disabled",!!n)):e.Widget.prototype._setOption.apply(this,arguments)},_mouseCapture:function(t,n){var r=this;if(this.reverting)return!1;if(this.options.disabled||this.options.type=="static")return!1;this._refreshItems(t);var i=null,s=e(t.target).parents().each(function(){if(e.data(this,r.widgetName+"-item")==r)return i=e(this),!1});e.data(t.target,r.widgetName+"-item")==r&&(i=e(t.target));if(!i)return!1;if(this.options.handle&&!n){var o=!1;e(this.options.handle,i).find("*").andSelf().each(function(){this==t.target&&(o=!0)});if(!o)return!1}return this.currentItem=i,this._removeCurrentsFromItems(),!0},_mouseStart:function(t,n,r){var i=this.options;this.currentContainer=this,this.refreshPositions(),this.helper=this._createHelper(t),this._cacheHelperProportions(),this._cacheMargins(),this.scrollParent=this.helper.scrollParent(),this.offset=this.currentItem.offset(),this.offset={top:this.offset.top-this.margins.top,left:this.offset.left-this.margins.left},e.extend(this.offset,{click:{left:t.pageX-this.offset.left,top:t.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()}),this.helper.css("position","absolute"),this.cssPosition=this.helper.css("position"),this.originalPosition=this._generatePosition(t),this.originalPageX=t.pageX,this.originalPageY=t.pageY,i.cursorAt&&this._adjustOffsetFromHelper(i.cursorAt),this.domPosition={prev:this.currentItem.prev()[0],parent:this.currentItem.parent()[0]},this.helper[0]!=this.currentItem[0]&&this.currentItem.hide(),this._createPlaceholder(),i.containment&&this._setContainment(),i.cursor&&(e("body").css("cursor")&&(this._storedCursor=e("body").css("cursor")),e("body").css("cursor",i.cursor)),i.opacity&&(this.helper.css("opacity")&&(this._storedOpacity=this.helper.css("opacity")),this.helper.css("opacity",i.opacity)),i.zIndex&&(this.helper.css("zIndex")&&(this._storedZIndex=this.helper.css("zIndex")),this.helper.css("zIndex",i.zIndex)),this.scrollParent[0]!=document&&this.scrollParent[0].tagName!="HTML"&&(this.overflowOffset=this.scrollParent.offset()),this._trigger("start",t,this._uiHash()),this._preserveHelperProportions||this._cacheHelperProportions();if(!r)for(var s=this.containers.length-1;s>=0;s--)this.containers[s]._trigger("activate",t,this._uiHash(this));return e.ui.ddmanager&&(e.ui.ddmanager.current=this),e.ui.ddmanager&&!i.dropBehaviour&&e.ui.ddmanager.prepareOffsets(this,t),this.dragging=!0,this.helper.addClass("ui-sortable-helper"),this._mouseDrag(t),!0},_mouseDrag:function(t){this.position=this._generatePosition(t),this.positionAbs=this._convertPositionTo("absolute"),this.lastPositionAbs||(this.lastPositionAbs=this.positionAbs);if(this.options.scroll){var n=this.options,r=!1;this.scrollParent[0]!=document&&this.scrollParent[0].tagName!="HTML"?(this.overflowOffset.top+this.scrollParent[0].offsetHeight-t.pageY<n.scrollSensitivity?this.scrollParent[0].scrollTop=r=this.scrollParent[0].scrollTop+n.scrollSpeed:t.pageY-this.overflowOffset.top<n.scrollSensitivity&&(this.scrollParent[0].scrollTop=r=this.scrollParent[0].scrollTop-n.scrollSpeed),this.overflowOffset.left+this.scrollParent[0].offsetWidth-t.pageX<n.scrollSensitivity?this.scrollParent[0].scrollLeft=r=this.scrollParent[0].scrollLeft+n.scrollSpeed:t.pageX-this.overflowOffset.left<n.scrollSensitivity&&(this.scrollParent[0].scrollLeft=r=this.scrollParent[0].scrollLeft-n.scrollSpeed)):(t.pageY-e(document).scrollTop()<n.scrollSensitivity?r=e(document).scrollTop(e(document).scrollTop()-n.scrollSpeed):e(window).height()-(t.pageY-e(document).scrollTop())<n.scrollSensitivity&&(r=e(document).scrollTop(e(document).scrollTop()+n.scrollSpeed)),t.pageX-e(document).scrollLeft()<n.scrollSensitivity?r=e(document).scrollLeft(e(document).scrollLeft()-n.scrollSpeed):e(window).width()-(t.pageX-e(document).scrollLeft())<n.scrollSensitivity&&(r=e(document).scrollLeft(e(document).scrollLeft()+n.scrollSpeed))),r!==!1&&e.ui.ddmanager&&!n.dropBehaviour&&e.ui.ddmanager.prepareOffsets(this,t)}this.positionAbs=this._convertPositionTo("absolute");if(!this.options.axis||this.options.axis!="y")this.helper[0].style.left=this.position.left+"px";if(!this.options.axis||this.options.axis!="x")this.helper[0].style.top=this.position.top+"px";for(var i=this.items.length-1;i>=0;i--){var s=this.items[i],o=s.item[0],u=this._intersectsWithPointer(s);if(!u)continue;if(s.instance!==this.currentContainer)continue;if(o!=this.currentItem[0]&&this.placeholder[u==1?"next":"prev"]()[0]!=o&&!e.contains(this.placeholder[0],o)&&(this.options.type=="semi-dynamic"?!e.contains(this.element[0],o):!0)){this.direction=u==1?"down":"up";if(this.options.tolerance!="pointer"&&!this._intersectsWithSides(s))break;this._rearrange(t,s),this._trigger("change",t,this._uiHash());break}}return this._contactContainers(t),e.ui.ddmanager&&e.ui.ddmanager.drag(this,t),this._trigger("sort",t,this._uiHash()),this.lastPositionAbs=this.positionAbs,!1},_mouseStop:function(t,n){if(!t)return;e.ui.ddmanager&&!this.options.dropBehaviour&&e.ui.ddmanager.drop(this,t);if(this.options.revert){var r=this,i=this.placeholder.offset();this.reverting=!0,e(this.helper).animate({left:i.left-this.offset.parent.left-this.margins.left+(this.offsetParent[0]==document.body?0:this.offsetParent[0].scrollLeft),top:i.top-this.offset.parent.top-this.margins.top+(this.offsetParent[0]==document.body?0:this.offsetParent[0].scrollTop)},parseInt(this.options.revert,10)||500,function(){r._clear(t)})}else this._clear(t,n);return!1},cancel:function(){if(this.dragging){this._mouseUp({target:null}),this.options.helper=="original"?this.currentItem.css(this._storedCSS).removeClass("ui-sortable-helper"):this.currentItem.show();for(var t=this.containers.length-1;t>=0;t--)this.containers[t]._trigger("deactivate",null,this._uiHash(this)),this.containers[t].containerCache.over&&(this.containers[t]._trigger("out",null,this._uiHash(this)),this.containers[t].containerCache.over=0)}return this.placeholder&&(this.placeholder[0].parentNode&&this.placeholder[0].parentNode.removeChild(this.placeholder[0]),this.options.helper!="original"&&this.helper&&this.helper[0].parentNode&&this.helper.remove(),e.extend(this,{helper:null,dragging:!1,reverting:!1,_noFinalSort:null}),this.domPosition.prev?e(this.domPosition.prev).after(this.currentItem):e(this.domPosition.parent).prepend(this.currentItem)),this},serialize:function(t){var n=this._getItemsAsjQuery(t&&t.connected),r=[];return t=t||{},e(n).each(function(){var n=(e(t.item||this).attr(t.attribute||"id")||"").match(t.expression||/(.+)[-=_](.+)/);n&&r.push((t.key||n[1]+"[]")+"="+(t.key&&t.expression?n[1]:n[2]))}),!r.length&&t.key&&r.push(t.key+"="),r.join("&")},toArray:function(t){var n=this._getItemsAsjQuery(t&&t.connected),r=[];return t=t||{},n.each(function(){r.push(e(t.item||this).attr(t.attribute||"id")||"")}),r},_intersectsWith:function(e){var t=this.positionAbs.left,n=t+this.helperProportions.width,r=this.positionAbs.top,i=r+this.helperProportions.height,s=e.left,o=s+e.width,u=e.top,a=u+e.height,f=this.offset.click.top,l=this.offset.click.left,c=r+f>u&&r+f<a&&t+l>s&&t+l<o;return this.options.tolerance=="pointer"||this.options.forcePointerForContainers||this.options.tolerance!="pointer"&&this.helperProportions[this.floating?"width":"height"]>e[this.floating?"width":"height"]?c:s<t+this.helperProportions.width/2&&n-this.helperProportions.width/2<o&&u<r+this.helperProportions.height/2&&i-this.helperProportions.height/2<a},_intersectsWithPointer:function(t){var n=this.options.axis==="x"||e.ui.isOverAxis(this.positionAbs.top+this.offset.click.top,t.top,t.height),r=this.options.axis==="y"||e.ui.isOverAxis(this.positionAbs.left+this.offset.click.left,t.left,t.width),i=n&&r,s=this._getDragVerticalDirection(),o=this._getDragHorizontalDirection();return i?this.floating?o&&o=="right"||s=="down"?2:1:s&&(s=="down"?2:1):!1},_intersectsWithSides:function(t){var n=e.ui.isOverAxis(this.positionAbs.top+this.offset.click.top,t.top+t.height/2,t.height),r=e.ui.isOverAxis(this.positionAbs.left+this.offset.click.left,t.left+t.width/2,t.width),i=this._getDragVerticalDirection(),s=this._getDragHorizontalDirection();return this.floating&&s?s=="right"&&r||s=="left"&&!r:i&&(i=="down"&&n||i=="up"&&!n)},_getDragVerticalDirection:function(){var e=this.positionAbs.top-this.lastPositionAbs.top;return e!=0&&(e>0?"down":"up")},_getDragHorizontalDirection:function(){var e=this.positionAbs.left-this.lastPositionAbs.left;return e!=0&&(e>0?"right":"left")},refresh:function(e){return this._refreshItems(e),this.refreshPositions(),this},_connectWith:function(){var e=this.options;return e.connectWith.constructor==String?[e.connectWith]:e.connectWith},_getItemsAsjQuery:function(t){var n=[],r=[],i=this._connectWith();if(i&&t)for(var s=i.length-1;s>=0;s--){var o=e(i[s]);for(var u=o.length-1;u>=0;u--){var a=e.data(o[u],this.widgetName);a&&a!=this&&!a.options.disabled&&r.push([e.isFunction(a.options.items)?a.options.items.call(a.element):e(a.options.items,a.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),a])}}r.push([e.isFunction(this.options.items)?this.options.items.call(this.element,null,{options:this.options,item:this.currentItem}):e(this.options.items,this.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),this]);for(var s=r.length-1;s>=0;s--)r[s][0].each(function(){n.push(this)});return e(n)},_removeCurrentsFromItems:function(){var t=this.currentItem.find(":data("+this.widgetName+"-item)");this.items=e.grep(this.items,function(e){for(var n=0;n<t.length;n++)if(t[n]==e.item[0])return!1;return!0})},_refreshItems:function(t){this.items=[],this.containers=[this];var n=this.items,r=[[e.isFunction(this.options.items)?this.options.items.call(this.element[0],t,{item:this.currentItem}):e(this.options.items,this.element),this]],i=this._connectWith();if(i&&this.ready)for(var s=i.length-1;s>=0;s--){var o=e(i[s]);for(var u=o.length-1;u>=0;u--){var a=e.data(o[u],this.widgetName);a&&a!=this&&!a.options.disabled&&(r.push([e.isFunction(a.options.items)?a.options.items.call(a.element[0],t,{item:this.currentItem}):e(a.options.items,a.element),a]),this.containers.push(a))}}for(var s=r.length-1;s>=0;s--){var f=r[s][1],l=r[s][0];for(var u=0,c=l.length;u<c;u++){var h=e(l[u]);h.data(this.widgetName+"-item",f),n.push({item:h,instance:f,width:0,height:0,left:0,top:0})}}},refreshPositions:function(t){this.offsetParent&&this.helper&&(this.offset.parent=this._getParentOffset());for(var n=this.items.length-1;n>=0;n--){var r=this.items[n];if(r.instance!=this.currentContainer&&this.currentContainer&&r.item[0]!=this.currentItem[0])continue;var i=this.options.toleranceElement?e(this.options.toleranceElement,r.item):r.item;t||(r.width=i.outerWidth(),r.height=i.outerHeight());var s=i.offset();r.left=s.left,r.top=s.top}if(this.options.custom&&this.options.custom.refreshContainers)this.options.custom.refreshContainers.call(this);else for(var n=this.containers.length-1;n>=0;n--){var s=this.containers[n].element.offset();this.containers[n].containerCache.left=s.left,this.containers[n].containerCache.top=s.top,this.containers[n].containerCache.width=this.containers[n].element.outerWidth(),this.containers[n].containerCache.height=this.containers[n].element.outerHeight()}return this},_createPlaceholder:function(t){t=t||this;var n=t.options;if(!n.placeholder||n.placeholder.constructor==String){var r=n.placeholder;n.placeholder={element:function(){var n=e(document.createElement(t.currentItem[0].nodeName)).addClass(r||t.currentItem[0].className+" ui-sortable-placeholder").removeClass("ui-sortable-helper")[0];return r||(n.style.visibility="hidden"),n},update:function(e,i){if(r&&!n.forcePlaceholderSize)return;i.height()||i.height(t.currentItem.innerHeight()-parseInt(t.currentItem.css("paddingTop")||0,10)-parseInt(t.currentItem.css("paddingBottom")||0,10)),i.width()||i.width(t.currentItem.innerWidth()-parseInt(t.currentItem.css("paddingLeft")||0,10)-parseInt(t.currentItem.css("paddingRight")||0,10))}}}t.placeholder=e(n.placeholder.element.call(t.element,t.currentItem)),t.currentItem.after(t.placeholder),n.placeholder.update(t,t.placeholder)},_contactContainers:function(t){var n=null,r=null;for(var i=this.containers.length-1;i>=0;i--){if(e.contains(this.currentItem[0],this.containers[i].element[0]))continue;if(this._intersectsWith(this.containers[i].containerCache)){if(n&&e.contains(this.containers[i].element[0],n.element[0]))continue;n=this.containers[i],r=i}else this.containers[i].containerCache.over&&(this.containers[i]._trigger("out",t,this._uiHash(this)),this.containers[i].containerCache.over=0)}if(!n)return;if(this.containers.length===1)this.containers[r]._trigger("over",t,this._uiHash(this)),this.containers[r].containerCache.over=1;else{var s=1e4,o=null,u=this.containers[r].floating?"left":"top",a=this.containers[r].floating?"width":"height",f=this.positionAbs[u]+this.offset.click[u];for(var l=this.items.length-1;l>=0;l--){if(!e.contains(this.containers[r].element[0],this.items[l].item[0]))continue;if(this.items[l].item[0]==this.currentItem[0])continue;var c=this.items[l].item.offset()[u],h=!1;Math.abs(c-f)>Math.abs(c+this.items[l][a]-f)&&(h=!0,c+=this.items[l][a]),Math.abs(c-f)<s&&(s=Math.abs(c-f),o=this.items[l],this.direction=h?"up":"down")}if(!o&&!this.options.dropOnEmpty)return;this.currentContainer=this.containers[r],o?this._rearrange(t,o,null,!0):this._rearrange(t,null,this.containers[r].element,!0),this._trigger("change",t,this._uiHash()),this.containers[r]._trigger("change",t,this._uiHash(this)),this.options.placeholder.update(this.currentContainer,this.placeholder),this.containers[r]._trigger("over",t,this._uiHash(this)),this.containers[r].containerCache.over=1}},_createHelper:function(t){var n=this.options,r=e.isFunction(n.helper)?e(n.helper.apply(this.element[0],[t,this.currentItem])):n.helper=="clone"?this.currentItem.clone():this.currentItem;return r.parents("body").length||e(n.appendTo!="parent"?n.appendTo:this.currentItem[0].parentNode)[0].appendChild(r[0]),r[0]==this.currentItem[0]&&(this._storedCSS={width:this.currentItem[0].style.width,height:this.currentItem[0].style.height,position:this.currentItem.css("position"),top:this.currentItem.css("top"),left:this.currentItem.css("left")}),(r[0].style.width==""||n.forceHelperSize)&&r.width(this.currentItem.width()),(r[0].style.height==""||n.forceHelperSize)&&r.height(this.currentItem.height()),r},_adjustOffsetFromHelper:function(t){typeof t=="string"&&(t=t.split(" ")),e.isArray(t)&&(t={left:+t[0],top:+t[1]||0}),"left"in t&&(this.offset.click.left=t.left+this.margins.left),"right"in t&&(this.offset.click.left=this.helperProportions.width-t.right+this.margins.left),"top"in t&&(this.offset.click.top=t.top+this.margins.top),"bottom"in t&&(this.offset.click.top=this.helperProportions.height-t.bottom+this.margins.top)},_getParentOffset:function(){this.offsetParent=this.helper.offsetParent();var t=this.offsetParent.offset();this.cssPosition=="absolute"&&this.scrollParent[0]!=document&&e.contains(this.scrollParent[0],this.offsetParent[0])&&(t.left+=this.scrollParent.scrollLeft(),t.top+=this.scrollParent.scrollTop());if(this.offsetParent[0]==document.body||this.offsetParent[0].tagName&&this.offsetParent[0].tagName.toLowerCase()=="html"&&e.ui.ie)t={top:0,left:0};return{top:t.top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:t.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if(this.cssPosition=="relative"){var e=this.currentItem.position();return{top:e.top-(parseInt(this.helper.css("top"),10)||0)+this.scrollParent.scrollTop(),left:e.left-(parseInt(this.helper.css("left"),10)||0)+this.scrollParent.scrollLeft()}}return{top:0,left:0}},_cacheMargins:function(){this.margins={left:parseInt(this.currentItem.css("marginLeft"),10)||0,top:parseInt(this.currentItem.css("marginTop"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(),height:this.helper.outerHeight()}},_setContainment:function(){var t=this.options;t.containment=="parent"&&(t.containment=this.helper[0].parentNode);if(t.containment=="document"||t.containment=="window")this.containment=[0-this.offset.relative.left-this.offset.parent.left,0-this.offset.relative.top-this.offset.parent.top,e(t.containment=="document"?document:window).width()-this.helperProportions.width-this.margins.left,(e(t.containment=="document"?document:window).height()||document.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top];if(!/^(document|window|parent)$/.test(t.containment)){var n=e(t.containment)[0],r=e(t.containment).offset(),i=e(n).css("overflow")!="hidden";this.containment=[r.left+(parseInt(e(n).css("borderLeftWidth"),10)||0)+(parseInt(e(n).css("paddingLeft"),10)||0)-this.margins.left,r.top+(parseInt(e(n).css("borderTopWidth"),10)||0)+(parseInt(e(n).css("paddingTop"),10)||0)-this.margins.top,r.left+(i?Math.max(n.scrollWidth,n.offsetWidth):n.offsetWidth)-(parseInt(e(n).css("borderLeftWidth"),10)||0)-(parseInt(e(n).css("paddingRight"),10)||0)-this.helperProportions.width-this.margins.left,r.top+(i?Math.max(n.scrollHeight,n.offsetHeight):n.offsetHeight)-(parseInt(e(n).css("borderTopWidth"),10)||0)-(parseInt(e(n).css("paddingBottom"),10)||0)-this.helperProportions.height-this.margins.top]}},_convertPositionTo:function(t,n){n||(n=this.position);var r=t=="absolute"?1:-1,i=this.options,s=this.cssPosition!="absolute"||this.scrollParent[0]!=document&&!!e.contains(this.scrollParent[0],this.offsetParent[0])?this.scrollParent:this.offsetParent,o=/(html|body)/i.test(s[0].tagName);return{top:n.top+this.offset.relative.top*r+this.offset.parent.top*r-(this.cssPosition=="fixed"?-this.scrollParent.scrollTop():o?0:s.scrollTop())*r,left:n.left+this.offset.relative.left*r+this.offset.parent.left*r-(this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():o?0:s.scrollLeft())*r}},_generatePosition:function(t){var n=this.options,r=this.cssPosition!="absolute"||this.scrollParent[0]!=document&&!!e.contains(this.scrollParent[0],this.offsetParent[0])?this.scrollParent:this.offsetParent,i=/(html|body)/i.test(r[0].tagName);this.cssPosition=="relative"&&(this.scrollParent[0]==document||this.scrollParent[0]==this.offsetParent[0])&&(this.offset.relative=this._getRelativeOffset());var s=t.pageX,o=t.pageY;if(this.originalPosition){this.containment&&(t.pageX-this.offset.click.left<this.containment[0]&&(s=this.containment[0]+this.offset.click.left),t.pageY-this.offset.click.top<this.containment[1]&&(o=this.containment[1]+this.offset.click.top),t.pageX-this.offset.click.left>this.containment[2]&&(s=this.containment[2]+this.offset.click.left),t.pageY-this.offset.click.top>this.containment[3]&&(o=this.containment[3]+this.offset.click.top));if(n.grid){var u=this.originalPageY+Math.round((o-this.originalPageY)/n.grid[1])*n.grid[1];o=this.containment?u-this.offset.click.top<this.containment[1]||u-this.offset.click.top>this.containment[3]?u-this.offset.click.top<this.containment[1]?u+n.grid[1]:u-n.grid[1]:u:u;var a=this.originalPageX+Math.round((s-this.originalPageX)/n.grid[0])*n.grid[0];s=this.containment?a-this.offset.click.left<this.containment[0]||a-this.offset.click.left>this.containment[2]?a-this.offset.click.left<this.containment[0]?a+n.grid[0]:a-n.grid[0]:a:a}}return{top:o-this.offset.click.top-this.offset.relative.top-this.offset.parent.top+(this.cssPosition=="fixed"?-this.scrollParent.scrollTop():i?0:r.scrollTop()),left:s-this.offset.click.left-this.offset.relative.left-this.offset.parent.left+(this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():i?0:r.scrollLeft())}},_rearrange:function(e,t,n,r){n?n[0].appendChild(this.placeholder[0]):t.item[0].parentNode.insertBefore(this.placeholder[0],this.direction=="down"?t.item[0]:t.item[0].nextSibling),this.counter=this.counter?++this.counter:1;var i=this.counter;this._delay(function(){i==this.counter&&this.refreshPositions(!r)})},_clear:function(t,n){this.reverting=!1;var r=[];!this._noFinalSort&&this.currentItem.parent().length&&this.placeholder.before(this.currentItem),this._noFinalSort=null;if(this.helper[0]==this.currentItem[0]){for(var i in this._storedCSS)if(this._storedCSS[i]=="auto"||this._storedCSS[i]=="static")this._storedCSS[i]="";this.currentItem.css(this._storedCSS).removeClass("ui-sortable-helper")}else this.currentItem.show();this.fromOutside&&!n&&r.push(function(e){this._trigger("receive",e,this._uiHash(this.fromOutside))}),(this.fromOutside||this.domPosition.prev!=this.currentItem.prev().not(".ui-sortable-helper")[0]||this.domPosition.parent!=this.currentItem.parent()[0])&&!n&&r.push(function(e){this._trigger("update",e,this._uiHash())}),this!==this.currentContainer&&(n||(r.push(function(e){this._trigger("remove",e,this._uiHash())}),r.push(function(e){return function(t){e._trigger("receive",t,this._uiHash(this))}}.call(this,this.currentContainer)),r.push(function(e){return function(t){e._trigger("update",t,this._uiHash(this))}}.call(this,this.currentContainer))));for(var i=this.containers.length-1;i>=0;i--)n||r.push(function(e){return function(t){e._trigger("deactivate",t,this._uiHash(this))}}.call(this,this.containers[i])),this.containers[i].containerCache.over&&(r.push(function(e){return function(t){e._trigger("out",t,this._uiHash(this))}}.call(this,this.containers[i])),this.containers[i].containerCache.over=0);this._storedCursor&&e("body").css("cursor",this._storedCursor),this._storedOpacity&&this.helper.css("opacity",this._storedOpacity),this._storedZIndex&&this.helper.css("zIndex",this._storedZIndex=="auto"?"":this._storedZIndex),this.dragging=!1;if(this.cancelHelperRemoval){if(!n){this._trigger("beforeStop",t,this._uiHash());for(var i=0;i<r.length;i++)r[i].call(this,t);this._trigger("stop",t,this._uiHash())}return this.fromOutside=!1,!1}n||this._trigger("beforeStop",t,this._uiHash()),this.placeholder[0].parentNode.removeChild(this.placeholder[0]),this.helper[0]!=this.currentItem[0]&&this.helper.remove(),this.helper=null;if(!n){for(var i=0;i<r.length;i++)r[i].call(this,t);this._trigger("stop",t,this._uiHash())}return this.fromOutside=!1,!0},_trigger:function(){e.Widget.prototype._trigger.apply(this,arguments)===!1&&this.cancel()},_uiHash:function(t){var n=t||this;return{helper:n.helper,placeholder:n.placeholder||e([]),position:n.position,originalPosition:n.originalPosition,offset:n.positionAbs,item:n.currentItem,sender:t?t.element:null}}})})(jQuery);(function(e){function t(e){return function(){var t=this.element.val();e.apply(this,arguments),this._refresh(),t!==this.element.val()&&this._trigger("change")}}e.widget("ui.spinner",{version:"1.9.1",defaultElement:"<input>",widgetEventPrefix:"spin",options:{culture:null,icons:{down:"ui-icon-triangle-1-s",up:"ui-icon-triangle-1-n"},incremental:!0,max:null,min:null,numberFormat:null,page:10,step:1,change:null,spin:null,start:null,stop:null},_create:function(){this._setOption("max",this.options.max),this._setOption("min",this.options.min),this._setOption("step",this.options.step),this._value(this.element.val(),!0),this._draw(),this._on(this._events),this._refresh(),this._on(this.window,{beforeunload:function(){this.element.removeAttr("autocomplete")}})},_getCreateOptions:function(){var t={},n=this.element;return e.each(["min","max","step"],function(e,r){var i=n.attr(r);i!==undefined&&i.length&&(t[r]=i)}),t},_events:{keydown:function(e){this._start(e)&&this._keydown(e)&&e.preventDefault()},keyup:"_stop",focus:function(){this.previous=this.element.val()},blur:function(e){if(this.cancelBlur){delete this.cancelBlur;return}this._refresh(),this.previous!==this.element.val()&&this._trigger("change",e)},mousewheel:function(e,t){if(!t)return;if(!this.spinning&&!this._start(e))return!1;this._spin((t>0?1:-1)*this.options.step,e),clearTimeout(this.mousewheelTimer),this.mousewheelTimer=this._delay(function(){this.spinning&&this._stop(e)},100),e.preventDefault()},"mousedown .ui-spinner-button":function(t){function r(){var e=this.element[0]===this.document[0].activeElement;e||(this.element.focus(),this.previous=n,this._delay(function(){this.previous=n}))}var n;n=this.element[0]===this.document[0].activeElement?this.previous:this.element.val(),t.preventDefault(),r.call(this),this.cancelBlur=!0,this._delay(function(){delete this.cancelBlur,r.call(this)});if(this._start(t)===!1)return;this._repeat(null,e(t.currentTarget).hasClass("ui-spinner-up")?1:-1,t)},"mouseup .ui-spinner-button":"_stop","mouseenter .ui-spinner-button":function(t){if(!e(t.currentTarget).hasClass("ui-state-active"))return;if(this._start(t)===!1)return!1;this._repeat(null,e(t.currentTarget).hasClass("ui-spinner-up")?1:-1,t)},"mouseleave .ui-spinner-button":"_stop"},_draw:function(){var e=this.uiSpinner=this.element.addClass("ui-spinner-input").attr("autocomplete","off").wrap(this._uiSpinnerHtml()).parent().append(this._buttonHtml());this.element.attr("role","spinbutton"),this.buttons=e.find(".ui-spinner-button").attr("tabIndex",-1).button().removeClass("ui-corner-all"),this.buttons.height()>Math.ceil(e.height()*.5)&&e.height()>0&&e.height(e.height()),this.options.disabled&&this.disable()},_keydown:function(t){var n=this.options,r=e.ui.keyCode;switch(t.keyCode){case r.UP:return this._repeat(null,1,t),!0;case r.DOWN:return this._repeat(null,-1,t),!0;case r.PAGE_UP:return this._repeat(null,n.page,t),!0;case r.PAGE_DOWN:return this._repeat(null,-n.page,t),!0}return!1},_uiSpinnerHtml:function(){return"<span class='ui-spinner ui-widget ui-widget-content ui-corner-all'></span>"},_buttonHtml:function(){return"<a class='ui-spinner-button ui-spinner-up ui-corner-tr'><span class='ui-icon "+this.options.icons.up+"'>▲</span>"+"</a>"+"<a class='ui-spinner-button ui-spinner-down ui-corner-br'>"+"<span class='ui-icon "+this.options.icons.down+"'>▼</span>"+"</a>"},_start:function(e){return!this.spinning&&this._trigger("start",e)===!1?!1:(this.counter||(this.counter=1),this.spinning=!0,!0)},_repeat:function(e,t,n){e=e||500,clearTimeout(this.timer),this.timer=this._delay(function(){this._repeat(40,t,n)},e),this._spin(t*this.options.step,n)},_spin:function(e,t){var n=this.value()||0;this.counter||(this.counter=1),n=this._adjustValue(n+e*this._increment(this.counter));if(!this.spinning||this._trigger("spin",t,{value:n})!==!1)this._value(n),this.counter++},_increment:function(t){var n=this.options.incremental;return n?e.isFunction(n)?n(t):Math.floor(t*t*t/5e4-t*t/500+17*t/200+1):1},_precision:function(){var e=this._precisionOf(this.options.step);return this.options.min!==null&&(e=Math.max(e,this._precisionOf(this.options.min))),e},_precisionOf:function(e){var t=e.toString(),n=t.indexOf(".");return n===-1?0:t.length-n-1},_adjustValue:function(e){var t,n,r=this.options;return t=r.min!==null?r.min:0,n=e-t,n=Math.round(n/r.step)*r.step,e=t+n,e=parseFloat(e.toFixed(this._precision())),r.max!==null&&e>r.max?r.max:r.min!==null&&e<r.min?r.min:e},_stop:function(e){if(!this.spinning)return;clearTimeout(this.timer),clearTimeout(this.mousewheelTimer),this.counter=0,this.spinning=!1,this._trigger("stop",e)},_setOption:function(e,t){if(e==="culture"||e==="numberFormat"){var n=this._parse(this.element.val());this.options[e]=t,this.element.val(this._format(n));return}(e==="max"||e==="min"||e==="step")&&typeof t=="string"&&(t=this._parse(t)),this._super(e,t),e==="disabled"&&(t?(this.element.prop("disabled",!0),this.buttons.button("disable")):(this.element.prop("disabled",!1),this.buttons.button("enable")))},_setOptions:t(function(e){this._super(e),this._value(this.element.val())}),_parse:function(e){return typeof e=="string"&&e!==""&&(e=window.Globalize&&this.options.numberFormat?Globalize.parseFloat(e,10,this.options.culture):+e),e===""||isNaN(e)?null:e},_format:function(e){return e===""?"":window.Globalize&&this.options.numberFormat?Globalize.format(e,this.options.numberFormat,this.options.culture):e},_refresh:function(){this.element.attr({"aria-valuemin":this.options.min,"aria-valuemax":this.options.max,"aria-valuenow":this._parse(this.element.val())})},_value:function(e,t){var n;e!==""&&(n=this._parse(e),n!==null&&(t||(n=this._adjustValue(n)),e=this._format(n))),this.element.val(e),this._refresh()},_destroy:function(){this.element.removeClass("ui-spinner-input").prop("disabled",!1).removeAttr("autocomplete").removeAttr("role").removeAttr("aria-valuemin").removeAttr("aria-valuemax").removeAttr("aria-valuenow"),this.uiSpinner.replaceWith(this.element)},stepUp:t(function(e){this._stepUp(e)}),_stepUp:function(e){this._spin((e||1)*this.options.step)},stepDown:t(function(e){this._stepDown(e)}),_stepDown:function(e){this._spin((e||1)*-this.options.step)},pageUp:t(function(e){this._stepUp((e||1)*this.options.page)}),pageDown:t(function(e){this._stepDown((e||1)*this.options.page)}),value:function(e){if(!arguments.length)return this._parse(this.element.val());t(this._value).call(this,e)},widget:function(){return this.uiSpinner}})})(jQuery);(function(e,t){function i(){return++n}function s(e){return e.hash.length>1&&e.href.replace(r,"")===location.href.replace(r,"")}var n=0,r=/#.*$/;e.widget("ui.tabs",{version:"1.9.1",delay:300,options:{active:null,collapsible:!1,event:"click",heightStyle:"content",hide:null,show:null,activate:null,beforeActivate:null,beforeLoad:null,load:null},_create:function(){var t=this,n=this.options,r=n.active,i=location.hash.substring(1);this.running=!1,this.element.addClass("ui-tabs ui-widget ui-widget-content ui-corner-all").toggleClass("ui-tabs-collapsible",n.collapsible).delegate(".ui-tabs-nav > li","mousedown"+this.eventNamespace,function(t){e(this).is(".ui-state-disabled")&&t.preventDefault()}).delegate(".ui-tabs-anchor","focus"+this.eventNamespace,function(){e(this).closest("li").is(".ui-state-disabled")&&this.blur()}),this._processTabs();if(r===null){i&&this.tabs.each(function(t,n){if(e(n).attr("aria-controls")===i)return r=t,!1}),r===null&&(r=this.tabs.index(this.tabs.filter(".ui-tabs-active")));if(r===null||r===-1)r=this.tabs.length?0:!1}r!==!1&&(r=this.tabs.index(this.tabs.eq(r)),r===-1&&(r=n.collapsible?!1:0)),n.active=r,!n.collapsible&&n.active===!1&&this.anchors.length&&(n.active=0),e.isArray(n.disabled)&&(n.disabled=e.unique(n.disabled.concat(e.map(this.tabs.filter(".ui-state-disabled"),function(e){return t.tabs.index(e)}))).sort()),this.options.active!==!1&&this.anchors.length?this.active=this._findActive(this.options.active):this.active=e(),this._refresh(),this.active.length&&this.load(n.active)},_getCreateEventData:function(){return{tab:this.active,panel:this.active.length?this._getPanelForTab(this.active):e()}},_tabKeydown:function(t){var n=e(this.document[0].activeElement).closest("li"),r=this.tabs.index(n),i=!0;if(this._handlePageNav(t))return;switch(t.keyCode){case e.ui.keyCode.RIGHT:case e.ui.keyCode.DOWN:r++;break;case e.ui.keyCode.UP:case e.ui.keyCode.LEFT:i=!1,r--;break;case e.ui.keyCode.END:r=this.anchors.length-1;break;case e.ui.keyCode.HOME:r=0;break;case e.ui.keyCode.SPACE:t.preventDefault(),clearTimeout(this.activating),this._activate(r);return;case e.ui.keyCode.ENTER:t.preventDefault(),clearTimeout(this.activating),this._activate(r===this.options.active?!1:r);return;default:return}t.preventDefault(),clearTimeout(this.activating),r=this._focusNextTab(r,i),t.ctrlKey||(n.attr("aria-selected","false"),this.tabs.eq(r).attr("aria-selected","true"),this.activating=this._delay(function(){this.option("active",r)},this.delay))},_panelKeydown:function(t){if(this._handlePageNav(t))return;t.ctrlKey&&t.keyCode===e.ui.keyCode.UP&&(t.preventDefault(),this.active.focus())},_handlePageNav:function(t){if(t.altKey&&t.keyCode===e.ui.keyCode.PAGE_UP)return this._activate(this._focusNextTab(this.options.active-1,!1)),!0;if(t.altKey&&t.keyCode===e.ui.keyCode.PAGE_DOWN)return this._activate(this._focusNextTab(this.options.active+1,!0)),!0},_findNextTab:function(t,n){function i(){return t>r&&(t=0),t<0&&(t=r),t}var r=this.tabs.length-1;while(e.inArray(i(),this.options.disabled)!==-1)t=n?t+1:t-1;return t},_focusNextTab:function(e,t){return e=this._findNextTab(e,t),this.tabs.eq(e).focus(),e},_setOption:function(e,t){if(e==="active"){this._activate(t);return}if(e==="disabled"){this._setupDisabled(t);return}this._super(e,t),e==="collapsible"&&(this.element.toggleClass("ui-tabs-collapsible",t),!t&&this.options.active===!1&&this._activate(0)),e==="event"&&this._setupEvents(t),e==="heightStyle"&&this._setupHeightStyle(t)},_tabId:function(e){return e.attr("aria-controls")||"ui-tabs-"+i()},_sanitizeSelector:function(e){return e?e.replace(/[!"$%&'()*+,.\/:;<=>?@\[\]\^`{|}~]/g,"\\$&"):""},refresh:function(){var t=this.options,n=this.tablist.children(":has(a[href])");t.disabled=e.map(n.filter(".ui-state-disabled"),function(e){return n.index(e)}),this._processTabs(),t.active===!1||!this.anchors.length?(t.active=!1,this.active=e()):this.active.length&&!e.contains(this.tablist[0],this.active[0])?this.tabs.length===t.disabled.length?(t.active=!1,this.active=e()):this._activate(this._findNextTab(Math.max(0,t.active-1),!1)):t.active=this.tabs.index(this.active),this._refresh()},_refresh:function(){this._setupDisabled(this.options.disabled),this._setupEvents(this.options.event),this._setupHeightStyle(this.options.heightStyle),this.tabs.not(this.active).attr({"aria-selected":"false",tabIndex:-1}),this.panels.not(this._getPanelForTab(this.active)).hide().attr({"aria-expanded":"false","aria-hidden":"true"}),this.active.length?(this.active.addClass("ui-tabs-active ui-state-active").attr({"aria-selected":"true",tabIndex:0}),this._getPanelForTab(this.active).show().attr({"aria-expanded":"true","aria-hidden":"false"})):this.tabs.eq(0).attr("tabIndex",0)},_processTabs:function(){var t=this;this.tablist=this._getList().addClass("ui-tabs-nav ui-helper-reset ui-helper-clearfix ui-widget-header ui-corner-all").attr("role","tablist"),this.tabs=this.tablist.find("> li:has(a[href])").addClass("ui-state-default ui-corner-top").attr({role:"tab",tabIndex:-1}),this.anchors=this.tabs.map(function(){return e("a",this)[0]}).addClass("ui-tabs-anchor").attr({role:"presentation",tabIndex:-1}),this.panels=e(),this.anchors.each(function(n,r){var i,o,u,a=e(r).uniqueId().attr("id"),f=e(r).closest("li"),l=f.attr("aria-controls");s(r)?(i=r.hash,o=t.element.find(t._sanitizeSelector(i))):(u=t._tabId(f),i="#"+u,o=t.element.find(i),o.length||(o=t._createPanel(u),o.insertAfter(t.panels[n-1]||t.tablist)),o.attr("aria-live","polite")),o.length&&(t.panels=t.panels.add(o)),l&&f.data("ui-tabs-aria-controls",l),f.attr({"aria-controls":i.substring(1),"aria-labelledby":a}),o.attr("aria-labelledby",a)}),this.panels.addClass("ui-tabs-panel ui-widget-content ui-corner-bottom").attr("role","tabpanel")},_getList:function(){return this.element.find("ol,ul").eq(0)},_createPanel:function(t){return e("<div>").attr("id",t).addClass("ui-tabs-panel ui-widget-content ui-corner-bottom").data("ui-tabs-destroy",!0)},_setupDisabled:function(t){e.isArray(t)&&(t.length?t.length===this.anchors.length&&(t=!0):t=!1);for(var n=0,r;r=this.tabs[n];n++)t===!0||e.inArray(n,t)!==-1?e(r).addClass("ui-state-disabled").attr("aria-disabled","true"):e(r).removeClass("ui-state-disabled").removeAttr("aria-disabled");this.options.disabled=t},_setupEvents:function(t){var n={click:function(e){e.preventDefault()}};t&&e.each(t.split(" "),function(e,t){n[t]="_eventHandler"}),this._off(this.anchors.add(this.tabs).add(this.panels)),this._on(this.anchors,n),this._on(this.tabs,{keydown:"_tabKeydown"}),this._on(this.panels,{keydown:"_panelKeydown"}),this._focusable(this.tabs),this._hoverable(this.tabs)},_setupHeightStyle:function(t){var n,r,i=this.element.parent();t==="fill"?(e.support.minHeight||(r=i.css("overflow"),i.css("overflow","hidden")),n=i.height(),this.element.siblings(":visible").each(function(){var t=e(this),r=t.css("position");if(r==="absolute"||r==="fixed")return;n-=t.outerHeight(!0)}),r&&i.css("overflow",r),this.element.children().not(this.panels).each(function(){n-=e(this).outerHeight(!0)}),this.panels.each(function(){e(this).height(Math.max(0,n-e(this).innerHeight()+e(this).height()))}).css("overflow","auto")):t==="auto"&&(n=0,this.panels.each(function(){n=Math.max(n,e(this).height("").height())}).height(n))},_eventHandler:function(t){var n=this.options,r=this.active,i=e(t.currentTarget),s=i.closest("li"),o=s[0]===r[0],u=o&&n.collapsible,a=u?e():this._getPanelForTab(s),f=r.length?this._getPanelForTab(r):e(),l={oldTab:r,oldPanel:f,newTab:u?e():s,newPanel:a};t.preventDefault();if(s.hasClass("ui-state-disabled")||s.hasClass("ui-tabs-loading")||this.running||o&&!n.collapsible||this._trigger("beforeActivate",t,l)===!1)return;n.active=u?!1:this.tabs.index(s),this.active=o?e():s,this.xhr&&this.xhr.abort(),!f.length&&!a.length&&e.error("jQuery UI Tabs: Mismatching fragment identifier."),a.length&&this.load(this.tabs.index(s),t),this._toggle(t,l)},_toggle:function(t,n){function o(){r.running=!1,r._trigger("activate",t,n)}function u(){n.newTab.closest("li").addClass("ui-tabs-active ui-state-active"),i.length&&r.options.show?r._show(i,r.options.show,o):(i.show(),o())}var r=this,i=n.newPanel,s=n.oldPanel;this.running=!0,s.length&&this.options.hide?this._hide(s,this.options.hide,function(){n.oldTab.closest("li").removeClass("ui-tabs-active ui-state-active"),u()}):(n.oldTab.closest("li").removeClass("ui-tabs-active ui-state-active"),s.hide(),u()),s.attr({"aria-expanded":"false","aria-hidden":"true"}),n.oldTab.attr("aria-selected","false"),i.length&&s.length?n.oldTab.attr("tabIndex",-1):i.length&&this.tabs.filter(function(){return e(this).attr("tabIndex")===0}).attr("tabIndex",-1),i.attr({"aria-expanded":"true","aria-hidden":"false"}),n.newTab.attr({"aria-selected":"true",tabIndex:0})},_activate:function(t){var n,r=this._findActive(t);if(r[0]===this.active[0])return;r.length||(r=this.active),n=r.find(".ui-tabs-anchor")[0],this._eventHandler({target:n,currentTarget:n,preventDefault:e.noop})},_findActive:function(t){return t===!1?e():this.tabs.eq(t)},_getIndex:function(e){return typeof e=="string"&&(e=this.anchors.index(this.anchors.filter("[href$='"+e+"']"))),e},_destroy:function(){this.xhr&&this.xhr.abort(),this.element.removeClass("ui-tabs ui-widget ui-widget-content ui-corner-all ui-tabs-collapsible"),this.tablist.removeClass("ui-tabs-nav ui-helper-reset ui-helper-clearfix ui-widget-header ui-corner-all").removeAttr("role"),this.anchors.removeClass("ui-tabs-anchor").removeAttr("role").removeAttr("tabIndex").removeData("href.tabs").removeData("load.tabs").removeUniqueId(),this.tabs.add(this.panels).each(function(){e.data(this,"ui-tabs-destroy")?e(this).remove():e(this).removeClass("ui-state-default ui-state-active ui-state-disabled ui-corner-top ui-corner-bottom ui-widget-content ui-tabs-active ui-tabs-panel").removeAttr("tabIndex").removeAttr("aria-live").removeAttr("aria-busy").removeAttr("aria-selected").removeAttr("aria-labelledby").removeAttr("aria-hidden").removeAttr("aria-expanded").removeAttr("role")}),this.tabs.each(function(){var t=e(this),n=t.data("ui-tabs-aria-controls");n?t.attr("aria-controls",n):t.removeAttr("aria-controls")}),this.options.heightStyle!=="content"&&this.panels.css("height","")},enable:function(n){var r=this.options.disabled;if(r===!1)return;n===t?r=!1:(n=this._getIndex(n),e.isArray(r)?r=e.map(r,function(e){return e!==n?e:null}):r=e.map(this.tabs,function(e,t){return t!==n?t:null})),this._setupDisabled(r)},disable:function(n){var r=this.options.disabled;if(r===!0)return;if(n===t)r=!0;else{n=this._getIndex(n);if(e.inArray(n,r)!==-1)return;e.isArray(r)?r=e.merge([n],r).sort():r=[n]}this._setupDisabled(r)},load:function(t,n){t=this._getIndex(t);var r=this,i=this.tabs.eq(t),o=i.find(".ui-tabs-anchor"),u=this._getPanelForTab(i),a={tab:i,panel:u};if(s(o[0]))return;this.xhr=e.ajax(this._ajaxSettings(o,n,a)),this.xhr&&this.xhr.statusText!=="canceled"&&(i.addClass("ui-tabs-loading"),u.attr("aria-busy","true"),this.xhr.success(function(e){setTimeout(function(){u.html(e),r._trigger("load",n,a)},1)}).complete(function(e,t){setTimeout(function(){t==="abort"&&r.panels.stop(!1,!0),i.removeClass("ui-tabs-loading"),u.removeAttr("aria-busy"),e===r.xhr&&delete r.xhr},1)}))},_ajaxSettings:function(t,n,r){var i=this;return{url:t.attr("href"),beforeSend:function(t,s){return i._trigger("beforeLoad",n,e.extend({jqXHR:t,ajaxSettings:s},r))}}},_getPanelForTab:function(t){var n=e(t).attr("aria-controls");return this.element.find(this._sanitizeSelector("#"+n))}}),e.uiBackCompat!==!1&&(e.ui.tabs.prototype._ui=function(e,t){return{tab:e,panel:t,index:this.anchors.index(e)}},e.widget("ui.tabs",e.ui.tabs,{url:function(e,t){this.anchors.eq(e).attr("href",t)}}),e.widget("ui.tabs",e.ui.tabs,{options:{ajaxOptions:null,cache:!1},_create:function(){this._super();var t=this;this._on({tabsbeforeload:function(n,r){if(e.data(r.tab[0],"cache.tabs")){n.preventDefault();return}r.jqXHR.success(function(){t.options.cache&&e.data(r.tab[0],"cache.tabs",!0)})}})},_ajaxSettings:function(t,n,r){var i=this.options.ajaxOptions;return e.extend({},i,{error:function(e,t){try{i.error(e,t,r.tab.closest("li").index(),r.tab[0])}catch(n){}}},this._superApply(arguments))},_setOption:function(e,t){e==="cache"&&t===!1&&this.anchors.removeData("cache.tabs"),this._super(e,t)},_destroy:function(){this.anchors.removeData("cache.tabs"),this._super()},url:function(e){this.anchors.eq(e).removeData("cache.tabs"),this._superApply(arguments)}}),e.widget("ui.tabs",e.ui.tabs,{abort:function(){this.xhr&&this.xhr.abort()}}),e.widget("ui.tabs",e.ui.tabs,{options:{spinner:"<em>Loading…</em>"},_create:function(){this._super(),this._on({tabsbeforeload:function(e,t){if(e.target!==this.element[0]||!this.options.spinner)return;var n=t.tab.find("span"),r=n.html();n.html(this.options.spinner),t.jqXHR.complete(function(){n.html(r)})}})}}),e.widget("ui.tabs",e.ui.tabs,{options:{enable:null,disable:null},enable:function(t){var n=this.options,r;if(t&&n.disabled===!0||e.isArray(n.disabled)&&e.inArray(t,n.disabled)!==-1)r=!0;this._superApply(arguments),r&&this._trigger("enable",null,this._ui(this.anchors[t],this.panels[t]))},disable:function(t){var n=this.options,r;if(t&&n.disabled===!1||e.isArray(n.disabled)&&e.inArray(t,n.disabled)===-1)r=!0;this._superApply(arguments),r&&this._trigger("disable",null,this._ui(this.anchors[t],this.panels[t]))}}),e.widget("ui.tabs",e.ui.tabs,{options:{add:null,remove:null,tabTemplate:"<li><a href='#{href}'><span>#{label}</span></a></li>"},add:function(n,r,i){i===t&&(i=this.anchors.length);var s,o,u=this.options,a=e(u.tabTemplate.replace(/#\{href\}/g,n).replace(/#\{label\}/g,r)),f=n.indexOf("#")?this._tabId(a):n.replace("#","");return a.addClass("ui-state-default ui-corner-top").data("ui-tabs-destroy",!0),a.attr("aria-controls",f),s=i>=this.tabs.length,o=this.element.find("#"+f),o.length||(o=this._createPanel(f),s?i>0?o.insertAfter(this.panels.eq(-1)):o.appendTo(this.element):o.insertBefore(this.panels[i])),o.addClass("ui-tabs-panel ui-widget-content ui-corner-bottom").hide(),s?a.appendTo(this.tablist):a.insertBefore(this.tabs[i]),u.disabled=e.map(u.disabled,function(e){return e>=i?++e:e}),this.refresh(),this.tabs.length===1&&u.active===!1&&this.option("active",0),this._trigger("add",null,this._ui(this.anchors[i],this.panels[i])),this},remove:function(t){t=this._getIndex(t);var n=this.options,r=this.tabs.eq(t).remove(),i=this._getPanelForTab(r).remove();return r.hasClass("ui-tabs-active")&&this.anchors.length>2&&this._activate(t+(t+1<this.anchors.length?1:-1)),n.disabled=e.map(e.grep(n.disabled,function(e){return e!==t}),function(e){return e>=t?--e:e}),this.refresh(),this._trigger("remove",null,this._ui(r.find("a")[0],i[0])),this}}),e.widget("ui.tabs",e.ui.tabs,{length:function(){return this.anchors.length}}),e.widget("ui.tabs",e.ui.tabs,{options:{idPrefix:"ui-tabs-"},_tabId:function(t){var n=t.is("li")?t.find("a[href]"):t;return n=n[0],e(n).closest("li").attr("aria-controls")||n.title&&n.title.replace(/\s/g,"_").replace(/[^\w\u00c0-\uFFFF\-]/g,"")||this.options.idPrefix+i()}}),e.widget("ui.tabs",e.ui.tabs,{options:{panelTemplate:"<div></div>"},_createPanel:function(t){return e(this.options.panelTemplate).attr("id",t).addClass("ui-tabs-panel ui-widget-content ui-corner-bottom").data("ui-tabs-destroy",!0)}}),e.widget("ui.tabs",e.ui.tabs,{_create:function(){var e=this.options;e.active===null&&e.selected!==t&&(e.active=e.selected===-1?!1:e.selected),this._super(),e.selected=e.active,e.selected===!1&&(e.selected=-1)},_setOption:function(e,t){if(e!=="selected")return this._super(e,t);var n=this.options;this._super("active",t===-1?!1:t),n.selected=n.active,n.selected===!1&&(n.selected=-1)},_eventHandler:function(){this._superApply(arguments),this.options.selected=this.options.active,this.options.selected===!1&&(this.options.selected=-1)}}),e.widget("ui.tabs",e.ui.tabs,{options:{show:null,select:null},_create:function(){this._super(),this.options.active!==!1&&this._trigger("show",null,this._ui(this.active.find(".ui-tabs-anchor")[0],this._getPanelForTab(this.active)[0]))},_trigger:function(e,t,n){var r=this._superApply(arguments);return r?(e==="beforeActivate"&&n.newTab.length?r=this._super("select",t,{tab:n.newTab.find(".ui-tabs-anchor")[0],panel:n.newPanel[0],index:n.newTab.closest("li").index()}):e==="activate"&&n.newTab.length&&(r=this._super("show",t,{tab:n.newTab.find(".ui-tabs-anchor")[0],panel:n.newPanel[0],index:n.newTab.closest("li").index()})),r):!1}}),e.widget("ui.tabs",e.ui.tabs,{select:function(e){e=this._getIndex(e);if(e===-1){if(!this.options.collapsible||this.options.selected===-1)return;e=this.options.selected}this.anchors.eq(e).trigger(this.options.event+this.eventNamespace)}}),function(){var t=0;e.widget("ui.tabs",e.ui.tabs,{options:{cookie:null},_create:function(){var e=this.options,t;e.active==null&&e.cookie&&(t=parseInt(this._cookie(),10),t===-1&&(t=!1),e.active=t),this._super()},_cookie:function(n){var r=[this.cookie||(this.cookie=this.options.cookie.name||"ui-tabs-"+ ++t)];return arguments.length&&(r.push(n===!1?-1:n),r.push(this.options.cookie)),e.cookie.apply(null,r)},_refresh:function(){this._super(),this.options.cookie&&this._cookie(this.options.active,this.options.cookie)},_eventHandler:function(){this._superApply(arguments),this.options.cookie&&this._cookie(this.options.active,this.options.cookie)},_destroy:function(){this._super(),this.options.cookie&&this._cookie(null,this.options.cookie)}})}(),e.widget("ui.tabs",e.ui.tabs,{_trigger:function(t,n,r){var i=e.extend({},r);return t==="load"&&(i.panel=i.panel[0],i.tab=i.tab.find(".ui-tabs-anchor")[0]),this._super(t,n,i)}}),e.widget("ui.tabs",e.ui.tabs,{options:{fx:null},_getFx:function(){var t,n,r=this.options.fx;return r&&(e.isArray(r)?(t=r[0],n=r[1]):t=n=r),r?{show:n,hide:t}:null},_toggle:function(e,t){function o(){n.running=!1,n._trigger("activate",e,t)}function u(){t.newTab.closest("li").addClass("ui-tabs-active ui-state-active"),r.length&&s.show?r.animate(s.show,s.show.duration,function(){o()}):(r.show(),o())}var n=this,r=t.newPanel,i=t.oldPanel,s=this._getFx();if(!s)return this._super(e,t);n.running=!0,i.length&&s.hide?i.animate(s.hide,s.hide.duration,function(){t.oldTab.closest("li").removeClass("ui-tabs-active ui-state-active"),u()}):(t.oldTab.closest("li").removeClass("ui-tabs-active ui-state-active"),i.hide(),u())}}))})(jQuery);(function(e){function n(t,n){var r=(t.attr("aria-describedby")||"").split(/\s+/);r.push(n),t.data("ui-tooltip-id",n).attr("aria-describedby",e.trim(r.join(" ")))}function r(t){var n=t.data("ui-tooltip-id"),r=(t.attr("aria-describedby")||"").split(/\s+/),i=e.inArray(n,r);i!==-1&&r.splice(i,1),t.removeData("ui-tooltip-id"),r=e.trim(r.join(" ")),r?t.attr("aria-describedby",r):t.removeAttr("aria-describedby")}var t=0;e.widget("ui.tooltip",{version:"1.9.1",options:{content:function(){return e(this).attr("title")},hide:!0,items:"[title]:not([disabled])",position:{my:"left top+15",at:"left bottom",collision:"flipfit flipfit"},show:!0,tooltipClass:null,track:!1,close:null,open:null},_create:function(){this._on({mouseover:"open",focusin:"open"}),this.tooltips={},this.parents={},this.options.disabled&&this._disable()},_setOption:function(t,n){var r=this;if(t==="disabled"){this[n?"_disable":"_enable"](),this.options[t]=n;return}this._super(t,n),t==="content"&&e.each(this.tooltips,function(e,t){r._updateContent(t)})},_disable:function(){var t=this;e.each(this.tooltips,function(n,r){var i=e.Event("blur");i.target=i.currentTarget=r[0],t.close(i,!0)}),this.element.find(this.options.items).andSelf().each(function(){var t=e(this);t.is("[title]")&&t.data("ui-tooltip-title",t.attr("title")).attr("title","")})},_enable:function(){this.element.find(this.options.items).andSelf().each(function(){var t=e(this);t.data("ui-tooltip-title")&&t.attr("title",t.data("ui-tooltip-title"))})},open:function(t){var n=this,r=e(t?t.target:this.element).closest(this.options.items);if(!r.length)return;if(this.options.track&&r.data("ui-tooltip-id")){this._find(r).position(e.extend({of:r},this.options.position)),this._off(this.document,"mousemove");return}r.attr("title")&&r.data("ui-tooltip-title",r.attr("title")),r.data("tooltip-open",!0),t&&t.type==="mouseover"&&r.parents().each(function(){var t;e(this).data("tooltip-open")&&(t=e.Event("blur"),t.target=t.currentTarget=this,n.close(t,!0)),this.title&&(e(this).uniqueId(),n.parents[this.id]={element:this,title:this.title},this.title="")}),this._updateContent(r,t)},_updateContent:function(e,t){var n,r=this.options.content,i=this;if(typeof r=="string")return this._open(t,e,r);n=r.call(e[0],function(n){if(!e.data("tooltip-open"))return;i._delay(function(){this._open(t,e,n)})}),n&&this._open(t,e,n)},_open:function(t,r,i){function f(e){a.of=e;if(s.is(":hidden"))return;s.position(a)}var s,o,u,a=e.extend({},this.options.position);if(!i)return;s=this._find(r);if(s.length){s.find(".ui-tooltip-content").html(i);return}r.is("[title]")&&(t&&t.type==="mouseover"?r.attr("title",""):r.removeAttr("title")),s=this._tooltip(r),n(r,s.attr("id")),s.find(".ui-tooltip-content").html(i),this.options.track&&t&&/^mouse/.test(t.originalEvent.type)?(this._on(this.document,{mousemove:f}),f(t)):s.position(e.extend({of:r},this.options.position)),s.hide(),this._show(s,this.options.show),this.options.show&&this.options.show.delay&&(u=setInterval(function(){s.is(":visible")&&(f(a.of),clearInterval(u))},e.fx.interval)),this._trigger("open",t,{tooltip:s}),o={keyup:function(t){if(t.keyCode===e.ui.keyCode.ESCAPE){var n=e.Event(t);n.currentTarget=r[0],this.close(n,!0)}},remove:function(){this._removeTooltip(s)}};if(!t||t.type==="mouseover")o.mouseleave="close";if(!t||t.type==="focusin")o.focusout="close";this._on(r,o)},close:function(t){var n=this,i=e(t?t.currentTarget:this.element),s=this._find(i);if(this.closing)return;i.data("ui-tooltip-title")&&i.attr("title",i.data("ui-tooltip-title")),r(i),s.stop(!0),this._hide(s,this.options.hide,function(){n._removeTooltip(e(this))}),i.removeData("tooltip-open"),this._off(i,"mouseleave focusout keyup"),i[0]!==this.element[0]&&this._off(i,"remove"),this._off(this.document,"mousemove"),t&&t.type==="mouseleave"&&e.each(this.parents,function(e,t){t.element.title=t.title,delete n.parents[e]}),this.closing=!0,this._trigger("close",t,{tooltip:s}),this.closing=!1},_tooltip:function(n){var r="ui-tooltip-"+t++,i=e("<div>").attr({id:r,role:"tooltip"}).addClass("ui-tooltip ui-widget ui-corner-all ui-widget-content "+(this.options.tooltipClass||""));return e("<div>").addClass("ui-tooltip-content").appendTo(i),i.appendTo(this.document[0].body),e.fn.bgiframe&&i.bgiframe(),this.tooltips[r]=n,i},_find:function(t){var n=t.data("ui-tooltip-id");return n?e("#"+n):e()},_removeTooltip:function(e){e.remove(),delete this.tooltips[e.attr("id")]},_destroy:function(){var t=this;e.each(this.tooltips,function(n,r){var i=e.Event("blur");i.target=i.currentTarget=r[0],t.close(i,!0),e("#"+n).remove(),r.data("ui-tooltip-title")&&(r.attr("title",r.data("ui-tooltip-title")),r.removeData("ui-tooltip-title"))})}})})(jQuery);
\ No newline at end of file
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_flat_0_aaaaaa_40x100.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_flat_0_aaaaaa_40x100.png
new file mode 100644
index 0000000..5b5dab2
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_flat_0_aaaaaa_40x100.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_flat_75_ffffff_40x100.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_flat_75_ffffff_40x100.png
new file mode 100644
index 0000000..ac8b229
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_flat_75_ffffff_40x100.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_55_fbf9ee_1x400.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_55_fbf9ee_1x400.png
new file mode 100644
index 0000000..ad3d634
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_55_fbf9ee_1x400.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_65_ffffff_1x400.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_65_ffffff_1x400.png
new file mode 100644
index 0000000..42ccba2
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_65_ffffff_1x400.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_75_dadada_1x400.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_75_dadada_1x400.png
new file mode 100644
index 0000000..5a46b47
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_75_dadada_1x400.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_75_e6e6e6_1x400.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_75_e6e6e6_1x400.png
new file mode 100644
index 0000000..86c2baa
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_75_e6e6e6_1x400.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_95_fef1ec_1x400.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_95_fef1ec_1x400.png
new file mode 100644
index 0000000..4443fdc
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_glass_95_fef1ec_1x400.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_highlight-soft_75_cccccc_1x100.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_highlight-soft_75_cccccc_1x100.png
new file mode 100644
index 0000000..7c9fa6c
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-bg_highlight-soft_75_cccccc_1x100.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_222222_256x240.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_222222_256x240.png
new file mode 100644
index 0000000..ee039dc
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_222222_256x240.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_2e83ff_256x240.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_2e83ff_256x240.png
new file mode 100644
index 0000000..45e8928
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_2e83ff_256x240.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_454545_256x240.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_454545_256x240.png
new file mode 100644
index 0000000..7ec70d1
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_454545_256x240.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_888888_256x240.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_888888_256x240.png
new file mode 100644
index 0000000..5ba708c
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_888888_256x240.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_cd0a0a_256x240.png b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_cd0a0a_256x240.png
new file mode 100644
index 0000000..7930a55
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/images/ui-icons_cd0a0a_256x240.png
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css
new file mode 100644
index 0000000..02b242e
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css
@@ -0,0 +1,473 @@
+/*! jQuery UI - v1.9.1 - 2012-10-25
+* http://jqueryui.com
+* Includes: jquery.ui.core.css, jquery.ui.accordion.css, jquery.ui.autocomplete.css, jquery.ui.button.css, jquery.ui.datepicker.css, jquery.ui.dialog.css, jquery.ui.menu.css, jquery.ui.progressbar.css, jquery.ui.resizable.css, jquery.ui.selectable.css, jquery.ui.slider.css, jquery.ui.spinner.css, jquery.ui.tabs.css, jquery.ui.tooltip.css, jquery.ui.theme.css
+* Copyright 2012 jQuery Foundation and other contributors; Licensed MIT */
+
+/* Layout helpers
+----------------------------------*/
+.ui-helper-hidden { display: none; }
+.ui-helper-hidden-accessible { position: absolute !important; clip: rect(1px 1px 1px 1px); clip: rect(1px,1px,1px,1px); }
+.ui-helper-reset { margin: 0; padding: 0; border: 0; outline: 0; line-height: 1.3; text-decoration: none; font-size: 100%; list-style: none; }
+.ui-helper-clearfix:before, .ui-helper-clearfix:after { content: ""; display: table; }
+.ui-helper-clearfix:after { clear: both; }
+.ui-helper-clearfix { zoom: 1; }
+.ui-helper-zfix { width: 100%; height: 100%; top: 0; left: 0; position: absolute; opacity: 0; filter:Alpha(Opacity=0); }
+
+
+/* Interaction Cues
+----------------------------------*/
+.ui-state-disabled { cursor: default !important; }
+
+
+/* Icons
+----------------------------------*/
+
+/* states and images */
+.ui-icon { display: block; text-indent: -99999px; overflow: hidden; background-repeat: no-repeat; }
+
+
+/* Misc visuals
+----------------------------------*/
+
+/* Overlays */
+.ui-widget-overlay { position: absolute; top: 0; left: 0; width: 100%; height: 100%; }
+
+.ui-accordion .ui-accordion-header { display: block; cursor: pointer; position: relative; margin-top: 2px; padding: .5em .5em .5em .7em; zoom: 1; }
+.ui-accordion .ui-accordion-icons { padding-left: 2.2em; }
+.ui-accordion .ui-accordion-noicons { padding-left: .7em; }
+.ui-accordion .ui-accordion-icons .ui-accordion-icons { padding-left: 2.2em; }
+.ui-accordion .ui-accordion-header .ui-accordion-header-icon { position: absolute; left: .5em; top: 50%; margin-top: -8px; }
+.ui-accordion .ui-accordion-content { padding: 1em 2.2em; border-top: 0; overflow: auto; zoom: 1; }
+
+.ui-autocomplete {
+ position: absolute;
+ top: 0; /* #8656 */
+ cursor: default;
+}
+
+/* workarounds */
+* html .ui-autocomplete { width:1px; } /* without this, the menu expands to 100% in IE6 */
+
+.ui-button { display: inline-block; position: relative; padding: 0; margin-right: .1em; cursor: pointer; text-align: center; zoom: 1; overflow: visible; } /* the overflow property removes extra width in IE */
+.ui-button, .ui-button:link, .ui-button:visited, .ui-button:hover, .ui-button:active { text-decoration: none; }
+.ui-button-icon-only { width: 2.2em; } /* to make room for the icon, a width needs to be set here */
+button.ui-button-icon-only { width: 2.4em; } /* button elements seem to need a little more width */
+.ui-button-icons-only { width: 3.4em; }
+button.ui-button-icons-only { width: 3.7em; }
+
+/*button text element */
+.ui-button .ui-button-text { display: block; line-height: 1.4; }
+.ui-button-text-only .ui-button-text { padding: .4em 1em; }
+.ui-button-icon-only .ui-button-text, .ui-button-icons-only .ui-button-text { padding: .4em; text-indent: -9999999px; }
+.ui-button-text-icon-primary .ui-button-text, .ui-button-text-icons .ui-button-text { padding: .4em 1em .4em 2.1em; }
+.ui-button-text-icon-secondary .ui-button-text, .ui-button-text-icons .ui-button-text { padding: .4em 2.1em .4em 1em; }
+.ui-button-text-icons .ui-button-text { padding-left: 2.1em; padding-right: 2.1em; }
+/* no icon support for input elements, provide padding by default */
+input.ui-button { padding: .4em 1em; }
+
+/*button icon element(s) */
+.ui-button-icon-only .ui-icon, .ui-button-text-icon-primary .ui-icon, .ui-button-text-icon-secondary .ui-icon, .ui-button-text-icons .ui-icon, .ui-button-icons-only .ui-icon { position: absolute; top: 50%; margin-top: -8px; }
+.ui-button-icon-only .ui-icon { left: 50%; margin-left: -8px; }
+.ui-button-text-icon-primary .ui-button-icon-primary, .ui-button-text-icons .ui-button-icon-primary, .ui-button-icons-only .ui-button-icon-primary { left: .5em; }
+.ui-button-text-icon-secondary .ui-button-icon-secondary, .ui-button-text-icons .ui-button-icon-secondary, .ui-button-icons-only .ui-button-icon-secondary { right: .5em; }
+.ui-button-text-icons .ui-button-icon-secondary, .ui-button-icons-only .ui-button-icon-secondary { right: .5em; }
+
+/*button sets*/
+.ui-buttonset { margin-right: 7px; }
+.ui-buttonset .ui-button { margin-left: 0; margin-right: -.3em; }
+
+/* workarounds */
+button.ui-button::-moz-focus-inner { border: 0; padding: 0; } /* reset extra padding in Firefox */
+
+.ui-datepicker { width: 17em; padding: .2em .2em 0; display: none; }
+.ui-datepicker .ui-datepicker-header { position:relative; padding:.2em 0; }
+.ui-datepicker .ui-datepicker-prev, .ui-datepicker .ui-datepicker-next { position:absolute; top: 2px; width: 1.8em; height: 1.8em; }
+.ui-datepicker .ui-datepicker-prev-hover, .ui-datepicker .ui-datepicker-next-hover { top: 1px; }
+.ui-datepicker .ui-datepicker-prev { left:2px; }
+.ui-datepicker .ui-datepicker-next { right:2px; }
+.ui-datepicker .ui-datepicker-prev-hover { left:1px; }
+.ui-datepicker .ui-datepicker-next-hover { right:1px; }
+.ui-datepicker .ui-datepicker-prev span, .ui-datepicker .ui-datepicker-next span { display: block; position: absolute; left: 50%; margin-left: -8px; top: 50%; margin-top: -8px; }
+.ui-datepicker .ui-datepicker-title { margin: 0 2.3em; line-height: 1.8em; text-align: center; }
+.ui-datepicker .ui-datepicker-title select { font-size:1em; margin:1px 0; }
+.ui-datepicker select.ui-datepicker-month-year {width: 100%;}
+.ui-datepicker select.ui-datepicker-month,
+.ui-datepicker select.ui-datepicker-year { width: 49%;}
+.ui-datepicker table {width: 100%; font-size: .9em; border-collapse: collapse; margin:0 0 .4em; }
+.ui-datepicker th { padding: .7em .3em; text-align: center; font-weight: bold; border: 0; }
+.ui-datepicker td { border: 0; padding: 1px; }
+.ui-datepicker td span, .ui-datepicker td a { display: block; padding: .2em; text-align: right; text-decoration: none; }
+.ui-datepicker .ui-datepicker-buttonpane { background-image: none; margin: .7em 0 0 0; padding:0 .2em; border-left: 0; border-right: 0; border-bottom: 0; }
+.ui-datepicker .ui-datepicker-buttonpane button { float: right; margin: .5em .2em .4em; cursor: pointer; padding: .2em .6em .3em .6em; width:auto; overflow:visible; }
+.ui-datepicker .ui-datepicker-buttonpane button.ui-datepicker-current { float:left; }
+
+/* with multiple calendars */
+.ui-datepicker.ui-datepicker-multi { width:auto; }
+.ui-datepicker-multi .ui-datepicker-group { float:left; }
+.ui-datepicker-multi .ui-datepicker-group table { width:95%; margin:0 auto .4em; }
+.ui-datepicker-multi-2 .ui-datepicker-group { width:50%; }
+.ui-datepicker-multi-3 .ui-datepicker-group { width:33.3%; }
+.ui-datepicker-multi-4 .ui-datepicker-group { width:25%; }
+.ui-datepicker-multi .ui-datepicker-group-last .ui-datepicker-header { border-left-width:0; }
+.ui-datepicker-multi .ui-datepicker-group-middle .ui-datepicker-header { border-left-width:0; }
+.ui-datepicker-multi .ui-datepicker-buttonpane { clear:left; }
+.ui-datepicker-row-break { clear:both; width:100%; font-size:0em; }
+
+/* RTL support */
+.ui-datepicker-rtl { direction: rtl; }
+.ui-datepicker-rtl .ui-datepicker-prev { right: 2px; left: auto; }
+.ui-datepicker-rtl .ui-datepicker-next { left: 2px; right: auto; }
+.ui-datepicker-rtl .ui-datepicker-prev:hover { right: 1px; left: auto; }
+.ui-datepicker-rtl .ui-datepicker-next:hover { left: 1px; right: auto; }
+.ui-datepicker-rtl .ui-datepicker-buttonpane { clear:right; }
+.ui-datepicker-rtl .ui-datepicker-buttonpane button { float: left; }
+.ui-datepicker-rtl .ui-datepicker-buttonpane button.ui-datepicker-current { float:right; }
+.ui-datepicker-rtl .ui-datepicker-group { float:right; }
+.ui-datepicker-rtl .ui-datepicker-group-last .ui-datepicker-header { border-right-width:0; border-left-width:1px; }
+.ui-datepicker-rtl .ui-datepicker-group-middle .ui-datepicker-header { border-right-width:0; border-left-width:1px; }
+
+/* IE6 IFRAME FIX (taken from datepicker 1.5.3 */
+.ui-datepicker-cover {
+ position: absolute; /*must have*/
+ z-index: -1; /*must have*/
+ filter: mask(); /*must have*/
+ top: -4px; /*must have*/
+ left: -4px; /*must have*/
+ width: 200px; /*must have*/
+ height: 200px; /*must have*/
+}
+.ui-dialog { position: absolute; padding: .2em; width: 300px; overflow: hidden; }
+.ui-dialog .ui-dialog-titlebar { padding: .4em 1em; position: relative; }
+.ui-dialog .ui-dialog-title { float: left; margin: .1em 16px .1em 0; }
+.ui-dialog .ui-dialog-titlebar-close { position: absolute; right: .3em; top: 50%; width: 19px; margin: -10px 0 0 0; padding: 1px; height: 18px; }
+.ui-dialog .ui-dialog-titlebar-close span { display: block; margin: 1px; }
+.ui-dialog .ui-dialog-titlebar-close:hover, .ui-dialog .ui-dialog-titlebar-close:focus { padding: 0; }
+.ui-dialog .ui-dialog-content { position: relative; border: 0; padding: .5em 1em; background: none; overflow: auto; zoom: 1; }
+.ui-dialog .ui-dialog-buttonpane { text-align: left; border-width: 1px 0 0 0; background-image: none; margin: .5em 0 0 0; padding: .3em 1em .5em .4em; }
+.ui-dialog .ui-dialog-buttonpane .ui-dialog-buttonset { float: right; }
+.ui-dialog .ui-dialog-buttonpane button { margin: .5em .4em .5em 0; cursor: pointer; }
+.ui-dialog .ui-resizable-se { width: 14px; height: 14px; right: 3px; bottom: 3px; }
+.ui-draggable .ui-dialog-titlebar { cursor: move; }
+
+.ui-menu { list-style:none; padding: 2px; margin: 0; display:block; outline: none; }
+.ui-menu .ui-menu { margin-top: -3px; position: absolute; }
+.ui-menu .ui-menu-item { margin: 0; padding: 0; zoom: 1; width: 100%; }
+.ui-menu .ui-menu-divider { margin: 5px -2px 5px -2px; height: 0; font-size: 0; line-height: 0; border-width: 1px 0 0 0; }
+.ui-menu .ui-menu-item a { text-decoration: none; display: block; padding: 2px .4em; line-height: 1.5; zoom: 1; font-weight: normal; }
+.ui-menu .ui-menu-item a.ui-state-focus,
+.ui-menu .ui-menu-item a.ui-state-active { font-weight: normal; margin: -1px; }
+
+.ui-menu .ui-state-disabled { font-weight: normal; margin: .4em 0 .2em; line-height: 1.5; }
+.ui-menu .ui-state-disabled a { cursor: default; }
+
+/* icon support */
+.ui-menu-icons { position: relative; }
+.ui-menu-icons .ui-menu-item a { position: relative; padding-left: 2em; }
+
+/* left-aligned */
+.ui-menu .ui-icon { position: absolute; top: .2em; left: .2em; }
+
+/* right-aligned */
+.ui-menu .ui-menu-icon { position: static; float: right; }
+
+.ui-progressbar { height:2em; text-align: left; overflow: hidden; }
+.ui-progressbar .ui-progressbar-value {margin: -1px; height:100%; }
+.ui-resizable { position: relative;}
+.ui-resizable-handle { position: absolute;font-size: 0.1px; display: block; }
+.ui-resizable-disabled .ui-resizable-handle, .ui-resizable-autohide .ui-resizable-handle { display: none; }
+.ui-resizable-n { cursor: n-resize; height: 7px; width: 100%; top: -5px; left: 0; }
+.ui-resizable-s { cursor: s-resize; height: 7px; width: 100%; bottom: -5px; left: 0; }
+.ui-resizable-e { cursor: e-resize; width: 7px; right: -5px; top: 0; height: 100%; }
+.ui-resizable-w { cursor: w-resize; width: 7px; left: -5px; top: 0; height: 100%; }
+.ui-resizable-se { cursor: se-resize; width: 12px; height: 12px; right: 1px; bottom: 1px; }
+.ui-resizable-sw { cursor: sw-resize; width: 9px; height: 9px; left: -5px; bottom: -5px; }
+.ui-resizable-nw { cursor: nw-resize; width: 9px; height: 9px; left: -5px; top: -5px; }
+.ui-resizable-ne { cursor: ne-resize; width: 9px; height: 9px; right: -5px; top: -5px;}
+.ui-selectable-helper { position: absolute; z-index: 100; border:1px dotted black; }
+
+.ui-slider { position: relative; text-align: left; }
+.ui-slider .ui-slider-handle { position: absolute; z-index: 2; width: 1.2em; height: 1.2em; cursor: default; }
+.ui-slider .ui-slider-range { position: absolute; z-index: 1; font-size: .7em; display: block; border: 0; background-position: 0 0; }
+
+.ui-slider-horizontal { height: .8em; }
+.ui-slider-horizontal .ui-slider-handle { top: -.3em; margin-left: -.6em; }
+.ui-slider-horizontal .ui-slider-range { top: 0; height: 100%; }
+.ui-slider-horizontal .ui-slider-range-min { left: 0; }
+.ui-slider-horizontal .ui-slider-range-max { right: 0; }
+
+.ui-slider-vertical { width: .8em; height: 100px; }
+.ui-slider-vertical .ui-slider-handle { left: -.3em; margin-left: 0; margin-bottom: -.6em; }
+.ui-slider-vertical .ui-slider-range { left: 0; width: 100%; }
+.ui-slider-vertical .ui-slider-range-min { bottom: 0; }
+.ui-slider-vertical .ui-slider-range-max { top: 0; }
+.ui-spinner { position:relative; display: inline-block; overflow: hidden; padding: 0; vertical-align: middle; }
+.ui-spinner-input { border: none; background: none; padding: 0; margin: .2em 0; vertical-align: middle; margin-left: .4em; margin-right: 22px; }
+.ui-spinner-button { width: 16px; height: 50%; font-size: .5em; padding: 0; margin: 0; text-align: center; position: absolute; cursor: default; display: block; overflow: hidden; right: 0; }
+.ui-spinner a.ui-spinner-button { border-top: none; border-bottom: none; border-right: none; } /* more specificity required here to overide default borders */
+.ui-spinner .ui-icon { position: absolute; margin-top: -8px; top: 50%; left: 0; } /* vertical centre icon */
+.ui-spinner-up { top: 0; }
+.ui-spinner-down { bottom: 0; }
+
+/* TR overrides */
+.ui-spinner .ui-icon-triangle-1-s {
+ /* need to fix icons sprite */
+ background-position:-65px -16px;
+}
+
+.ui-tabs { position: relative; padding: .2em; zoom: 1; } /* position: relative prevents IE scroll bug (element with position: relative inside container with overflow: auto appear as "fixed") */
+.ui-tabs .ui-tabs-nav { margin: 0; padding: .2em .2em 0; }
+.ui-tabs .ui-tabs-nav li { list-style: none; float: left; position: relative; top: 0; margin: 1px .2em 0 0; border-bottom: 0; padding: 0; white-space: nowrap; }
+.ui-tabs .ui-tabs-nav li a { float: left; padding: .5em 1em; text-decoration: none; }
+.ui-tabs .ui-tabs-nav li.ui-tabs-active { margin-bottom: -1px; padding-bottom: 1px; }
+.ui-tabs .ui-tabs-nav li.ui-tabs-active a, .ui-tabs .ui-tabs-nav li.ui-state-disabled a, .ui-tabs .ui-tabs-nav li.ui-tabs-loading a { cursor: text; }
+.ui-tabs .ui-tabs-nav li a, .ui-tabs-collapsible .ui-tabs-nav li.ui-tabs-active a { cursor: pointer; } /* first selector in group seems obsolete, but required to overcome bug in Opera applying cursor: text overall if defined elsewhere... */
+.ui-tabs .ui-tabs-panel { display: block; border-width: 0; padding: 1em 1.4em; background: none; }
+
+.ui-tooltip {
+ padding: 8px;
+ position: absolute;
+ z-index: 9999;
+ max-width: 300px;
+ -webkit-box-shadow: 0 0 5px #aaa;
+ box-shadow: 0 0 5px #aaa;
+}
+/* Fades and background-images don't work well together in IE6, drop the image */
+* html .ui-tooltip {
+ background-image: none;
+}
+body .ui-tooltip { border-width: 2px; }
+
+/* Component containers
+----------------------------------*/
+.ui-widget { font-family: Verdana,Arial,sans-serif/*{ffDefault}*/; font-size: 1.1em/*{fsDefault}*/; }
+.ui-widget .ui-widget { font-size: 1em; }
+.ui-widget input, .ui-widget select, .ui-widget textarea, .ui-widget button { font-family: Verdana,Arial,sans-serif/*{ffDefault}*/; font-size: 1em; }
+.ui-widget-content { border: 1px solid #aaaaaa/*{borderColorContent}*/; background: #ffffff/*{bgColorContent}*/ url(images/ui-bg_flat_75_ffffff_40x100.png)/*{bgImgUrlContent}*/ 50%/*{bgContentXPos}*/ 50%/*{bgContentYPos}*/ repeat-x/*{bgContentRepeat}*/; color: #222222/*{fcContent}*/; }
+.ui-widget-content a { color: #222222/*{fcContent}*/; }
+.ui-widget-header { border: 1px solid #aaaaaa/*{borderColorHeader}*/; background: #cccccc/*{bgColorHeader}*/ url(images/ui-bg_highlight-soft_75_cccccc_1x100.png)/*{bgImgUrlHeader}*/ 50%/*{bgHeaderXPos}*/ 50%/*{bgHeaderYPos}*/ repeat-x/*{bgHeaderRepeat}*/; color: #222222/*{fcHeader}*/; font-weight: bold; }
+.ui-widget-header a { color: #222222/*{fcHeader}*/; }
+
+/* Interaction states
+----------------------------------*/
+.ui-state-default, .ui-widget-content .ui-state-default, .ui-widget-header .ui-state-default { border: 1px solid #d3d3d3/*{borderColorDefault}*/; background: #e6e6e6/*{bgColorDefault}*/ url(images/ui-bg_glass_75_e6e6e6_1x400.png)/*{bgImgUrlDefault}*/ 50%/*{bgDefaultXPos}*/ 50%/*{bgDefaultYPos}*/ repeat-x/*{bgDefaultRepeat}*/; font-weight: normal/*{fwDefault}*/; color: #555555/*{fcDefault}*/; }
+.ui-state-default a, .ui-state-default a:link, .ui-state-default a:visited { color: #555555/*{fcDefault}*/; text-decoration: none; }
+.ui-state-hover, .ui-widget-content .ui-state-hover, .ui-widget-header .ui-state-hover, .ui-state-focus, .ui-widget-content .ui-state-focus, .ui-widget-header .ui-state-focus { border: 1px solid #999999/*{borderColorHover}*/; background: #dadada/*{bgColorHover}*/ url(images/ui-bg_glass_75_dadada_1x400.png)/*{bgImgUrlHover}*/ 50%/*{bgHoverXPos}*/ 50%/*{bgHoverYPos}*/ repeat-x/*{bgHoverRepeat}*/; font-weight: normal/*{fwDefault}*/; color: #212121/*{fcHover}*/; }
+.ui-state-hover a, .ui-state-hover a:hover, .ui-state-hover a:link, .ui-state-hover a:visited { color: #212121/*{fcHover}*/; text-decoration: none; }
+.ui-state-active, .ui-widget-content .ui-state-active, .ui-widget-header .ui-state-active { border: 1px solid #aaaaaa/*{borderColorActive}*/; background: #ffffff/*{bgColorActive}*/ url(images/ui-bg_glass_65_ffffff_1x400.png)/*{bgImgUrlActive}*/ 50%/*{bgActiveXPos}*/ 50%/*{bgActiveYPos}*/ repeat-x/*{bgActiveRepeat}*/; font-weight: normal/*{fwDefault}*/; color: #212121/*{fcActive}*/; }
+.ui-state-active a, .ui-state-active a:link, .ui-state-active a:visited { color: #212121/*{fcActive}*/; text-decoration: none; }
+
+/* Interaction Cues
+----------------------------------*/
+.ui-state-highlight, .ui-widget-content .ui-state-highlight, .ui-widget-header .ui-state-highlight {border: 1px solid #fcefa1/*{borderColorHighlight}*/; background: #fbf9ee/*{bgColorHighlight}*/ url(images/ui-bg_glass_55_fbf9ee_1x400.png)/*{bgImgUrlHighlight}*/ 50%/*{bgHighlightXPos}*/ 50%/*{bgHighlightYPos}*/ repeat-x/*{bgHighlightRepeat}*/; color: #363636/*{fcHighlight}*/; }
+.ui-state-highlight a, .ui-widget-content .ui-state-highlight a,.ui-widget-header .ui-state-highlight a { color: #363636/*{fcHighlight}*/; }
+.ui-state-error, .ui-widget-content .ui-state-error, .ui-widget-header .ui-state-error {border: 1px solid #cd0a0a/*{borderColorError}*/; background: #fef1ec/*{bgColorError}*/ url(images/ui-bg_glass_95_fef1ec_1x400.png)/*{bgImgUrlError}*/ 50%/*{bgErrorXPos}*/ 50%/*{bgErrorYPos}*/ repeat-x/*{bgErrorRepeat}*/; color: #cd0a0a/*{fcError}*/; }
+.ui-state-error a, .ui-widget-content .ui-state-error a, .ui-widget-header .ui-state-error a { color: #cd0a0a/*{fcError}*/; }
+.ui-state-error-text, .ui-widget-content .ui-state-error-text, .ui-widget-header .ui-state-error-text { color: #cd0a0a/*{fcError}*/; }
+.ui-priority-primary, .ui-widget-content .ui-priority-primary, .ui-widget-header .ui-priority-primary { font-weight: bold; }
+.ui-priority-secondary, .ui-widget-content .ui-priority-secondary, .ui-widget-header .ui-priority-secondary { opacity: .7; filter:Alpha(Opacity=70); font-weight: normal; }
+.ui-state-disabled, .ui-widget-content .ui-state-disabled, .ui-widget-header .ui-state-disabled { opacity: .35; filter:Alpha(Opacity=35); background-image: none; }
+.ui-state-disabled .ui-icon { filter:Alpha(Opacity=35); } /* For IE8 - See #6059 */
+
+/* Icons
+----------------------------------*/
+
+/* states and images */
+.ui-icon { width: 16px; height: 16px; background-image: url(images/ui-icons_222222_256x240.png)/*{iconsContent}*/; }
+.ui-widget-content .ui-icon {background-image: url(images/ui-icons_222222_256x240.png)/*{iconsContent}*/; }
+.ui-widget-header .ui-icon {background-image: url(images/ui-icons_222222_256x240.png)/*{iconsHeader}*/; }
+.ui-state-default .ui-icon { background-image: url(images/ui-icons_888888_256x240.png)/*{iconsDefault}*/; }
+.ui-state-hover .ui-icon, .ui-state-focus .ui-icon {background-image: url(images/ui-icons_454545_256x240.png)/*{iconsHover}*/; }
+.ui-state-active .ui-icon {background-image: url(images/ui-icons_454545_256x240.png)/*{iconsActive}*/; }
+.ui-state-highlight .ui-icon {background-image: url(images/ui-icons_2e83ff_256x240.png)/*{iconsHighlight}*/; }
+.ui-state-error .ui-icon, .ui-state-error-text .ui-icon {background-image: url(images/ui-icons_cd0a0a_256x240.png)/*{iconsError}*/; }
+
+/* positioning */
+.ui-icon-carat-1-n { background-position: 0 0; }
+.ui-icon-carat-1-ne { background-position: -16px 0; }
+.ui-icon-carat-1-e { background-position: -32px 0; }
+.ui-icon-carat-1-se { background-position: -48px 0; }
+.ui-icon-carat-1-s { background-position: -64px 0; }
+.ui-icon-carat-1-sw { background-position: -80px 0; }
+.ui-icon-carat-1-w { background-position: -96px 0; }
+.ui-icon-carat-1-nw { background-position: -112px 0; }
+.ui-icon-carat-2-n-s { background-position: -128px 0; }
+.ui-icon-carat-2-e-w { background-position: -144px 0; }
+.ui-icon-triangle-1-n { background-position: 0 -16px; }
+.ui-icon-triangle-1-ne { background-position: -16px -16px; }
+.ui-icon-triangle-1-e { background-position: -32px -16px; }
+.ui-icon-triangle-1-se { background-position: -48px -16px; }
+.ui-icon-triangle-1-s { background-position: -64px -16px; }
+.ui-icon-triangle-1-sw { background-position: -80px -16px; }
+.ui-icon-triangle-1-w { background-position: -96px -16px; }
+.ui-icon-triangle-1-nw { background-position: -112px -16px; }
+.ui-icon-triangle-2-n-s { background-position: -128px -16px; }
+.ui-icon-triangle-2-e-w { background-position: -144px -16px; }
+.ui-icon-arrow-1-n { background-position: 0 -32px; }
+.ui-icon-arrow-1-ne { background-position: -16px -32px; }
+.ui-icon-arrow-1-e { background-position: -32px -32px; }
+.ui-icon-arrow-1-se { background-position: -48px -32px; }
+.ui-icon-arrow-1-s { background-position: -64px -32px; }
+.ui-icon-arrow-1-sw { background-position: -80px -32px; }
+.ui-icon-arrow-1-w { background-position: -96px -32px; }
+.ui-icon-arrow-1-nw { background-position: -112px -32px; }
+.ui-icon-arrow-2-n-s { background-position: -128px -32px; }
+.ui-icon-arrow-2-ne-sw { background-position: -144px -32px; }
+.ui-icon-arrow-2-e-w { background-position: -160px -32px; }
+.ui-icon-arrow-2-se-nw { background-position: -176px -32px; }
+.ui-icon-arrowstop-1-n { background-position: -192px -32px; }
+.ui-icon-arrowstop-1-e { background-position: -208px -32px; }
+.ui-icon-arrowstop-1-s { background-position: -224px -32px; }
+.ui-icon-arrowstop-1-w { background-position: -240px -32px; }
+.ui-icon-arrowthick-1-n { background-position: 0 -48px; }
+.ui-icon-arrowthick-1-ne { background-position: -16px -48px; }
+.ui-icon-arrowthick-1-e { background-position: -32px -48px; }
+.ui-icon-arrowthick-1-se { background-position: -48px -48px; }
+.ui-icon-arrowthick-1-s { background-position: -64px -48px; }
+.ui-icon-arrowthick-1-sw { background-position: -80px -48px; }
+.ui-icon-arrowthick-1-w { background-position: -96px -48px; }
+.ui-icon-arrowthick-1-nw { background-position: -112px -48px; }
+.ui-icon-arrowthick-2-n-s { background-position: -128px -48px; }
+.ui-icon-arrowthick-2-ne-sw { background-position: -144px -48px; }
+.ui-icon-arrowthick-2-e-w { background-position: -160px -48px; }
+.ui-icon-arrowthick-2-se-nw { background-position: -176px -48px; }
+.ui-icon-arrowthickstop-1-n { background-position: -192px -48px; }
+.ui-icon-arrowthickstop-1-e { background-position: -208px -48px; }
+.ui-icon-arrowthickstop-1-s { background-position: -224px -48px; }
+.ui-icon-arrowthickstop-1-w { background-position: -240px -48px; }
+.ui-icon-arrowreturnthick-1-w { background-position: 0 -64px; }
+.ui-icon-arrowreturnthick-1-n { background-position: -16px -64px; }
+.ui-icon-arrowreturnthick-1-e { background-position: -32px -64px; }
+.ui-icon-arrowreturnthick-1-s { background-position: -48px -64px; }
+.ui-icon-arrowreturn-1-w { background-position: -64px -64px; }
+.ui-icon-arrowreturn-1-n { background-position: -80px -64px; }
+.ui-icon-arrowreturn-1-e { background-position: -96px -64px; }
+.ui-icon-arrowreturn-1-s { background-position: -112px -64px; }
+.ui-icon-arrowrefresh-1-w { background-position: -128px -64px; }
+.ui-icon-arrowrefresh-1-n { background-position: -144px -64px; }
+.ui-icon-arrowrefresh-1-e { background-position: -160px -64px; }
+.ui-icon-arrowrefresh-1-s { background-position: -176px -64px; }
+.ui-icon-arrow-4 { background-position: 0 -80px; }
+.ui-icon-arrow-4-diag { background-position: -16px -80px; }
+.ui-icon-extlink { background-position: -32px -80px; }
+.ui-icon-newwin { background-position: -48px -80px; }
+.ui-icon-refresh { background-position: -64px -80px; }
+.ui-icon-shuffle { background-position: -80px -80px; }
+.ui-icon-transfer-e-w { background-position: -96px -80px; }
+.ui-icon-transferthick-e-w { background-position: -112px -80px; }
+.ui-icon-folder-collapsed { background-position: 0 -96px; }
+.ui-icon-folder-open { background-position: -16px -96px; }
+.ui-icon-document { background-position: -32px -96px; }
+.ui-icon-document-b { background-position: -48px -96px; }
+.ui-icon-note { background-position: -64px -96px; }
+.ui-icon-mail-closed { background-position: -80px -96px; }
+.ui-icon-mail-open { background-position: -96px -96px; }
+.ui-icon-suitcase { background-position: -112px -96px; }
+.ui-icon-comment { background-position: -128px -96px; }
+.ui-icon-person { background-position: -144px -96px; }
+.ui-icon-print { background-position: -160px -96px; }
+.ui-icon-trash { background-position: -176px -96px; }
+.ui-icon-locked { background-position: -192px -96px; }
+.ui-icon-unlocked { background-position: -208px -96px; }
+.ui-icon-bookmark { background-position: -224px -96px; }
+.ui-icon-tag { background-position: -240px -96px; }
+.ui-icon-home { background-position: 0 -112px; }
+.ui-icon-flag { background-position: -16px -112px; }
+.ui-icon-calendar { background-position: -32px -112px; }
+.ui-icon-cart { background-position: -48px -112px; }
+.ui-icon-pencil { background-position: -64px -112px; }
+.ui-icon-clock { background-position: -80px -112px; }
+.ui-icon-disk { background-position: -96px -112px; }
+.ui-icon-calculator { background-position: -112px -112px; }
+.ui-icon-zoomin { background-position: -128px -112px; }
+.ui-icon-zoomout { background-position: -144px -112px; }
+.ui-icon-search { background-position: -160px -112px; }
+.ui-icon-wrench { background-position: -176px -112px; }
+.ui-icon-gear { background-position: -192px -112px; }
+.ui-icon-heart { background-position: -208px -112px; }
+.ui-icon-star { background-position: -224px -112px; }
+.ui-icon-link { background-position: -240px -112px; }
+.ui-icon-cancel { background-position: 0 -128px; }
+.ui-icon-plus { background-position: -16px -128px; }
+.ui-icon-plusthick { background-position: -32px -128px; }
+.ui-icon-minus { background-position: -48px -128px; }
+.ui-icon-minusthick { background-position: -64px -128px; }
+.ui-icon-close { background-position: -80px -128px; }
+.ui-icon-closethick { background-position: -96px -128px; }
+.ui-icon-key { background-position: -112px -128px; }
+.ui-icon-lightbulb { background-position: -128px -128px; }
+.ui-icon-scissors { background-position: -144px -128px; }
+.ui-icon-clipboard { background-position: -160px -128px; }
+.ui-icon-copy { background-position: -176px -128px; }
+.ui-icon-contact { background-position: -192px -128px; }
+.ui-icon-image { background-position: -208px -128px; }
+.ui-icon-video { background-position: -224px -128px; }
+.ui-icon-script { background-position: -240px -128px; }
+.ui-icon-alert { background-position: 0 -144px; }
+.ui-icon-info { background-position: -16px -144px; }
+.ui-icon-notice { background-position: -32px -144px; }
+.ui-icon-help { background-position: -48px -144px; }
+.ui-icon-check { background-position: -64px -144px; }
+.ui-icon-bullet { background-position: -80px -144px; }
+.ui-icon-radio-on { background-position: -96px -144px; }
+.ui-icon-radio-off { background-position: -112px -144px; }
+.ui-icon-pin-w { background-position: -128px -144px; }
+.ui-icon-pin-s { background-position: -144px -144px; }
+.ui-icon-play { background-position: 0 -160px; }
+.ui-icon-pause { background-position: -16px -160px; }
+.ui-icon-seek-next { background-position: -32px -160px; }
+.ui-icon-seek-prev { background-position: -48px -160px; }
+.ui-icon-seek-end { background-position: -64px -160px; }
+.ui-icon-seek-start { background-position: -80px -160px; }
+/* ui-icon-seek-first is deprecated, use ui-icon-seek-start instead */
+.ui-icon-seek-first { background-position: -80px -160px; }
+.ui-icon-stop { background-position: -96px -160px; }
+.ui-icon-eject { background-position: -112px -160px; }
+.ui-icon-volume-off { background-position: -128px -160px; }
+.ui-icon-volume-on { background-position: -144px -160px; }
+.ui-icon-power { background-position: 0 -176px; }
+.ui-icon-signal-diag { background-position: -16px -176px; }
+.ui-icon-signal { background-position: -32px -176px; }
+.ui-icon-battery-0 { background-position: -48px -176px; }
+.ui-icon-battery-1 { background-position: -64px -176px; }
+.ui-icon-battery-2 { background-position: -80px -176px; }
+.ui-icon-battery-3 { background-position: -96px -176px; }
+.ui-icon-circle-plus { background-position: 0 -192px; }
+.ui-icon-circle-minus { background-position: -16px -192px; }
+.ui-icon-circle-close { background-position: -32px -192px; }
+.ui-icon-circle-triangle-e { background-position: -48px -192px; }
+.ui-icon-circle-triangle-s { background-position: -64px -192px; }
+.ui-icon-circle-triangle-w { background-position: -80px -192px; }
+.ui-icon-circle-triangle-n { background-position: -96px -192px; }
+.ui-icon-circle-arrow-e { background-position: -112px -192px; }
+.ui-icon-circle-arrow-s { background-position: -128px -192px; }
+.ui-icon-circle-arrow-w { background-position: -144px -192px; }
+.ui-icon-circle-arrow-n { background-position: -160px -192px; }
+.ui-icon-circle-zoomin { background-position: -176px -192px; }
+.ui-icon-circle-zoomout { background-position: -192px -192px; }
+.ui-icon-circle-check { background-position: -208px -192px; }
+.ui-icon-circlesmall-plus { background-position: 0 -208px; }
+.ui-icon-circlesmall-minus { background-position: -16px -208px; }
+.ui-icon-circlesmall-close { background-position: -32px -208px; }
+.ui-icon-squaresmall-plus { background-position: -48px -208px; }
+.ui-icon-squaresmall-minus { background-position: -64px -208px; }
+.ui-icon-squaresmall-close { background-position: -80px -208px; }
+.ui-icon-grip-dotted-vertical { background-position: 0 -224px; }
+.ui-icon-grip-dotted-horizontal { background-position: -16px -224px; }
+.ui-icon-grip-solid-vertical { background-position: -32px -224px; }
+.ui-icon-grip-solid-horizontal { background-position: -48px -224px; }
+.ui-icon-gripsmall-diagonal-se { background-position: -64px -224px; }
+.ui-icon-grip-diagonal-se { background-position: -80px -224px; }
+
+
+/* Misc visuals
+----------------------------------*/
+
+/* Corner radius */
+.ui-corner-all, .ui-corner-top, .ui-corner-left, .ui-corner-tl { -moz-border-radius-topleft: 4px/*{cornerRadius}*/; -webkit-border-top-left-radius: 4px/*{cornerRadius}*/; -khtml-border-top-left-radius: 4px/*{cornerRadius}*/; border-top-left-radius: 4px/*{cornerRadius}*/; }
+.ui-corner-all, .ui-corner-top, .ui-corner-right, .ui-corner-tr { -moz-border-radius-topright: 4px/*{cornerRadius}*/; -webkit-border-top-right-radius: 4px/*{cornerRadius}*/; -khtml-border-top-right-radius: 4px/*{cornerRadius}*/; border-top-right-radius: 4px/*{cornerRadius}*/; }
+.ui-corner-all, .ui-corner-bottom, .ui-corner-left, .ui-corner-bl { -moz-border-radius-bottomleft: 4px/*{cornerRadius}*/; -webkit-border-bottom-left-radius: 4px/*{cornerRadius}*/; -khtml-border-bottom-left-radius: 4px/*{cornerRadius}*/; border-bottom-left-radius: 4px/*{cornerRadius}*/; }
+.ui-corner-all, .ui-corner-bottom, .ui-corner-right, .ui-corner-br { -moz-border-radius-bottomright: 4px/*{cornerRadius}*/; -webkit-border-bottom-right-radius: 4px/*{cornerRadius}*/; -khtml-border-bottom-right-radius: 4px/*{cornerRadius}*/; border-bottom-right-radius: 4px/*{cornerRadius}*/; }
+
+/* Overlays */
+.ui-widget-overlay { background: #aaaaaa/*{bgColorOverlay}*/ url(images/ui-bg_flat_0_aaaaaa_40x100.png)/*{bgImgUrlOverlay}*/ 50%/*{bgOverlayXPos}*/ 50%/*{bgOverlayYPos}*/ repeat-x/*{bgOverlayRepeat}*/; opacity: .3;filter:Alpha(Opacity=30)/*{opacityOverlay}*/; }
+.ui-widget-shadow { margin: -8px/*{offsetTopShadow}*/ 0 0 -8px/*{offsetLeftShadow}*/; padding: 8px/*{thicknessShadow}*/; background: #aaaaaa/*{bgColorShadow}*/ url(images/ui-bg_flat_0_aaaaaa_40x100.png)/*{bgImgUrlShadow}*/ 50%/*{bgShadowXPos}*/ 50%/*{bgShadowYPos}*/ repeat-x/*{bgShadowRepeat}*/; opacity: .3;filter:Alpha(Opacity=30)/*{opacityShadow}*/; -moz-border-radius: 8px/*{cornerRadiusShadow}*/; -khtml-border-radius: 8px/*{cornerRadiusShadow}*/; -webkit-border-radius: 8px/*{cornerRadiusShadow}*/; border-radius: 8px/*{cornerRadiusShadow}*/; }
\ No newline at end of file
diff --git a/slider-core/src/main/resources/webapps/static/jt/jquery.jstree.js b/slider-core/src/main/resources/webapps/static/jt/jquery.jstree.js
new file mode 100644
index 0000000..8996ac0
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/jt/jquery.jstree.js
@@ -0,0 +1,4544 @@
+/*
+ * jsTree 1.0-rc3
+ * http://jstree.com/
+ *
+ * Copyright (c) 2010 Ivan Bozhanov (vakata.com)
+ *
+ * Licensed same as jquery - under the terms of either the MIT License or the GPL Version 2 License
+ * http://www.opensource.org/licenses/mit-license.php
+ * http://www.gnu.org/licenses/gpl.html
+ *
+ * $Date$
+ * $Revision$
+ */
+
+/*jslint browser: true, onevar: true, undef: true, bitwise: true, strict: true */
+/*global window : false, clearInterval: false, clearTimeout: false, document: false, setInterval: false, setTimeout: false, jQuery: false, navigator: false, XSLTProcessor: false, DOMParser: false, XMLSerializer: false*/
+
+"use strict";
+
+// top wrapper to prevent multiple inclusion (is this OK?)
+(function () { if(jQuery && jQuery.jstree) { return; }
+ var is_ie6 = false, is_ie7 = false, is_ff2 = false;
+
+/*
+ * jsTree core
+ */
+(function ($) {
+ // Common functions not related to jsTree
+ // decided to move them to a `vakata` "namespace"
+ $.vakata = {};
+ // CSS related functions
+ $.vakata.css = {
+ get_css : function(rule_name, delete_flag, sheet) {
+ rule_name = rule_name.toLowerCase();
+ var css_rules = sheet.cssRules || sheet.rules,
+ j = 0;
+ do {
+ if(css_rules.length && j > css_rules.length + 5) { return false; }
+ if(css_rules[j].selectorText && css_rules[j].selectorText.toLowerCase() == rule_name) {
+ if(delete_flag === true) {
+ if(sheet.removeRule) { sheet.removeRule(j); }
+ if(sheet.deleteRule) { sheet.deleteRule(j); }
+ return true;
+ }
+ else { return css_rules[j]; }
+ }
+ }
+ while (css_rules[++j]);
+ return false;
+ },
+ add_css : function(rule_name, sheet) {
+ if($.jstree.css.get_css(rule_name, false, sheet)) { return false; }
+ if(sheet.insertRule) { sheet.insertRule(rule_name + ' { }', 0); } else { sheet.addRule(rule_name, null, 0); }
+ return $.vakata.css.get_css(rule_name);
+ },
+ remove_css : function(rule_name, sheet) {
+ return $.vakata.css.get_css(rule_name, true, sheet);
+ },
+ add_sheet : function(opts) {
+ var tmp = false, is_new = true;
+ if(opts.str) {
+ if(opts.title) { tmp = $("style[id='" + opts.title + "-stylesheet']")[0]; }
+ if(tmp) { is_new = false; }
+ else {
+ tmp = document.createElement("style");
+ tmp.setAttribute('type',"text/css");
+ if(opts.title) { tmp.setAttribute("id", opts.title + "-stylesheet"); }
+ }
+ if(tmp.styleSheet) {
+ if(is_new) {
+ document.getElementsByTagName("head")[0].appendChild(tmp);
+ tmp.styleSheet.cssText = opts.str;
+ }
+ else {
+ tmp.styleSheet.cssText = tmp.styleSheet.cssText + " " + opts.str;
+ }
+ }
+ else {
+ tmp.appendChild(document.createTextNode(opts.str));
+ document.getElementsByTagName("head")[0].appendChild(tmp);
+ }
+ return tmp.sheet || tmp.styleSheet;
+ }
+ if(opts.url) {
+ if(document.createStyleSheet) {
+ try { tmp = document.createStyleSheet(opts.url); } catch (e) { }
+ }
+ else {
+ tmp = document.createElement('link');
+ tmp.rel = 'stylesheet';
+ tmp.type = 'text/css';
+ tmp.media = "all";
+ tmp.href = opts.url;
+ document.getElementsByTagName("head")[0].appendChild(tmp);
+ return tmp.styleSheet;
+ }
+ }
+ }
+ };
+
+ // private variables
+ var instances = [], // instance array (used by $.jstree.reference/create/focused)
+ focused_instance = -1, // the index in the instance array of the currently focused instance
+ plugins = {}, // list of included plugins
+ prepared_move = {}; // for the move_node function
+
+ // jQuery plugin wrapper (thanks to jquery UI widget function)
+ $.fn.jstree = function (settings) {
+ var isMethodCall = (typeof settings == 'string'), // is this a method call like $().jstree("open_node")
+ args = Array.prototype.slice.call(arguments, 1),
+ returnValue = this;
+
+ // if a method call execute the method on all selected instances
+ if(isMethodCall) {
+ if(settings.substring(0, 1) == '_') { return returnValue; }
+ this.each(function() {
+ var instance = instances[$.data(this, "jstree-instance-id")],
+ methodValue = (instance && $.isFunction(instance[settings])) ? instance[settings].apply(instance, args) : instance;
+ if(typeof methodValue !== "undefined" && (settings.indexOf("is_") === 0 || (methodValue !== true && methodValue !== false))) { returnValue = methodValue; return false; }
+ });
+ }
+ else {
+ this.each(function() {
+ // extend settings and allow for multiple hashes and $.data
+ var instance_id = $.data(this, "jstree-instance-id"),
+ a = [],
+ b = settings ? $.extend({}, true, settings) : {},
+ c = $(this),
+ s = false,
+ t = [];
+ a = a.concat(args);
+ if(c.data("jstree")) { a.push(c.data("jstree")); }
+ b = a.length ? $.extend.apply(null, [true, b].concat(a)) : b;
+
+ // if an instance already exists, destroy it first
+ if(typeof instance_id !== "undefined" && instances[instance_id]) { instances[instance_id].destroy(); }
+ // push a new empty object to the instances array
+ instance_id = parseInt(instances.push({}),10) - 1;
+ // store the jstree instance id to the container element
+ $.data(this, "jstree-instance-id", instance_id);
+ // clean up all plugins
+ b.plugins = $.isArray(b.plugins) ? b.plugins : $.jstree.defaults.plugins.slice();
+ b.plugins.unshift("core");
+ // only unique plugins
+ b.plugins = b.plugins.sort().join(",,").replace(/(,|^)([^,]+)(,,\2)+(,|$)/g,"$1$2$4").replace(/,,+/g,",").replace(/,$/,"").split(",");
+
+ // extend defaults with passed data
+ s = $.extend(true, {}, $.jstree.defaults, b);
+ s.plugins = b.plugins;
+ $.each(plugins, function (i, val) {
+ if($.inArray(i, s.plugins) === -1) { s[i] = null; delete s[i]; }
+ else { t.push(i); }
+ });
+ s.plugins = t;
+
+ // push the new object to the instances array (at the same time set the default classes to the container) and init
+ instances[instance_id] = new $.jstree._instance(instance_id, $(this).addClass("jstree jstree-" + instance_id), s);
+ // init all activated plugins for this instance
+ $.each(instances[instance_id]._get_settings().plugins, function (i, val) { instances[instance_id].data[val] = {}; });
+ $.each(instances[instance_id]._get_settings().plugins, function (i, val) { if(plugins[val]) { plugins[val].__init.apply(instances[instance_id]); } });
+ // initialize the instance
+ setTimeout(function() { instances[instance_id].init(); }, 0);
+ });
+ }
+ // return the jquery selection (or if it was a method call that returned a value - the returned value)
+ return returnValue;
+ };
+ // object to store exposed functions and objects
+ $.jstree = {
+ defaults : {
+ plugins : []
+ },
+ _focused : function () { return instances[focused_instance] || null; },
+ _reference : function (needle) {
+ // get by instance id
+ if(instances[needle]) { return instances[needle]; }
+ // get by DOM (if still no luck - return null
+ var o = $(needle);
+ if(!o.length && typeof needle === "string") { o = $("#" + needle); }
+ if(!o.length) { return null; }
+ return instances[o.closest(".jstree").data("jstree-instance-id")] || null;
+ },
+ _instance : function (index, container, settings) {
+ // for plugins to store data in
+ this.data = { core : {} };
+ this.get_settings = function () { return $.extend(true, {}, settings); };
+ this._get_settings = function () { return settings; };
+ this.get_index = function () { return index; };
+ this.get_container = function () { return container; };
+ this.get_container_ul = function () { return container.children("ul:eq(0)"); };
+ this._set_settings = function (s) {
+ settings = $.extend(true, {}, settings, s);
+ };
+ },
+ _fn : { },
+ plugin : function (pname, pdata) {
+ pdata = $.extend({}, {
+ __init : $.noop,
+ __destroy : $.noop,
+ _fn : {},
+ defaults : false
+ }, pdata);
+ plugins[pname] = pdata;
+
+ $.jstree.defaults[pname] = pdata.defaults;
+ $.each(pdata._fn, function (i, val) {
+ val.plugin = pname;
+ val.old = $.jstree._fn[i];
+ $.jstree._fn[i] = function () {
+ var rslt,
+ func = val,
+ args = Array.prototype.slice.call(arguments),
+ evnt = new $.Event("before.jstree"),
+ rlbk = false;
+
+ if(this.data.core.locked === true && i !== "unlock" && i !== "is_locked") { return; }
+
+ // Check if function belongs to the included plugins of this instance
+ do {
+ if(func && func.plugin && $.inArray(func.plugin, this._get_settings().plugins) !== -1) { break; }
+ func = func.old;
+ } while(func);
+ if(!func) { return; }
+
+ // context and function to trigger events, then finally call the function
+ if(i.indexOf("_") === 0) {
+ rslt = func.apply(this, args);
+ }
+ else {
+ rslt = this.get_container().triggerHandler(evnt, { "func" : i, "inst" : this, "args" : args, "plugin" : func.plugin });
+ if(rslt === false) { return; }
+ if(typeof rslt !== "undefined") { args = rslt; }
+
+ rslt = func.apply(
+ $.extend({}, this, {
+ __callback : function (data) {
+ this.get_container().triggerHandler( i + '.jstree', { "inst" : this, "args" : args, "rslt" : data, "rlbk" : rlbk });
+ },
+ __rollback : function () {
+ rlbk = this.get_rollback();
+ return rlbk;
+ },
+ __call_old : function (replace_arguments) {
+ return func.old.apply(this, (replace_arguments ? Array.prototype.slice.call(arguments, 1) : args ) );
+ }
+ }), args);
+ }
+
+ // return the result
+ return rslt;
+ };
+ $.jstree._fn[i].old = val.old;
+ $.jstree._fn[i].plugin = pname;
+ });
+ },
+ rollback : function (rb) {
+ if(rb) {
+ if(!$.isArray(rb)) { rb = [ rb ]; }
+ $.each(rb, function (i, val) {
+ instances[val.i].set_rollback(val.h, val.d);
+ });
+ }
+ }
+ };
+ // set the prototype for all instances
+ $.jstree._fn = $.jstree._instance.prototype = {};
+
+ // load the css when DOM is ready
+ $(function() {
+ // code is copied from jQuery ($.browser is deprecated + there is a bug in IE)
+ var u = navigator.userAgent.toLowerCase(),
+ v = (u.match( /.+?(?:rv|it|ra|ie)[\/: ]([\d.]+)/ ) || [0,'0'])[1],
+ css_string = '' +
+ '.jstree ul, .jstree li { display:block; margin:0 0 0 0; padding:0 0 0 0; list-style-type:none; } ' +
+ '.jstree li { display:block; min-height:18px; line-height:18px; white-space:nowrap; margin-left:18px; min-width:18px; } ' +
+ '.jstree-rtl li { margin-left:0; margin-right:18px; } ' +
+ '.jstree > ul > li { margin-left:0px; } ' +
+ '.jstree-rtl > ul > li { margin-right:0px; } ' +
+ '.jstree ins { display:inline-block; text-decoration:none; width:18px; height:18px; margin:0 0 0 0; padding:0; } ' +
+ '.jstree a { display:inline-block; line-height:16px; height:16px; color:black; white-space:nowrap; text-decoration:none; padding:1px 2px; margin:0; } ' +
+ '.jstree a:focus { outline: none; } ' +
+ '.jstree a > ins { height:16px; width:16px; } ' +
+ '.jstree a > .jstree-icon { margin-right:3px; } ' +
+ '.jstree-rtl a > .jstree-icon { margin-left:3px; margin-right:0; } ' +
+ 'li.jstree-open > ul { display:block; } ' +
+ 'li.jstree-closed > ul { display:none; } ';
+ // Correct IE 6 (does not support the > CSS selector)
+ if(/msie/.test(u) && parseInt(v, 10) == 6) {
+ is_ie6 = true;
+
+ // fix image flicker and lack of caching
+ try {
+ document.execCommand("BackgroundImageCache", false, true);
+ } catch (err) { }
+
+ css_string += '' +
+ '.jstree li { height:18px; margin-left:0; margin-right:0; } ' +
+ '.jstree li li { margin-left:18px; } ' +
+ '.jstree-rtl li li { margin-left:0px; margin-right:18px; } ' +
+ 'li.jstree-open ul { display:block; } ' +
+ 'li.jstree-closed ul { display:none !important; } ' +
+ '.jstree li a { display:inline; border-width:0 !important; padding:0px 2px !important; } ' +
+ '.jstree li a ins { height:16px; width:16px; margin-right:3px; } ' +
+ '.jstree-rtl li a ins { margin-right:0px; margin-left:3px; } ';
+ }
+ // Correct IE 7 (shifts anchor nodes onhover)
+ if(/msie/.test(u) && parseInt(v, 10) == 7) {
+ is_ie7 = true;
+ css_string += '.jstree li a { border-width:0 !important; padding:0px 2px !important; } ';
+ }
+ // correct ff2 lack of display:inline-block
+ if(!/compatible/.test(u) && /mozilla/.test(u) && parseFloat(v, 10) < 1.9) {
+ is_ff2 = true;
+ css_string += '' +
+ '.jstree ins { display:-moz-inline-box; } ' +
+ '.jstree li { line-height:12px; } ' + // WHY??
+ '.jstree a { display:-moz-inline-box; } ' +
+ '.jstree .jstree-no-icons .jstree-checkbox { display:-moz-inline-stack !important; } ';
+ /* this shouldn't be here as it is theme specific */
+ }
+ // the default stylesheet
+ $.vakata.css.add_sheet({ str : css_string, title : "jstree" });
+ });
+
+ // core functions (open, close, create, update, delete)
+ $.jstree.plugin("core", {
+ __init : function () {
+ this.data.core.locked = false;
+ this.data.core.to_open = this.get_settings().core.initially_open;
+ this.data.core.to_load = this.get_settings().core.initially_load;
+ },
+ defaults : {
+ html_titles : false,
+ animation : 500,
+ initially_open : [],
+ initially_load : [],
+ open_parents : true,
+ notify_plugins : true,
+ rtl : false,
+ load_open : false,
+ strings : {
+ loading : "Loading ...",
+ new_node : "New node",
+ multiple_selection : "Multiple selection"
+ }
+ },
+ _fn : {
+ init : function () {
+ this.set_focus();
+ if(this._get_settings().core.rtl) {
+ this.get_container().addClass("jstree-rtl").css("direction", "rtl");
+ }
+ this.get_container().html("<ul><li class='jstree-last jstree-leaf'><ins> </ins><a class='jstree-loading' href='#'><ins class='jstree-icon'> </ins>" + this._get_string("loading") + "</a></li></ul>");
+ this.data.core.li_height = this.get_container_ul().find("li.jstree-closed, li.jstree-leaf").eq(0).height() || 18;
+
+ this.get_container()
+ .delegate("li > ins", "click.jstree", $.proxy(function (event) {
+ var trgt = $(event.target);
+ if(trgt.is("ins") && event.pageY - trgt.offset().top < this.data.core.li_height) { this.toggle_node(trgt); }
+ }, this))
+ .bind("mousedown.jstree", $.proxy(function () {
+ this.set_focus(); // This used to be setTimeout(set_focus,0) - why?
+ }, this))
+ .bind("dblclick.jstree", function (event) {
+ var sel;
+ if(document.selection && document.selection.empty) { document.selection.empty(); }
+ else {
+ if(window.getSelection) {
+ sel = window.getSelection();
+ try {
+ sel.removeAllRanges();
+ sel.collapse();
+ } catch (err) { }
+ }
+ }
+ });
+ if(this._get_settings().core.notify_plugins) {
+ this.get_container()
+ .bind("load_node.jstree", $.proxy(function (e, data) {
+ var o = this._get_node(data.rslt.obj),
+ t = this;
+ if(o === -1) { o = this.get_container_ul(); }
+ if(!o.length) { return; }
+ o.find("li").each(function () {
+ var th = $(this);
+ if(th.data("jstree")) {
+ $.each(th.data("jstree"), function (plugin, values) {
+ if(t.data[plugin] && $.isFunction(t["_" + plugin + "_notify"])) {
+ t["_" + plugin + "_notify"].call(t, th, values);
+ }
+ });
+ }
+ });
+ }, this));
+ }
+ if(this._get_settings().core.load_open) {
+ this.get_container()
+ .bind("load_node.jstree", $.proxy(function (e, data) {
+ var o = this._get_node(data.rslt.obj),
+ t = this;
+ if(o === -1) { o = this.get_container_ul(); }
+ if(!o.length) { return; }
+ o.find("li.jstree-open:not(:has(ul))").each(function () {
+ t.load_node(this, $.noop, $.noop);
+ });
+ }, this));
+ }
+ this.__callback();
+ this.load_node(-1, function () { this.loaded(); this.reload_nodes(); });
+ },
+ destroy : function () {
+ var i,
+ n = this.get_index(),
+ s = this._get_settings(),
+ _this = this;
+
+ $.each(s.plugins, function (i, val) {
+ try { plugins[val].__destroy.apply(_this); } catch(err) { }
+ });
+ this.__callback();
+ // set focus to another instance if this one is focused
+ if(this.is_focused()) {
+ for(i in instances) {
+ if(instances.hasOwnProperty(i) && i != n) {
+ instances[i].set_focus();
+ break;
+ }
+ }
+ }
+ // if no other instance found
+ if(n === focused_instance) { focused_instance = -1; }
+ // remove all traces of jstree in the DOM (only the ones set using jstree*) and cleans all events
+ this.get_container()
+ .unbind(".jstree")
+ .undelegate(".jstree")
+ .removeData("jstree-instance-id")
+ .find("[class^='jstree']")
+ .andSelf()
+ .attr("class", function () { return this.className.replace(/jstree[^ ]*|$/ig,''); });
+ $(document)
+ .unbind(".jstree-" + n)
+ .undelegate(".jstree-" + n);
+ // remove the actual data
+ instances[n] = null;
+ delete instances[n];
+ },
+
+ _core_notify : function (n, data) {
+ if(data.opened) {
+ this.open_node(n, false, true);
+ }
+ },
+
+ lock : function () {
+ this.data.core.locked = true;
+ this.get_container().children("ul").addClass("jstree-locked").css("opacity","0.7");
+ this.__callback({});
+ },
+ unlock : function () {
+ this.data.core.locked = false;
+ this.get_container().children("ul").removeClass("jstree-locked").css("opacity","1");
+ this.__callback({});
+ },
+ is_locked : function () { return this.data.core.locked; },
+ save_opened : function () {
+ var _this = this;
+ this.data.core.to_open = [];
+ this.get_container_ul().find("li.jstree-open").each(function () {
+ if(this.id) { _this.data.core.to_open.push("#" + this.id.toString().replace(/^#/,"").replace(/\\\//g,"/").replace(/\//g,"\\\/").replace(/\\\./g,".").replace(/\./g,"\\.").replace(/\:/g,"\\:")); }
+ });
+ this.__callback(_this.data.core.to_open);
+ },
+ save_loaded : function () { },
+ reload_nodes : function (is_callback) {
+ var _this = this,
+ done = true,
+ current = [],
+ remaining = [];
+ if(!is_callback) {
+ this.data.core.reopen = false;
+ this.data.core.refreshing = true;
+ this.data.core.to_open = $.map($.makeArray(this.data.core.to_open), function (n) { return "#" + n.toString().replace(/^#/,"").replace(/\\\//g,"/").replace(/\//g,"\\\/").replace(/\\\./g,".").replace(/\./g,"\\.").replace(/\:/g,"\\:"); });
+ this.data.core.to_load = $.map($.makeArray(this.data.core.to_load), function (n) { return "#" + n.toString().replace(/^#/,"").replace(/\\\//g,"/").replace(/\//g,"\\\/").replace(/\\\./g,".").replace(/\./g,"\\.").replace(/\:/g,"\\:"); });
+ if(this.data.core.to_open.length) {
+ this.data.core.to_load = this.data.core.to_load.concat(this.data.core.to_open);
+ }
+ }
+ if(this.data.core.to_load.length) {
+ $.each(this.data.core.to_load, function (i, val) {
+ if(val == "#") { return true; }
+ if($(val).length) { current.push(val); }
+ else { remaining.push(val); }
+ });
+ if(current.length) {
+ this.data.core.to_load = remaining;
+ $.each(current, function (i, val) {
+ if(!_this._is_loaded(val)) {
+ _this.load_node(val, function () { _this.reload_nodes(true); }, function () { _this.reload_nodes(true); });
+ done = false;
+ }
+ });
+ }
+ }
+ if(this.data.core.to_open.length) {
+ $.each(this.data.core.to_open, function (i, val) {
+ _this.open_node(val, false, true);
+ });
+ }
+ if(done) {
+ // TODO: find a more elegant approach to syncronizing returning requests
+ if(this.data.core.reopen) { clearTimeout(this.data.core.reopen); }
+ this.data.core.reopen = setTimeout(function () { _this.__callback({}, _this); }, 50);
+ this.data.core.refreshing = false;
+ this.reopen();
+ }
+ },
+ reopen : function () {
+ var _this = this;
+ if(this.data.core.to_open.length) {
+ $.each(this.data.core.to_open, function (i, val) {
+ _this.open_node(val, false, true);
+ });
+ }
+ this.__callback({});
+ },
+ refresh : function (obj) {
+ var _this = this;
+ this.save_opened();
+ if(!obj) { obj = -1; }
+ obj = this._get_node(obj);
+ if(!obj) { obj = -1; }
+ if(obj !== -1) { obj.children("UL").remove(); }
+ else { this.get_container_ul().empty(); }
+ this.load_node(obj, function () { _this.__callback({ "obj" : obj}); _this.reload_nodes(); });
+ },
+ // Dummy function to fire after the first load (so that there is a jstree.loaded event)
+ loaded : function () {
+ this.__callback();
+ },
+ // deal with focus
+ set_focus : function () {
+ if(this.is_focused()) { return; }
+ var f = $.jstree._focused();
+ if(f) { f.unset_focus(); }
+
+ this.get_container().addClass("jstree-focused");
+ focused_instance = this.get_index();
+ this.__callback();
+ },
+ is_focused : function () {
+ return focused_instance == this.get_index();
+ },
+ unset_focus : function () {
+ if(this.is_focused()) {
+ this.get_container().removeClass("jstree-focused");
+ focused_instance = -1;
+ }
+ this.__callback();
+ },
+
+ // traverse
+ _get_node : function (obj) {
+ var $obj = $(obj, this.get_container());
+ if($obj.is(".jstree") || obj == -1) { return -1; }
+ $obj = $obj.closest("li", this.get_container());
+ return $obj.length ? $obj : false;
+ },
+ _get_next : function (obj, strict) {
+ obj = this._get_node(obj);
+ if(obj === -1) { return this.get_container().find("> ul > li:first-child"); }
+ if(!obj.length) { return false; }
+ if(strict) { return (obj.nextAll("li").size() > 0) ? obj.nextAll("li:eq(0)") : false; }
+
+ if(obj.hasClass("jstree-open")) { return obj.find("li:eq(0)"); }
+ else if(obj.nextAll("li").size() > 0) { return obj.nextAll("li:eq(0)"); }
+ else { return obj.parentsUntil(".jstree","li").next("li").eq(0); }
+ },
+ _get_prev : function (obj, strict) {
+ obj = this._get_node(obj);
+ if(obj === -1) { return this.get_container().find("> ul > li:last-child"); }
+ if(!obj.length) { return false; }
+ if(strict) { return (obj.prevAll("li").length > 0) ? obj.prevAll("li:eq(0)") : false; }
+
+ if(obj.prev("li").length) {
+ obj = obj.prev("li").eq(0);
+ while(obj.hasClass("jstree-open")) { obj = obj.children("ul:eq(0)").children("li:last"); }
+ return obj;
+ }
+ else { var o = obj.parentsUntil(".jstree","li:eq(0)"); return o.length ? o : false; }
+ },
+ _get_parent : function (obj) {
+ obj = this._get_node(obj);
+ if(obj == -1 || !obj.length) { return false; }
+ var o = obj.parentsUntil(".jstree", "li:eq(0)");
+ return o.length ? o : -1;
+ },
+ _get_children : function (obj) {
+ obj = this._get_node(obj);
+ if(obj === -1) { return this.get_container().children("ul:eq(0)").children("li"); }
+ if(!obj.length) { return false; }
+ return obj.children("ul:eq(0)").children("li");
+ },
+ get_path : function (obj, id_mode) {
+ var p = [],
+ _this = this;
+ obj = this._get_node(obj);
+ if(obj === -1 || !obj || !obj.length) { return false; }
+ obj.parentsUntil(".jstree", "li").each(function () {
+ p.push( id_mode ? this.id : _this.get_text(this) );
+ });
+ p.reverse();
+ p.push( id_mode ? obj.attr("id") : this.get_text(obj) );
+ return p;
+ },
+
+ // string functions
+ _get_string : function (key) {
+ return this._get_settings().core.strings[key] || key;
+ },
+
+ is_open : function (obj) { obj = this._get_node(obj); return obj && obj !== -1 && obj.hasClass("jstree-open"); },
+ is_closed : function (obj) { obj = this._get_node(obj); return obj && obj !== -1 && obj.hasClass("jstree-closed"); },
+ is_leaf : function (obj) { obj = this._get_node(obj); return obj && obj !== -1 && obj.hasClass("jstree-leaf"); },
+ correct_state : function (obj) {
+ obj = this._get_node(obj);
+ if(!obj || obj === -1) { return false; }
+ obj.removeClass("jstree-closed jstree-open").addClass("jstree-leaf").children("ul").remove();
+ this.__callback({ "obj" : obj });
+ },
+ // open/close
+ open_node : function (obj, callback, skip_animation) {
+ obj = this._get_node(obj);
+ if(!obj.length) { return false; }
+ if(!obj.hasClass("jstree-closed")) { if(callback) { callback.call(); } return false; }
+ var s = skip_animation || is_ie6 ? 0 : this._get_settings().core.animation,
+ t = this;
+ if(!this._is_loaded(obj)) {
+ obj.children("a").addClass("jstree-loading");
+ this.load_node(obj, function () { t.open_node(obj, callback, skip_animation); }, callback);
+ }
+ else {
+ if(this._get_settings().core.open_parents) {
+ obj.parentsUntil(".jstree",".jstree-closed").each(function () {
+ t.open_node(this, false, true);
+ });
+ }
+ if(s) { obj.children("ul").css("display","none"); }
+ obj.removeClass("jstree-closed").addClass("jstree-open").children("a").removeClass("jstree-loading");
+ if(s) { obj.children("ul").stop(true, true).slideDown(s, function () { this.style.display = ""; t.after_open(obj); }); }
+ else { t.after_open(obj); }
+ this.__callback({ "obj" : obj });
+ if(callback) { callback.call(); }
+ }
+ },
+ after_open : function (obj) { this.__callback({ "obj" : obj }); },
+ close_node : function (obj, skip_animation) {
+ obj = this._get_node(obj);
+ var s = skip_animation || is_ie6 ? 0 : this._get_settings().core.animation,
+ t = this;
+ if(!obj.length || !obj.hasClass("jstree-open")) { return false; }
+ if(s) { obj.children("ul").attr("style","display:block !important"); }
+ obj.removeClass("jstree-open").addClass("jstree-closed");
+ if(s) { obj.children("ul").stop(true, true).slideUp(s, function () { this.style.display = ""; t.after_close(obj); }); }
+ else { t.after_close(obj); }
+ this.__callback({ "obj" : obj });
+ },
+ after_close : function (obj) { this.__callback({ "obj" : obj }); },
+ toggle_node : function (obj) {
+ obj = this._get_node(obj);
+ if(obj.hasClass("jstree-closed")) { return this.open_node(obj); }
+ if(obj.hasClass("jstree-open")) { return this.close_node(obj); }
+ },
+ open_all : function (obj, do_animation, original_obj) {
+ obj = obj ? this._get_node(obj) : -1;
+ if(!obj || obj === -1) { obj = this.get_container_ul(); }
+ if(original_obj) {
+ obj = obj.find("li.jstree-closed");
+ }
+ else {
+ original_obj = obj;
+ if(obj.is(".jstree-closed")) { obj = obj.find("li.jstree-closed").andSelf(); }
+ else { obj = obj.find("li.jstree-closed"); }
+ }
+ var _this = this;
+ obj.each(function () {
+ var __this = this;
+ if(!_this._is_loaded(this)) { _this.open_node(this, function() { _this.open_all(__this, do_animation, original_obj); }, !do_animation); }
+ else { _this.open_node(this, false, !do_animation); }
+ });
+ // so that callback is fired AFTER all nodes are open
+ if(original_obj.find('li.jstree-closed').length === 0) { this.__callback({ "obj" : original_obj }); }
+ },
+ close_all : function (obj, do_animation) {
+ var _this = this;
+ obj = obj ? this._get_node(obj) : this.get_container();
+ if(!obj || obj === -1) { obj = this.get_container_ul(); }
+ obj.find("li.jstree-open").andSelf().each(function () { _this.close_node(this, !do_animation); });
+ this.__callback({ "obj" : obj });
+ },
+ clean_node : function (obj) {
+ obj = obj && obj != -1 ? $(obj) : this.get_container_ul();
+ obj = obj.is("li") ? obj.find("li").andSelf() : obj.find("li");
+ obj.removeClass("jstree-last")
+ .filter("li:last-child").addClass("jstree-last").end()
+ .filter(":has(li)")
+ .not(".jstree-open").removeClass("jstree-leaf").addClass("jstree-closed");
+ obj.not(".jstree-open, .jstree-closed").addClass("jstree-leaf").children("ul").remove();
+ this.__callback({ "obj" : obj });
+ },
+ // rollback
+ get_rollback : function () {
+ this.__callback();
+ return { i : this.get_index(), h : this.get_container().children("ul").clone(true), d : this.data };
+ },
+ set_rollback : function (html, data) {
+ this.get_container().empty().append(html);
+ this.data = data;
+ this.__callback();
+ },
+ // Dummy functions to be overwritten by any datastore plugin included
+ load_node : function (obj, s_call, e_call) { this.__callback({ "obj" : obj }); },
+ _is_loaded : function (obj) { return true; },
+
+ // Basic operations: create
+ create_node : function (obj, position, js, callback, is_loaded) {
+ obj = this._get_node(obj);
+ position = typeof position === "undefined" ? "last" : position;
+ var d = $("<li />"),
+ s = this._get_settings().core,
+ tmp;
+
+ if(obj !== -1 && !obj.length) { return false; }
+ if(!is_loaded && !this._is_loaded(obj)) { this.load_node(obj, function () { this.create_node(obj, position, js, callback, true); }); return false; }
+
+ this.__rollback();
+
+ if(typeof js === "string") { js = { "data" : js }; }
+ if(!js) { js = {}; }
+ if(js.attr) { d.attr(js.attr); }
+ if(js.metadata) { d.data(js.metadata); }
+ if(js.state) { d.addClass("jstree-" + js.state); }
+ if(!js.data) { js.data = this._get_string("new_node"); }
+ if(!$.isArray(js.data)) { tmp = js.data; js.data = []; js.data.push(tmp); }
+ $.each(js.data, function (i, m) {
+ tmp = $("<a />");
+ if($.isFunction(m)) { m = m.call(this, js); }
+ if(typeof m == "string") { tmp.attr('href','#')[ s.html_titles ? "html" : "text" ](m); }
+ else {
+ if(!m.attr) { m.attr = {}; }
+ if(!m.attr.href) { m.attr.href = '#'; }
+ tmp.attr(m.attr)[ s.html_titles ? "html" : "text" ](m.title);
+ if(m.language) { tmp.addClass(m.language); }
+ }
+ tmp.prepend("<ins class='jstree-icon'> </ins>");
+ if(m.icon) {
+ if(m.icon.indexOf("/") === -1) { tmp.children("ins").addClass(m.icon); }
+ else { tmp.children("ins").css("background","url('" + m.icon + "') center center no-repeat"); }
+ }
+ d.append(tmp);
+ });
+ d.prepend("<ins class='jstree-icon'> </ins>");
+ if(obj === -1) {
+ obj = this.get_container();
+ if(position === "before") { position = "first"; }
+ if(position === "after") { position = "last"; }
+ }
+ switch(position) {
+ case "before": obj.before(d); tmp = this._get_parent(obj); break;
+ case "after" : obj.after(d); tmp = this._get_parent(obj); break;
+ case "inside":
+ case "first" :
+ if(!obj.children("ul").length) { obj.append("<ul />"); }
+ obj.children("ul").prepend(d);
+ tmp = obj;
+ break;
+ case "last":
+ if(!obj.children("ul").length) { obj.append("<ul />"); }
+ obj.children("ul").append(d);
+ tmp = obj;
+ break;
+ default:
+ if(!obj.children("ul").length) { obj.append("<ul />"); }
+ if(!position) { position = 0; }
+ tmp = obj.children("ul").children("li").eq(position);
+ if(tmp.length) { tmp.before(d); }
+ else { obj.children("ul").append(d); }
+ tmp = obj;
+ break;
+ }
+ if(tmp === -1 || tmp.get(0) === this.get_container().get(0)) { tmp = -1; }
+ this.clean_node(tmp);
+ this.__callback({ "obj" : d, "parent" : tmp });
+ if(callback) { callback.call(this, d); }
+ return d;
+ },
+ // Basic operations: rename (deal with text)
+ get_text : function (obj) {
+ obj = this._get_node(obj);
+ if(!obj.length) { return false; }
+ var s = this._get_settings().core.html_titles;
+ obj = obj.children("a:eq(0)");
+ if(s) {
+ obj = obj.clone();
+ obj.children("INS").remove();
+ return obj.html();
+ }
+ else {
+ obj = obj.contents().filter(function() { return this.nodeType == 3; })[0];
+ return obj.nodeValue;
+ }
+ },
+ set_text : function (obj, val) {
+ obj = this._get_node(obj);
+ if(!obj.length) { return false; }
+ obj = obj.children("a:eq(0)");
+ if(this._get_settings().core.html_titles) {
+ var tmp = obj.children("INS").clone();
+ obj.html(val).prepend(tmp);
+ this.__callback({ "obj" : obj, "name" : val });
+ return true;
+ }
+ else {
+ obj = obj.contents().filter(function() { return this.nodeType == 3; })[0];
+ this.__callback({ "obj" : obj, "name" : val });
+ return (obj.nodeValue = val);
+ }
+ },
+ rename_node : function (obj, val) {
+ obj = this._get_node(obj);
+ this.__rollback();
+ if(obj && obj.length && this.set_text.apply(this, Array.prototype.slice.call(arguments))) { this.__callback({ "obj" : obj, "name" : val }); }
+ },
+ // Basic operations: deleting nodes
+ delete_node : function (obj) {
+ obj = this._get_node(obj);
+ if(!obj.length) { return false; }
+ this.__rollback();
+ var p = this._get_parent(obj), prev = $([]), t = this;
+ obj.each(function () {
+ prev = prev.add(t._get_prev(this));
+ });
+ obj = obj.detach();
+ if(p !== -1 && p.find("> ul > li").length === 0) {
+ p.removeClass("jstree-open jstree-closed").addClass("jstree-leaf");
+ }
+ this.clean_node(p);
+ this.__callback({ "obj" : obj, "prev" : prev, "parent" : p });
+ return obj;
+ },
+ prepare_move : function (o, r, pos, cb, is_cb) {
+ var p = {};
+
+ p.ot = $.jstree._reference(o) || this;
+ p.o = p.ot._get_node(o);
+ p.r = r === - 1 ? -1 : this._get_node(r);
+ p.p = (typeof pos === "undefined" || pos === false) ? "last" : pos; // TODO: move to a setting
+ if(!is_cb && prepared_move.o && prepared_move.o[0] === p.o[0] && prepared_move.r[0] === p.r[0] && prepared_move.p === p.p) {
+ this.__callback(prepared_move);
+ if(cb) { cb.call(this, prepared_move); }
+ return;
+ }
+ p.ot = $.jstree._reference(p.o) || this;
+ p.rt = $.jstree._reference(p.r) || this; // r === -1 ? p.ot : $.jstree._reference(p.r) || this
+ if(p.r === -1 || !p.r) {
+ p.cr = -1;
+ switch(p.p) {
+ case "first":
+ case "before":
+ case "inside":
+ p.cp = 0;
+ break;
+ case "after":
+ case "last":
+ p.cp = p.rt.get_container().find(" > ul > li").length;
+ break;
+ default:
+ p.cp = p.p;
+ break;
+ }
+ }
+ else {
+ if(!/^(before|after)$/.test(p.p) && !this._is_loaded(p.r)) {
+ return this.load_node(p.r, function () { this.prepare_move(o, r, pos, cb, true); });
+ }
+ switch(p.p) {
+ case "before":
+ p.cp = p.r.index();
+ p.cr = p.rt._get_parent(p.r);
+ break;
+ case "after":
+ p.cp = p.r.index() + 1;
+ p.cr = p.rt._get_parent(p.r);
+ break;
+ case "inside":
+ case "first":
+ p.cp = 0;
+ p.cr = p.r;
+ break;
+ case "last":
+ p.cp = p.r.find(" > ul > li").length;
+ p.cr = p.r;
+ break;
+ default:
+ p.cp = p.p;
+ p.cr = p.r;
+ break;
+ }
+ }
+ p.np = p.cr == -1 ? p.rt.get_container() : p.cr;
+ p.op = p.ot._get_parent(p.o);
+ p.cop = p.o.index();
+ if(p.op === -1) { p.op = p.ot ? p.ot.get_container() : this.get_container(); }
+ if(!/^(before|after)$/.test(p.p) && p.op && p.np && p.op[0] === p.np[0] && p.o.index() < p.cp) { p.cp++; }
+ //if(p.p === "before" && p.op && p.np && p.op[0] === p.np[0] && p.o.index() < p.cp) { p.cp--; }
+ p.or = p.np.find(" > ul > li:nth-child(" + (p.cp + 1) + ")");
+ prepared_move = p;
+ this.__callback(prepared_move);
+ if(cb) { cb.call(this, prepared_move); }
+ },
+ check_move : function () {
+ var obj = prepared_move, ret = true, r = obj.r === -1 ? this.get_container() : obj.r;
+ if(!obj || !obj.o || obj.or[0] === obj.o[0]) { return false; }
+ if(obj.op && obj.np && obj.op[0] === obj.np[0] && obj.cp - 1 === obj.o.index()) { return false; }
+ obj.o.each(function () {
+ if(r.parentsUntil(".jstree", "li").andSelf().index(this) !== -1) { ret = false; return false; }
+ });
+ return ret;
+ },
+ move_node : function (obj, ref, position, is_copy, is_prepared, skip_check) {
+ if(!is_prepared) {
+ return this.prepare_move(obj, ref, position, function (p) {
+ this.move_node(p, false, false, is_copy, true, skip_check);
+ });
+ }
+ if(is_copy) {
+ prepared_move.cy = true;
+ }
+ if(!skip_check && !this.check_move()) { return false; }
+
+ this.__rollback();
+ var o = false;
+ if(is_copy) {
+ o = obj.o.clone(true);
+ o.find("*[id]").andSelf().each(function () {
+ if(this.id) { this.id = "copy_" + this.id; }
+ });
+ }
+ else { o = obj.o; }
+
+ if(obj.or.length) { obj.or.before(o); }
+ else {
+ if(!obj.np.children("ul").length) { $("<ul />").appendTo(obj.np); }
+ obj.np.children("ul:eq(0)").append(o);
+ }
+
+ try {
+ obj.ot.clean_node(obj.op);
+ obj.rt.clean_node(obj.np);
+ if(!obj.op.find("> ul > li").length) {
+ obj.op.removeClass("jstree-open jstree-closed").addClass("jstree-leaf").children("ul").remove();
+ }
+ } catch (e) { }
+
+ if(is_copy) {
+ prepared_move.cy = true;
+ prepared_move.oc = o;
+ }
+ this.__callback(prepared_move);
+ return prepared_move;
+ },
+ _get_move : function () { return prepared_move; }
+ }
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree ui plugin
+ * This plugins handles selecting/deselecting/hovering/dehovering nodes
+ */
+(function ($) {
+ var scrollbar_width, e1, e2;
+ $(function() {
+ if (/msie/.test(navigator.userAgent.toLowerCase())) {
+ e1 = $('<textarea cols="10" rows="2"></textarea>').css({ position: 'absolute', top: -1000, left: 0 }).appendTo('body');
+ e2 = $('<textarea cols="10" rows="2" style="overflow: hidden;"></textarea>').css({ position: 'absolute', top: -1000, left: 0 }).appendTo('body');
+ scrollbar_width = e1.width() - e2.width();
+ e1.add(e2).remove();
+ }
+ else {
+ e1 = $('<div />').css({ width: 100, height: 100, overflow: 'auto', position: 'absolute', top: -1000, left: 0 })
+ .prependTo('body').append('<div />').find('div').css({ width: '100%', height: 200 });
+ scrollbar_width = 100 - e1.width();
+ e1.parent().remove();
+ }
+ });
+ $.jstree.plugin("ui", {
+ __init : function () {
+ this.data.ui.selected = $();
+ this.data.ui.last_selected = false;
+ this.data.ui.hovered = null;
+ this.data.ui.to_select = this.get_settings().ui.initially_select;
+
+ this.get_container()
+ .delegate("a", "click.jstree", $.proxy(function (event) {
+ event.preventDefault();
+ event.currentTarget.blur();
+ if(!$(event.currentTarget).hasClass("jstree-loading")) {
+ this.select_node(event.currentTarget, true, event);
+ }
+ }, this))
+ .delegate("a", "mouseenter.jstree", $.proxy(function (event) {
+ if(!$(event.currentTarget).hasClass("jstree-loading")) {
+ this.hover_node(event.target);
+ }
+ }, this))
+ .delegate("a", "mouseleave.jstree", $.proxy(function (event) {
+ if(!$(event.currentTarget).hasClass("jstree-loading")) {
+ this.dehover_node(event.target);
+ }
+ }, this))
+ .bind("reopen.jstree", $.proxy(function () {
+ this.reselect();
+ }, this))
+ .bind("get_rollback.jstree", $.proxy(function () {
+ this.dehover_node();
+ this.save_selected();
+ }, this))
+ .bind("set_rollback.jstree", $.proxy(function () {
+ this.reselect();
+ }, this))
+ .bind("close_node.jstree", $.proxy(function (event, data) {
+ var s = this._get_settings().ui,
+ obj = this._get_node(data.rslt.obj),
+ clk = (obj && obj.length) ? obj.children("ul").find("a.jstree-clicked") : $(),
+ _this = this;
+ if(s.selected_parent_close === false || !clk.length) { return; }
+ clk.each(function () {
+ _this.deselect_node(this);
+ if(s.selected_parent_close === "select_parent") { _this.select_node(obj); }
+ });
+ }, this))
+ .bind("delete_node.jstree", $.proxy(function (event, data) {
+ var s = this._get_settings().ui.select_prev_on_delete,
+ obj = this._get_node(data.rslt.obj),
+ clk = (obj && obj.length) ? obj.find("a.jstree-clicked") : [],
+ _this = this;
+ clk.each(function () { _this.deselect_node(this); });
+ if(s && clk.length) {
+ data.rslt.prev.each(function () {
+ if(this.parentNode) { _this.select_node(this); return false; /* if return false is removed all prev nodes will be selected */}
+ });
+ }
+ }, this))
+ .bind("move_node.jstree", $.proxy(function (event, data) {
+ if(data.rslt.cy) {
+ data.rslt.oc.find("a.jstree-clicked").removeClass("jstree-clicked");
+ }
+ }, this));
+ },
+ defaults : {
+ select_limit : -1, // 0, 1, 2 ... or -1 for unlimited
+ select_multiple_modifier : "ctrl", // on, or ctrl, shift, alt
+ select_range_modifier : "shift",
+ selected_parent_close : "select_parent", // false, "deselect", "select_parent"
+ selected_parent_open : true,
+ select_prev_on_delete : true,
+ disable_selecting_children : false,
+ initially_select : []
+ },
+ _fn : {
+ _get_node : function (obj, allow_multiple) {
+ if(typeof obj === "undefined" || obj === null) { return allow_multiple ? this.data.ui.selected : this.data.ui.last_selected; }
+ var $obj = $(obj, this.get_container());
+ if($obj.is(".jstree") || obj == -1) { return -1; }
+ $obj = $obj.closest("li", this.get_container());
+ return $obj.length ? $obj : false;
+ },
+ _ui_notify : function (n, data) {
+ if(data.selected) {
+ this.select_node(n, false);
+ }
+ },
+ save_selected : function () {
+ var _this = this;
+ this.data.ui.to_select = [];
+ this.data.ui.selected.each(function () { if(this.id) { _this.data.ui.to_select.push("#" + this.id.toString().replace(/^#/,"").replace(/\\\//g,"/").replace(/\//g,"\\\/").replace(/\\\./g,".").replace(/\./g,"\\.").replace(/\:/g,"\\:")); } });
+ this.__callback(this.data.ui.to_select);
+ },
+ reselect : function () {
+ var _this = this,
+ s = this.data.ui.to_select;
+ s = $.map($.makeArray(s), function (n) { return "#" + n.toString().replace(/^#/,"").replace(/\\\//g,"/").replace(/\//g,"\\\/").replace(/\\\./g,".").replace(/\./g,"\\.").replace(/\:/g,"\\:"); });
+ // this.deselect_all(); WHY deselect, breaks plugin state notifier?
+ $.each(s, function (i, val) { if(val && val !== "#") { _this.select_node(val); } });
+ this.data.ui.selected = this.data.ui.selected.filter(function () { return this.parentNode; });
+ this.__callback();
+ },
+ refresh : function (obj) {
+ this.save_selected();
+ return this.__call_old();
+ },
+ hover_node : function (obj) {
+ obj = this._get_node(obj);
+ if(!obj.length) { return false; }
+ //if(this.data.ui.hovered && obj.get(0) === this.data.ui.hovered.get(0)) { return; }
+ if(!obj.hasClass("jstree-hovered")) { this.dehover_node(); }
+ this.data.ui.hovered = obj.children("a").addClass("jstree-hovered").parent();
+ this._fix_scroll(obj);
+ this.__callback({ "obj" : obj });
+ },
+ dehover_node : function () {
+ var obj = this.data.ui.hovered, p;
+ if(!obj || !obj.length) { return false; }
+ p = obj.children("a").removeClass("jstree-hovered").parent();
+ if(this.data.ui.hovered[0] === p[0]) { this.data.ui.hovered = null; }
+ this.__callback({ "obj" : obj });
+ },
+ select_node : function (obj, check, e) {
+ obj = this._get_node(obj);
+ if(obj == -1 || !obj || !obj.length) { return false; }
+ var s = this._get_settings().ui,
+ is_multiple = (s.select_multiple_modifier == "on" || (s.select_multiple_modifier !== false && e && e[s.select_multiple_modifier + "Key"])),
+ is_range = (s.select_range_modifier !== false && e && e[s.select_range_modifier + "Key"] && this.data.ui.last_selected && this.data.ui.last_selected[0] !== obj[0] && this.data.ui.last_selected.parent()[0] === obj.parent()[0]),
+ is_selected = this.is_selected(obj),
+ proceed = true,
+ t = this;
+ if(check) {
+ if(s.disable_selecting_children && is_multiple &&
+ (
+ (obj.parentsUntil(".jstree","li").children("a.jstree-clicked").length) ||
+ (obj.children("ul").find("a.jstree-clicked:eq(0)").length)
+ )
+ ) {
+ return false;
+ }
+ proceed = false;
+ switch(!0) {
+ case (is_range):
+ this.data.ui.last_selected.addClass("jstree-last-selected");
+ obj = obj[ obj.index() < this.data.ui.last_selected.index() ? "nextUntil" : "prevUntil" ](".jstree-last-selected").andSelf();
+ if(s.select_limit == -1 || obj.length < s.select_limit) {
+ this.data.ui.last_selected.removeClass("jstree-last-selected");
+ this.data.ui.selected.each(function () {
+ if(this !== t.data.ui.last_selected[0]) { t.deselect_node(this); }
+ });
+ is_selected = false;
+ proceed = true;
+ }
+ else {
+ proceed = false;
+ }
+ break;
+ case (is_selected && !is_multiple):
+ this.deselect_all();
+ is_selected = false;
+ proceed = true;
+ break;
+ case (!is_selected && !is_multiple):
+ if(s.select_limit == -1 || s.select_limit > 0) {
+ this.deselect_all();
+ proceed = true;
+ }
+ break;
+ case (is_selected && is_multiple):
+ this.deselect_node(obj);
+ break;
+ case (!is_selected && is_multiple):
+ if(s.select_limit == -1 || this.data.ui.selected.length + 1 <= s.select_limit) {
+ proceed = true;
+ }
+ break;
+ }
+ }
+ if(proceed && !is_selected) {
+ if(!is_range) { this.data.ui.last_selected = obj; }
+ obj.children("a").addClass("jstree-clicked");
+ if(s.selected_parent_open) {
+ obj.parents(".jstree-closed").each(function () { t.open_node(this, false, true); });
+ }
+ this.data.ui.selected = this.data.ui.selected.add(obj);
+ this._fix_scroll(obj.eq(0));
+ this.__callback({ "obj" : obj, "e" : e });
+ }
+ },
+ _fix_scroll : function (obj) {
+ var c = this.get_container()[0], t;
+ if(c.scrollHeight > c.offsetHeight) {
+ obj = this._get_node(obj);
+ if(!obj || obj === -1 || !obj.length || !obj.is(":visible")) { return; }
+ t = obj.offset().top - this.get_container().offset().top;
+ if(t < 0) {
+ c.scrollTop = c.scrollTop + t - 1;
+ }
+ if(t + this.data.core.li_height + (c.scrollWidth > c.offsetWidth ? scrollbar_width : 0) > c.offsetHeight) {
+ c.scrollTop = c.scrollTop + (t - c.offsetHeight + this.data.core.li_height + 1 + (c.scrollWidth > c.offsetWidth ? scrollbar_width : 0));
+ }
+ }
+ },
+ deselect_node : function (obj) {
+ obj = this._get_node(obj);
+ if(!obj.length) { return false; }
+ if(this.is_selected(obj)) {
+ obj.children("a").removeClass("jstree-clicked");
+ this.data.ui.selected = this.data.ui.selected.not(obj);
+ if(this.data.ui.last_selected.get(0) === obj.get(0)) { this.data.ui.last_selected = this.data.ui.selected.eq(0); }
+ this.__callback({ "obj" : obj });
+ }
+ },
+ toggle_select : function (obj) {
+ obj = this._get_node(obj);
+ if(!obj.length) { return false; }
+ if(this.is_selected(obj)) { this.deselect_node(obj); }
+ else { this.select_node(obj); }
+ },
+ is_selected : function (obj) { return this.data.ui.selected.index(this._get_node(obj)) >= 0; },
+ get_selected : function (context) {
+ return context ? $(context).find("a.jstree-clicked").parent() : this.data.ui.selected;
+ },
+ deselect_all : function (context) {
+ var ret = context ? $(context).find("a.jstree-clicked").parent() : this.get_container().find("a.jstree-clicked").parent();
+ ret.children("a.jstree-clicked").removeClass("jstree-clicked");
+ this.data.ui.selected = $([]);
+ this.data.ui.last_selected = false;
+ this.__callback({ "obj" : ret });
+ }
+ }
+ });
+ // include the selection plugin by default
+ $.jstree.defaults.plugins.push("ui");
+})(jQuery);
+//*/
+
+/*
+ * jsTree CRRM plugin
+ * Handles creating/renaming/removing/moving nodes by user interaction.
+ */
+(function ($) {
+ $.jstree.plugin("crrm", {
+ __init : function () {
+ this.get_container()
+ .bind("move_node.jstree", $.proxy(function (e, data) {
+ if(this._get_settings().crrm.move.open_onmove) {
+ var t = this;
+ data.rslt.np.parentsUntil(".jstree").andSelf().filter(".jstree-closed").each(function () {
+ t.open_node(this, false, true);
+ });
+ }
+ }, this));
+ },
+ defaults : {
+ input_width_limit : 200,
+ move : {
+ always_copy : false, // false, true or "multitree"
+ open_onmove : true,
+ default_position : "last",
+ check_move : function (m) { return true; }
+ }
+ },
+ _fn : {
+ _show_input : function (obj, callback) {
+ obj = this._get_node(obj);
+ var rtl = this._get_settings().core.rtl,
+ w = this._get_settings().crrm.input_width_limit,
+ w1 = obj.children("ins").width(),
+ w2 = obj.find("> a:visible > ins").width() * obj.find("> a:visible > ins").length,
+ t = this.get_text(obj),
+ h1 = $("<div />", { css : { "position" : "absolute", "top" : "-200px", "left" : (rtl ? "0px" : "-1000px"), "visibility" : "hidden" } }).appendTo("body"),
+ h2 = obj.css("position","relative").append(
+ $("<input />", {
+ "value" : t,
+ "class" : "jstree-rename-input",
+ // "size" : t.length,
+ "css" : {
+ "padding" : "0",
+ "border" : "1px solid silver",
+ "position" : "absolute",
+ "left" : (rtl ? "auto" : (w1 + w2 + 4) + "px"),
+ "right" : (rtl ? (w1 + w2 + 4) + "px" : "auto"),
+ "top" : "0px",
+ "height" : (this.data.core.li_height - 2) + "px",
+ "lineHeight" : (this.data.core.li_height - 2) + "px",
+ "width" : "150px" // will be set a bit further down
+ },
+ "blur" : $.proxy(function () {
+ var i = obj.children(".jstree-rename-input"),
+ v = i.val();
+ if(v === "") { v = t; }
+ h1.remove();
+ i.remove(); // rollback purposes
+ this.set_text(obj,t); // rollback purposes
+ this.rename_node(obj, v);
+ callback.call(this, obj, v, t);
+ obj.css("position","");
+ }, this),
+ "keyup" : function (event) {
+ var key = event.keyCode || event.which;
+ if(key == 27) { this.value = t; this.blur(); return; }
+ else if(key == 13) { this.blur(); return; }
+ else {
+ h2.width(Math.min(h1.text("pW" + this.value).width(),w));
+ }
+ },
+ "keypress" : function(event) {
+ var key = event.keyCode || event.which;
+ if(key == 13) { return false; }
+ }
+ })
+ ).children(".jstree-rename-input");
+ this.set_text(obj, "");
+ h1.css({
+ fontFamily : h2.css('fontFamily') || '',
+ fontSize : h2.css('fontSize') || '',
+ fontWeight : h2.css('fontWeight') || '',
+ fontStyle : h2.css('fontStyle') || '',
+ fontStretch : h2.css('fontStretch') || '',
+ fontVariant : h2.css('fontVariant') || '',
+ letterSpacing : h2.css('letterSpacing') || '',
+ wordSpacing : h2.css('wordSpacing') || ''
+ });
+ h2.width(Math.min(h1.text("pW" + h2[0].value).width(),w))[0].select();
+ },
+ rename : function (obj) {
+ obj = this._get_node(obj);
+ this.__rollback();
+ var f = this.__callback;
+ this._show_input(obj, function (obj, new_name, old_name) {
+ f.call(this, { "obj" : obj, "new_name" : new_name, "old_name" : old_name });
+ });
+ },
+ create : function (obj, position, js, callback, skip_rename) {
+ var t, _this = this;
+ obj = this._get_node(obj);
+ if(!obj) { obj = -1; }
+ this.__rollback();
+ t = this.create_node(obj, position, js, function (t) {
+ var p = this._get_parent(t),
+ pos = $(t).index();
+ if(callback) { callback.call(this, t); }
+ if(p.length && p.hasClass("jstree-closed")) { this.open_node(p, false, true); }
+ if(!skip_rename) {
+ this._show_input(t, function (obj, new_name, old_name) {
+ _this.__callback({ "obj" : obj, "name" : new_name, "parent" : p, "position" : pos });
+ });
+ }
+ else { _this.__callback({ "obj" : t, "name" : this.get_text(t), "parent" : p, "position" : pos }); }
+ });
+ return t;
+ },
+ remove : function (obj) {
+ obj = this._get_node(obj, true);
+ var p = this._get_parent(obj), prev = this._get_prev(obj);
+ this.__rollback();
+ obj = this.delete_node(obj);
+ if(obj !== false) { this.__callback({ "obj" : obj, "prev" : prev, "parent" : p }); }
+ },
+ check_move : function () {
+ if(!this.__call_old()) { return false; }
+ var s = this._get_settings().crrm.move;
+ if(!s.check_move.call(this, this._get_move())) { return false; }
+ return true;
+ },
+ move_node : function (obj, ref, position, is_copy, is_prepared, skip_check) {
+ var s = this._get_settings().crrm.move;
+ if(!is_prepared) {
+ if(typeof position === "undefined") { position = s.default_position; }
+ if(position === "inside" && !s.default_position.match(/^(before|after)$/)) { position = s.default_position; }
+ return this.__call_old(true, obj, ref, position, is_copy, false, skip_check);
+ }
+ // if the move is already prepared
+ if(s.always_copy === true || (s.always_copy === "multitree" && obj.rt.get_index() !== obj.ot.get_index() )) {
+ is_copy = true;
+ }
+ this.__call_old(true, obj, ref, position, is_copy, true, skip_check);
+ },
+
+ cut : function (obj) {
+ obj = this._get_node(obj, true);
+ if(!obj || !obj.length) { return false; }
+ this.data.crrm.cp_nodes = false;
+ this.data.crrm.ct_nodes = obj;
+ this.__callback({ "obj" : obj });
+ },
+ copy : function (obj) {
+ obj = this._get_node(obj, true);
+ if(!obj || !obj.length) { return false; }
+ this.data.crrm.ct_nodes = false;
+ this.data.crrm.cp_nodes = obj;
+ this.__callback({ "obj" : obj });
+ },
+ paste : function (obj) {
+ obj = this._get_node(obj);
+ if(!obj || !obj.length) { return false; }
+ var nodes = this.data.crrm.ct_nodes ? this.data.crrm.ct_nodes : this.data.crrm.cp_nodes;
+ if(!this.data.crrm.ct_nodes && !this.data.crrm.cp_nodes) { return false; }
+ if(this.data.crrm.ct_nodes) { this.move_node(this.data.crrm.ct_nodes, obj); this.data.crrm.ct_nodes = false; }
+ if(this.data.crrm.cp_nodes) { this.move_node(this.data.crrm.cp_nodes, obj, false, true); }
+ this.__callback({ "obj" : obj, "nodes" : nodes });
+ }
+ }
+ });
+ // include the crr plugin by default
+ // $.jstree.defaults.plugins.push("crrm");
+})(jQuery);
+//*/
+
+/*
+ * jsTree themes plugin
+ * Handles loading and setting themes, as well as detecting path to themes, etc.
+ */
+(function ($) {
+ var themes_loaded = [];
+ // this variable stores the path to the themes folder - if left as false - it will be autodetected
+ $.jstree._themes = false;
+ $.jstree.plugin("themes", {
+ __init : function () {
+ this.get_container()
+ .bind("init.jstree", $.proxy(function () {
+ var s = this._get_settings().themes;
+ this.data.themes.dots = s.dots;
+ this.data.themes.icons = s.icons;
+ this.set_theme(s.theme, s.url);
+ }, this))
+ .bind("loaded.jstree", $.proxy(function () {
+ // bound here too, as simple HTML tree's won't honor dots & icons otherwise
+ if(!this.data.themes.dots) { this.hide_dots(); }
+ else { this.show_dots(); }
+ if(!this.data.themes.icons) { this.hide_icons(); }
+ else { this.show_icons(); }
+ }, this));
+ },
+ defaults : {
+ theme : "default",
+ url : false,
+ dots : true,
+ icons : true
+ },
+ _fn : {
+ set_theme : function (theme_name, theme_url) {
+ if(!theme_name) { return false; }
+ if(!theme_url) { theme_url = $.jstree._themes + theme_name + '/style.css'; }
+ if($.inArray(theme_url, themes_loaded) == -1) {
+ $.vakata.css.add_sheet({ "url" : theme_url });
+ themes_loaded.push(theme_url);
+ }
+ if(this.data.themes.theme != theme_name) {
+ this.get_container().removeClass('jstree-' + this.data.themes.theme);
+ this.data.themes.theme = theme_name;
+ }
+ this.get_container().addClass('jstree-' + theme_name);
+ if(!this.data.themes.dots) { this.hide_dots(); }
+ else { this.show_dots(); }
+ if(!this.data.themes.icons) { this.hide_icons(); }
+ else { this.show_icons(); }
+ this.__callback();
+ },
+ get_theme : function () { return this.data.themes.theme; },
+
+ show_dots : function () { this.data.themes.dots = true; this.get_container().children("ul").removeClass("jstree-no-dots"); },
+ hide_dots : function () { this.data.themes.dots = false; this.get_container().children("ul").addClass("jstree-no-dots"); },
+ toggle_dots : function () { if(this.data.themes.dots) { this.hide_dots(); } else { this.show_dots(); } },
+
+ show_icons : function () { this.data.themes.icons = true; this.get_container().children("ul").removeClass("jstree-no-icons"); },
+ hide_icons : function () { this.data.themes.icons = false; this.get_container().children("ul").addClass("jstree-no-icons"); },
+ toggle_icons: function () { if(this.data.themes.icons) { this.hide_icons(); } else { this.show_icons(); } }
+ }
+ });
+ // autodetect themes path
+ $(function () {
+ if($.jstree._themes === false) {
+ $("script").each(function () {
+ if(this.src.toString().match(/jquery\.jstree[^\/]*?\.js(\?.*)?$/)) {
+ $.jstree._themes = this.src.toString().replace(/jquery\.jstree[^\/]*?\.js(\?.*)?$/, "") + 'themes/';
+ return false;
+ }
+ });
+ }
+ if($.jstree._themes === false) { $.jstree._themes = "themes/"; }
+ });
+ // include the themes plugin by default
+ $.jstree.defaults.plugins.push("themes");
+})(jQuery);
+//*/
+
+/*
+ * jsTree hotkeys plugin
+ * Enables keyboard navigation for all tree instances
+ * Depends on the jstree ui & jquery hotkeys plugins
+ */
+(function ($) {
+ var bound = [];
+ function exec(i, event) {
+ var f = $.jstree._focused(), tmp;
+ if(f && f.data && f.data.hotkeys && f.data.hotkeys.enabled) {
+ tmp = f._get_settings().hotkeys[i];
+ if(tmp) { return tmp.call(f, event); }
+ }
+ }
+ $.jstree.plugin("hotkeys", {
+ __init : function () {
+ if(typeof $.hotkeys === "undefined") { throw "jsTree hotkeys: jQuery hotkeys plugin not included."; }
+ if(!this.data.ui) { throw "jsTree hotkeys: jsTree UI plugin not included."; }
+ $.each(this._get_settings().hotkeys, function (i, v) {
+ if(v !== false && $.inArray(i, bound) == -1) {
+ $(document).bind("keydown", i, function (event) { return exec(i, event); });
+ bound.push(i);
+ }
+ });
+ this.get_container()
+ .bind("lock.jstree", $.proxy(function () {
+ if(this.data.hotkeys.enabled) { this.data.hotkeys.enabled = false; this.data.hotkeys.revert = true; }
+ }, this))
+ .bind("unlock.jstree", $.proxy(function () {
+ if(this.data.hotkeys.revert) { this.data.hotkeys.enabled = true; }
+ }, this));
+ this.enable_hotkeys();
+ },
+ defaults : {
+ "up" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected || -1;
+ this.hover_node(this._get_prev(o));
+ return false;
+ },
+ "ctrl+up" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected || -1;
+ this.hover_node(this._get_prev(o));
+ return false;
+ },
+ "shift+up" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected || -1;
+ this.hover_node(this._get_prev(o));
+ return false;
+ },
+ "down" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected || -1;
+ this.hover_node(this._get_next(o));
+ return false;
+ },
+ "ctrl+down" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected || -1;
+ this.hover_node(this._get_next(o));
+ return false;
+ },
+ "shift+down" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected || -1;
+ this.hover_node(this._get_next(o));
+ return false;
+ },
+ "left" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected;
+ if(o) {
+ if(o.hasClass("jstree-open")) { this.close_node(o); }
+ else { this.hover_node(this._get_prev(o)); }
+ }
+ return false;
+ },
+ "ctrl+left" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected;
+ if(o) {
+ if(o.hasClass("jstree-open")) { this.close_node(o); }
+ else { this.hover_node(this._get_prev(o)); }
+ }
+ return false;
+ },
+ "shift+left" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected;
+ if(o) {
+ if(o.hasClass("jstree-open")) { this.close_node(o); }
+ else { this.hover_node(this._get_prev(o)); }
+ }
+ return false;
+ },
+ "right" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected;
+ if(o && o.length) {
+ if(o.hasClass("jstree-closed")) { this.open_node(o); }
+ else { this.hover_node(this._get_next(o)); }
+ }
+ return false;
+ },
+ "ctrl+right" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected;
+ if(o && o.length) {
+ if(o.hasClass("jstree-closed")) { this.open_node(o); }
+ else { this.hover_node(this._get_next(o)); }
+ }
+ return false;
+ },
+ "shift+right" : function () {
+ var o = this.data.ui.hovered || this.data.ui.last_selected;
+ if(o && o.length) {
+ if(o.hasClass("jstree-closed")) { this.open_node(o); }
+ else { this.hover_node(this._get_next(o)); }
+ }
+ return false;
+ },
+ "space" : function () {
+ if(this.data.ui.hovered) { this.data.ui.hovered.children("a:eq(0)").click(); }
+ return false;
+ },
+ "ctrl+space" : function (event) {
+ event.type = "click";
+ if(this.data.ui.hovered) { this.data.ui.hovered.children("a:eq(0)").trigger(event); }
+ return false;
+ },
+ "shift+space" : function (event) {
+ event.type = "click";
+ if(this.data.ui.hovered) { this.data.ui.hovered.children("a:eq(0)").trigger(event); }
+ return false;
+ },
+ "f2" : function () { this.rename(this.data.ui.hovered || this.data.ui.last_selected); },
+ "del" : function () { this.remove(this.data.ui.hovered || this._get_node(null)); }
+ },
+ _fn : {
+ enable_hotkeys : function () {
+ this.data.hotkeys.enabled = true;
+ },
+ disable_hotkeys : function () {
+ this.data.hotkeys.enabled = false;
+ }
+ }
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree JSON plugin
+ * The JSON data store. Datastores are build by overriding the `load_node` and `_is_loaded` functions.
+ */
+(function ($) {
+ $.jstree.plugin("json_data", {
+ __init : function() {
+ var s = this._get_settings().json_data;
+ if(s.progressive_unload) {
+ this.get_container().bind("after_close.jstree", function (e, data) {
+ data.rslt.obj.children("ul").remove();
+ });
+ }
+ },
+ defaults : {
+ // `data` can be a function:
+ // * accepts two arguments - node being loaded and a callback to pass the result to
+ // * will be executed in the current tree's scope & ajax won't be supported
+ data : false,
+ ajax : false,
+ correct_state : true,
+ progressive_render : false,
+ progressive_unload : false
+ },
+ _fn : {
+ load_node : function (obj, s_call, e_call) { var _this = this; this.load_node_json(obj, function () { _this.__callback({ "obj" : _this._get_node(obj) }); s_call.call(this); }, e_call); },
+ _is_loaded : function (obj) {
+ var s = this._get_settings().json_data;
+ obj = this._get_node(obj);
+ return obj == -1 || !obj || (!s.ajax && !s.progressive_render && !$.isFunction(s.data)) || obj.is(".jstree-open, .jstree-leaf") || obj.children("ul").children("li").length > 0;
+ },
+ refresh : function (obj) {
+ obj = this._get_node(obj);
+ var s = this._get_settings().json_data;
+ if(obj && obj !== -1 && s.progressive_unload && ($.isFunction(s.data) || !!s.ajax)) {
+ obj.removeData("jstree-children");
+ }
+ return this.__call_old();
+ },
+ load_node_json : function (obj, s_call, e_call) {
+ var s = this.get_settings().json_data, d,
+ error_func = function () {},
+ success_func = function () {};
+ obj = this._get_node(obj);
+
+ if(obj && obj !== -1 && (s.progressive_render || s.progressive_unload) && !obj.is(".jstree-open, .jstree-leaf") && obj.children("ul").children("li").length === 0 && obj.data("jstree-children")) {
+ d = this._parse_json(obj.data("jstree-children"), obj);
+ if(d) {
+ obj.append(d);
+ if(!s.progressive_unload) { obj.removeData("jstree-children"); }
+ }
+ this.clean_node(obj);
+ if(s_call) { s_call.call(this); }
+ return;
+ }
+
+ if(obj && obj !== -1) {
+ if(obj.data("jstree-is-loading")) { return; }
+ else { obj.data("jstree-is-loading",true); }
+ }
+ switch(!0) {
+ case (!s.data && !s.ajax): throw "Neither data nor ajax settings supplied.";
+ // function option added here for easier model integration (also supporting async - see callback)
+ case ($.isFunction(s.data)):
+ s.data.call(this, obj, $.proxy(function (d) {
+ d = this._parse_json(d, obj);
+ if(!d) {
+ if(obj === -1 || !obj) {
+ if(s.correct_state) { this.get_container().children("ul").empty(); }
+ }
+ else {
+ obj.children("a.jstree-loading").removeClass("jstree-loading");
+ obj.removeData("jstree-is-loading");
+ if(s.correct_state) { this.correct_state(obj); }
+ }
+ if(e_call) { e_call.call(this); }
+ }
+ else {
+ if(obj === -1 || !obj) { this.get_container().children("ul").empty().append(d.children()); }
+ else { obj.append(d).children("a.jstree-loading").removeClass("jstree-loading"); obj.removeData("jstree-is-loading"); }
+ this.clean_node(obj);
+ if(s_call) { s_call.call(this); }
+ }
+ }, this));
+ break;
+ case (!!s.data && !s.ajax) || (!!s.data && !!s.ajax && (!obj || obj === -1)):
+ if(!obj || obj == -1) {
+ d = this._parse_json(s.data, obj);
+ if(d) {
+ this.get_container().children("ul").empty().append(d.children());
+ this.clean_node();
+ }
+ else {
+ if(s.correct_state) { this.get_container().children("ul").empty(); }
+ }
+ }
+ if(s_call) { s_call.call(this); }
+ break;
+ case (!s.data && !!s.ajax) || (!!s.data && !!s.ajax && obj && obj !== -1):
+ error_func = function (x, t, e) {
+ var ef = this.get_settings().json_data.ajax.error;
+ if(ef) { ef.call(this, x, t, e); }
+ if(obj != -1 && obj.length) {
+ obj.children("a.jstree-loading").removeClass("jstree-loading");
+ obj.removeData("jstree-is-loading");
+ if(t === "success" && s.correct_state) { this.correct_state(obj); }
+ }
+ else {
+ if(t === "success" && s.correct_state) { this.get_container().children("ul").empty(); }
+ }
+ if(e_call) { e_call.call(this); }
+ };
+ success_func = function (d, t, x) {
+ var sf = this.get_settings().json_data.ajax.success;
+ if(sf) { d = sf.call(this,d,t,x) || d; }
+ if(d === "" || (d && d.toString && d.toString().replace(/^[\s\n]+$/,"") === "") || (!$.isArray(d) && !$.isPlainObject(d))) {
+ return error_func.call(this, x, t, "");
+ }
+ d = this._parse_json(d, obj);
+ if(d) {
+ if(obj === -1 || !obj) { this.get_container().children("ul").empty().append(d.children()); }
+ else { obj.append(d).children("a.jstree-loading").removeClass("jstree-loading"); obj.removeData("jstree-is-loading"); }
+ this.clean_node(obj);
+ if(s_call) { s_call.call(this); }
+ }
+ else {
+ if(obj === -1 || !obj) {
+ if(s.correct_state) {
+ this.get_container().children("ul").empty();
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ else {
+ obj.children("a.jstree-loading").removeClass("jstree-loading");
+ obj.removeData("jstree-is-loading");
+ if(s.correct_state) {
+ this.correct_state(obj);
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ }
+ };
+ s.ajax.context = this;
+ s.ajax.error = error_func;
+ s.ajax.success = success_func;
+ if(!s.ajax.dataType) { s.ajax.dataType = "json"; }
+ if($.isFunction(s.ajax.url)) { s.ajax.url = s.ajax.url.call(this, obj); }
+ if($.isFunction(s.ajax.data)) { s.ajax.data = s.ajax.data.call(this, obj); }
+ $.ajax(s.ajax);
+ break;
+ }
+ },
+ _parse_json : function (js, obj, is_callback) {
+ var d = false,
+ p = this._get_settings(),
+ s = p.json_data,
+ t = p.core.html_titles,
+ tmp, i, j, ul1, ul2;
+
+ if(!js) { return d; }
+ if(s.progressive_unload && obj && obj !== -1) {
+ obj.data("jstree-children", d);
+ }
+ if($.isArray(js)) {
+ d = $();
+ if(!js.length) { return false; }
+ for(i = 0, j = js.length; i < j; i++) {
+ tmp = this._parse_json(js[i], obj, true);
+ if(tmp.length) { d = d.add(tmp); }
+ }
+ }
+ else {
+ if(typeof js == "string") { js = { data : js }; }
+ if(!js.data && js.data !== "") { return d; }
+ d = $("<li />");
+ if(js.attr) { d.attr(js.attr); }
+ if(js.metadata) { d.data(js.metadata); }
+ if(js.state) { d.addClass("jstree-" + js.state); }
+ if(!$.isArray(js.data)) { tmp = js.data; js.data = []; js.data.push(tmp); }
+ $.each(js.data, function (i, m) {
+ tmp = $("<a />");
+ if($.isFunction(m)) { m = m.call(this, js); }
+ if(typeof m == "string") { tmp.attr('href','#')[ t ? "html" : "text" ](m); }
+ else {
+ if(!m.attr) { m.attr = {}; }
+ if(!m.attr.href) { m.attr.href = '#'; }
+ tmp.attr(m.attr)[ t ? "html" : "text" ](m.title);
+ if(m.language) { tmp.addClass(m.language); }
+ }
+ tmp.prepend("<ins class='jstree-icon'> </ins>");
+ if(!m.icon && js.icon) { m.icon = js.icon; }
+ if(m.icon) {
+ if(m.icon.indexOf("/") === -1) { tmp.children("ins").addClass(m.icon); }
+ else { tmp.children("ins").css("background","url('" + m.icon + "') center center no-repeat"); }
+ }
+ d.append(tmp);
+ });
+ d.prepend("<ins class='jstree-icon'> </ins>");
+ if(js.children) {
+ if(s.progressive_render && js.state !== "open") {
+ d.addClass("jstree-closed").data("jstree-children", js.children);
+ }
+ else {
+ if(s.progressive_unload) { d.data("jstree-children", js.children); }
+ if($.isArray(js.children) && js.children.length) {
+ tmp = this._parse_json(js.children, obj, true);
+ if(tmp.length) {
+ ul2 = $("<ul />");
+ ul2.append(tmp);
+ d.append(ul2);
+ }
+ }
+ }
+ }
+ }
+ if(!is_callback) {
+ ul1 = $("<ul />");
+ ul1.append(d);
+ d = ul1;
+ }
+ return d;
+ },
+ get_json : function (obj, li_attr, a_attr, is_callback) {
+ var result = [],
+ s = this._get_settings(),
+ _this = this,
+ tmp1, tmp2, li, a, t, lang;
+ obj = this._get_node(obj);
+ if(!obj || obj === -1) { obj = this.get_container().find("> ul > li"); }
+ li_attr = $.isArray(li_attr) ? li_attr : [ "id", "class" ];
+ if(!is_callback && this.data.types) { li_attr.push(s.types.type_attr); }
+ a_attr = $.isArray(a_attr) ? a_attr : [ ];
+
+ obj.each(function () {
+ li = $(this);
+ tmp1 = { data : [] };
+ if(li_attr.length) { tmp1.attr = { }; }
+ $.each(li_attr, function (i, v) {
+ tmp2 = li.attr(v);
+ if(tmp2 && tmp2.length && tmp2.replace(/jstree[^ ]*/ig,'').length) {
+ tmp1.attr[v] = (" " + tmp2).replace(/ jstree[^ ]*/ig,'').replace(/\s+$/ig," ").replace(/^ /,"").replace(/ $/,"");
+ }
+ });
+ if(li.hasClass("jstree-open")) { tmp1.state = "open"; }
+ if(li.hasClass("jstree-closed")) { tmp1.state = "closed"; }
+ if(li.data()) { tmp1.metadata = li.data(); }
+ a = li.children("a");
+ a.each(function () {
+ t = $(this);
+ if(
+ a_attr.length ||
+ $.inArray("languages", s.plugins) !== -1 ||
+ t.children("ins").get(0).style.backgroundImage.length ||
+ (t.children("ins").get(0).className && t.children("ins").get(0).className.replace(/jstree[^ ]*|$/ig,'').length)
+ ) {
+ lang = false;
+ if($.inArray("languages", s.plugins) !== -1 && $.isArray(s.languages) && s.languages.length) {
+ $.each(s.languages, function (l, lv) {
+ if(t.hasClass(lv)) {
+ lang = lv;
+ return false;
+ }
+ });
+ }
+ tmp2 = { attr : { }, title : _this.get_text(t, lang) };
+ $.each(a_attr, function (k, z) {
+ tmp2.attr[z] = (" " + (t.attr(z) || "")).replace(/ jstree[^ ]*/ig,'').replace(/\s+$/ig," ").replace(/^ /,"").replace(/ $/,"");
+ });
+ if($.inArray("languages", s.plugins) !== -1 && $.isArray(s.languages) && s.languages.length) {
+ $.each(s.languages, function (k, z) {
+ if(t.hasClass(z)) { tmp2.language = z; return true; }
+ });
+ }
+ if(t.children("ins").get(0).className.replace(/jstree[^ ]*|$/ig,'').replace(/^\s+$/ig,"").length) {
+ tmp2.icon = t.children("ins").get(0).className.replace(/jstree[^ ]*|$/ig,'').replace(/\s+$/ig," ").replace(/^ /,"").replace(/ $/,"");
+ }
+ if(t.children("ins").get(0).style.backgroundImage.length) {
+ tmp2.icon = t.children("ins").get(0).style.backgroundImage.replace("url(","").replace(")","");
+ }
+ }
+ else {
+ tmp2 = _this.get_text(t);
+ }
+ if(a.length > 1) { tmp1.data.push(tmp2); }
+ else { tmp1.data = tmp2; }
+ });
+ li = li.find("> ul > li");
+ if(li.length) { tmp1.children = _this.get_json(li, li_attr, a_attr, true); }
+ result.push(tmp1);
+ });
+ return result;
+ }
+ }
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree languages plugin
+ * Adds support for multiple language versions in one tree
+ * This basically allows for many titles coexisting in one node, but only one of them being visible at any given time
+ * This is useful for maintaining the same structure in many languages (hence the name of the plugin)
+ */
+(function ($) {
+ $.jstree.plugin("languages", {
+ __init : function () { this._load_css(); },
+ defaults : [],
+ _fn : {
+ set_lang : function (i) {
+ var langs = this._get_settings().languages,
+ st = false,
+ selector = ".jstree-" + this.get_index() + ' a';
+ if(!$.isArray(langs) || langs.length === 0) { return false; }
+ if($.inArray(i,langs) == -1) {
+ if(!!langs[i]) { i = langs[i]; }
+ else { return false; }
+ }
+ if(i == this.data.languages.current_language) { return true; }
+ st = $.vakata.css.get_css(selector + "." + this.data.languages.current_language, false, this.data.languages.language_css);
+ if(st !== false) { st.style.display = "none"; }
+ st = $.vakata.css.get_css(selector + "." + i, false, this.data.languages.language_css);
+ if(st !== false) { st.style.display = ""; }
+ this.data.languages.current_language = i;
+ this.__callback(i);
+ return true;
+ },
+ get_lang : function () {
+ return this.data.languages.current_language;
+ },
+ _get_string : function (key, lang) {
+ var langs = this._get_settings().languages,
+ s = this._get_settings().core.strings;
+ if($.isArray(langs) && langs.length) {
+ lang = (lang && $.inArray(lang,langs) != -1) ? lang : this.data.languages.current_language;
+ }
+ if(s[lang] && s[lang][key]) { return s[lang][key]; }
+ if(s[key]) { return s[key]; }
+ return key;
+ },
+ get_text : function (obj, lang) {
+ obj = this._get_node(obj) || this.data.ui.last_selected;
+ if(!obj.size()) { return false; }
+ var langs = this._get_settings().languages,
+ s = this._get_settings().core.html_titles;
+ if($.isArray(langs) && langs.length) {
+ lang = (lang && $.inArray(lang,langs) != -1) ? lang : this.data.languages.current_language;
+ obj = obj.children("a." + lang);
+ }
+ else { obj = obj.children("a:eq(0)"); }
+ if(s) {
+ obj = obj.clone();
+ obj.children("INS").remove();
+ return obj.html();
+ }
+ else {
+ obj = obj.contents().filter(function() { return this.nodeType == 3; })[0];
+ return obj.nodeValue;
+ }
+ },
+ set_text : function (obj, val, lang) {
+ obj = this._get_node(obj) || this.data.ui.last_selected;
+ if(!obj.size()) { return false; }
+ var langs = this._get_settings().languages,
+ s = this._get_settings().core.html_titles,
+ tmp;
+ if($.isArray(langs) && langs.length) {
+ lang = (lang && $.inArray(lang,langs) != -1) ? lang : this.data.languages.current_language;
+ obj = obj.children("a." + lang);
+ }
+ else { obj = obj.children("a:eq(0)"); }
+ if(s) {
+ tmp = obj.children("INS").clone();
+ obj.html(val).prepend(tmp);
+ this.__callback({ "obj" : obj, "name" : val, "lang" : lang });
+ return true;
+ }
+ else {
+ obj = obj.contents().filter(function() { return this.nodeType == 3; })[0];
+ this.__callback({ "obj" : obj, "name" : val, "lang" : lang });
+ return (obj.nodeValue = val);
+ }
+ },
+ _load_css : function () {
+ var langs = this._get_settings().languages,
+ str = "/* languages css */",
+ selector = ".jstree-" + this.get_index() + ' a',
+ ln;
+ if($.isArray(langs) && langs.length) {
+ this.data.languages.current_language = langs[0];
+ for(ln = 0; ln < langs.length; ln++) {
+ str += selector + "." + langs[ln] + " {";
+ if(langs[ln] != this.data.languages.current_language) { str += " display:none; "; }
+ str += " } ";
+ }
+ this.data.languages.language_css = $.vakata.css.add_sheet({ 'str' : str, 'title' : "jstree-languages" });
+ }
+ },
+ create_node : function (obj, position, js, callback) {
+ var t = this.__call_old(true, obj, position, js, function (t) {
+ var langs = this._get_settings().languages,
+ a = t.children("a"),
+ ln;
+ if($.isArray(langs) && langs.length) {
+ for(ln = 0; ln < langs.length; ln++) {
+ if(!a.is("." + langs[ln])) {
+ t.append(a.eq(0).clone().removeClass(langs.join(" ")).addClass(langs[ln]));
+ }
+ }
+ a.not("." + langs.join(", .")).remove();
+ }
+ if(callback) { callback.call(this, t); }
+ });
+ return t;
+ }
+ }
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree cookies plugin
+ * Stores the currently opened/selected nodes in a cookie and then restores them
+ * Depends on the jquery.cookie plugin
+ */
+(function ($) {
+ $.jstree.plugin("cookies", {
+ __init : function () {
+ if(typeof $.cookie === "undefined") { throw "jsTree cookie: jQuery cookie plugin not included."; }
+
+ var s = this._get_settings().cookies,
+ tmp;
+ if(!!s.save_loaded) {
+ tmp = $.cookie(s.save_loaded);
+ if(tmp && tmp.length) { this.data.core.to_load = tmp.split(","); }
+ }
+ if(!!s.save_opened) {
+ tmp = $.cookie(s.save_opened);
+ if(tmp && tmp.length) { this.data.core.to_open = tmp.split(","); }
+ }
+ if(!!s.save_selected) {
+ tmp = $.cookie(s.save_selected);
+ if(tmp && tmp.length && this.data.ui) { this.data.ui.to_select = tmp.split(","); }
+ }
+ this.get_container()
+ .one( ( this.data.ui ? "reselect" : "reopen" ) + ".jstree", $.proxy(function () {
+ this.get_container()
+ .bind("open_node.jstree close_node.jstree select_node.jstree deselect_node.jstree", $.proxy(function (e) {
+ if(this._get_settings().cookies.auto_save) { this.save_cookie((e.handleObj.namespace + e.handleObj.type).replace("jstree","")); }
+ }, this));
+ }, this));
+ },
+ defaults : {
+ save_loaded : "jstree_load",
+ save_opened : "jstree_open",
+ save_selected : "jstree_select",
+ auto_save : true,
+ cookie_options : {}
+ },
+ _fn : {
+ save_cookie : function (c) {
+ if(this.data.core.refreshing) { return; }
+ var s = this._get_settings().cookies;
+ if(!c) { // if called manually and not by event
+ if(s.save_loaded) {
+ this.save_loaded();
+ $.cookie(s.save_loaded, this.data.core.to_load.join(","), s.cookie_options);
+ }
+ if(s.save_opened) {
+ this.save_opened();
+ $.cookie(s.save_opened, this.data.core.to_open.join(","), s.cookie_options);
+ }
+ if(s.save_selected && this.data.ui) {
+ this.save_selected();
+ $.cookie(s.save_selected, this.data.ui.to_select.join(","), s.cookie_options);
+ }
+ return;
+ }
+ switch(c) {
+ case "open_node":
+ case "close_node":
+ if(!!s.save_opened) {
+ this.save_opened();
+ $.cookie(s.save_opened, this.data.core.to_open.join(","), s.cookie_options);
+ }
+ if(!!s.save_loaded) {
+ this.save_loaded();
+ $.cookie(s.save_loaded, this.data.core.to_load.join(","), s.cookie_options);
+ }
+ break;
+ case "select_node":
+ case "deselect_node":
+ if(!!s.save_selected && this.data.ui) {
+ this.save_selected();
+ $.cookie(s.save_selected, this.data.ui.to_select.join(","), s.cookie_options);
+ }
+ break;
+ }
+ }
+ }
+ });
+ // include cookies by default
+ // $.jstree.defaults.plugins.push("cookies");
+})(jQuery);
+//*/
+
+/*
+ * jsTree sort plugin
+ * Sorts items alphabetically (or using any other function)
+ */
+(function ($) {
+ $.jstree.plugin("sort", {
+ __init : function () {
+ this.get_container()
+ .bind("load_node.jstree", $.proxy(function (e, data) {
+ var obj = this._get_node(data.rslt.obj);
+ obj = obj === -1 ? this.get_container().children("ul") : obj.children("ul");
+ this.sort(obj);
+ }, this))
+ .bind("rename_node.jstree create_node.jstree create.jstree", $.proxy(function (e, data) {
+ this.sort(data.rslt.obj.parent());
+ }, this))
+ .bind("move_node.jstree", $.proxy(function (e, data) {
+ var m = data.rslt.np == -1 ? this.get_container() : data.rslt.np;
+ this.sort(m.children("ul"));
+ }, this));
+ },
+ defaults : function (a, b) { return this.get_text(a) > this.get_text(b) ? 1 : -1; },
+ _fn : {
+ sort : function (obj) {
+ var s = this._get_settings().sort,
+ t = this;
+ obj.append($.makeArray(obj.children("li")).sort($.proxy(s, t)));
+ obj.find("> li > ul").each(function() { t.sort($(this)); });
+ this.clean_node(obj);
+ }
+ }
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree DND plugin
+ * Drag and drop plugin for moving/copying nodes
+ */
+(function ($) {
+ var o = false,
+ r = false,
+ m = false,
+ ml = false,
+ sli = false,
+ sti = false,
+ dir1 = false,
+ dir2 = false,
+ last_pos = false;
+ $.vakata.dnd = {
+ is_down : false,
+ is_drag : false,
+ helper : false,
+ scroll_spd : 10,
+ init_x : 0,
+ init_y : 0,
+ threshold : 5,
+ helper_left : 5,
+ helper_top : 10,
+ user_data : {},
+
+ drag_start : function (e, data, html) {
+ if($.vakata.dnd.is_drag) { $.vakata.drag_stop({}); }
+ try {
+ e.currentTarget.unselectable = "on";
+ e.currentTarget.onselectstart = function() { return false; };
+ if(e.currentTarget.style) { e.currentTarget.style.MozUserSelect = "none"; }
+ } catch(err) { }
+ $.vakata.dnd.init_x = e.pageX;
+ $.vakata.dnd.init_y = e.pageY;
+ $.vakata.dnd.user_data = data;
+ $.vakata.dnd.is_down = true;
+ $.vakata.dnd.helper = $("<div id='vakata-dragged' />").html(html); //.fadeTo(10,0.25);
+ $(document).bind("mousemove", $.vakata.dnd.drag);
+ $(document).bind("mouseup", $.vakata.dnd.drag_stop);
+ return false;
+ },
+ drag : function (e) {
+ if(!$.vakata.dnd.is_down) { return; }
+ if(!$.vakata.dnd.is_drag) {
+ if(Math.abs(e.pageX - $.vakata.dnd.init_x) > 5 || Math.abs(e.pageY - $.vakata.dnd.init_y) > 5) {
+ $.vakata.dnd.helper.appendTo("body");
+ $.vakata.dnd.is_drag = true;
+ $(document).triggerHandler("drag_start.vakata", { "event" : e, "data" : $.vakata.dnd.user_data });
+ }
+ else { return; }
+ }
+
+ // maybe use a scrolling parent element instead of document?
+ if(e.type === "mousemove") { // thought of adding scroll in order to move the helper, but mouse poisition is n/a
+ var d = $(document), t = d.scrollTop(), l = d.scrollLeft();
+ if(e.pageY - t < 20) {
+ if(sti && dir1 === "down") { clearInterval(sti); sti = false; }
+ if(!sti) { dir1 = "up"; sti = setInterval(function () { $(document).scrollTop($(document).scrollTop() - $.vakata.dnd.scroll_spd); }, 150); }
+ }
+ else {
+ if(sti && dir1 === "up") { clearInterval(sti); sti = false; }
+ }
+ if($(window).height() - (e.pageY - t) < 20) {
+ if(sti && dir1 === "up") { clearInterval(sti); sti = false; }
+ if(!sti) { dir1 = "down"; sti = setInterval(function () { $(document).scrollTop($(document).scrollTop() + $.vakata.dnd.scroll_spd); }, 150); }
+ }
+ else {
+ if(sti && dir1 === "down") { clearInterval(sti); sti = false; }
+ }
+
+ if(e.pageX - l < 20) {
+ if(sli && dir2 === "right") { clearInterval(sli); sli = false; }
+ if(!sli) { dir2 = "left"; sli = setInterval(function () { $(document).scrollLeft($(document).scrollLeft() - $.vakata.dnd.scroll_spd); }, 150); }
+ }
+ else {
+ if(sli && dir2 === "left") { clearInterval(sli); sli = false; }
+ }
+ if($(window).width() - (e.pageX - l) < 20) {
+ if(sli && dir2 === "left") { clearInterval(sli); sli = false; }
+ if(!sli) { dir2 = "right"; sli = setInterval(function () { $(document).scrollLeft($(document).scrollLeft() + $.vakata.dnd.scroll_spd); }, 150); }
+ }
+ else {
+ if(sli && dir2 === "right") { clearInterval(sli); sli = false; }
+ }
+ }
+
+ $.vakata.dnd.helper.css({ left : (e.pageX + $.vakata.dnd.helper_left) + "px", top : (e.pageY + $.vakata.dnd.helper_top) + "px" });
+ $(document).triggerHandler("drag.vakata", { "event" : e, "data" : $.vakata.dnd.user_data });
+ },
+ drag_stop : function (e) {
+ if(sli) { clearInterval(sli); }
+ if(sti) { clearInterval(sti); }
+ $(document).unbind("mousemove", $.vakata.dnd.drag);
+ $(document).unbind("mouseup", $.vakata.dnd.drag_stop);
+ $(document).triggerHandler("drag_stop.vakata", { "event" : e, "data" : $.vakata.dnd.user_data });
+ $.vakata.dnd.helper.remove();
+ $.vakata.dnd.init_x = 0;
+ $.vakata.dnd.init_y = 0;
+ $.vakata.dnd.user_data = {};
+ $.vakata.dnd.is_down = false;
+ $.vakata.dnd.is_drag = false;
+ }
+ };
+ $(function() {
+ var css_string = '#vakata-dragged { display:block; margin:0 0 0 0; padding:4px 4px 4px 24px; position:absolute; top:-2000px; line-height:16px; z-index:10000; } ';
+ $.vakata.css.add_sheet({ str : css_string, title : "vakata" });
+ });
+
+ $.jstree.plugin("dnd", {
+ __init : function () {
+ this.data.dnd = {
+ active : false,
+ after : false,
+ inside : false,
+ before : false,
+ off : false,
+ prepared : false,
+ w : 0,
+ to1 : false,
+ to2 : false,
+ cof : false,
+ cw : false,
+ ch : false,
+ i1 : false,
+ i2 : false,
+ mto : false
+ };
+ this.get_container()
+ .bind("mouseenter.jstree", $.proxy(function (e) {
+ if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree) {
+ if(this.data.themes) {
+ m.attr("class", "jstree-" + this.data.themes.theme);
+ if(ml) { ml.attr("class", "jstree-" + this.data.themes.theme); }
+ $.vakata.dnd.helper.attr("class", "jstree-dnd-helper jstree-" + this.data.themes.theme);
+ }
+ //if($(e.currentTarget).find("> ul > li").length === 0) {
+ if(e.currentTarget === e.target && $.vakata.dnd.user_data.obj && $($.vakata.dnd.user_data.obj).length && $($.vakata.dnd.user_data.obj).parents(".jstree:eq(0)")[0] !== e.target) { // node should not be from the same tree
+ var tr = $.jstree._reference(e.target), dc;
+ if(tr.data.dnd.foreign) {
+ dc = tr._get_settings().dnd.drag_check.call(this, { "o" : o, "r" : tr.get_container(), is_root : true });
+ if(dc === true || dc.inside === true || dc.before === true || dc.after === true) {
+ $.vakata.dnd.helper.children("ins").attr("class","jstree-ok");
+ }
+ }
+ else {
+ tr.prepare_move(o, tr.get_container(), "last");
+ if(tr.check_move()) {
+ $.vakata.dnd.helper.children("ins").attr("class","jstree-ok");
+ }
+ }
+ }
+ }
+ }, this))
+ .bind("mouseup.jstree", $.proxy(function (e) {
+ //if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree && $(e.currentTarget).find("> ul > li").length === 0) {
+ if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree && e.currentTarget === e.target && $.vakata.dnd.user_data.obj && $($.vakata.dnd.user_data.obj).length && $($.vakata.dnd.user_data.obj).parents(".jstree:eq(0)")[0] !== e.target) { // node should not be from the same tree
+ var tr = $.jstree._reference(e.currentTarget), dc;
+ if(tr.data.dnd.foreign) {
+ dc = tr._get_settings().dnd.drag_check.call(this, { "o" : o, "r" : tr.get_container(), is_root : true });
+ if(dc === true || dc.inside === true || dc.before === true || dc.after === true) {
+ tr._get_settings().dnd.drag_finish.call(this, { "o" : o, "r" : tr.get_container(), is_root : true });
+ }
+ }
+ else {
+ tr.move_node(o, tr.get_container(), "last", e[tr._get_settings().dnd.copy_modifier + "Key"]);
+ }
+ }
+ }, this))
+ .bind("mouseleave.jstree", $.proxy(function (e) {
+ if(e.relatedTarget && e.relatedTarget.id && e.relatedTarget.id === "jstree-marker-line") {
+ return false;
+ }
+ if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree) {
+ if(this.data.dnd.i1) { clearInterval(this.data.dnd.i1); }
+ if(this.data.dnd.i2) { clearInterval(this.data.dnd.i2); }
+ if(this.data.dnd.to1) { clearTimeout(this.data.dnd.to1); }
+ if(this.data.dnd.to2) { clearTimeout(this.data.dnd.to2); }
+ if($.vakata.dnd.helper.children("ins").hasClass("jstree-ok")) {
+ $.vakata.dnd.helper.children("ins").attr("class","jstree-invalid");
+ }
+ }
+ }, this))
+ .bind("mousemove.jstree", $.proxy(function (e) {
+ if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree) {
+ var cnt = this.get_container()[0];
+
+ // Horizontal scroll
+ if(e.pageX + 24 > this.data.dnd.cof.left + this.data.dnd.cw) {
+ if(this.data.dnd.i1) { clearInterval(this.data.dnd.i1); }
+ this.data.dnd.i1 = setInterval($.proxy(function () { this.scrollLeft += $.vakata.dnd.scroll_spd; }, cnt), 100);
+ }
+ else if(e.pageX - 24 < this.data.dnd.cof.left) {
+ if(this.data.dnd.i1) { clearInterval(this.data.dnd.i1); }
+ this.data.dnd.i1 = setInterval($.proxy(function () { this.scrollLeft -= $.vakata.dnd.scroll_spd; }, cnt), 100);
+ }
+ else {
+ if(this.data.dnd.i1) { clearInterval(this.data.dnd.i1); }
+ }
+
+ // Vertical scroll
+ if(e.pageY + 24 > this.data.dnd.cof.top + this.data.dnd.ch) {
+ if(this.data.dnd.i2) { clearInterval(this.data.dnd.i2); }
+ this.data.dnd.i2 = setInterval($.proxy(function () { this.scrollTop += $.vakata.dnd.scroll_spd; }, cnt), 100);
+ }
+ else if(e.pageY - 24 < this.data.dnd.cof.top) {
+ if(this.data.dnd.i2) { clearInterval(this.data.dnd.i2); }
+ this.data.dnd.i2 = setInterval($.proxy(function () { this.scrollTop -= $.vakata.dnd.scroll_spd; }, cnt), 100);
+ }
+ else {
+ if(this.data.dnd.i2) { clearInterval(this.data.dnd.i2); }
+ }
+
+ }
+ }, this))
+ .bind("scroll.jstree", $.proxy(function (e) {
+ if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree && m && ml) {
+ m.hide();
+ ml.hide();
+ }
+ }, this))
+ .delegate("a", "mousedown.jstree", $.proxy(function (e) {
+ if(e.which === 1) {
+ this.start_drag(e.currentTarget, e);
+ return false;
+ }
+ }, this))
+ .delegate("a", "mouseenter.jstree", $.proxy(function (e) {
+ if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree) {
+ this.dnd_enter(e.currentTarget);
+ }
+ }, this))
+ .delegate("a", "mousemove.jstree", $.proxy(function (e) {
+ if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree) {
+ if(!r || !r.length || r.children("a")[0] !== e.currentTarget) {
+ this.dnd_enter(e.currentTarget);
+ }
+ if(typeof this.data.dnd.off.top === "undefined") { this.data.dnd.off = $(e.target).offset(); }
+ this.data.dnd.w = (e.pageY - (this.data.dnd.off.top || 0)) % this.data.core.li_height;
+ if(this.data.dnd.w < 0) { this.data.dnd.w += this.data.core.li_height; }
+ this.dnd_show();
+ }
+ }, this))
+ .delegate("a", "mouseleave.jstree", $.proxy(function (e) {
+ if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree) {
+ if(e.relatedTarget && e.relatedTarget.id && e.relatedTarget.id === "jstree-marker-line") {
+ return false;
+ }
+ if(m) { m.hide(); }
+ if(ml) { ml.hide(); }
+ /*
+ var ec = $(e.currentTarget).closest("li"),
+ er = $(e.relatedTarget).closest("li");
+ if(er[0] !== ec.prev()[0] && er[0] !== ec.next()[0]) {
+ if(m) { m.hide(); }
+ if(ml) { ml.hide(); }
+ }
+ */
+ this.data.dnd.mto = setTimeout(
+ (function (t) { return function () { t.dnd_leave(e); }; })(this),
+ 0);
+ }
+ }, this))
+ .delegate("a", "mouseup.jstree", $.proxy(function (e) {
+ if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree) {
+ this.dnd_finish(e);
+ }
+ }, this));
+
+ $(document)
+ .bind("drag_stop.vakata", $.proxy(function () {
+ if(this.data.dnd.to1) { clearTimeout(this.data.dnd.to1); }
+ if(this.data.dnd.to2) { clearTimeout(this.data.dnd.to2); }
+ if(this.data.dnd.i1) { clearInterval(this.data.dnd.i1); }
+ if(this.data.dnd.i2) { clearInterval(this.data.dnd.i2); }
+ this.data.dnd.after = false;
+ this.data.dnd.before = false;
+ this.data.dnd.inside = false;
+ this.data.dnd.off = false;
+ this.data.dnd.prepared = false;
+ this.data.dnd.w = false;
+ this.data.dnd.to1 = false;
+ this.data.dnd.to2 = false;
+ this.data.dnd.i1 = false;
+ this.data.dnd.i2 = false;
+ this.data.dnd.active = false;
+ this.data.dnd.foreign = false;
+ if(m) { m.css({ "top" : "-2000px" }); }
+ if(ml) { ml.css({ "top" : "-2000px" }); }
+ }, this))
+ .bind("drag_start.vakata", $.proxy(function (e, data) {
+ if(data.data.jstree) {
+ var et = $(data.event.target);
+ if(et.closest(".jstree").hasClass("jstree-" + this.get_index())) {
+ this.dnd_enter(et);
+ }
+ }
+ }, this));
+ /*
+ .bind("keydown.jstree-" + this.get_index() + " keyup.jstree-" + this.get_index(), $.proxy(function(e) {
+ if($.vakata.dnd.is_drag && $.vakata.dnd.user_data.jstree && !this.data.dnd.foreign) {
+ var h = $.vakata.dnd.helper.children("ins");
+ if(e[this._get_settings().dnd.copy_modifier + "Key"] && h.hasClass("jstree-ok")) {
+ h.parent().html(h.parent().html().replace(/ \(Copy\)$/, "") + " (Copy)");
+ }
+ else {
+ h.parent().html(h.parent().html().replace(/ \(Copy\)$/, ""));
+ }
+ }
+ }, this)); */
+
+
+
+ var s = this._get_settings().dnd;
+ if(s.drag_target) {
+ $(document)
+ .delegate(s.drag_target, "mousedown.jstree-" + this.get_index(), $.proxy(function (e) {
+ o = e.target;
+ $.vakata.dnd.drag_start(e, { jstree : true, obj : e.target }, "<ins class='jstree-icon'></ins>" + $(e.target).text() );
+ if(this.data.themes) {
+ if(m) { m.attr("class", "jstree-" + this.data.themes.theme); }
+ if(ml) { ml.attr("class", "jstree-" + this.data.themes.theme); }
+ $.vakata.dnd.helper.attr("class", "jstree-dnd-helper jstree-" + this.data.themes.theme);
+ }
+ $.vakata.dnd.helper.children("ins").attr("class","jstree-invalid");
+ var cnt = this.get_container();
+ this.data.dnd.cof = cnt.offset();
+ this.data.dnd.cw = parseInt(cnt.width(),10);
+ this.data.dnd.ch = parseInt(cnt.height(),10);
+ this.data.dnd.foreign = true;
+ e.preventDefault();
+ }, this));
+ }
+ if(s.drop_target) {
+ $(document)
+ .delegate(s.drop_target, "mouseenter.jstree-" + this.get_index(), $.proxy(function (e) {
+ if(this.data.dnd.active && this._get_settings().dnd.drop_check.call(this, { "o" : o, "r" : $(e.target), "e" : e })) {
+ $.vakata.dnd.helper.children("ins").attr("class","jstree-ok");
+ }
+ }, this))
+ .delegate(s.drop_target, "mouseleave.jstree-" + this.get_index(), $.proxy(function (e) {
+ if(this.data.dnd.active) {
+ $.vakata.dnd.helper.children("ins").attr("class","jstree-invalid");
+ }
+ }, this))
+ .delegate(s.drop_target, "mouseup.jstree-" + this.get_index(), $.proxy(function (e) {
+ if(this.data.dnd.active && $.vakata.dnd.helper.children("ins").hasClass("jstree-ok")) {
+ this._get_settings().dnd.drop_finish.call(this, { "o" : o, "r" : $(e.target), "e" : e });
+ }
+ }, this));
+ }
+ },
+ defaults : {
+ copy_modifier : "ctrl",
+ check_timeout : 100,
+ open_timeout : 500,
+ drop_target : ".jstree-drop",
+ drop_check : function (data) { return true; },
+ drop_finish : $.noop,
+ drag_target : ".jstree-draggable",
+ drag_finish : $.noop,
+ drag_check : function (data) { return { after : false, before : false, inside : true }; }
+ },
+ _fn : {
+ dnd_prepare : function () {
+ if(!r || !r.length) { return; }
+ this.data.dnd.off = r.offset();
+ if(this._get_settings().core.rtl) {
+ this.data.dnd.off.right = this.data.dnd.off.left + r.width();
+ }
+ if(this.data.dnd.foreign) {
+ var a = this._get_settings().dnd.drag_check.call(this, { "o" : o, "r" : r });
+ this.data.dnd.after = a.after;
+ this.data.dnd.before = a.before;
+ this.data.dnd.inside = a.inside;
+ this.data.dnd.prepared = true;
+ return this.dnd_show();
+ }
+ this.prepare_move(o, r, "before");
+ this.data.dnd.before = this.check_move();
+ this.prepare_move(o, r, "after");
+ this.data.dnd.after = this.check_move();
+ if(this._is_loaded(r)) {
+ this.prepare_move(o, r, "inside");
+ this.data.dnd.inside = this.check_move();
+ }
+ else {
+ this.data.dnd.inside = false;
+ }
+ this.data.dnd.prepared = true;
+ return this.dnd_show();
+ },
+ dnd_show : function () {
+ if(!this.data.dnd.prepared) { return; }
+ var o = ["before","inside","after"],
+ r = false,
+ rtl = this._get_settings().core.rtl,
+ pos;
+ if(this.data.dnd.w < this.data.core.li_height/3) { o = ["before","inside","after"]; }
+ else if(this.data.dnd.w <= this.data.core.li_height*2/3) {
+ o = this.data.dnd.w < this.data.core.li_height/2 ? ["inside","before","after"] : ["inside","after","before"];
+ }
+ else { o = ["after","inside","before"]; }
+ $.each(o, $.proxy(function (i, val) {
+ if(this.data.dnd[val]) {
+ $.vakata.dnd.helper.children("ins").attr("class","jstree-ok");
+ r = val;
+ return false;
+ }
+ }, this));
+ if(r === false) { $.vakata.dnd.helper.children("ins").attr("class","jstree-invalid"); }
+
+ pos = rtl ? (this.data.dnd.off.right - 18) : (this.data.dnd.off.left + 10);
+ switch(r) {
+ case "before":
+ m.css({ "left" : pos + "px", "top" : (this.data.dnd.off.top - 6) + "px" }).show();
+ if(ml) { ml.css({ "left" : (pos + 8) + "px", "top" : (this.data.dnd.off.top - 1) + "px" }).show(); }
+ break;
+ case "after":
+ m.css({ "left" : pos + "px", "top" : (this.data.dnd.off.top + this.data.core.li_height - 6) + "px" }).show();
+ if(ml) { ml.css({ "left" : (pos + 8) + "px", "top" : (this.data.dnd.off.top + this.data.core.li_height - 1) + "px" }).show(); }
+ break;
+ case "inside":
+ m.css({ "left" : pos + ( rtl ? -4 : 4) + "px", "top" : (this.data.dnd.off.top + this.data.core.li_height/2 - 5) + "px" }).show();
+ if(ml) { ml.hide(); }
+ break;
+ default:
+ m.hide();
+ if(ml) { ml.hide(); }
+ break;
+ }
+ last_pos = r;
+ return r;
+ },
+ dnd_open : function () {
+ this.data.dnd.to2 = false;
+ this.open_node(r, $.proxy(this.dnd_prepare,this), true);
+ },
+ dnd_finish : function (e) {
+ if(this.data.dnd.foreign) {
+ if(this.data.dnd.after || this.data.dnd.before || this.data.dnd.inside) {
+ this._get_settings().dnd.drag_finish.call(this, { "o" : o, "r" : r, "p" : last_pos });
+ }
+ }
+ else {
+ this.dnd_prepare();
+ this.move_node(o, r, last_pos, e[this._get_settings().dnd.copy_modifier + "Key"]);
+ }
+ o = false;
+ r = false;
+ m.hide();
+ if(ml) { ml.hide(); }
+ },
+ dnd_enter : function (obj) {
+ if(this.data.dnd.mto) {
+ clearTimeout(this.data.dnd.mto);
+ this.data.dnd.mto = false;
+ }
+ var s = this._get_settings().dnd;
+ this.data.dnd.prepared = false;
+ r = this._get_node(obj);
+ if(s.check_timeout) {
+ // do the calculations after a minimal timeout (users tend to drag quickly to the desired location)
+ if(this.data.dnd.to1) { clearTimeout(this.data.dnd.to1); }
+ this.data.dnd.to1 = setTimeout($.proxy(this.dnd_prepare, this), s.check_timeout);
+ }
+ else {
+ this.dnd_prepare();
+ }
+ if(s.open_timeout) {
+ if(this.data.dnd.to2) { clearTimeout(this.data.dnd.to2); }
+ if(r && r.length && r.hasClass("jstree-closed")) {
+ // if the node is closed - open it, then recalculate
+ this.data.dnd.to2 = setTimeout($.proxy(this.dnd_open, this), s.open_timeout);
+ }
+ }
+ else {
+ if(r && r.length && r.hasClass("jstree-closed")) {
+ this.dnd_open();
+ }
+ }
+ },
+ dnd_leave : function (e) {
+ this.data.dnd.after = false;
+ this.data.dnd.before = false;
+ this.data.dnd.inside = false;
+ $.vakata.dnd.helper.children("ins").attr("class","jstree-invalid");
+ m.hide();
+ if(ml) { ml.hide(); }
+ if(r && r[0] === e.target.parentNode) {
+ if(this.data.dnd.to1) {
+ clearTimeout(this.data.dnd.to1);
+ this.data.dnd.to1 = false;
+ }
+ if(this.data.dnd.to2) {
+ clearTimeout(this.data.dnd.to2);
+ this.data.dnd.to2 = false;
+ }
+ }
+ },
+ start_drag : function (obj, e) {
+ o = this._get_node(obj);
+ if(this.data.ui && this.is_selected(o)) { o = this._get_node(null, true); }
+ var dt = o.length > 1 ? this._get_string("multiple_selection") : this.get_text(o),
+ cnt = this.get_container();
+ if(!this._get_settings().core.html_titles) { dt = dt.replace(/</ig,"<").replace(/>/ig,">"); }
+ $.vakata.dnd.drag_start(e, { jstree : true, obj : o }, "<ins class='jstree-icon'></ins>" + dt );
+ if(this.data.themes) {
+ if(m) { m.attr("class", "jstree-" + this.data.themes.theme); }
+ if(ml) { ml.attr("class", "jstree-" + this.data.themes.theme); }
+ $.vakata.dnd.helper.attr("class", "jstree-dnd-helper jstree-" + this.data.themes.theme);
+ }
+ this.data.dnd.cof = cnt.offset();
+ this.data.dnd.cw = parseInt(cnt.width(),10);
+ this.data.dnd.ch = parseInt(cnt.height(),10);
+ this.data.dnd.active = true;
+ }
+ }
+ });
+ $(function() {
+ var css_string = '' +
+ '#vakata-dragged ins { display:block; text-decoration:none; width:16px; height:16px; margin:0 0 0 0; padding:0; position:absolute; top:4px; left:4px; ' +
+ ' -moz-border-radius:4px; border-radius:4px; -webkit-border-radius:4px; ' +
+ '} ' +
+ '#vakata-dragged .jstree-ok { background:green; } ' +
+ '#vakata-dragged .jstree-invalid { background:red; } ' +
+ '#jstree-marker { padding:0; margin:0; font-size:12px; overflow:hidden; height:12px; width:8px; position:absolute; top:-30px; z-index:10001; background-repeat:no-repeat; display:none; background-color:transparent; text-shadow:1px 1px 1px white; color:black; line-height:10px; } ' +
+ '#jstree-marker-line { padding:0; margin:0; line-height:0%; font-size:1px; overflow:hidden; height:1px; width:100px; position:absolute; top:-30px; z-index:10000; background-repeat:no-repeat; display:none; background-color:#456c43; ' +
+ ' cursor:pointer; border:1px solid #eeeeee; border-left:0; -moz-box-shadow: 0px 0px 2px #666; -webkit-box-shadow: 0px 0px 2px #666; box-shadow: 0px 0px 2px #666; ' +
+ ' -moz-border-radius:1px; border-radius:1px; -webkit-border-radius:1px; ' +
+ '}' +
+ '';
+ $.vakata.css.add_sheet({ str : css_string, title : "jstree" });
+ m = $("<div />").attr({ id : "jstree-marker" }).hide().html("»")
+ .bind("mouseleave mouseenter", function (e) {
+ m.hide();
+ ml.hide();
+ e.preventDefault();
+ e.stopImmediatePropagation();
+ return false;
+ })
+ .appendTo("body");
+ ml = $("<div />").attr({ id : "jstree-marker-line" }).hide()
+ .bind("mouseup", function (e) {
+ if(r && r.length) {
+ r.children("a").trigger(e);
+ e.preventDefault();
+ e.stopImmediatePropagation();
+ return false;
+ }
+ })
+ .bind("mouseleave", function (e) {
+ var rt = $(e.relatedTarget);
+ if(rt.is(".jstree") || rt.closest(".jstree").length === 0) {
+ if(r && r.length) {
+ r.children("a").trigger(e);
+ m.hide();
+ ml.hide();
+ e.preventDefault();
+ e.stopImmediatePropagation();
+ return false;
+ }
+ }
+ })
+ .appendTo("body");
+ $(document).bind("drag_start.vakata", function (e, data) {
+ if(data.data.jstree) { m.show(); if(ml) { ml.show(); } }
+ });
+ $(document).bind("drag_stop.vakata", function (e, data) {
+ if(data.data.jstree) { m.hide(); if(ml) { ml.hide(); } }
+ });
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree checkbox plugin
+ * Inserts checkboxes in front of every node
+ * Depends on the ui plugin
+ * DOES NOT WORK NICELY WITH MULTITREE DRAG'N'DROP
+ */
+(function ($) {
+ $.jstree.plugin("checkbox", {
+ __init : function () {
+ this.data.checkbox.noui = this._get_settings().checkbox.override_ui;
+ if(this.data.ui && this.data.checkbox.noui) {
+ this.select_node = this.deselect_node = this.deselect_all = $.noop;
+ this.get_selected = this.get_checked;
+ }
+
+ this.get_container()
+ .bind("open_node.jstree create_node.jstree clean_node.jstree refresh.jstree", $.proxy(function (e, data) {
+ this._prepare_checkboxes(data.rslt.obj);
+ }, this))
+ .bind("loaded.jstree", $.proxy(function (e) {
+ this._prepare_checkboxes();
+ }, this))
+ .delegate( (this.data.ui && this.data.checkbox.noui ? "a" : "ins.jstree-checkbox") , "click.jstree", $.proxy(function (e) {
+ e.preventDefault();
+ if(this._get_node(e.target).hasClass("jstree-checked")) { this.uncheck_node(e.target); }
+ else { this.check_node(e.target); }
+ if(this.data.ui && this.data.checkbox.noui) {
+ this.save_selected();
+ if(this.data.cookies) { this.save_cookie("select_node"); }
+ }
+ else {
+ e.stopImmediatePropagation();
+ return false;
+ }
+ }, this));
+ },
+ defaults : {
+ override_ui : false,
+ two_state : false,
+ real_checkboxes : false,
+ checked_parent_open : true,
+ real_checkboxes_names : function (n) { return [ ("check_" + (n[0].id || Math.ceil(Math.random() * 10000))) , 1]; }
+ },
+ __destroy : function () {
+ this.get_container()
+ .find("input.jstree-real-checkbox").removeClass("jstree-real-checkbox").end()
+ .find("ins.jstree-checkbox").remove();
+ },
+ _fn : {
+ _checkbox_notify : function (n, data) {
+ if(data.checked) {
+ this.check_node(n, false);
+ }
+ },
+ _prepare_checkboxes : function (obj) {
+ obj = !obj || obj == -1 ? this.get_container().find("> ul > li") : this._get_node(obj);
+ if(obj === false) { return; } // added for removing root nodes
+ var c, _this = this, t, ts = this._get_settings().checkbox.two_state, rc = this._get_settings().checkbox.real_checkboxes, rcn = this._get_settings().checkbox.real_checkboxes_names;
+ obj.each(function () {
+ t = $(this);
+ c = t.is("li") && (t.hasClass("jstree-checked") || (rc && t.children(":checked").length)) ? "jstree-checked" : "jstree-unchecked";
+ t.find("li").andSelf().each(function () {
+ var $t = $(this), nm;
+ $t.children("a" + (_this.data.languages ? "" : ":eq(0)") ).not(":has(.jstree-checkbox)").prepend("<ins class='jstree-checkbox'> </ins>").parent().not(".jstree-checked, .jstree-unchecked").addClass( ts ? "jstree-unchecked" : c );
+ if(rc) {
+ if(!$t.children(":checkbox").length) {
+ nm = rcn.call(_this, $t);
+ $t.prepend("<input type='checkbox' class='jstree-real-checkbox' id='" + nm[0] + "' name='" + nm[0] + "' value='" + nm[1] + "' />");
+ }
+ else {
+ $t.children(":checkbox").addClass("jstree-real-checkbox");
+ }
+ if(c === "jstree-checked") {
+ $t.children(":checkbox").attr("checked","checked");
+ }
+ }
+ if(c === "jstree-checked" && !ts) {
+ $t.find("li").addClass("jstree-checked");
+ }
+ });
+ });
+ if(!ts) {
+ if(obj.length === 1 && obj.is("li")) { this._repair_state(obj); }
+ if(obj.is("li")) { obj.each(function () { _this._repair_state(this); }); }
+ else { obj.find("> ul > li").each(function () { _this._repair_state(this); }); }
+ obj.find(".jstree-checked").parent().parent().each(function () { _this._repair_state(this); });
+ }
+ },
+ change_state : function (obj, state) {
+ obj = this._get_node(obj);
+ var coll = false, rc = this._get_settings().checkbox.real_checkboxes;
+ if(!obj || obj === -1) { return false; }
+ state = (state === false || state === true) ? state : obj.hasClass("jstree-checked");
+ if(this._get_settings().checkbox.two_state) {
+ if(state) {
+ obj.removeClass("jstree-checked").addClass("jstree-unchecked");
+ if(rc) { obj.children(":checkbox").removeAttr("checked"); }
+ }
+ else {
+ obj.removeClass("jstree-unchecked").addClass("jstree-checked");
+ if(rc) { obj.children(":checkbox").attr("checked","checked"); }
+ }
+ }
+ else {
+ if(state) {
+ coll = obj.find("li").andSelf();
+ if(!coll.filter(".jstree-checked, .jstree-undetermined").length) { return false; }
+ coll.removeClass("jstree-checked jstree-undetermined").addClass("jstree-unchecked");
+ if(rc) { coll.children(":checkbox").removeAttr("checked"); }
+ }
+ else {
+ coll = obj.find("li").andSelf();
+ if(!coll.filter(".jstree-unchecked, .jstree-undetermined").length) { return false; }
+ coll.removeClass("jstree-unchecked jstree-undetermined").addClass("jstree-checked");
+ if(rc) { coll.children(":checkbox").attr("checked","checked"); }
+ if(this.data.ui) { this.data.ui.last_selected = obj; }
+ this.data.checkbox.last_selected = obj;
+ }
+ obj.parentsUntil(".jstree", "li").each(function () {
+ var $this = $(this);
+ if(state) {
+ if($this.children("ul").children("li.jstree-checked, li.jstree-undetermined").length) {
+ $this.parentsUntil(".jstree", "li").andSelf().removeClass("jstree-checked jstree-unchecked").addClass("jstree-undetermined");
+ if(rc) { $this.parentsUntil(".jstree", "li").andSelf().children(":checkbox").removeAttr("checked"); }
+ return false;
+ }
+ else {
+ $this.removeClass("jstree-checked jstree-undetermined").addClass("jstree-unchecked");
+ if(rc) { $this.children(":checkbox").removeAttr("checked"); }
+ }
+ }
+ else {
+ if($this.children("ul").children("li.jstree-unchecked, li.jstree-undetermined").length) {
+ $this.parentsUntil(".jstree", "li").andSelf().removeClass("jstree-checked jstree-unchecked").addClass("jstree-undetermined");
+ if(rc) { $this.parentsUntil(".jstree", "li").andSelf().children(":checkbox").removeAttr("checked"); }
+ return false;
+ }
+ else {
+ $this.removeClass("jstree-unchecked jstree-undetermined").addClass("jstree-checked");
+ if(rc) { $this.children(":checkbox").attr("checked","checked"); }
+ }
+ }
+ });
+ }
+ if(this.data.ui && this.data.checkbox.noui) { this.data.ui.selected = this.get_checked(); }
+ this.__callback(obj);
+ return true;
+ },
+ check_node : function (obj) {
+ if(this.change_state(obj, false)) {
+ obj = this._get_node(obj);
+ if(this._get_settings().checkbox.checked_parent_open) {
+ var t = this;
+ obj.parents(".jstree-closed").each(function () { t.open_node(this, false, true); });
+ }
+ this.__callback({ "obj" : obj });
+ }
+ },
+ uncheck_node : function (obj) {
+ if(this.change_state(obj, true)) { this.__callback({ "obj" : this._get_node(obj) }); }
+ },
+ check_all : function () {
+ var _this = this,
+ coll = this._get_settings().checkbox.two_state ? this.get_container_ul().find("li") : this.get_container_ul().children("li");
+ coll.each(function () {
+ _this.change_state(this, false);
+ });
+ this.__callback();
+ },
+ uncheck_all : function () {
+ var _this = this,
+ coll = this._get_settings().checkbox.two_state ? this.get_container_ul().find("li") : this.get_container_ul().children("li");
+ coll.each(function () {
+ _this.change_state(this, true);
+ });
+ this.__callback();
+ },
+
+ is_checked : function(obj) {
+ obj = this._get_node(obj);
+ return obj.length ? obj.is(".jstree-checked") : false;
+ },
+ get_checked : function (obj, get_all) {
+ obj = !obj || obj === -1 ? this.get_container() : this._get_node(obj);
+ return get_all || this._get_settings().checkbox.two_state ? obj.find(".jstree-checked") : obj.find("> ul > .jstree-checked, .jstree-undetermined > ul > .jstree-checked");
+ },
+ get_unchecked : function (obj, get_all) {
+ obj = !obj || obj === -1 ? this.get_container() : this._get_node(obj);
+ return get_all || this._get_settings().checkbox.two_state ? obj.find(".jstree-unchecked") : obj.find("> ul > .jstree-unchecked, .jstree-undetermined > ul > .jstree-unchecked");
+ },
+
+ show_checkboxes : function () { this.get_container().children("ul").removeClass("jstree-no-checkboxes"); },
+ hide_checkboxes : function () { this.get_container().children("ul").addClass("jstree-no-checkboxes"); },
+
+ _repair_state : function (obj) {
+ obj = this._get_node(obj);
+ if(!obj.length) { return; }
+ var rc = this._get_settings().checkbox.real_checkboxes,
+ a = obj.find("> ul > .jstree-checked").length,
+ b = obj.find("> ul > .jstree-undetermined").length,
+ c = obj.find("> ul > li").length;
+ if(c === 0) { if(obj.hasClass("jstree-undetermined")) { this.change_state(obj, false); } }
+ else if(a === 0 && b === 0) { this.change_state(obj, true); }
+ else if(a === c) { this.change_state(obj, false); }
+ else {
+ obj.parentsUntil(".jstree","li").andSelf().removeClass("jstree-checked jstree-unchecked").addClass("jstree-undetermined");
+ if(rc) { obj.parentsUntil(".jstree", "li").andSelf().children(":checkbox").removeAttr("checked"); }
+ }
+ },
+ reselect : function () {
+ if(this.data.ui && this.data.checkbox.noui) {
+ var _this = this,
+ s = this.data.ui.to_select;
+ s = $.map($.makeArray(s), function (n) { return "#" + n.toString().replace(/^#/,"").replace(/\\\//g,"/").replace(/\//g,"\\\/").replace(/\\\./g,".").replace(/\./g,"\\.").replace(/\:/g,"\\:"); });
+ this.deselect_all();
+ $.each(s, function (i, val) { _this.check_node(val); });
+ this.__callback();
+ }
+ else {
+ this.__call_old();
+ }
+ },
+ save_loaded : function () {
+ var _this = this;
+ this.data.core.to_load = [];
+ this.get_container_ul().find("li.jstree-closed.jstree-undetermined").each(function () {
+ if(this.id) { _this.data.core.to_load.push("#" + this.id); }
+ });
+ }
+ }
+ });
+ $(function() {
+ var css_string = '.jstree .jstree-real-checkbox { display:none; } ';
+ $.vakata.css.add_sheet({ str : css_string, title : "jstree" });
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree XML plugin
+ * The XML data store. Datastores are build by overriding the `load_node` and `_is_loaded` functions.
+ */
+(function ($) {
+ $.vakata.xslt = function (xml, xsl, callback) {
+ var rs = "", xm, xs, processor, support;
+ // TODO: IE9 no XSLTProcessor, no document.recalc
+ if(document.recalc) {
+ xm = document.createElement('xml');
+ xs = document.createElement('xml');
+ xm.innerHTML = xml;
+ xs.innerHTML = xsl;
+ $("body").append(xm).append(xs);
+ setTimeout( (function (xm, xs, callback) {
+ return function () {
+ callback.call(null, xm.transformNode(xs.XMLDocument));
+ setTimeout( (function (xm, xs) { return function () { $(xm).remove(); $(xs).remove(); }; })(xm, xs), 200);
+ };
+ })(xm, xs, callback), 100);
+ return true;
+ }
+ if(typeof window.DOMParser !== "undefined" && typeof window.XMLHttpRequest !== "undefined" && typeof window.XSLTProcessor === "undefined") {
+ xml = new DOMParser().parseFromString(xml, "text/xml");
+ xsl = new DOMParser().parseFromString(xsl, "text/xml");
+ // alert(xml.transformNode());
+ // callback.call(null, new XMLSerializer().serializeToString(rs));
+
+ }
+ if(typeof window.DOMParser !== "undefined" && typeof window.XMLHttpRequest !== "undefined" && typeof window.XSLTProcessor !== "undefined") {
+ processor = new XSLTProcessor();
+ support = $.isFunction(processor.transformDocument) ? (typeof window.XMLSerializer !== "undefined") : true;
+ if(!support) { return false; }
+ xml = new DOMParser().parseFromString(xml, "text/xml");
+ xsl = new DOMParser().parseFromString(xsl, "text/xml");
+ if($.isFunction(processor.transformDocument)) {
+ rs = document.implementation.createDocument("", "", null);
+ processor.transformDocument(xml, xsl, rs, null);
+ callback.call(null, new XMLSerializer().serializeToString(rs));
+ return true;
+ }
+ else {
+ processor.importStylesheet(xsl);
+ rs = processor.transformToFragment(xml, document);
+ callback.call(null, $("<div />").append(rs).html());
+ return true;
+ }
+ }
+ return false;
+ };
+ var xsl = {
+ 'nest' : '<' + '?xml version="1.0" encoding="utf-8" ?>' +
+ '<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" >' +
+ '<xsl:output method="html" encoding="utf-8" omit-xml-declaration="yes" standalone="no" indent="no" media-type="text/html" />' +
+ '<xsl:template match="/">' +
+ ' <xsl:call-template name="nodes">' +
+ ' <xsl:with-param name="node" select="/root" />' +
+ ' </xsl:call-template>' +
+ '</xsl:template>' +
+ '<xsl:template name="nodes">' +
+ ' <xsl:param name="node" />' +
+ ' <ul>' +
+ ' <xsl:for-each select="$node/item">' +
+ ' <xsl:variable name="children" select="count(./item) > 0" />' +
+ ' <li>' +
+ ' <xsl:attribute name="class">' +
+ ' <xsl:if test="position() = last()">jstree-last </xsl:if>' +
+ ' <xsl:choose>' +
+ ' <xsl:when test="@state = \'open\'">jstree-open </xsl:when>' +
+ ' <xsl:when test="$children or @hasChildren or @state = \'closed\'">jstree-closed </xsl:when>' +
+ ' <xsl:otherwise>jstree-leaf </xsl:otherwise>' +
+ ' </xsl:choose>' +
+ ' <xsl:value-of select="@class" />' +
+ ' </xsl:attribute>' +
+ ' <xsl:for-each select="@*">' +
+ ' <xsl:if test="name() != \'class\' and name() != \'state\' and name() != \'hasChildren\'">' +
+ ' <xsl:attribute name="{name()}"><xsl:value-of select="." /></xsl:attribute>' +
+ ' </xsl:if>' +
+ ' </xsl:for-each>' +
+ ' <ins class="jstree-icon"><xsl:text> </xsl:text></ins>' +
+ ' <xsl:for-each select="content/name">' +
+ ' <a>' +
+ ' <xsl:attribute name="href">' +
+ ' <xsl:choose>' +
+ ' <xsl:when test="@href"><xsl:value-of select="@href" /></xsl:when>' +
+ ' <xsl:otherwise>#</xsl:otherwise>' +
+ ' </xsl:choose>' +
+ ' </xsl:attribute>' +
+ ' <xsl:attribute name="class"><xsl:value-of select="@lang" /> <xsl:value-of select="@class" /></xsl:attribute>' +
+ ' <xsl:attribute name="style"><xsl:value-of select="@style" /></xsl:attribute>' +
+ ' <xsl:for-each select="@*">' +
+ ' <xsl:if test="name() != \'style\' and name() != \'class\' and name() != \'href\'">' +
+ ' <xsl:attribute name="{name()}"><xsl:value-of select="." /></xsl:attribute>' +
+ ' </xsl:if>' +
+ ' </xsl:for-each>' +
+ ' <ins>' +
+ ' <xsl:attribute name="class">jstree-icon ' +
+ ' <xsl:if test="string-length(attribute::icon) > 0 and not(contains(@icon,\'/\'))"><xsl:value-of select="@icon" /></xsl:if>' +
+ ' </xsl:attribute>' +
+ ' <xsl:if test="string-length(attribute::icon) > 0 and contains(@icon,\'/\')"><xsl:attribute name="style">background:url(<xsl:value-of select="@icon" />) center center no-repeat;</xsl:attribute></xsl:if>' +
+ ' <xsl:text> </xsl:text>' +
+ ' </ins>' +
+ ' <xsl:copy-of select="./child::node()" />' +
+ ' </a>' +
+ ' </xsl:for-each>' +
+ ' <xsl:if test="$children or @hasChildren"><xsl:call-template name="nodes"><xsl:with-param name="node" select="current()" /></xsl:call-template></xsl:if>' +
+ ' </li>' +
+ ' </xsl:for-each>' +
+ ' </ul>' +
+ '</xsl:template>' +
+ '</xsl:stylesheet>',
+
+ 'flat' : '<' + '?xml version="1.0" encoding="utf-8" ?>' +
+ '<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" >' +
+ '<xsl:output method="html" encoding="utf-8" omit-xml-declaration="yes" standalone="no" indent="no" media-type="text/xml" />' +
+ '<xsl:template match="/">' +
+ ' <ul>' +
+ ' <xsl:for-each select="//item[not(@parent_id) or @parent_id=0 or not(@parent_id = //item/@id)]">' + /* the last `or` may be removed */
+ ' <xsl:call-template name="nodes">' +
+ ' <xsl:with-param name="node" select="." />' +
+ ' <xsl:with-param name="is_last" select="number(position() = last())" />' +
+ ' </xsl:call-template>' +
+ ' </xsl:for-each>' +
+ ' </ul>' +
+ '</xsl:template>' +
+ '<xsl:template name="nodes">' +
+ ' <xsl:param name="node" />' +
+ ' <xsl:param name="is_last" />' +
+ ' <xsl:variable name="children" select="count(//item[@parent_id=$node/attribute::id]) > 0" />' +
+ ' <li>' +
+ ' <xsl:attribute name="class">' +
+ ' <xsl:if test="$is_last = true()">jstree-last </xsl:if>' +
+ ' <xsl:choose>' +
+ ' <xsl:when test="@state = \'open\'">jstree-open </xsl:when>' +
+ ' <xsl:when test="$children or @hasChildren or @state = \'closed\'">jstree-closed </xsl:when>' +
+ ' <xsl:otherwise>jstree-leaf </xsl:otherwise>' +
+ ' </xsl:choose>' +
+ ' <xsl:value-of select="@class" />' +
+ ' </xsl:attribute>' +
+ ' <xsl:for-each select="@*">' +
+ ' <xsl:if test="name() != \'parent_id\' and name() != \'hasChildren\' and name() != \'class\' and name() != \'state\'">' +
+ ' <xsl:attribute name="{name()}"><xsl:value-of select="." /></xsl:attribute>' +
+ ' </xsl:if>' +
+ ' </xsl:for-each>' +
+ ' <ins class="jstree-icon"><xsl:text> </xsl:text></ins>' +
+ ' <xsl:for-each select="content/name">' +
+ ' <a>' +
+ ' <xsl:attribute name="href">' +
+ ' <xsl:choose>' +
+ ' <xsl:when test="@href"><xsl:value-of select="@href" /></xsl:when>' +
+ ' <xsl:otherwise>#</xsl:otherwise>' +
+ ' </xsl:choose>' +
+ ' </xsl:attribute>' +
+ ' <xsl:attribute name="class"><xsl:value-of select="@lang" /> <xsl:value-of select="@class" /></xsl:attribute>' +
+ ' <xsl:attribute name="style"><xsl:value-of select="@style" /></xsl:attribute>' +
+ ' <xsl:for-each select="@*">' +
+ ' <xsl:if test="name() != \'style\' and name() != \'class\' and name() != \'href\'">' +
+ ' <xsl:attribute name="{name()}"><xsl:value-of select="." /></xsl:attribute>' +
+ ' </xsl:if>' +
+ ' </xsl:for-each>' +
+ ' <ins>' +
+ ' <xsl:attribute name="class">jstree-icon ' +
+ ' <xsl:if test="string-length(attribute::icon) > 0 and not(contains(@icon,\'/\'))"><xsl:value-of select="@icon" /></xsl:if>' +
+ ' </xsl:attribute>' +
+ ' <xsl:if test="string-length(attribute::icon) > 0 and contains(@icon,\'/\')"><xsl:attribute name="style">background:url(<xsl:value-of select="@icon" />) center center no-repeat;</xsl:attribute></xsl:if>' +
+ ' <xsl:text> </xsl:text>' +
+ ' </ins>' +
+ ' <xsl:copy-of select="./child::node()" />' +
+ ' </a>' +
+ ' </xsl:for-each>' +
+ ' <xsl:if test="$children">' +
+ ' <ul>' +
+ ' <xsl:for-each select="//item[@parent_id=$node/attribute::id]">' +
+ ' <xsl:call-template name="nodes">' +
+ ' <xsl:with-param name="node" select="." />' +
+ ' <xsl:with-param name="is_last" select="number(position() = last())" />' +
+ ' </xsl:call-template>' +
+ ' </xsl:for-each>' +
+ ' </ul>' +
+ ' </xsl:if>' +
+ ' </li>' +
+ '</xsl:template>' +
+ '</xsl:stylesheet>'
+ },
+ escape_xml = function(string) {
+ return string
+ .toString()
+ .replace(/&/g, '&')
+ .replace(/</g, '<')
+ .replace(/>/g, '>')
+ .replace(/"/g, '"')
+ .replace(/'/g, ''');
+ };
+ $.jstree.plugin("xml_data", {
+ defaults : {
+ data : false,
+ ajax : false,
+ xsl : "flat",
+ clean_node : false,
+ correct_state : true,
+ get_skip_empty : false,
+ get_include_preamble : true
+ },
+ _fn : {
+ load_node : function (obj, s_call, e_call) { var _this = this; this.load_node_xml(obj, function () { _this.__callback({ "obj" : _this._get_node(obj) }); s_call.call(this); }, e_call); },
+ _is_loaded : function (obj) {
+ var s = this._get_settings().xml_data;
+ obj = this._get_node(obj);
+ return obj == -1 || !obj || (!s.ajax && !$.isFunction(s.data)) || obj.is(".jstree-open, .jstree-leaf") || obj.children("ul").children("li").size() > 0;
+ },
+ load_node_xml : function (obj, s_call, e_call) {
+ var s = this.get_settings().xml_data,
+ error_func = function () {},
+ success_func = function () {};
+
+ obj = this._get_node(obj);
+ if(obj && obj !== -1) {
+ if(obj.data("jstree-is-loading")) { return; }
+ else { obj.data("jstree-is-loading",true); }
+ }
+ switch(!0) {
+ case (!s.data && !s.ajax): throw "Neither data nor ajax settings supplied.";
+ case ($.isFunction(s.data)):
+ s.data.call(this, obj, $.proxy(function (d) {
+ this.parse_xml(d, $.proxy(function (d) {
+ if(d) {
+ d = d.replace(/ ?xmlns="[^"]*"/ig, "");
+ if(d.length > 10) {
+ d = $(d);
+ if(obj === -1 || !obj) { this.get_container().children("ul").empty().append(d.children()); }
+ else { obj.children("a.jstree-loading").removeClass("jstree-loading"); obj.append(d); obj.removeData("jstree-is-loading"); }
+ if(s.clean_node) { this.clean_node(obj); }
+ if(s_call) { s_call.call(this); }
+ }
+ else {
+ if(obj && obj !== -1) {
+ obj.children("a.jstree-loading").removeClass("jstree-loading");
+ obj.removeData("jstree-is-loading");
+ if(s.correct_state) {
+ this.correct_state(obj);
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ else {
+ if(s.correct_state) {
+ this.get_container().children("ul").empty();
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ }
+ }
+ }, this));
+ }, this));
+ break;
+ case (!!s.data && !s.ajax) || (!!s.data && !!s.ajax && (!obj || obj === -1)):
+ if(!obj || obj == -1) {
+ this.parse_xml(s.data, $.proxy(function (d) {
+ if(d) {
+ d = d.replace(/ ?xmlns="[^"]*"/ig, "");
+ if(d.length > 10) {
+ d = $(d);
+ this.get_container().children("ul").empty().append(d.children());
+ if(s.clean_node) { this.clean_node(obj); }
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ else {
+ if(s.correct_state) {
+ this.get_container().children("ul").empty();
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ }, this));
+ }
+ break;
+ case (!s.data && !!s.ajax) || (!!s.data && !!s.ajax && obj && obj !== -1):
+ error_func = function (x, t, e) {
+ var ef = this.get_settings().xml_data.ajax.error;
+ if(ef) { ef.call(this, x, t, e); }
+ if(obj !== -1 && obj.length) {
+ obj.children("a.jstree-loading").removeClass("jstree-loading");
+ obj.removeData("jstree-is-loading");
+ if(t === "success" && s.correct_state) { this.correct_state(obj); }
+ }
+ else {
+ if(t === "success" && s.correct_state) { this.get_container().children("ul").empty(); }
+ }
+ if(e_call) { e_call.call(this); }
+ };
+ success_func = function (d, t, x) {
+ d = x.responseText;
+ var sf = this.get_settings().xml_data.ajax.success;
+ if(sf) { d = sf.call(this,d,t,x) || d; }
+ if(d === "" || (d && d.toString && d.toString().replace(/^[\s\n]+$/,"") === "")) {
+ return error_func.call(this, x, t, "");
+ }
+ this.parse_xml(d, $.proxy(function (d) {
+ if(d) {
+ d = d.replace(/ ?xmlns="[^"]*"/ig, "");
+ if(d.length > 10) {
+ d = $(d);
+ if(obj === -1 || !obj) { this.get_container().children("ul").empty().append(d.children()); }
+ else { obj.children("a.jstree-loading").removeClass("jstree-loading"); obj.append(d); obj.removeData("jstree-is-loading"); }
+ if(s.clean_node) { this.clean_node(obj); }
+ if(s_call) { s_call.call(this); }
+ }
+ else {
+ if(obj && obj !== -1) {
+ obj.children("a.jstree-loading").removeClass("jstree-loading");
+ obj.removeData("jstree-is-loading");
+ if(s.correct_state) {
+ this.correct_state(obj);
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ else {
+ if(s.correct_state) {
+ this.get_container().children("ul").empty();
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ }
+ }
+ }, this));
+ };
+ s.ajax.context = this;
+ s.ajax.error = error_func;
+ s.ajax.success = success_func;
+ if(!s.ajax.dataType) { s.ajax.dataType = "xml"; }
+ if($.isFunction(s.ajax.url)) { s.ajax.url = s.ajax.url.call(this, obj); }
+ if($.isFunction(s.ajax.data)) { s.ajax.data = s.ajax.data.call(this, obj); }
+ $.ajax(s.ajax);
+ break;
+ }
+ },
+ parse_xml : function (xml, callback) {
+ var s = this._get_settings().xml_data;
+ $.vakata.xslt(xml, xsl[s.xsl], callback);
+ },
+ get_xml : function (tp, obj, li_attr, a_attr, is_callback) {
+ var result = "",
+ s = this._get_settings(),
+ _this = this,
+ tmp1, tmp2, li, a, lang;
+ if(!tp) { tp = "flat"; }
+ if(!is_callback) { is_callback = 0; }
+ obj = this._get_node(obj);
+ if(!obj || obj === -1) { obj = this.get_container().find("> ul > li"); }
+ li_attr = $.isArray(li_attr) ? li_attr : [ "id", "class" ];
+ if(!is_callback && this.data.types && $.inArray(s.types.type_attr, li_attr) === -1) { li_attr.push(s.types.type_attr); }
+
+ a_attr = $.isArray(a_attr) ? a_attr : [ ];
+
+ if(!is_callback) {
+ if(s.xml_data.get_include_preamble) {
+ result += '<' + '?xml version="1.0" encoding="UTF-8"?' + '>';
+ }
+ result += "<root>";
+ }
+ obj.each(function () {
+ result += "<item";
+ li = $(this);
+ $.each(li_attr, function (i, v) {
+ var t = li.attr(v);
+ if(!s.xml_data.get_skip_empty || typeof t !== "undefined") {
+ result += " " + v + "=\"" + escape_xml((" " + (t || "")).replace(/ jstree[^ ]*/ig,'').replace(/\s+$/ig," ").replace(/^ /,"").replace(/ $/,"")) + "\"";
+ }
+ });
+ if(li.hasClass("jstree-open")) { result += " state=\"open\""; }
+ if(li.hasClass("jstree-closed")) { result += " state=\"closed\""; }
+ if(tp === "flat") { result += " parent_id=\"" + escape_xml(is_callback) + "\""; }
+ result += ">";
+ result += "<content>";
+ a = li.children("a");
+ a.each(function () {
+ tmp1 = $(this);
+ lang = false;
+ result += "<name";
+ if($.inArray("languages", s.plugins) !== -1) {
+ $.each(s.languages, function (k, z) {
+ if(tmp1.hasClass(z)) { result += " lang=\"" + escape_xml(z) + "\""; lang = z; return false; }
+ });
+ }
+ if(a_attr.length) {
+ $.each(a_attr, function (k, z) {
+ var t = tmp1.attr(z);
+ if(!s.xml_data.get_skip_empty || typeof t !== "undefined") {
+ result += " " + z + "=\"" + escape_xml((" " + t || "").replace(/ jstree[^ ]*/ig,'').replace(/\s+$/ig," ").replace(/^ /,"").replace(/ $/,"")) + "\"";
+ }
+ });
+ }
+ if(tmp1.children("ins").get(0).className.replace(/jstree[^ ]*|$/ig,'').replace(/^\s+$/ig,"").length) {
+ result += ' icon="' + escape_xml(tmp1.children("ins").get(0).className.replace(/jstree[^ ]*|$/ig,'').replace(/\s+$/ig," ").replace(/^ /,"").replace(/ $/,"")) + '"';
+ }
+ if(tmp1.children("ins").get(0).style.backgroundImage.length) {
+ result += ' icon="' + escape_xml(tmp1.children("ins").get(0).style.backgroundImage.replace("url(","").replace(")","").replace(/'/ig,"").replace(/"/ig,"")) + '"';
+ }
+ result += ">";
+ result += "<![CDATA[" + _this.get_text(tmp1, lang) + "]]>";
+ result += "</name>";
+ });
+ result += "</content>";
+ tmp2 = li[0].id || true;
+ li = li.find("> ul > li");
+ if(li.length) { tmp2 = _this.get_xml(tp, li, li_attr, a_attr, tmp2); }
+ else { tmp2 = ""; }
+ if(tp == "nest") { result += tmp2; }
+ result += "</item>";
+ if(tp == "flat") { result += tmp2; }
+ });
+ if(!is_callback) { result += "</root>"; }
+ return result;
+ }
+ }
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree search plugin
+ * Enables both sync and async search on the tree
+ * DOES NOT WORK WITH JSON PROGRESSIVE RENDER
+ */
+(function ($) {
+ $.expr[':'].jstree_contains = function(a,i,m){
+ return (a.textContent || a.innerText || "").toLowerCase().indexOf(m[3].toLowerCase())>=0;
+ };
+ $.expr[':'].jstree_title_contains = function(a,i,m) {
+ return (a.getAttribute("title") || "").toLowerCase().indexOf(m[3].toLowerCase())>=0;
+ };
+ $.jstree.plugin("search", {
+ __init : function () {
+ this.data.search.str = "";
+ this.data.search.result = $();
+ if(this._get_settings().search.show_only_matches) {
+ this.get_container()
+ .bind("search.jstree", function (e, data) {
+ $(this).children("ul").find("li").hide().removeClass("jstree-last");
+ data.rslt.nodes.parentsUntil(".jstree").andSelf().show()
+ .filter("ul").each(function () { $(this).children("li:visible").eq(-1).addClass("jstree-last"); });
+ })
+ .bind("clear_search.jstree", function () {
+ $(this).children("ul").find("li").css("display","").end().end().jstree("clean_node", -1);
+ });
+ }
+ },
+ defaults : {
+ ajax : false,
+ search_method : "jstree_contains", // for case insensitive - jstree_contains
+ show_only_matches : false
+ },
+ _fn : {
+ search : function (str, skip_async) {
+ if($.trim(str) === "") { this.clear_search(); return; }
+ var s = this.get_settings().search,
+ t = this,
+ error_func = function () { },
+ success_func = function () { };
+ this.data.search.str = str;
+
+ if(!skip_async && s.ajax !== false && this.get_container_ul().find("li.jstree-closed:not(:has(ul)):eq(0)").length > 0) {
+ this.search.supress_callback = true;
+ error_func = function () { };
+ success_func = function (d, t, x) {
+ var sf = this.get_settings().search.ajax.success;
+ if(sf) { d = sf.call(this,d,t,x) || d; }
+ this.data.search.to_open = d;
+ this._search_open();
+ };
+ s.ajax.context = this;
+ s.ajax.error = error_func;
+ s.ajax.success = success_func;
+ if($.isFunction(s.ajax.url)) { s.ajax.url = s.ajax.url.call(this, str); }
+ if($.isFunction(s.ajax.data)) { s.ajax.data = s.ajax.data.call(this, str); }
+ if(!s.ajax.data) { s.ajax.data = { "search_string" : str }; }
+ if(!s.ajax.dataType || /^json/.exec(s.ajax.dataType)) { s.ajax.dataType = "json"; }
+ $.ajax(s.ajax);
+ return;
+ }
+ if(this.data.search.result.length) { this.clear_search(); }
+ this.data.search.result = this.get_container().find("a" + (this.data.languages ? "." + this.get_lang() : "" ) + ":" + (s.search_method) + "(" + this.data.search.str + ")");
+ this.data.search.result.addClass("jstree-search").parent().parents(".jstree-closed").each(function () {
+ t.open_node(this, false, true);
+ });
+ this.__callback({ nodes : this.data.search.result, str : str });
+ },
+ clear_search : function (str) {
+ this.data.search.result.removeClass("jstree-search");
+ this.__callback(this.data.search.result);
+ this.data.search.result = $();
+ },
+ _search_open : function (is_callback) {
+ var _this = this,
+ done = true,
+ current = [],
+ remaining = [];
+ if(this.data.search.to_open.length) {
+ $.each(this.data.search.to_open, function (i, val) {
+ if(val == "#") { return true; }
+ if($(val).length && $(val).is(".jstree-closed")) { current.push(val); }
+ else { remaining.push(val); }
+ });
+ if(current.length) {
+ this.data.search.to_open = remaining;
+ $.each(current, function (i, val) {
+ _this.open_node(val, function () { _this._search_open(true); });
+ });
+ done = false;
+ }
+ }
+ if(done) { this.search(this.data.search.str, true); }
+ }
+ }
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree contextmenu plugin
+ */
+(function ($) {
+ $.vakata.context = {
+ hide_on_mouseleave : false,
+
+ cnt : $("<div id='vakata-contextmenu' />"),
+ vis : false,
+ tgt : false,
+ par : false,
+ func : false,
+ data : false,
+ rtl : false,
+ show : function (s, t, x, y, d, p, rtl) {
+ $.vakata.context.rtl = !!rtl;
+ var html = $.vakata.context.parse(s), h, w;
+ if(!html) { return; }
+ $.vakata.context.vis = true;
+ $.vakata.context.tgt = t;
+ $.vakata.context.par = p || t || null;
+ $.vakata.context.data = d || null;
+ $.vakata.context.cnt
+ .html(html)
+ .css({ "visibility" : "hidden", "display" : "block", "left" : 0, "top" : 0 });
+
+ if($.vakata.context.hide_on_mouseleave) {
+ $.vakata.context.cnt
+ .one("mouseleave", function(e) { $.vakata.context.hide(); });
+ }
+
+ h = $.vakata.context.cnt.height();
+ w = $.vakata.context.cnt.width();
+ if(x + w > $(document).width()) {
+ x = $(document).width() - (w + 5);
+ $.vakata.context.cnt.find("li > ul").addClass("right");
+ }
+ if(y + h > $(document).height()) {
+ y = y - (h + t[0].offsetHeight);
+ $.vakata.context.cnt.find("li > ul").addClass("bottom");
+ }
+
+ $.vakata.context.cnt
+ .css({ "left" : x, "top" : y })
+ .find("li:has(ul)")
+ .bind("mouseenter", function (e) {
+ var w = $(document).width(),
+ h = $(document).height(),
+ ul = $(this).children("ul").show();
+ if(w !== $(document).width()) { ul.toggleClass("right"); }
+ if(h !== $(document).height()) { ul.toggleClass("bottom"); }
+ })
+ .bind("mouseleave", function (e) {
+ $(this).children("ul").hide();
+ })
+ .end()
+ .css({ "visibility" : "visible" })
+ .show();
+ $(document).triggerHandler("context_show.vakata");
+ },
+ hide : function () {
+ $.vakata.context.vis = false;
+ $.vakata.context.cnt.attr("class","").css({ "visibility" : "hidden" });
+ $(document).triggerHandler("context_hide.vakata");
+ },
+ parse : function (s, is_callback) {
+ if(!s) { return false; }
+ var str = "",
+ tmp = false,
+ was_sep = true;
+ if(!is_callback) { $.vakata.context.func = {}; }
+ str += "<ul>";
+ $.each(s, function (i, val) {
+ if(!val) { return true; }
+ $.vakata.context.func[i] = val.action;
+ if(!was_sep && val.separator_before) {
+ str += "<li class='vakata-separator vakata-separator-before'></li>";
+ }
+ was_sep = false;
+ str += "<li class='" + (val._class || "") + (val._disabled ? " jstree-contextmenu-disabled " : "") + "'><ins ";
+ if(val.icon && val.icon.indexOf("/") === -1) { str += " class='" + val.icon + "' "; }
+ if(val.icon && val.icon.indexOf("/") !== -1) { str += " style='background:url(" + val.icon + ") center center no-repeat;' "; }
+ str += "> </ins><a href='#' rel='" + i + "'>";
+ if(val.submenu) {
+ str += "<span style='float:" + ($.vakata.context.rtl ? "left" : "right") + ";'>»</span>";
+ }
+ str += val.label + "</a>";
+ if(val.submenu) {
+ tmp = $.vakata.context.parse(val.submenu, true);
+ if(tmp) { str += tmp; }
+ }
+ str += "</li>";
+ if(val.separator_after) {
+ str += "<li class='vakata-separator vakata-separator-after'></li>";
+ was_sep = true;
+ }
+ });
+ str = str.replace(/<li class\='vakata-separator vakata-separator-after'\><\/li\>$/,"");
+ str += "</ul>";
+ $(document).triggerHandler("context_parse.vakata");
+ return str.length > 10 ? str : false;
+ },
+ exec : function (i) {
+ if($.isFunction($.vakata.context.func[i])) {
+ // if is string - eval and call it!
+ $.vakata.context.func[i].call($.vakata.context.data, $.vakata.context.par);
+ return true;
+ }
+ else { return false; }
+ }
+ };
+ $(function () {
+ var css_string = '' +
+ '#vakata-contextmenu { display:block; visibility:hidden; left:0; top:-200px; position:absolute; margin:0; padding:0; min-width:180px; background:#ebebeb; border:1px solid silver; z-index:10000; *width:180px; } ' +
+ '#vakata-contextmenu ul { min-width:180px; *width:180px; } ' +
+ '#vakata-contextmenu ul, #vakata-contextmenu li { margin:0; padding:0; list-style-type:none; display:block; } ' +
+ '#vakata-contextmenu li { line-height:20px; min-height:20px; position:relative; padding:0px; } ' +
+ '#vakata-contextmenu li a { padding:1px 6px; line-height:17px; display:block; text-decoration:none; margin:1px 1px 0 1px; } ' +
+ '#vakata-contextmenu li ins { float:left; width:16px; height:16px; text-decoration:none; margin-right:2px; } ' +
+ '#vakata-contextmenu li a:hover, #vakata-contextmenu li.vakata-hover > a { background:gray; color:white; } ' +
+ '#vakata-contextmenu li ul { display:none; position:absolute; top:-2px; left:100%; background:#ebebeb; border:1px solid gray; } ' +
+ '#vakata-contextmenu .right { right:100%; left:auto; } ' +
+ '#vakata-contextmenu .bottom { bottom:-1px; top:auto; } ' +
+ '#vakata-contextmenu li.vakata-separator { min-height:0; height:1px; line-height:1px; font-size:1px; overflow:hidden; margin:0 2px; background:silver; /* border-top:1px solid #fefefe; */ padding:0; } ';
+ $.vakata.css.add_sheet({ str : css_string, title : "vakata" });
+ $.vakata.context.cnt
+ .delegate("a","click", function (e) { e.preventDefault(); })
+ .delegate("a","mouseup", function (e) {
+ if(!$(this).parent().hasClass("jstree-contextmenu-disabled") && $.vakata.context.exec($(this).attr("rel"))) {
+ $.vakata.context.hide();
+ }
+ else { $(this).blur(); }
+ })
+ .delegate("a","mouseover", function () {
+ $.vakata.context.cnt.find(".vakata-hover").removeClass("vakata-hover");
+ })
+ .appendTo("body");
+ $(document).bind("mousedown", function (e) { if($.vakata.context.vis && !$.contains($.vakata.context.cnt[0], e.target)) { $.vakata.context.hide(); } });
+ if(typeof $.hotkeys !== "undefined") {
+ $(document)
+ .bind("keydown", "up", function (e) {
+ if($.vakata.context.vis) {
+ var o = $.vakata.context.cnt.find("ul:visible").last().children(".vakata-hover").removeClass("vakata-hover").prevAll("li:not(.vakata-separator)").first();
+ if(!o.length) { o = $.vakata.context.cnt.find("ul:visible").last().children("li:not(.vakata-separator)").last(); }
+ o.addClass("vakata-hover");
+ e.stopImmediatePropagation();
+ e.preventDefault();
+ }
+ })
+ .bind("keydown", "down", function (e) {
+ if($.vakata.context.vis) {
+ var o = $.vakata.context.cnt.find("ul:visible").last().children(".vakata-hover").removeClass("vakata-hover").nextAll("li:not(.vakata-separator)").first();
+ if(!o.length) { o = $.vakata.context.cnt.find("ul:visible").last().children("li:not(.vakata-separator)").first(); }
+ o.addClass("vakata-hover");
+ e.stopImmediatePropagation();
+ e.preventDefault();
+ }
+ })
+ .bind("keydown", "right", function (e) {
+ if($.vakata.context.vis) {
+ $.vakata.context.cnt.find(".vakata-hover").children("ul").show().children("li:not(.vakata-separator)").removeClass("vakata-hover").first().addClass("vakata-hover");
+ e.stopImmediatePropagation();
+ e.preventDefault();
+ }
+ })
+ .bind("keydown", "left", function (e) {
+ if($.vakata.context.vis) {
+ $.vakata.context.cnt.find(".vakata-hover").children("ul").hide().children(".vakata-separator").removeClass("vakata-hover");
+ e.stopImmediatePropagation();
+ e.preventDefault();
+ }
+ })
+ .bind("keydown", "esc", function (e) {
+ $.vakata.context.hide();
+ e.preventDefault();
+ })
+ .bind("keydown", "space", function (e) {
+ $.vakata.context.cnt.find(".vakata-hover").last().children("a").click();
+ e.preventDefault();
+ });
+ }
+ });
+
+ $.jstree.plugin("contextmenu", {
+ __init : function () {
+ this.get_container()
+ .delegate("a", "contextmenu.jstree", $.proxy(function (e) {
+ e.preventDefault();
+ if(!$(e.currentTarget).hasClass("jstree-loading")) {
+ this.show_contextmenu(e.currentTarget, e.pageX, e.pageY);
+ }
+ }, this))
+ .delegate("a", "click.jstree", $.proxy(function (e) {
+ if(this.data.contextmenu) {
+ $.vakata.context.hide();
+ }
+ }, this))
+ .bind("destroy.jstree", $.proxy(function () {
+ // TODO: move this to descruct method
+ if(this.data.contextmenu) {
+ $.vakata.context.hide();
+ }
+ }, this));
+ $(document).bind("context_hide.vakata", $.proxy(function () { this.data.contextmenu = false; }, this));
+ },
+ defaults : {
+ select_node : false, // requires UI plugin
+ show_at_node : true,
+ items : { // Could be a function that should return an object like this one
+ "create" : {
+ "separator_before" : false,
+ "separator_after" : true,
+ "label" : "Create",
+ "action" : function (obj) { this.create(obj); }
+ },
+ "rename" : {
+ "separator_before" : false,
+ "separator_after" : false,
+ "label" : "Rename",
+ "action" : function (obj) { this.rename(obj); }
+ },
+ "remove" : {
+ "separator_before" : false,
+ "icon" : false,
+ "separator_after" : false,
+ "label" : "Delete",
+ "action" : function (obj) { if(this.is_selected(obj)) { this.remove(); } else { this.remove(obj); } }
+ },
+ "ccp" : {
+ "separator_before" : true,
+ "icon" : false,
+ "separator_after" : false,
+ "label" : "Edit",
+ "action" : false,
+ "submenu" : {
+ "cut" : {
+ "separator_before" : false,
+ "separator_after" : false,
+ "label" : "Cut",
+ "action" : function (obj) { this.cut(obj); }
+ },
+ "copy" : {
+ "separator_before" : false,
+ "icon" : false,
+ "separator_after" : false,
+ "label" : "Copy",
+ "action" : function (obj) { this.copy(obj); }
+ },
+ "paste" : {
+ "separator_before" : false,
+ "icon" : false,
+ "separator_after" : false,
+ "label" : "Paste",
+ "action" : function (obj) { this.paste(obj); }
+ }
+ }
+ }
+ }
+ },
+ _fn : {
+ show_contextmenu : function (obj, x, y) {
+ obj = this._get_node(obj);
+ var s = this.get_settings().contextmenu,
+ a = obj.children("a:visible:eq(0)"),
+ o = false,
+ i = false;
+ if(s.select_node && this.data.ui && !this.is_selected(obj)) {
+ this.deselect_all();
+ this.select_node(obj, true);
+ }
+ if(s.show_at_node || typeof x === "undefined" || typeof y === "undefined") {
+ o = a.offset();
+ x = o.left;
+ y = o.top + this.data.core.li_height;
+ }
+ i = obj.data("jstree") && obj.data("jstree").contextmenu ? obj.data("jstree").contextmenu : s.items;
+ if($.isFunction(i)) { i = i.call(this, obj); }
+ this.data.contextmenu = true;
+ $.vakata.context.show(i, a, x, y, this, obj, this._get_settings().core.rtl);
+ if(this.data.themes) { $.vakata.context.cnt.attr("class", "jstree-" + this.data.themes.theme + "-context"); }
+ }
+ }
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree types plugin
+ * Adds support types of nodes
+ * You can set an attribute on each li node, that represents its type.
+ * According to the type setting the node may get custom icon/validation rules
+ */
+(function ($) {
+ $.jstree.plugin("types", {
+ __init : function () {
+ var s = this._get_settings().types;
+ this.data.types.attach_to = [];
+ this.get_container()
+ .bind("init.jstree", $.proxy(function () {
+ var types = s.types,
+ attr = s.type_attr,
+ icons_css = "",
+ _this = this;
+
+ $.each(types, function (i, tp) {
+ $.each(tp, function (k, v) {
+ if(!/^(max_depth|max_children|icon|valid_children)$/.test(k)) { _this.data.types.attach_to.push(k); }
+ });
+ if(!tp.icon) { return true; }
+ if( tp.icon.image || tp.icon.position) {
+ if(i == "default") { icons_css += '.jstree-' + _this.get_index() + ' a > .jstree-icon { '; }
+ else { icons_css += '.jstree-' + _this.get_index() + ' li[' + attr + '="' + i + '"] > a > .jstree-icon { '; }
+ if(tp.icon.image) { icons_css += ' background-image:url(' + tp.icon.image + '); '; }
+ if(tp.icon.position){ icons_css += ' background-position:' + tp.icon.position + '; '; }
+ else { icons_css += ' background-position:0 0; '; }
+ icons_css += '} ';
+ }
+ });
+ if(icons_css !== "") { $.vakata.css.add_sheet({ 'str' : icons_css, title : "jstree-types" }); }
+ }, this))
+ .bind("before.jstree", $.proxy(function (e, data) {
+ var s, t,
+ o = this._get_settings().types.use_data ? this._get_node(data.args[0]) : false,
+ d = o && o !== -1 && o.length ? o.data("jstree") : false;
+ if(d && d.types && d.types[data.func] === false) { e.stopImmediatePropagation(); return false; }
+ if($.inArray(data.func, this.data.types.attach_to) !== -1) {
+ if(!data.args[0] || (!data.args[0].tagName && !data.args[0].jquery)) { return; }
+ s = this._get_settings().types.types;
+ t = this._get_type(data.args[0]);
+ if(
+ (
+ (s[t] && typeof s[t][data.func] !== "undefined") ||
+ (s["default"] && typeof s["default"][data.func] !== "undefined")
+ ) && this._check(data.func, data.args[0]) === false
+ ) {
+ e.stopImmediatePropagation();
+ return false;
+ }
+ }
+ }, this));
+ if(is_ie6) {
+ this.get_container()
+ .bind("load_node.jstree set_type.jstree", $.proxy(function (e, data) {
+ var r = data && data.rslt && data.rslt.obj && data.rslt.obj !== -1 ? this._get_node(data.rslt.obj).parent() : this.get_container_ul(),
+ c = false,
+ s = this._get_settings().types;
+ $.each(s.types, function (i, tp) {
+ if(tp.icon && (tp.icon.image || tp.icon.position)) {
+ c = i === "default" ? r.find("li > a > .jstree-icon") : r.find("li[" + s.type_attr + "='" + i + "'] > a > .jstree-icon");
+ if(tp.icon.image) { c.css("backgroundImage","url(" + tp.icon.image + ")"); }
+ c.css("backgroundPosition", tp.icon.position || "0 0");
+ }
+ });
+ }, this));
+ }
+ },
+ defaults : {
+ // defines maximum number of root nodes (-1 means unlimited, -2 means disable max_children checking)
+ max_children : -1,
+ // defines the maximum depth of the tree (-1 means unlimited, -2 means disable max_depth checking)
+ max_depth : -1,
+ // defines valid node types for the root nodes
+ valid_children : "all",
+
+ // whether to use $.data
+ use_data : false,
+ // where is the type stores (the rel attribute of the LI element)
+ type_attr : "rel",
+ // a list of types
+ types : {
+ // the default type
+ "default" : {
+ "max_children" : -1,
+ "max_depth" : -1,
+ "valid_children": "all"
+
+ // Bound functions - you can bind any other function here (using boolean or function)
+ //"select_node" : true
+ }
+ }
+ },
+ _fn : {
+ _types_notify : function (n, data) {
+ if(data.type && this._get_settings().types.use_data) {
+ this.set_type(data.type, n);
+ }
+ },
+ _get_type : function (obj) {
+ obj = this._get_node(obj);
+ return (!obj || !obj.length) ? false : obj.attr(this._get_settings().types.type_attr) || "default";
+ },
+ set_type : function (str, obj) {
+ obj = this._get_node(obj);
+ var ret = (!obj.length || !str) ? false : obj.attr(this._get_settings().types.type_attr, str);
+ if(ret) { this.__callback({ obj : obj, type : str}); }
+ return ret;
+ },
+ _check : function (rule, obj, opts) {
+ obj = this._get_node(obj);
+ var v = false, t = this._get_type(obj), d = 0, _this = this, s = this._get_settings().types, data = false;
+ if(obj === -1) {
+ if(!!s[rule]) { v = s[rule]; }
+ else { return; }
+ }
+ else {
+ if(t === false) { return; }
+ data = s.use_data ? obj.data("jstree") : false;
+ if(data && data.types && typeof data.types[rule] !== "undefined") { v = data.types[rule]; }
+ else if(!!s.types[t] && typeof s.types[t][rule] !== "undefined") { v = s.types[t][rule]; }
+ else if(!!s.types["default"] && typeof s.types["default"][rule] !== "undefined") { v = s.types["default"][rule]; }
+ }
+ if($.isFunction(v)) { v = v.call(this, obj); }
+ if(rule === "max_depth" && obj !== -1 && opts !== false && s.max_depth !== -2 && v !== 0) {
+ // also include the node itself - otherwise if root node it is not checked
+ obj.children("a:eq(0)").parentsUntil(".jstree","li").each(function (i) {
+ // check if current depth already exceeds global tree depth
+ if(s.max_depth !== -1 && s.max_depth - (i + 1) <= 0) { v = 0; return false; }
+ d = (i === 0) ? v : _this._check(rule, this, false);
+ // check if current node max depth is already matched or exceeded
+ if(d !== -1 && d - (i + 1) <= 0) { v = 0; return false; }
+ // otherwise - set the max depth to the current value minus current depth
+ if(d >= 0 && (d - (i + 1) < v || v < 0) ) { v = d - (i + 1); }
+ // if the global tree depth exists and it minus the nodes calculated so far is less than `v` or `v` is unlimited
+ if(s.max_depth >= 0 && (s.max_depth - (i + 1) < v || v < 0) ) { v = s.max_depth - (i + 1); }
+ });
+ }
+ return v;
+ },
+ check_move : function () {
+ if(!this.__call_old()) { return false; }
+ var m = this._get_move(),
+ s = m.rt._get_settings().types,
+ mc = m.rt._check("max_children", m.cr),
+ md = m.rt._check("max_depth", m.cr),
+ vc = m.rt._check("valid_children", m.cr),
+ ch = 0, d = 1, t;
+
+ if(vc === "none") { return false; }
+ if($.isArray(vc) && m.ot && m.ot._get_type) {
+ m.o.each(function () {
+ if($.inArray(m.ot._get_type(this), vc) === -1) { d = false; return false; }
+ });
+ if(d === false) { return false; }
+ }
+ if(s.max_children !== -2 && mc !== -1) {
+ ch = m.cr === -1 ? this.get_container().find("> ul > li").not(m.o).length : m.cr.find("> ul > li").not(m.o).length;
+ if(ch + m.o.length > mc) { return false; }
+ }
+ if(s.max_depth !== -2 && md !== -1) {
+ d = 0;
+ if(md === 0) { return false; }
+ if(typeof m.o.d === "undefined") {
+ // TODO: deal with progressive rendering and async when checking max_depth (how to know the depth of the moved node)
+ t = m.o;
+ while(t.length > 0) {
+ t = t.find("> ul > li");
+ d ++;
+ }
+ m.o.d = d;
+ }
+ if(md - m.o.d < 0) { return false; }
+ }
+ return true;
+ },
+ create_node : function (obj, position, js, callback, is_loaded, skip_check) {
+ if(!skip_check && (is_loaded || this._is_loaded(obj))) {
+ var p = (typeof position == "string" && position.match(/^before|after$/i) && obj !== -1) ? this._get_parent(obj) : this._get_node(obj),
+ s = this._get_settings().types,
+ mc = this._check("max_children", p),
+ md = this._check("max_depth", p),
+ vc = this._check("valid_children", p),
+ ch;
+ if(typeof js === "string") { js = { data : js }; }
+ if(!js) { js = {}; }
+ if(vc === "none") { return false; }
+ if($.isArray(vc)) {
+ if(!js.attr || !js.attr[s.type_attr]) {
+ if(!js.attr) { js.attr = {}; }
+ js.attr[s.type_attr] = vc[0];
+ }
+ else {
+ if($.inArray(js.attr[s.type_attr], vc) === -1) { return false; }
+ }
+ }
+ if(s.max_children !== -2 && mc !== -1) {
+ ch = p === -1 ? this.get_container().find("> ul > li").length : p.find("> ul > li").length;
+ if(ch + 1 > mc) { return false; }
+ }
+ if(s.max_depth !== -2 && md !== -1 && (md - 1) < 0) { return false; }
+ }
+ return this.__call_old(true, obj, position, js, callback, is_loaded, skip_check);
+ }
+ }
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree HTML plugin
+ * The HTML data store. Datastores are build by replacing the `load_node` and `_is_loaded` functions.
+ */
+(function ($) {
+ $.jstree.plugin("html_data", {
+ __init : function () {
+ // this used to use html() and clean the whitespace, but this way any attached data was lost
+ this.data.html_data.original_container_html = this.get_container().find(" > ul > li").clone(true);
+ // remove white space from LI node - otherwise nodes appear a bit to the right
+ this.data.html_data.original_container_html.find("li").andSelf().contents().filter(function() { return this.nodeType == 3; }).remove();
+ },
+ defaults : {
+ data : false,
+ ajax : false,
+ correct_state : true
+ },
+ _fn : {
+ load_node : function (obj, s_call, e_call) { var _this = this; this.load_node_html(obj, function () { _this.__callback({ "obj" : _this._get_node(obj) }); s_call.call(this); }, e_call); },
+ _is_loaded : function (obj) {
+ obj = this._get_node(obj);
+ return obj == -1 || !obj || (!this._get_settings().html_data.ajax && !$.isFunction(this._get_settings().html_data.data)) || obj.is(".jstree-open, .jstree-leaf") || obj.children("ul").children("li").size() > 0;
+ },
+ load_node_html : function (obj, s_call, e_call) {
+ var d,
+ s = this.get_settings().html_data,
+ error_func = function () {},
+ success_func = function () {};
+ obj = this._get_node(obj);
+ if(obj && obj !== -1) {
+ if(obj.data("jstree-is-loading")) { return; }
+ else { obj.data("jstree-is-loading",true); }
+ }
+ switch(!0) {
+ case ($.isFunction(s.data)):
+ s.data.call(this, obj, $.proxy(function (d) {
+ if(d && d !== "" && d.toString && d.toString().replace(/^[\s\n]+$/,"") !== "") {
+ d = $(d);
+ if(!d.is("ul")) { d = $("<ul />").append(d); }
+ if(obj == -1 || !obj) { this.get_container().children("ul").empty().append(d.children()).find("li, a").filter(function () { return !this.firstChild || !this.firstChild.tagName || this.firstChild.tagName !== "INS"; }).prepend("<ins class='jstree-icon'> </ins>").end().filter("a").children("ins:first-child").not(".jstree-icon").addClass("jstree-icon"); }
+ else { obj.children("a.jstree-loading").removeClass("jstree-loading"); obj.append(d).children("ul").find("li, a").filter(function () { return !this.firstChild || !this.firstChild.tagName || this.firstChild.tagName !== "INS"; }).prepend("<ins class='jstree-icon'> </ins>").end().filter("a").children("ins:first-child").not(".jstree-icon").addClass("jstree-icon"); obj.removeData("jstree-is-loading"); }
+ this.clean_node(obj);
+ if(s_call) { s_call.call(this); }
+ }
+ else {
+ if(obj && obj !== -1) {
+ obj.children("a.jstree-loading").removeClass("jstree-loading");
+ obj.removeData("jstree-is-loading");
+ if(s.correct_state) {
+ this.correct_state(obj);
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ else {
+ if(s.correct_state) {
+ this.get_container().children("ul").empty();
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ }
+ }, this));
+ break;
+ case (!s.data && !s.ajax):
+ if(!obj || obj == -1) {
+ this.get_container()
+ .children("ul").empty()
+ .append(this.data.html_data.original_container_html)
+ .find("li, a").filter(function () { return !this.firstChild || !this.firstChild.tagName || this.firstChild.tagName !== "INS"; }).prepend("<ins class='jstree-icon'> </ins>").end()
+ .filter("a").children("ins:first-child").not(".jstree-icon").addClass("jstree-icon");
+ this.clean_node();
+ }
+ if(s_call) { s_call.call(this); }
+ break;
+ case (!!s.data && !s.ajax) || (!!s.data && !!s.ajax && (!obj || obj === -1)):
+ if(!obj || obj == -1) {
+ d = $(s.data);
+ if(!d.is("ul")) { d = $("<ul />").append(d); }
+ this.get_container()
+ .children("ul").empty().append(d.children())
+ .find("li, a").filter(function () { return !this.firstChild || !this.firstChild.tagName || this.firstChild.tagName !== "INS"; }).prepend("<ins class='jstree-icon'> </ins>").end()
+ .filter("a").children("ins:first-child").not(".jstree-icon").addClass("jstree-icon");
+ this.clean_node();
+ }
+ if(s_call) { s_call.call(this); }
+ break;
+ case (!s.data && !!s.ajax) || (!!s.data && !!s.ajax && obj && obj !== -1):
+ obj = this._get_node(obj);
+ error_func = function (x, t, e) {
+ var ef = this.get_settings().html_data.ajax.error;
+ if(ef) { ef.call(this, x, t, e); }
+ if(obj != -1 && obj.length) {
+ obj.children("a.jstree-loading").removeClass("jstree-loading");
+ obj.removeData("jstree-is-loading");
+ if(t === "success" && s.correct_state) { this.correct_state(obj); }
+ }
+ else {
+ if(t === "success" && s.correct_state) { this.get_container().children("ul").empty(); }
+ }
+ if(e_call) { e_call.call(this); }
+ };
+ success_func = function (d, t, x) {
+ var sf = this.get_settings().html_data.ajax.success;
+ if(sf) { d = sf.call(this,d,t,x) || d; }
+ if(d === "" || (d && d.toString && d.toString().replace(/^[\s\n]+$/,"") === "")) {
+ return error_func.call(this, x, t, "");
+ }
+ if(d) {
+ d = $(d);
+ if(!d.is("ul")) { d = $("<ul />").append(d); }
+ if(obj == -1 || !obj) { this.get_container().children("ul").empty().append(d.children()).find("li, a").filter(function () { return !this.firstChild || !this.firstChild.tagName || this.firstChild.tagName !== "INS"; }).prepend("<ins class='jstree-icon'> </ins>").end().filter("a").children("ins:first-child").not(".jstree-icon").addClass("jstree-icon"); }
+ else { obj.children("a.jstree-loading").removeClass("jstree-loading"); obj.append(d).children("ul").find("li, a").filter(function () { return !this.firstChild || !this.firstChild.tagName || this.firstChild.tagName !== "INS"; }).prepend("<ins class='jstree-icon'> </ins>").end().filter("a").children("ins:first-child").not(".jstree-icon").addClass("jstree-icon"); obj.removeData("jstree-is-loading"); }
+ this.clean_node(obj);
+ if(s_call) { s_call.call(this); }
+ }
+ else {
+ if(obj && obj !== -1) {
+ obj.children("a.jstree-loading").removeClass("jstree-loading");
+ obj.removeData("jstree-is-loading");
+ if(s.correct_state) {
+ this.correct_state(obj);
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ else {
+ if(s.correct_state) {
+ this.get_container().children("ul").empty();
+ if(s_call) { s_call.call(this); }
+ }
+ }
+ }
+ };
+ s.ajax.context = this;
+ s.ajax.error = error_func;
+ s.ajax.success = success_func;
+ if(!s.ajax.dataType) { s.ajax.dataType = "html"; }
+ if($.isFunction(s.ajax.url)) { s.ajax.url = s.ajax.url.call(this, obj); }
+ if($.isFunction(s.ajax.data)) { s.ajax.data = s.ajax.data.call(this, obj); }
+ $.ajax(s.ajax);
+ break;
+ }
+ }
+ }
+ });
+ // include the HTML data plugin by default
+ $.jstree.defaults.plugins.push("html_data");
+})(jQuery);
+//*/
+
+/*
+ * jsTree themeroller plugin
+ * Adds support for jQuery UI themes. Include this at the end of your plugins list, also make sure "themes" is not included.
+ */
+(function ($) {
+ $.jstree.plugin("themeroller", {
+ __init : function () {
+ var s = this._get_settings().themeroller;
+ this.get_container()
+ .addClass("ui-widget-content")
+ .addClass("jstree-themeroller")
+ .delegate("a","mouseenter.jstree", function (e) {
+ if(!$(e.currentTarget).hasClass("jstree-loading")) {
+ $(this).addClass(s.item_h);
+ }
+ })
+ .delegate("a","mouseleave.jstree", function () {
+ $(this).removeClass(s.item_h);
+ })
+ .bind("init.jstree", $.proxy(function (e, data) {
+ data.inst.get_container().find("> ul > li > .jstree-loading > ins").addClass("ui-icon-refresh");
+ this._themeroller(data.inst.get_container().find("> ul > li"));
+ }, this))
+ .bind("open_node.jstree create_node.jstree", $.proxy(function (e, data) {
+ this._themeroller(data.rslt.obj);
+ }, this))
+ .bind("loaded.jstree refresh.jstree", $.proxy(function (e) {
+ this._themeroller();
+ }, this))
+ .bind("close_node.jstree", $.proxy(function (e, data) {
+ this._themeroller(data.rslt.obj);
+ }, this))
+ .bind("delete_node.jstree", $.proxy(function (e, data) {
+ this._themeroller(data.rslt.parent);
+ }, this))
+ .bind("correct_state.jstree", $.proxy(function (e, data) {
+ data.rslt.obj
+ .children("ins.jstree-icon").removeClass(s.opened + " " + s.closed + " ui-icon").end()
+ .find("> a > ins.ui-icon")
+ .filter(function() {
+ return this.className.toString()
+ .replace(s.item_clsd,"").replace(s.item_open,"").replace(s.item_leaf,"")
+ .indexOf("ui-icon-") === -1;
+ }).removeClass(s.item_open + " " + s.item_clsd).addClass(s.item_leaf || "jstree-no-icon");
+ }, this))
+ .bind("select_node.jstree", $.proxy(function (e, data) {
+ data.rslt.obj.children("a").addClass(s.item_a);
+ }, this))
+ .bind("deselect_node.jstree deselect_all.jstree", $.proxy(function (e, data) {
+ this.get_container()
+ .find("a." + s.item_a).removeClass(s.item_a).end()
+ .find("a.jstree-clicked").addClass(s.item_a);
+ }, this))
+ .bind("dehover_node.jstree", $.proxy(function (e, data) {
+ data.rslt.obj.children("a").removeClass(s.item_h);
+ }, this))
+ .bind("hover_node.jstree", $.proxy(function (e, data) {
+ this.get_container()
+ .find("a." + s.item_h).not(data.rslt.obj).removeClass(s.item_h);
+ data.rslt.obj.children("a").addClass(s.item_h);
+ }, this))
+ .bind("move_node.jstree", $.proxy(function (e, data) {
+ this._themeroller(data.rslt.o);
+ this._themeroller(data.rslt.op);
+ }, this));
+ },
+ __destroy : function () {
+ var s = this._get_settings().themeroller,
+ c = [ "ui-icon" ];
+ $.each(s, function (i, v) {
+ v = v.split(" ");
+ if(v.length) { c = c.concat(v); }
+ });
+ this.get_container()
+ .removeClass("ui-widget-content")
+ .find("." + c.join(", .")).removeClass(c.join(" "));
+ },
+ _fn : {
+ _themeroller : function (obj) {
+ var s = this._get_settings().themeroller;
+ obj = !obj || obj == -1 ? this.get_container_ul() : this._get_node(obj).parent();
+ obj
+ .find("li.jstree-closed")
+ .children("ins.jstree-icon").removeClass(s.opened).addClass("ui-icon " + s.closed).end()
+ .children("a").addClass(s.item)
+ .children("ins.jstree-icon").addClass("ui-icon")
+ .filter(function() {
+ return this.className.toString()
+ .replace(s.item_clsd,"").replace(s.item_open,"").replace(s.item_leaf,"")
+ .indexOf("ui-icon-") === -1;
+ }).removeClass(s.item_leaf + " " + s.item_open).addClass(s.item_clsd || "jstree-no-icon")
+ .end()
+ .end()
+ .end()
+ .end()
+ .find("li.jstree-open")
+ .children("ins.jstree-icon").removeClass(s.closed).addClass("ui-icon " + s.opened).end()
+ .children("a").addClass(s.item)
+ .children("ins.jstree-icon").addClass("ui-icon")
+ .filter(function() {
+ return this.className.toString()
+ .replace(s.item_clsd,"").replace(s.item_open,"").replace(s.item_leaf,"")
+ .indexOf("ui-icon-") === -1;
+ }).removeClass(s.item_leaf + " " + s.item_clsd).addClass(s.item_open || "jstree-no-icon")
+ .end()
+ .end()
+ .end()
+ .end()
+ .find("li.jstree-leaf")
+ .children("ins.jstree-icon").removeClass(s.closed + " ui-icon " + s.opened).end()
+ .children("a").addClass(s.item)
+ .children("ins.jstree-icon").addClass("ui-icon")
+ .filter(function() {
+ return this.className.toString()
+ .replace(s.item_clsd,"").replace(s.item_open,"").replace(s.item_leaf,"")
+ .indexOf("ui-icon-") === -1;
+ }).removeClass(s.item_clsd + " " + s.item_open).addClass(s.item_leaf || "jstree-no-icon");
+ }
+ },
+ defaults : {
+ "opened" : "ui-icon-triangle-1-se",
+ "closed" : "ui-icon-triangle-1-e",
+ "item" : "ui-state-default",
+ "item_h" : "ui-state-hover",
+ "item_a" : "ui-state-active",
+ "item_open" : "ui-icon-folder-open",
+ "item_clsd" : "ui-icon-folder-collapsed",
+ "item_leaf" : "ui-icon-document"
+ }
+ });
+ $(function() {
+ var css_string = '' +
+ '.jstree-themeroller .ui-icon { overflow:visible; } ' +
+ '.jstree-themeroller a { padding:0 2px; } ' +
+ '.jstree-themeroller .jstree-no-icon { display:none; }';
+ $.vakata.css.add_sheet({ str : css_string, title : "jstree" });
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree unique plugin
+ * Forces different names amongst siblings (still a bit experimental)
+ * NOTE: does not check language versions (it will not be possible to have nodes with the same title, even in different languages)
+ */
+(function ($) {
+ $.jstree.plugin("unique", {
+ __init : function () {
+ this.get_container()
+ .bind("before.jstree", $.proxy(function (e, data) {
+ var nms = [], res = true, p, t;
+ if(data.func == "move_node") {
+ // obj, ref, position, is_copy, is_prepared, skip_check
+ if(data.args[4] === true) {
+ if(data.args[0].o && data.args[0].o.length) {
+ data.args[0].o.children("a").each(function () { nms.push($(this).text().replace(/^\s+/g,"")); });
+ res = this._check_unique(nms, data.args[0].np.find("> ul > li").not(data.args[0].o), "move_node");
+ }
+ }
+ }
+ if(data.func == "create_node") {
+ // obj, position, js, callback, is_loaded
+ if(data.args[4] || this._is_loaded(data.args[0])) {
+ p = this._get_node(data.args[0]);
+ if(data.args[1] && (data.args[1] === "before" || data.args[1] === "after")) {
+ p = this._get_parent(data.args[0]);
+ if(!p || p === -1) { p = this.get_container(); }
+ }
+ if(typeof data.args[2] === "string") { nms.push(data.args[2]); }
+ else if(!data.args[2] || !data.args[2].data) { nms.push(this._get_string("new_node")); }
+ else { nms.push(data.args[2].data); }
+ res = this._check_unique(nms, p.find("> ul > li"), "create_node");
+ }
+ }
+ if(data.func == "rename_node") {
+ // obj, val
+ nms.push(data.args[1]);
+ t = this._get_node(data.args[0]);
+ p = this._get_parent(t);
+ if(!p || p === -1) { p = this.get_container(); }
+ res = this._check_unique(nms, p.find("> ul > li").not(t), "rename_node");
+ }
+ if(!res) {
+ e.stopPropagation();
+ return false;
+ }
+ }, this));
+ },
+ defaults : {
+ error_callback : $.noop
+ },
+ _fn : {
+ _check_unique : function (nms, p, func) {
+ var cnms = [];
+ p.children("a").each(function () { cnms.push($(this).text().replace(/^\s+/g,"")); });
+ if(!cnms.length || !nms.length) { return true; }
+ cnms = cnms.sort().join(",,").replace(/(,|^)([^,]+)(,,\2)+(,|$)/g,"$1$2$4").replace(/,,+/g,",").replace(/,$/,"").split(",");
+ if((cnms.length + nms.length) != cnms.concat(nms).sort().join(",,").replace(/(,|^)([^,]+)(,,\2)+(,|$)/g,"$1$2$4").replace(/,,+/g,",").replace(/,$/,"").split(",").length) {
+ this._get_settings().unique.error_callback.call(null, nms, p, func);
+ return false;
+ }
+ return true;
+ },
+ check_move : function () {
+ if(!this.__call_old()) { return false; }
+ var p = this._get_move(), nms = [];
+ if(p.o && p.o.length) {
+ p.o.children("a").each(function () { nms.push($(this).text().replace(/^\s+/g,"")); });
+ return this._check_unique(nms, p.np.find("> ul > li").not(p.o), "check_move");
+ }
+ return true;
+ }
+ }
+ });
+})(jQuery);
+//*/
+
+/*
+ * jsTree wholerow plugin
+ * Makes select and hover work on the entire width of the node
+ * MAY BE HEAVY IN LARGE DOM
+ */
+(function ($) {
+ $.jstree.plugin("wholerow", {
+ __init : function () {
+ if(!this.data.ui) { throw "jsTree wholerow: jsTree UI plugin not included."; }
+ this.data.wholerow.html = false;
+ this.data.wholerow.to = false;
+ this.get_container()
+ .bind("init.jstree", $.proxy(function (e, data) {
+ this._get_settings().core.animation = 0;
+ }, this))
+ .bind("open_node.jstree create_node.jstree clean_node.jstree loaded.jstree", $.proxy(function (e, data) {
+ this._prepare_wholerow_span( data && data.rslt && data.rslt.obj ? data.rslt.obj : -1 );
+ }, this))
+ .bind("search.jstree clear_search.jstree reopen.jstree after_open.jstree after_close.jstree create_node.jstree delete_node.jstree clean_node.jstree", $.proxy(function (e, data) {
+ if(this.data.to) { clearTimeout(this.data.to); }
+ this.data.to = setTimeout( (function (t, o) { return function() { t._prepare_wholerow_ul(o); }; })(this, data && data.rslt && data.rslt.obj ? data.rslt.obj : -1), 0);
+ }, this))
+ .bind("deselect_all.jstree", $.proxy(function (e, data) {
+ this.get_container().find(" > .jstree-wholerow .jstree-clicked").removeClass("jstree-clicked " + (this.data.themeroller ? this._get_settings().themeroller.item_a : "" ));
+ }, this))
+ .bind("select_node.jstree deselect_node.jstree ", $.proxy(function (e, data) {
+ data.rslt.obj.each(function () {
+ var ref = data.inst.get_container().find(" > .jstree-wholerow li:visible:eq(" + ( parseInt((($(this).offset().top - data.inst.get_container().offset().top + data.inst.get_container()[0].scrollTop) / data.inst.data.core.li_height),10)) + ")");
+ // ref.children("a")[e.type === "select_node" ? "addClass" : "removeClass"]("jstree-clicked");
+ ref.children("a").attr("class",data.rslt.obj.children("a").attr("class"));
+ });
+ }, this))
+ .bind("hover_node.jstree dehover_node.jstree", $.proxy(function (e, data) {
+ this.get_container().find(" > .jstree-wholerow .jstree-hovered").removeClass("jstree-hovered " + (this.data.themeroller ? this._get_settings().themeroller.item_h : "" ));
+ if(e.type === "hover_node") {
+ var ref = this.get_container().find(" > .jstree-wholerow li:visible:eq(" + ( parseInt(((data.rslt.obj.offset().top - this.get_container().offset().top + this.get_container()[0].scrollTop) / this.data.core.li_height),10)) + ")");
+ // ref.children("a").addClass("jstree-hovered");
+ ref.children("a").attr("class",data.rslt.obj.children(".jstree-hovered").attr("class"));
+ }
+ }, this))
+ .delegate(".jstree-wholerow-span, ins.jstree-icon, li", "click.jstree", function (e) {
+ var n = $(e.currentTarget);
+ if(e.target.tagName === "A" || (e.target.tagName === "INS" && n.closest("li").is(".jstree-open, .jstree-closed"))) { return; }
+ n.closest("li").children("a:visible:eq(0)").click();
+ e.stopImmediatePropagation();
+ })
+ .delegate("li", "mouseover.jstree", $.proxy(function (e) {
+ e.stopImmediatePropagation();
+ if($(e.currentTarget).children(".jstree-hovered, .jstree-clicked").length) { return false; }
+ this.hover_node(e.currentTarget);
+ return false;
+ }, this))
+ .delegate("li", "mouseleave.jstree", $.proxy(function (e) {
+ if($(e.currentTarget).children("a").hasClass("jstree-hovered").length) { return; }
+ this.dehover_node(e.currentTarget);
+ }, this));
+ if(is_ie7 || is_ie6) {
+ $.vakata.css.add_sheet({ str : ".jstree-" + this.get_index() + " { position:relative; } ", title : "jstree" });
+ }
+ },
+ defaults : {
+ },
+ __destroy : function () {
+ this.get_container().children(".jstree-wholerow").remove();
+ this.get_container().find(".jstree-wholerow-span").remove();
+ },
+ _fn : {
+ _prepare_wholerow_span : function (obj) {
+ obj = !obj || obj == -1 ? this.get_container().find("> ul > li") : this._get_node(obj);
+ if(obj === false) { return; } // added for removing root nodes
+ obj.each(function () {
+ $(this).find("li").andSelf().each(function () {
+ var $t = $(this);
+ if($t.children(".jstree-wholerow-span").length) { return true; }
+ $t.prepend("<span class='jstree-wholerow-span' style='width:" + ($t.parentsUntil(".jstree","li").length * 18) + "px;'> </span>");
+ });
+ });
+ },
+ _prepare_wholerow_ul : function () {
+ var o = this.get_container().children("ul").eq(0), h = o.html();
+ o.addClass("jstree-wholerow-real");
+ if(this.data.wholerow.last_html !== h) {
+ this.data.wholerow.last_html = h;
+ this.get_container().children(".jstree-wholerow").remove();
+ this.get_container().append(
+ o.clone().removeClass("jstree-wholerow-real")
+ .wrapAll("<div class='jstree-wholerow' />").parent()
+ .width(o.parent()[0].scrollWidth)
+ .css("top", (o.height() + ( is_ie7 ? 5 : 0)) * -1 )
+ .find("li[id]").each(function () { this.removeAttribute("id"); }).end()
+ );
+ }
+ }
+ }
+ });
+ $(function() {
+ var css_string = '' +
+ '.jstree .jstree-wholerow-real { position:relative; z-index:1; } ' +
+ '.jstree .jstree-wholerow-real li { cursor:pointer; } ' +
+ '.jstree .jstree-wholerow-real a { border-left-color:transparent !important; border-right-color:transparent !important; } ' +
+ '.jstree .jstree-wholerow { position:relative; z-index:0; height:0; } ' +
+ '.jstree .jstree-wholerow ul, .jstree .jstree-wholerow li { width:100%; } ' +
+ '.jstree .jstree-wholerow, .jstree .jstree-wholerow ul, .jstree .jstree-wholerow li, .jstree .jstree-wholerow a { margin:0 !important; padding:0 !important; } ' +
+ '.jstree .jstree-wholerow, .jstree .jstree-wholerow ul, .jstree .jstree-wholerow li { background:transparent !important; }' +
+ '.jstree .jstree-wholerow ins, .jstree .jstree-wholerow span, .jstree .jstree-wholerow input { display:none !important; }' +
+ '.jstree .jstree-wholerow a, .jstree .jstree-wholerow a:hover { text-indent:-9999px; !important; width:100%; padding:0 !important; border-right-width:0px !important; border-left-width:0px !important; } ' +
+ '.jstree .jstree-wholerow-span { position:absolute; left:0; margin:0px; padding:0; height:18px; border-width:0; padding:0; z-index:0; }';
+ if(is_ff2) {
+ css_string += '' +
+ '.jstree .jstree-wholerow a { display:block; height:18px; margin:0; padding:0; border:0; } ' +
+ '.jstree .jstree-wholerow-real a { border-color:transparent !important; } ';
+ }
+ if(is_ie7 || is_ie6) {
+ css_string += '' +
+ '.jstree .jstree-wholerow, .jstree .jstree-wholerow li, .jstree .jstree-wholerow ul, .jstree .jstree-wholerow a { margin:0; padding:0; line-height:18px; } ' +
+ '.jstree .jstree-wholerow a { display:block; height:18px; line-height:18px; overflow:hidden; } ';
+ }
+ $.vakata.css.add_sheet({ str : css_string, title : "jstree" });
+ });
+})(jQuery);
+//*/
+
+/*
+* jsTree model plugin
+* This plugin gets jstree to use a class model to retrieve data, creating great dynamism
+*/
+(function ($) {
+ var nodeInterface = ["getChildren","getChildrenCount","getAttr","getName","getProps"],
+ validateInterface = function(obj, inter) {
+ var valid = true;
+ obj = obj || {};
+ inter = [].concat(inter);
+ $.each(inter, function (i, v) {
+ if(!$.isFunction(obj[v])) { valid = false; return false; }
+ });
+ return valid;
+ };
+ $.jstree.plugin("model", {
+ __init : function () {
+ if(!this.data.json_data) { throw "jsTree model: jsTree json_data plugin not included."; }
+ this._get_settings().json_data.data = function (n, b) {
+ var obj = (n == -1) ? this._get_settings().model.object : n.data("jstree_model");
+ if(!validateInterface(obj, nodeInterface)) { return b.call(null, false); }
+ if(this._get_settings().model.async) {
+ obj.getChildren($.proxy(function (data) {
+ this.model_done(data, b);
+ }, this));
+ }
+ else {
+ this.model_done(obj.getChildren(), b);
+ }
+ };
+ },
+ defaults : {
+ object : false,
+ id_prefix : false,
+ async : false
+ },
+ _fn : {
+ model_done : function (data, callback) {
+ var ret = [],
+ s = this._get_settings(),
+ _this = this;
+
+ if(!$.isArray(data)) { data = [data]; }
+ $.each(data, function (i, nd) {
+ var r = nd.getProps() || {};
+ r.attr = nd.getAttr() || {};
+ if(nd.getChildrenCount()) { r.state = "closed"; }
+ r.data = nd.getName();
+ if(!$.isArray(r.data)) { r.data = [r.data]; }
+ if(_this.data.types && $.isFunction(nd.getType)) {
+ r.attr[s.types.type_attr] = nd.getType();
+ }
+ if(r.attr.id && s.model.id_prefix) { r.attr.id = s.model.id_prefix + r.attr.id; }
+ if(!r.metadata) { r.metadata = { }; }
+ r.metadata.jstree_model = nd;
+ ret.push(r);
+ });
+ callback.call(null, ret);
+ }
+ }
+ });
+})(jQuery);
+//*/
+
+})();
\ No newline at end of file
diff --git a/slider-core/src/main/resources/webapps/static/yarn.css b/slider-core/src/main/resources/webapps/static/yarn.css
new file mode 100644
index 0000000..ba12acb
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/yarn.css
@@ -0,0 +1,63 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/* Styles for YARN */
+* { margin: 0; border: 0 }
+html, body { height: 100% }
+body { padding: 0; font: 90% sans-serif }
+.content { padding-right: 1em }
+.content h1, .content h2, .content h3 { margin: 0 0 0.3em; }
+/*.content h1, .content h2, .content h3 { margin: 0 0 0.3em; font-weight: normal }*/
+table { border-collapse: collapse; border-spacing: 0; width: 100% }
+table br { display: none }
+table.info th { text-align: right }
+.info-wrap { margin: 0 0 1em }
+th, td { padding: 0.2em 0.5em 0 }
+td.table { padding: 0 }
+.ui-dialog, .shadow {
+ -moz-box-shadow: 0 8px 38px #000;
+ -webkit-box-shadow: 0 8px 38px #000;
+ box-shadow: 0 8px 38px #000 }
+/* styles for common objects */
+#logo { float: left; position: relative; line-height: 0.5em; top: -1.3em }
+#user { float: right; position: relative; top: -1.5em; font-size: 0.77em }
+#header { padding: 1.5em 0.5em; text-align: center }
+#nav h3 { padding: 0 0 0 1.6em }
+#nav ul {
+ padding: 0.3em 1em 0.8em 2em; line-height: 0.5em; list-style: none;
+ line-height: 1.2em; font-size: 90% }
+#themeswitcher { margin: 1em 0.25em }
+#footer { padding: 1em; text-align: center }
+.bold { font-weight: bold; }
+
+
+div.role-info { width: 700px; padding-left: 15px; padding-top: 15px; margin-bottom: 2em; }
+
+div.role-stats-containers { width: 350px; margin-left: 2em; margin-bottom: 2em; }
+div.role-stats-containers table { margin-left: 0.5em; }
+div.role-stats-wrap { margin-bottom: 2em; margin-left: 2em; width: 200px; }
+div.role-stats-wrap table { margin-left: 0.5em; width: 200px }
+div.role-options-wrap { margin-left: 2em; margin-bottom: 2em; width: 350px; }
+div.role-options-wrap table { margin-left: 0.5em; }
+
+.no-table-contents { padding-left: 0.5em; }
+
+h1.index_header { margin-bottom: 15px; }
+div.general_info { margin-bottom: 15px; }
+div.provder_info { margin-bottom: 15px; }
+
+div.cluster_json pre { white-space: pre-wrap; }
diff --git a/slider-core/src/main/resources/webapps/static/yarn.dt.plugins.js b/slider-core/src/main/resources/webapps/static/yarn.dt.plugins.js
new file mode 100644
index 0000000..d0bde29
--- /dev/null
+++ b/slider-core/src/main/resources/webapps/static/yarn.dt.plugins.js
@@ -0,0 +1,146 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+if (!jQuery.fn.dataTableExt.fnVersionCheck("1.7.5")) {
+ alert("These plugins requires dataTables 1.7.5+");
+}
+
+// don't filter on hidden html elements for an sType of title-numeric
+$.fn.dataTableExt.ofnSearch['title-numeric'] = function ( sData ) {
+ return sData.replace(/\n/g," ").replace( /<.*?>/g, "" );
+}
+
+// 'title-numeric' sort type
+jQuery.fn.dataTableExt.oSort['title-numeric-asc'] = function(a,b) {
+ var x = a.match(/title=["']?(-?\d+\.?\d*)/)[1];
+ var y = b.match(/title=["']?(-?\d+\.?\d*)/)[1];
+ x = parseFloat( x );
+ y = parseFloat( y );
+ return ((x < y) ? -1 : ((x > y) ? 1 : 0));
+};
+
+jQuery.fn.dataTableExt.oSort['title-numeric-desc'] = function(a,b) {
+ var x = a.match(/title=["']?(-?\d+\.?\d*)/)[1];
+ var y = b.match(/title=["']?(-?\d+\.?\d*)/)[1];
+ x = parseFloat( x );
+ y = parseFloat( y );
+ return ((x < y) ? 1 : ((x > y) ? -1 : 0));
+};
+
+jQuery.fn.dataTableExt.oApi.fnSetFilteringDelay = function ( oSettings, iDelay ) {
+ var
+ _that = this,
+ iDelay = (typeof iDelay == 'undefined') ? 250 : iDelay;
+
+ this.each( function ( i ) {
+ $.fn.dataTableExt.iApiIndex = i;
+ var
+ $this = this,
+ oTimerId = null,
+ sPreviousSearch = null,
+ anControl = $( 'input', _that.fnSettings().aanFeatures.f );
+
+ anControl.unbind( 'keyup' ).bind( 'keyup', function() {
+ var $$this = $this;
+
+ if (sPreviousSearch === null || sPreviousSearch != anControl.val()) {
+ window.clearTimeout(oTimerId);
+ sPreviousSearch = anControl.val();
+ oSettings.oApi._fnProcessingDisplay(oSettings, true);
+ oTimerId = window.setTimeout(function() {
+ $.fn.dataTableExt.iApiIndex = i;
+ _that.fnFilter( anControl.val() );
+ oSettings.oApi._fnProcessingDisplay(oSettings, false);
+ }, iDelay);
+ }
+ });
+ return this;
+ } );
+ return this;
+}
+
+function renderHadoopDate(data, type, full) {
+ if (type === 'display' || type === 'filter') {
+ if(data === '0') {
+ return "N/A";
+ }
+ return new Date(parseInt(data)).toUTCString();
+ }
+ // 'sort', 'type' and undefined all just use the number
+ // If date is 0, then for purposes of sorting it should be consider max_int
+ return data === '0' ? '9007199254740992' : data;
+}
+
+function renderHadoopElapsedTime(data, type, full) {
+ if (type === 'display' || type === 'filter') {
+ var timeDiff = parseInt(data);
+ if(timeDiff < 0)
+ return "N/A";
+
+ var hours = Math.floor(timeDiff / (60*60*1000));
+ var rem = (timeDiff % (60*60*1000));
+ var minutes = Math.floor(rem / (60*1000));
+ rem = rem % (60*1000);
+ var seconds = Math.floor(rem / 1000);
+
+ var toReturn = "";
+ if (hours != 0){
+ toReturn += hours;
+ toReturn += "hrs, ";
+ }
+ if (minutes != 0){
+ toReturn += minutes;
+ toReturn += "mins, ";
+ }
+ toReturn += seconds;
+ toReturn += "sec";
+ return toReturn;
+ }
+ // 'sort', 'type' and undefined all just use the number
+ return data;
+}
+
+function parseHadoopID(data, type, full) {
+ if (type === 'display') {
+ return data;
+ }
+ //Return the visible string rather than the entire HTML tag
+ if (type === 'filter') {
+ return data.split('>')[1].split('<')[0];
+ }
+ //Parse the ID for 'sort', 'type' and undefined
+ //The number after the last '_' and before the end tag '<'
+ var splits = data.split('_');
+ return splits[parseInt(splits.length-1)].split('<')[0];
+}
+
+//JSON array element is "20000 attempt_1360183373897_0001_m_000002_0"
+function parseHadoopAttemptID(data, type, full) {
+ if (type === 'display' || type === 'filter') {
+ return data.split(' ')[1];
+ }
+ //For sorting use the order as defined in the JSON element
+ return data.split(' ')[0];
+}
+
+function parseHadoopProgress(data, type, full) {
+ if (type === 'display') {
+ return data;
+ }
+ //Return the title attribute for 'sort', 'filter', 'type' and undefined
+ return data.split("'")[1];
+}
\ No newline at end of file
diff --git a/slider-core/src/scripts/slider_keytabs.sh b/slider-core/src/scripts/slider_keytabs.sh
new file mode 100644
index 0000000..f0a8fc2
--- /dev/null
+++ b/slider-core/src/scripts/slider_keytabs.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script exists to create the keytab set for a node on the cluster
+# including hbase and ZK alongside then YARN cores.
+
+# usage
+# keytabs <realm> <hostname>
+# validate the args
+
+num_vars=$#
+if [[ $num_vars < 2 ]]
+then
+ echo "Usage: $0 <realm> <hostname>"
+ exit -2
+fi
+
+realm="$1"
+hostname="$2"
+dest="."
+
+kadmin=kadmin.local
+
+${kadmin} <<EOF
+addprinc -randkey hdfs/${hostname}@${realm}
+addprinc -randkey yarn/${hostname}@${realm}
+addprinc -randkey HTTP/${hostname}@${realm}
+addprinc -randkey hbase/${hostname}@${realm}
+addprinc -randkey zookeeper/${hostname}@${realm}
+
+ktadd -norandkey -k ${dest}/hdfs.keytab \
+ hdfs/${hostname}@${realm} \
+ HTTP/${hostname}@${realm}
+
+ktadd -norandkey -k ${dest}/yarn.keytab \
+ yarn/${hostname}@${realm} \
+ HTTP/${hostname}@${realm}
+
+ktadd -norandkey -k ${dest}/hbase.keytab \
+ hbase/${hostname}@${realm}
+
+ktadd -norandkey -k ${dest}/zookeeper.keytab \
+ zookeeper/${hostname}@${realm}
+EOF
+
+exitcode=$?
+if [[ $exitcode != 0 ]]
+then
+ echo "keytab generation from ${kadmin} failed with exit code $exitcode"
+ exit $exitcode
+else
+ echo "keytab files for ${hostname}@${realm} created"
+fi
diff --git a/slider-core/src/scripts/yarnservice.py b/slider-core/src/scripts/yarnservice.py
new file mode 100644
index 0000000..1208c28
--- /dev/null
+++ b/slider-core/src/scripts/yarnservice.py
@@ -0,0 +1,383 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Launches a yarn service
+
+WORK IN PROGRESS, IGNORE
+
+This is as work in progress project to build as new launcher script for
+any Hadoop service
+A key feature here is that the configs are defined in JSON files -
+files that are read in the order passed down, and merged into each other.
+
+The final merged file is used to define the java command to execute
+-and hadoop XML files.
+
+
+It uses a JSON config file
+ --jfile configuration file (JSON format)
+ -class classname
+ -Dname=value -arbitrary value to pass down to the JVM
+ --java: any JVM arg
+ -javaX: javaX value
+
+
+ after an -- , all following commands are passed straight down to the invoked process.
+ # -xJ name=value JVM options. No: this is just another param
+ -xF file file to load next. Files are loaded in order.
+ -xD name=value again, values are loaded in order
+ -xU undefine
+ -xX main class, 'eXecute'
+
+ -- end of arguments
+
+
+"""
+
+import sys
+# see : http://simplejson.readthedocs.org/en/latest/
+# and install w/ easy_install simplejson
+import simplejson
+
+KEY_JFILE = "-xF"
+KEY_DEF = "-xD"
+KEY_UNDEF = "-xU"
+KEY_EXEC = "-xX"
+KEY_ARGS = "--"
+
+COMMANDS = [KEY_JFILE, KEY_DEF, KEY_EXEC]
+
+#
+
+def debug(string) :
+ print string
+
+
+def pop_required_arg(arglist, previousArg) :
+ """
+ Pop the first element off the list and return it.
+ If the list is empty, raise an exception about a missing argument after the $previousArgument
+ """
+ if not len(arglist) :
+ raise Exception, "Missing required parameter after %s" % previousArg
+ head = arglist[0]
+ del arglist[0]
+ return head
+
+
+def parse_one_jfile(filename) :
+ """
+ read in the given config file
+ """
+ parsed = simplejson.load(open(filename, "r"))
+ return parsed
+
+# hand down sys.argv:
+def extract_jfiles(args) :
+ """ takes a list of arg strings and separates them into jfile references
+ and other arguments.
+ """
+ l = len(args)
+ stripped = []
+ jfiles = []
+ index = 0
+ while index < l :
+ elt = args[index]
+ index += 1
+ if KEY_JFILE == elt :
+ # a match
+ if index == l :
+ #overshoot
+ raise Exception("Missing filename after " + KEY_JFILE)
+ filename = args[index]
+ debug("jfile " + filename)
+ jfiles.append(filename)
+ index += 1
+ else :
+ stripped.append(elt)
+ return jfiles, stripped
+
+
+def extract_args(args) :
+ """
+ Take a list of args, parse them or fail, generating a dictionary of actions
+ Return: dictionary and all leftover arguments
+ """
+ jfiles = []
+ execs = []
+ defs = []
+ remainder = []
+ while len(args) :
+ # the next call cannot fail, because of the len(args)
+ arg = pop_required_arg(args, "")
+ if KEY_JFILE == arg :
+ jfiles.append(pop_required_arg(args, KEY_JFILE))
+ elif KEY_DEF == arg :
+ defs.append((KEY_DEF, pop_required_arg(args, KEY_DEF)))
+ elif KEY_UNDEF == arg :
+ defs.append((KEY_UNDEF, pop_required_arg(args, KEY_UNDEF)))
+ elif KEY_EXEC == arg :
+ execs.append(pop_required_arg(args, KEY_EXEC))
+ elif KEY_ARGS == arg :
+ remainder += args
+ args = []
+ else :
+ remainder.append(arg)
+ #build the action list
+ actions = {
+ KEY_JFILE : jfiles,
+ KEY_EXEC : execs,
+ KEY_DEF : defs,
+ KEY_ARGS : remainder
+ }
+ #end of the run, there's a dictionary and a list of unparsed values
+ return actions
+
+
+def get(conf, key, defVal) :
+ if conf.has_key(key) :
+ return conf[key]
+ else :
+ return defVal
+
+
+def merge_json(conf, json) :
+ """ merge in a json dict with the existing one
+ in: configuration dict, json dict
+ out: configuration'
+ """
+ for (key, val) in json.items() :
+ if key in conf :
+ #there's a match, do a more detailed merge
+ oldval = conf[key]
+ if type(oldval) == dict and type(val) == dict :
+ # two dictionary instances -merge
+ merge_json(oldval, val)
+ else :
+ conf[key] = val
+ else :
+ conf[key] = val
+ return conf
+
+
+def merge_jfile(conf, filename) :
+ json = parse_one_jfile(filename)
+ return merge_json(conf, json)
+
+
+def merge_jfile_list(conf, jfiles) :
+ """ merge a list of jfiles on top of a conf dict
+ """
+ for jfile in jfiles :
+ conf = merge_jfile(conf, jfile)
+ return conf
+
+
+def split_to_keyval_tuple(param) :
+ """
+ Split a key=value string into the (key,value) tuple
+ * an exception is raised on any string "=value"
+ * if there is no string: exception.
+ * a key only definition maps to (key, None)
+ * a "key=" definition maps to (key, "")
+ """
+ if not len(param) :
+ raise Exception, "Empty string cannot be a key=value definition"
+ equalsPos = param.find("=")
+ if equalsPos < 0 :
+ return param, None
+ elif not equalsPos :
+ raise Exception, "no key in argument %s" % param
+ else :
+ key = param[:(equalsPos - 1)]
+ value = param[(equalsPos + 1) :]
+ return key, value
+
+
+def recursive_define(conf, path, value) :
+ if not len(path) :
+ #fallen off the end of the world
+ return
+ entry = path[0]
+ if len(path) == 1 :
+ #end of list, apply it.
+ conf[entry] = value
+ else :
+ #there's 1+ elements below, yet there's a subdir here.
+ if conf.has_key(entry) and type(conf[entry]) == dict :
+ #it's a subdir, simple: recurse.
+ recursive_define(conf[entry], path[1 :], value)
+ else :
+ #either there is an entry that isn't a conf, or its not there. Same outcome.
+ subconf = {}
+ conf[entry] = subconf
+ recursive_define(subconf, path[1 :], value)
+
+def recursive_undef(conf, path) :
+ if not len(path) :
+ #fallen off the end of the world
+ return
+ entry = path[0]
+ if len(path) == 1 :
+ #end of list, apply it.
+ del conf[entry]
+ else :
+ #there's 1+ elements below, yet there's a subdir here.
+ if conf.has_key(entry) and type(conf[entry]) == dict :
+ #it's a subdir, simple: recurse.
+ recursive_undef(conf[entry], path[1 :])
+ else :
+ #either there is an entry that isn't a conf, or its not there. Same outcome.
+ pass
+
+def apply_action(conf, action, key, value) :
+ """
+ Apply either a def or undef action; splitting the key into a path and running through it.
+ """
+ keypath = key.split("/")
+ #now have a split key,
+ if KEY_DEF == action :
+ recursive_define(conf, keypath, value)
+ elif KEY_UNDEF == action :
+ recursive_undef(conf, keypath)
+
+
+def apply_local_definitions(conf, definitions) :
+ """
+ Run through the definition actions and apply them one by one
+ """
+ for defn in definitions :
+ # split into key=value; no value -> empty string
+ (action, param) = defn
+ if KEY_DEF == action :
+ (key, val) = split_to_keyval_tuple(param)
+ apply_action(conf, KEY_DEF, key, val)
+
+ return conf
+
+
+#def parse_args(conf, args) :
+# """
+# split an arg string, parse the jfiles & merge over the conf
+# (configuration, args[]) -> (conf', stripped, jfiles[])
+# """
+# (jfiles, stripped) = extract_jfiles(args)
+#
+# actions = extract_args(args)
+# jfiles = actions[KEY_JFILE]
+# conf = merge_jfile_list(conf, jfiles)
+# return conf, actions
+
+
+def print_conf(conf) :
+ """ dump the configuration to the console
+ """
+ print "{"
+ for (key, val) in conf.items() :
+ if type(val) == dict :
+ print key
+ print_conf(val)
+ else :
+ print "" + key + " => " + str(val)
+ print "}"
+
+
+def list_to_str(l, spacer) :
+ result = ""
+ for elt in l :
+ if len(result) > 0 :
+ result += spacer
+ result += elt
+ return result
+
+
+def list_to_hxml_str(l) :
+ return list_to_str(l, ",")
+
+
+def export_kv_xml(output, key, value) :
+ line = "<property><name>" + key + "</name><value>" + str(value) + "</value>\n"
+ print line
+ output.write(line)
+
+
+def export_to_hadoop_xml(output, conf) :
+ """ export the conf to hadoop XML
+ dictionaries are skipped.
+ """
+ output.write("<configuration>\n")
+ for (key, value) in conf.items() :
+ if type(value) is list :
+ # list print
+ export_kv_xml(output, key, list_to_hxml_str(value))
+ else :
+ if type(value) is dict :
+ print "skipping dict " + key
+ else :
+ export_kv_xml(output, key, value)
+ output.write("</configuration>\n")
+
+
+def start(conf, stripped_args) :
+ """
+ start the process by grabbing exec/args for the arguments
+ """
+ ex = conf["exec"]
+ args = []
+ jsonargs = get(ex, "args", [])
+ args.extend(jsonargs)
+ args.extend(stripped_args)
+ classname = get(ex, "classname", "")
+ if not len(classname) :
+ raise Exception, "No classname supplied"
+ classname = get(ex, "classname", "")
+ commandline = ["java"]
+ classpath = []
+ jvmargs = []
+ commandline.extend(jvmargs)
+ commandline.append("-classpath")
+ commandline.append(list_to_str(classpath, ":"))
+ commandline.append("org.apache.hadoop.yarn.service.launcher.ServiceLauncher")
+ commandline.append(classname)
+ commandline.extend(args)
+ print "ready to exec : %s" % commandline
+
+
+def main() :
+# (conf, stripped, jfiles) = parse_args({}, sys.argv[1 :])
+ actions = extract_args(sys.argv[1 :])
+ jfiles = actions[KEY_JFILE]
+ conf = merge_jfile_list({}, jfiles)
+ apply_local_definitions(conf, actions[KEY_DEF])
+ exec_args = actions[KEY_ARGS]
+
+ print_conf(conf)
+ # if len(stripped) > 0 :
+ #got an output file
+ # filename = stripped[0]
+ # print "Writing XML configuration to " + filename
+ # output = open(filename, "w")
+ # export_to_hadoop_xml(output, conf["site"])
+ start(conf, exec_args)
+
+
+if __name__ == "__main__" :
+ main()
+
+
diff --git a/slider-core/src/site/site.xml b/slider-core/src/site/site.xml
new file mode 100644
index 0000000..8dd81d4
--- /dev/null
+++ b/slider-core/src/site/site.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project name="Slider">
+<!--
+
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-fluido-skin</artifactId>
+ <version>1.3.0</version>
+ </skin>
+-->
+<!--
+
+ <custom>
+ <fluidoSkin>
+ <topBarEnabled>true</topBarEnabled>
+ <sideBarEnabled>false</sideBarEnabled>
+ </fluidoSkin>
+ </custom>
+-->
+
+ <version position="right"/>
+
+ <body>
+ <menu ref="reports"/>
+
+
+
+ </body>
+
+
+</project>
\ No newline at end of file
diff --git a/slider-core/src/test/app_packages/test_command_log/configuration/cl-site.xml b/slider-core/src/test/app_packages/test_command_log/configuration/cl-site.xml
new file mode 100644
index 0000000..977c8bc
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/configuration/cl-site.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>logfile.location</name>
+ <value>/tmp/logfiles/operations.log</value>
+ <description>The location where the log file is stored.</description>
+ </property>
+ <property>
+ <name>datetime.format</name>
+ <value>%A, %d. %B %Y %I:%M%p</value>
+ <description>The format to use.
+ </description>
+ </property>
+</configuration>
diff --git a/slider-core/src/test/app_packages/test_command_log/metainfo.xml b/slider-core/src/test/app_packages/test_command_log/metainfo.xml
new file mode 100644
index 0000000..f5fde30
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/metainfo.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>TEST_COMMAND_LOG</name>
+ <comment>
+ When started it creates a new log file and stores all commands in the
+ log file. When stopped it renames the file.
+ </comment>
+ <version>0.1.0</version>
+ <components>
+ <component>
+ <name>COMMAND_LOGGER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/cl.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/command_log.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ </service>
+ </services>
+</metainfo>
diff --git a/slider-core/src/test/app_packages/test_command_log/package/files/command_log_10.tar b/slider-core/src/test/app_packages/test_command_log/package/files/command_log_10.tar
new file mode 100644
index 0000000..b8231d1
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/package/files/command_log_10.tar
Binary files differ
diff --git a/slider-core/src/test/app_packages/test_command_log/package/scripts/cl.py b/slider-core/src/test/app_packages/test_command_log/package/scripts/cl.py
new file mode 100644
index 0000000..4b7b91b
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/package/scripts/cl.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+import os
+from datetime import datetime
+from resource_management import *
+from resource_management.core.base import Fail
+
+
+class HbaseMaster(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+
+ env.set_params(params)
+
+ def start(self, env):
+ import params
+
+ env.set_params(params)
+ self.configure(env)
+ self.rename_file(env)
+ self.ensure_file(env)
+ self.check_and_log(env, "Starting instance.")
+
+ def stop(self, env):
+ import params
+
+ env.set_params(params)
+ self.check_and_log(env, "Stopping instance.")
+ self.rename_file(env)
+
+ def status(self, env):
+ import params
+
+ env.set_params(params)
+ self.check_and_log(env, "Status check.")
+
+ def check_and_log(self, env, message):
+ import params
+
+ file_location = params.file_location
+ datetime_format = params.datetime_format
+ if not os.path.isfile(file_location) or not os.access(file_location,
+ os.W_OK):
+ raise Fail("File does not exist or not writable. %s" % file_location)
+ with open(file_location, "a") as logfile:
+ logfile.write("Time: " + datetime.utcnow().strftime(datetime_format) + "\n")
+ logfile.write("Log: " + message + "\n")
+ logfile.write("---------------\n")
+
+ def rename_file(self, env):
+ import params
+
+ file_location = params.file_location
+ if os.path.isfile(file_location) and \
+ os.access(file_location, os.W_OK):
+ new_file_name = \
+ file_location + "." + datetime.utcnow().strftime("%d_%m_%y_%I_%M_%S")
+ os.rename(file_location, new_file_name)
+
+ def ensure_file(self, env):
+ import params
+
+ file_location = params.file_location
+ TemplateConfig( file_location,
+ owner = params.app_user,
+ template_tag = None
+ )
+
+if __name__ == "__main__":
+ HbaseMaster().execute()
diff --git a/slider-core/src/test/app_packages/test_command_log/package/scripts/params.py b/slider-core/src/test/app_packages/test_command_log/package/scripts/params.py
new file mode 100644
index 0000000..af200c1
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/package/scripts/params.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+
+container_id = config['configurations']['global']['container_id']
+application_id = config['configurations']['global']['application_id']
+app_user = config['configurations']['global']['app_user']
+
+datetime_format = config['configurations']['cl-site']['datetime.format']
+file_location = config['configurations']['cl-site']['logfile.location']
diff --git a/slider-core/src/test/app_packages/test_command_log/package/templates/operations.log.j2 b/slider-core/src/test/app_packages/test_command_log/package/templates/operations.log.j2
new file mode 100644
index 0000000..8d6e879
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/package/templates/operations.log.j2
@@ -0,0 +1,22 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+This is a log for all operations on a Yarn container.
+Container Id: {{container_id}}
+Application id: {{application_id}}
+---------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
new file mode 100644
index 0000000..4ad94cf
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXMLConfKeysForTesting
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.agent.AgentKeys
+import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.junit.BeforeClass
+
+/**
+ * test base for agent clusters
+ */
+@CompileStatic
+@Slf4j
+public abstract class AgentMiniClusterTestBase
+ extends YarnZKMiniClusterTestBase {
+ private static File agentConf
+ private static File agentDef
+ private static File imagePath
+
+
+ @BeforeClass
+ public static void createSubConfFiles() {
+ File destDir = new File("target/agent_minicluster_testbase")
+ destDir.mkdirs()
+ agentConf = new File(destDir, "agentconf")
+ agentConf.createNewFile()
+ agentDef = new File(destDir, "agentdef")
+ agentDef.createNewFile()
+ imagePath = new File(destDir, "imagePath.tar.gz")
+ imagePath.createNewFile()
+
+ }
+ @Override
+ public String getTestConfigurationPath() {
+ return "src/main/resources/" + AgentKeys.CONF_RESOURCE;
+ }
+
+ @Override
+ void setup() {
+ super.setup()
+ def testConf = getTestConfiguration()
+ }
+
+ /**
+ * Teardown kills region servers
+ */
+ @Override
+ void teardown() {
+ super.teardown();
+ if (teardownKillall) {
+
+ }
+ }
+
+ @Override
+ String getApplicationHomeKey() {
+ return SliderXMLConfKeysForTesting.KEY_TEST_AGENT_HOME;
+ }
+
+ @Override
+ String getArchiveKey() {
+ return SliderXMLConfKeysForTesting.KEY_TEST_AGENT_TAR;
+ }
+
+ /**
+ return a mock home dir -this test case does not intend to run any agent
+ */
+ @Override
+ List<String> getImageCommands() {
+ [Arguments.ARG_IMAGE, imagePath.toURI().toString()]
+ }
+
+/**
+ * Create an AM without a master
+ * @param clustername AM name
+ * @param size # of nodes
+ * @param deleteExistingData should any existing cluster data be deleted
+ * @param blockUntilRunning block until the AM is running
+ * @return launcher which will have executed the command.
+ */
+ public ServiceLauncher<SliderClient> createMasterlessAM(
+ String clustername,
+ int size,
+ boolean deleteExistingData,
+ boolean blockUntilRunning) {
+ return createCluster(clustername,
+ [:],
+ [
+
+ ],
+ deleteExistingData,
+ blockUntilRunning,
+ [
+ (AgentKeys.APP_DEF): agentConf.toURI().toString(),
+ (AgentKeys.AGENT_CONF): agentDef.toURI().toString()
+ ])
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
new file mode 100644
index 0000000..9cc4931
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationId
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.server.services.curator.CuratorServiceInstance
+import org.apache.slider.server.services.curator.RegistryBinderService
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
+ @Test
+ public void testStandaloneAgentAM() throws Throwable {
+
+
+ describe "create a masterless AM then get the service and look it up via the AM"
+
+ //launch fake master
+ String clustername = "test_standalone_agent_am"
+ createMiniCluster(clustername, configuration, 1, true)
+ ServiceLauncher<SliderClient> launcher
+ launcher = createMasterlessAM(clustername, 0, true, false)
+ SliderClient client = launcher.service
+ addToTeardown(client);
+
+ ApplicationReport report = waitForClusterLive(client)
+ logReport(report)
+ List<ApplicationReport> apps = client.applications;
+
+ //get some of its status
+ dumpClusterStatus(client, "masterless application status")
+ List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
+ SliderKeys.COMPONENT_AM)
+ assert clusterNodes.size() == 1
+
+ ClusterNode masterNode = clusterNodes[0]
+ log.info("Master node = ${masterNode}");
+
+ List<ClusterNode> nodes
+ String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM)
+ assert uuids.length == 1;
+ nodes = client.listClusterNodes(uuids);
+ assert nodes.size() == 1;
+ describe "AM Node UUID=${uuids[0]}"
+
+ nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM)
+ assert nodes.size() == 1;
+ nodes = listNodesInRole(client, "")
+ assert nodes.size() == 1;
+ assert nodes[0].role == SliderKeys.COMPONENT_AM
+
+
+
+
+ String username = client.username
+ def serviceRegistryClient = client.YARNRegistryClient
+ describe("list of all applications")
+ logApplications(apps)
+ describe("apps of user $username")
+ List<ApplicationReport> userInstances = serviceRegistryClient.listInstances()
+ logApplications(userInstances)
+ assert userInstances.size() == 1
+ describe("named app $clustername")
+ ApplicationReport instance = serviceRegistryClient.findInstance(clustername)
+ logReport(instance)
+ assert instance != null
+
+ //switch to the ZK-based registry
+
+ describe "service registry names"
+ RegistryBinderService<ServiceInstanceData> registry = client.registry
+ def names = registry.queryForNames();
+ dumpRegistryNames(names)
+ describe "service registry instance IDs"
+
+ def instanceIds = client.listRegistryInstanceIDs()
+
+ log.info("number of instanceIds: ${instanceIds.size()}")
+ instanceIds.each { String it -> log.info(it) }
+
+ describe "service registry slider instances"
+ List<CuratorServiceInstance<ServiceInstanceData>> instances = client.listRegistryInstances(
+ )
+ instances.each { CuratorServiceInstance<ServiceInstanceData> svc ->
+ log.info svc.toString()
+ }
+ describe "end list service registry slider instances"
+
+ describe "teardown of cluster instance #1"
+ //now kill that cluster
+ assert 0 == clusterActionFreeze(client, clustername)
+ //list it & See if it is still there
+ ApplicationReport oldInstance = serviceRegistryClient.findInstance(
+ clustername)
+ assert oldInstance != null
+ assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
+
+ //create another AM
+ launcher = createMasterlessAM(clustername, 0, true, true)
+ client = launcher.service
+ ApplicationId i2AppID = client.applicationId
+
+ //expect 2 in the list
+ userInstances = serviceRegistryClient.listInstances()
+ logApplications(userInstances)
+ assert userInstances.size() == 2
+
+ //but when we look up an instance, we get the new App ID
+ ApplicationReport instance2 = serviceRegistryClient.findInstance(
+ clustername)
+ assert i2AppID == instance2.applicationId
+
+
+
+ describe("attempting to create instance #3")
+ //now try to create instance #3, and expect an in-use failure
+ try {
+ createMasterlessAM(clustername, 0, false, true)
+ fail("expected a failure, got a masterless AM")
+ } catch (SliderException e) {
+ assertFailureClusterInUse(e);
+ }
+
+ describe("Stopping instance #2")
+
+ //now stop that cluster
+ assert 0 == clusterActionFreeze(client, clustername)
+
+ logApplications(client.listSliderInstances(username))
+
+ //verify it is down
+ ApplicationReport reportFor = client.getApplicationReport(i2AppID)
+
+ //downgrade this to a fail
+// Assume.assumeTrue(YarnApplicationState.FINISHED <= report.yarnApplicationState)
+ assert YarnApplicationState.FINISHED <= reportFor.yarnApplicationState
+
+
+ ApplicationReport instance3 = serviceRegistryClient.findInstance(
+ clustername)
+ assert instance3.yarnApplicationState >= YarnApplicationState.FINISHED
+
+
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
new file mode 100644
index 0000000..7a4500a
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.core.registry.info.CustomRegistryConstants
+import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.apache.slider.server.services.curator.CuratorServiceInstance
+import org.apache.slider.server.services.curator.RegistryBinderService
+import org.junit.Test
+
+/**
+ * create masterless AMs and work with them. This is faster than
+ * bringing up full clusters
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneRegistryAM extends AgentMiniClusterTestBase {
+
+
+ public static final String YARN_SITE = "yarn-site.xml"
+
+ @Test
+ public void testRegistryAM() throws Throwable {
+
+
+ describe "create a masterless AM then perform registry operations on it"
+
+ //launch fake master
+ String clustername = "test_standalone_registry_am"
+ createMiniCluster(clustername, configuration, 1, true)
+ ServiceLauncher<SliderClient> launcher
+ launcher = createMasterlessAM(clustername, 0, true, false)
+ SliderClient client = launcher.service
+ addToTeardown(client);
+
+ ApplicationReport report = waitForClusterLive(client)
+ logReport(report)
+ List<ApplicationReport> apps = client.applications;
+
+ List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
+ SliderKeys.COMPONENT_AM)
+ assert clusterNodes.size() == 1
+
+ ClusterNode masterNode = clusterNodes[0]
+ log.info("Master node = ${masterNode}");
+
+ List<ClusterNode> nodes
+ String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM)
+ assert uuids.length == 1;
+ nodes = client.listClusterNodes(uuids);
+ assert nodes.size() == 1;
+ describe "AM Node UUID=${uuids[0]}"
+
+ nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM)
+ assert nodes.size() == 1;
+ nodes = listNodesInRole(client, "")
+ assert nodes.size() == 1;
+ ClusterNode master = nodes[0]
+ assert master.role == SliderKeys.COMPONENT_AM
+
+
+
+
+ String username = client.username
+ def serviceRegistryClient = client.YARNRegistryClient
+ describe("list of all applications")
+ logApplications(apps)
+ describe("apps of user $username")
+ List<ApplicationReport> userInstances = serviceRegistryClient.listInstances()
+ logApplications(userInstances)
+ assert userInstances.size() == 1
+ describe("named app $clustername")
+ ApplicationReport instance = serviceRegistryClient.findInstance(clustername)
+ logReport(instance)
+ assert instance != null
+
+ //switch to the ZK-based registry
+
+ describe "service registry names"
+ RegistryBinderService<ServiceInstanceData> registry = client.registry
+ def names = registry.queryForNames();
+ dumpRegistryNames(names)
+
+ List<String> instanceIds = client.listRegistryInstanceIDs()
+
+
+ dumpRegistryInstanceIDs(instanceIds)
+ assert instanceIds.size() == 1
+
+ List<CuratorServiceInstance<ServiceInstanceData>> instances = client.listRegistryInstances(
+ )
+ dumpRegistryInstances(instances)
+
+ assert instances.size() == 1
+
+ def amInstance = instances[0]
+ def serviceInstanceData = amInstance.payload
+
+ def externalEndpoints = serviceInstanceData.externalView.endpoints
+
+ def endpoint = externalEndpoints.get(CustomRegistryConstants.PUBLISHER_REST_API)
+ assert endpoint != null
+ def publisherURL = endpoint.asURL()
+ def publisher = publisherURL.toString()
+ describe("Publisher")
+
+ def publishedJSON = GET(publisherURL)
+ log.info(publishedJSON)
+ JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<PublishedConfigSet>(
+ PublishedConfigSet)
+ def configSet = serDeser.fromJson(publishedJSON)
+ assert configSet.size() >= 1
+ assert configSet.contains(YARN_SITE)
+ def publishedYarnSite = configSet.get(YARN_SITE)
+
+
+ def yarnSitePublisher = appendToURL(publisher, YARN_SITE)
+ def yarnSiteXML = appendToURL(yarnSitePublisher, "xml")
+
+
+ String confXML = GET(yarnSiteXML)
+ log.info("Conf XML at $yarnSiteXML = \n $confXML")
+
+ String confJSON = GET(yarnSitePublisher, "json")
+
+ // hit the registry web page
+
+ def registryEndpoint = externalEndpoints.get(CustomRegistryConstants.REGISTRY_REST_API)
+ assert registryEndpoint != null
+ def registryURL = registryEndpoint.asURL()
+ describe("Registry WADL @ $registryURL")
+
+ describe("Registry List")
+ log.info(GET(registryURL, RestPaths.REGISTRY_SERVICE ))
+
+
+
+ describe "teardown of cluster"
+ //now kill that cluster
+ assert 0 == clusterActionFreeze(client, clustername)
+ //list it & See if it is still there
+ ApplicationReport oldInstance = serviceRegistryClient.findInstance(
+ clustername)
+ assert oldInstance != null
+ assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
+
+
+ sleep(20000)
+
+ instances = client.listRegistryInstances()
+ assert instances.size() == 0
+
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/api/TestRPCBinding.java b/slider-core/src/test/groovy/org/apache/slider/api/TestRPCBinding.java
new file mode 100644
index 0000000..32e5824
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/api/TestRPCBinding.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.server.appmaster.rpc.RpcBinder;
+import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPB;
+import org.junit.Test;
+
+import java.net.InetSocketAddress;
+
+/**
+ * Tests RPC work
+ */
+public class TestRPCBinding {
+
+ @Test
+ public void testRegistration() throws Throwable {
+ Configuration conf = new Configuration();
+ RpcBinder.registerSliderAPI(conf);
+ assert RpcBinder.verifyBondedToProtobuf(conf, SliderClusterProtocolPB.class);
+ }
+
+ @Test
+ public void testGetProxy() throws Throwable {
+ Configuration conf = new Configuration();
+ InetSocketAddress saddr= new InetSocketAddress("127.0.0.1",9000);
+ SliderClusterProtocol proxy =
+ RpcBinder.connectToServer(saddr, null, conf, 1000);
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
new file mode 100644
index 0000000..088fdb3
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import groovy.transform.CompileStatic
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.junit.Test
+
+/**
+ * Test the argument parsing/validation logic
+ */
+@CompileStatic
+class TestClientBadArgs extends ServiceLauncherBaseTest {
+ @Test
+ public void testNoAction() throws Throwable {
+ launchExpectingException(SliderClient,
+ new Configuration(),
+ ErrorStrings.ERROR_NO_ACTION,
+ [])
+
+ }
+
+ @Test
+ public void testUnknownAction() throws Throwable {
+ launchExpectingException(SliderClient,
+ new Configuration(),
+ "not-a-known-action",
+ ["not-a-known-action"])
+ }
+
+ @Test
+ public void testActionWithoutEnoughArgs() throws Throwable {
+ launchExpectingException(SliderClient,
+ new Configuration(),
+ ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS,
+ [SliderActions.ACTION_THAW])
+ }
+
+ @Test
+ public void testActionWithTooManyArgs() throws Throwable {
+ launchExpectingException(SliderClient,
+ new Configuration(),
+ ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
+ [SliderActions.ACTION_HELP,
+ "hello, world"])
+ }
+
+ @Test
+ public void testBadImageArg() throws Throwable {
+ launchExpectingException(SliderClient,
+ new Configuration(),
+ "Unknown option: --image",
+ [SliderActions.ACTION_HELP,
+ Arguments.ARG_IMAGE])
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy
new file mode 100644
index 0000000..1e03030
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.ClientArgs
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.junit.Test
+
+/**
+ * Test bad argument handling
+ */
+//@CompileStatic
+class TestClientBasicArgs extends ServiceLauncherBaseTest {
+
+ /**
+ * help should print out help string and then succeed
+ * @throws Throwable
+ */
+ @Test
+ public void testHelp() throws Throwable {
+ ServiceLauncher launcher = launch(SliderClient,
+ SliderUtils.createConfiguration(),
+ [ClientArgs.ACTION_HELP])
+ assert 0 == launcher.serviceExitCode
+ }
+
+ @Test
+ public void testNoArgs() throws Throwable {
+ try {
+ ServiceLauncher launcher = launch(SliderClient,
+ SliderUtils.createConfiguration(),
+ [])
+ assert SliderExitCodes.EXIT_COMMAND_ARGUMENT_ERROR == launcher.serviceExitCode
+ } catch (BadCommandArgumentsException ignored) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testListUnknownHost() throws Throwable {
+ try {
+ ServiceLauncher launcher = launch(SliderClient,
+ SliderUtils.createConfiguration(),
+ [
+ ClientArgs.ACTION_LIST,
+ "cluster",
+ Arguments.ARG_MANAGER,
+ "unknownhost.example.org:80"])
+ fail("expected an exception, got a launcher with exit code $launcher.serviceExitCode")
+ } catch (UnknownHostException expected) {
+ //expected
+ }
+
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
new file mode 100644
index 0000000..12e4a43
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
@@ -0,0 +1,520 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import com.google.common.collect.Maps
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hdfs.DFSConfigKeys
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.*
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.junit.Assert
+import org.junit.Test
+
+/**
+ * Test handling of common arguments, specifically how things get split up
+ */
+@CompileStatic
+@Slf4j
+
+class TestCommonArgParsing implements SliderActions, Arguments {
+
+
+ public static final String CLUSTERNAME = "clustername"
+
+ @Test
+ public void testCreateActionArgs() throws Throwable {
+ ClientArgs clientArgs = createClientArgs([ACTION_CREATE, 'cluster1'])
+ assert clientArgs.clusterName == 'cluster1'
+ }
+
+ @Test
+ public void testCreateFailsNoClustername() throws Throwable {
+ assertParseFails([ACTION_CREATE])
+ }
+
+ @Test
+ public void testCreateFailsTwoClusternames() throws Throwable {
+ assertParseFails([
+ ACTION_CREATE,
+ "c1",
+ "c2",
+ ])
+ }
+
+ @Test
+ public void testHelp() throws Throwable {
+ ClientArgs clientArgs = createClientArgs([ACTION_HELP])
+ assert clientArgs.clusterName == null
+ }
+
+ @Test
+ public void testSliderBasePath() throws Throwable {
+ ClientArgs clientArgs = createClientArgs([ACTION_LIST, "--basepath", "/projects/slider/clusters"])
+ assert clientArgs.basePath == new Path("/projects/slider/clusters")
+ }
+
+ @Test
+ public void testNoSliderBasePath() throws Throwable {
+ ClientArgs clientArgs = createClientArgs([ACTION_LIST])
+ assert clientArgs.basePath == null
+ }
+
+ @Test
+ public void testListNoClusternames() throws Throwable {
+ ClientArgs clientArgs = createClientArgs([ACTION_LIST])
+ assert clientArgs.clusterName == null
+ }
+
+ @Test
+ public void testListNoClusternamesDefinition() throws Throwable {
+ ClientArgs clientArgs = createClientArgs(
+ [ACTION_LIST,
+ ARG_DEFINE,
+ 'fs.default.FS=file://localhost',
+ ])
+ assert clientArgs.clusterName == null
+ }
+
+ @Test
+ public void testList1Clustername() throws Throwable {
+ ClientArgs ca = createClientArgs([ACTION_LIST, 'cluster1'])
+ assert ca.clusterName == 'cluster1'
+ assert ca.coreAction instanceof ActionListArgs
+ }
+
+ @Test
+ public void testListFailsTwoClusternames() throws Throwable {
+ assertParseFails([
+ ACTION_LIST,
+ "c1",
+ "c2",
+ ])
+ }
+
+ @Test
+ public void testDefinitions() throws Throwable {
+ ClientArgs ca = createClientArgs([
+ ACTION_CREATE,
+ CLUSTERNAME,
+ "-D","yarn.resourcemanager.principal=yarn/server@LOCAL",
+ "-D","dfs.datanode.kerberos.principal=hdfs/server@LOCAL",
+ ])
+ Configuration conf = new Configuration(false)
+ ca.applyDefinitions(conf)
+ assert ca.clusterName == CLUSTERNAME
+ assert conf.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH) == null
+ SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
+ SliderUtils.verifyPrincipalSet(
+ conf,
+ DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY);
+
+ }
+
+ @Test
+ public void testDefinitionsSettingBaseSliderDir() throws Throwable {
+ ClientArgs ca = createClientArgs([
+ ACTION_CREATE,
+ CLUSTERNAME,
+ "--basepath", "/projects/slider/clusters",
+ "-D","yarn.resourcemanager.principal=yarn/server@LOCAL",
+ "-D","dfs.datanode.kerberos.principal=hdfs/server@LOCAL",
+ ])
+ Configuration conf = new Configuration(false)
+ ca.applyDefinitions(conf)
+ assert ca.clusterName == CLUSTERNAME
+ assert conf.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH) == "/projects/slider/clusters"
+ SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
+ SliderUtils.verifyPrincipalSet(conf, DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY);
+
+ }
+
+ /**
+ * Test a thaw command
+ * @throws Throwable
+ */
+ @Test
+ public void testComplexThaw() throws Throwable {
+ ClientArgs ca = createClientArgs([
+ ACTION_THAW,
+ "--manager", "rhel:8032", "--filesystem", "hdfs://rhel:9090",
+ "-S","java.security.krb5.realm=LOCAL","-S", "java.security.krb5.kdc=rhel",
+ "-D","yarn.resourcemanager.principal=yarn/rhel@LOCAL",
+ "-D","namenode.resourcemanager.principal=hdfs/rhel@LOCAL",
+ "cl1"
+ ])
+ assert "cl1" == ca.clusterName
+ assert ca.coreAction instanceof ActionThawArgs
+
+ }
+
+ /**
+ * Test a force kill command where the app comes at the end of the line
+ * @throws Throwable
+ *
+ */
+ @Test
+ public void testStatusSplit() throws Throwable {
+
+ String appId = "application_1381252124398_0013"
+ ClientArgs ca = createClientArgs([
+ ACTION_STATUS,
+ "--manager", "rhel:8032",
+ "--filesystem", "hdfs://rhel:9090",
+ "-S","java.security.krb5.realm=LOCAL",
+ "-S", "java.security.krb5.kdc=rhel",
+ "-D","yarn.resourcemanager.principal=yarn/rhel@LOCAL",
+ "-D","namenode.resourcemanager.principal=hdfs/rhel@LOCAL",
+ appId
+ ])
+ assert appId == ca.clusterName
+ }
+
+ @Test
+ public void testFreezeFailsNoArg() throws Throwable {
+ assertParseFails([
+ ACTION_FREEZE,
+ ])
+ }
+
+ @Test
+ public void testFreezeWorks1Arg() throws Throwable {
+ ClientArgs ca = createClientArgs([
+ ACTION_FREEZE,
+ CLUSTERNAME,
+ ])
+ assert ca.clusterName == CLUSTERNAME
+ assert ca.coreAction instanceof ActionFreezeArgs
+ }
+
+ @Test
+ public void testFreezeFails2Arg() throws Throwable {
+ assertParseFails([
+ ACTION_FREEZE, "cluster", "cluster2"
+ ])
+ }
+
+ @Test
+ public void testFreezeForceWaitAndMessage() throws Throwable {
+ ClientArgs ca = createClientArgs([
+ ACTION_FREEZE, CLUSTERNAME,
+ ARG_FORCE,
+ ARG_WAIT, "0",
+ ARG_MESSAGE, "explanation"
+ ])
+ assert ca.clusterName == CLUSTERNAME
+ assert ca.coreAction instanceof ActionFreezeArgs
+ ActionFreezeArgs freezeArgs = (ActionFreezeArgs) ca.coreAction;
+ assert freezeArgs.message == "explanation"
+ assert freezeArgs.force;
+ }
+
+ @Test
+ public void testGetConfFailsNoArg() throws Throwable {
+ assertParseFails([
+ ACTION_GETCONF,
+ ])
+ }
+
+ @Test
+ public void testGetConfWorks1Arg() throws Throwable {
+ ClientArgs ca = createClientArgs([
+ ACTION_GETCONF,
+ CLUSTERNAME,
+ ])
+ assert ca.clusterName == CLUSTERNAME
+ assert ca.coreAction instanceof ActionGetConfArgs
+ }
+
+ @Test
+ public void testGetConfWorksOut() throws Throwable {
+ ClientArgs ca = createClientArgs([
+ ACTION_GETCONF,
+ CLUSTERNAME,
+ ARG_FORMAT,"xml",
+ ARG_OUTPUT,"file.xml"
+ ])
+ assert ca.clusterName == CLUSTERNAME
+ assert ca.coreAction instanceof ActionGetConfArgs
+ assert ca.actionGetConfArgs.format == "xml"
+ assert ca.actionGetConfArgs.output == "file.xml"
+ }
+
+ @Test
+ public void testGetStatusWorks1Arg() throws Throwable {
+ ClientArgs ca = createClientArgs([
+ ACTION_STATUS,
+ CLUSTERNAME,
+ ])
+ assert ca.clusterName == CLUSTERNAME
+ assert ca.coreAction instanceof ActionStatusArgs
+ }
+
+ @Test
+ public void testExistsWorks1Arg() throws Throwable {
+ ClientArgs ca = createClientArgs([
+ ACTION_EXISTS,
+ CLUSTERNAME,
+ ARG_LIVE
+ ])
+ assert ca.clusterName == CLUSTERNAME
+ assert ca.coreAction instanceof ActionExistsArgs
+ assert ca.actionExistsArgs.live
+ }
+
+ @Test
+ public void testDestroy1Arg() throws Throwable {
+ ClientArgs ca = createClientArgs([
+ ACTION_DESTROY,
+ CLUSTERNAME,
+ ])
+ assert ca.clusterName == CLUSTERNAME
+ assert ca.coreAction instanceof ActionDestroyArgs
+ }
+
+ /**
+ * Assert that a pass fails with a BadCommandArgumentsException
+ * @param argsList
+ */
+
+ private void assertParseFails(List argsList) {
+ try {
+ ClientArgs clientArgs = createClientArgs(argsList)
+ Assert.fail("exected an exception, got $clientArgs")
+ } catch (BadCommandArgumentsException ignored) {
+ //expected
+ }
+ }
+
+ /**
+ * build and parse client args, after adding the base args list
+ * @param argsList
+ */
+ public ClientArgs createClientArgs(List argsList) {
+ def serviceArgs = new ClientArgs(argsList + baseArgs())
+ serviceArgs.parse()
+ serviceArgs
+ }
+
+ public ActionCreateArgs createAction(List argsList) {
+ def ca = createClientArgs(argsList)
+ assert ca.action == ACTION_CREATE
+ ActionCreateArgs args = ca.actionCreateArgs
+ assert args != null
+ return args
+ }
+
+ /**
+ * build the list of base arguments for all operations
+ * @return the base arguments
+ */
+ private def baseArgs() {
+ return [
+
+ ]
+ }
+
+
+ @Test
+ public void testCreateWaitTime() throws Throwable {
+ ActionCreateArgs createArgs = createAction([
+ ACTION_CREATE, 'cluster1',
+ ARG_WAIT, "600"
+ ])
+ assert 600 == createArgs.getWaittime()
+ }
+
+
+ @Test
+ public void testSingleRoleArg() throws Throwable {
+ def createArgs = createAction([
+ ACTION_CREATE, 'cluster1',
+ ARG_COMPONENT,"master","5",
+ ])
+ def tuples = createArgs.componentTuples;
+ assert tuples.size() == 2;
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
+ assert roleMap["master"] == "5"
+ }
+
+ @Test
+ public void testNoRoleArg() throws Throwable {
+ ActionCreateArgs createArgs = createAction([
+ ACTION_CREATE, 'cluster1',
+ ])
+ def tuples = createArgs.componentTuples;
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
+ assert roleMap["master"] == null
+ }
+
+
+ @Test
+ public void testMultiRoleArgBuild() throws Throwable {
+ def ca = createClientArgs([
+ ACTION_BUILD, 'cluster1',
+ ARG_COMPONENT, "master", "1",
+ ARG_COMPONENT, "worker", "2",
+ ])
+ assert ca.action == ACTION_BUILD
+ assert ca.coreAction instanceof ActionBuildArgs
+ assert ca.buildingActionArgs instanceof ActionBuildArgs
+ AbstractClusterBuildingActionArgs args = ca.actionBuildArgs
+ def tuples = args.componentTuples;
+ assert tuples.size() == 4;
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
+ assert roleMap["master"] == "1"
+ assert roleMap["worker"] == "2"
+ }
+
+ @Test
+ public void testFlexArgs() throws Throwable {
+ def ca = createClientArgs([
+ ACTION_FLEX, 'cluster1',
+ ARG_COMPONENT, "master", "1",
+ ARG_COMPONENT, "worker", "2",
+ ])
+ assert ca.coreAction instanceof ActionFlexArgs
+ def tuples = ca.actionFlexArgs.componentTuples;
+ assert tuples.size() == 4;
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
+ assert roleMap["master"] == "1"
+ assert roleMap["worker"] == "2"
+ }
+
+ @Test
+ public void testDuplicateRole() throws Throwable {
+ ActionCreateArgs createArgs = createAction([
+ ACTION_CREATE, 'cluster1',
+ ARG_COMPONENT, "master", "1",
+ ARG_COMPONENT, "master", "2",
+ ])
+ def tuples = createArgs.componentTuples;
+ assert tuples.size() == 4;
+ try {
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap(
+ "roles",
+ tuples);
+ Assert.fail("got a role map $roleMap not a failure");
+ } catch (BadCommandArgumentsException expected) {
+ assert expected.message.contains(ErrorStrings.ERROR_DUPLICATE_ENTRY)
+ }
+ }
+
+ @Test
+ public void testOddRoleCount() throws Throwable {
+ ActionCreateArgs createArgs = createAction([
+ ACTION_CREATE, 'cluster1',
+ ARG_COMPONENT,"master","1",
+ ARG_COMPONENT,"master","2",
+ ])
+ List<String> tuples = createArgs.componentTuples
+ tuples += "loggers";
+ assert tuples.size() == 5;
+ try {
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
+ Assert.fail("got a role map $roleMap not a failure");
+ } catch (BadCommandArgumentsException expected) {
+ assert expected.message.contains(ErrorStrings.ERROR_PARSE_FAILURE)
+ }
+ }
+
+ /**
+ * Create some role-opt client args, so that multiple tests can use it
+ * @return the args
+ */
+ public ActionCreateArgs createRoleOptClientArgs() {
+ ActionCreateArgs createArgs = createAction([
+ ACTION_CREATE, 'cluster1',
+ ARG_COMPONENT, "master", "1",
+ ARG_COMP_OPT, "master", "cheese", "swiss",
+ ARG_COMP_OPT, "master", "env.CHEESE", "cheddar",
+ ARG_COMP_OPT, "master", ResourceKeys.YARN_CORES, 3,
+
+ ARG_COMPONENT, "worker", "2",
+ ARG_COMP_OPT, "worker", ResourceKeys.YARN_CORES, 2,
+ ARG_COMP_OPT, "worker", RoleKeys.JVM_HEAP, "65536",
+ ARG_COMP_OPT, "worker", "env.CHEESE", "stilton",
+ ])
+ return createArgs
+ }
+
+ @Test
+ public void testRoleOptionParse() throws Throwable {
+ ActionCreateArgs createArgs = createRoleOptClientArgs()
+ def tripleMaps = createArgs.compOptionMap
+ def workerOpts = tripleMaps["worker"];
+ assert workerOpts.size() == 3
+ assert workerOpts[ResourceKeys.YARN_CORES] == "2"
+ assert workerOpts[RoleKeys.JVM_HEAP] == "65536"
+
+ def masterOpts = tripleMaps["master"];
+ assert masterOpts.size() == 3
+ assert masterOpts[ResourceKeys.YARN_CORES] == "3"
+
+ }
+
+ @Test
+ public void testRoleOptionsMerge() throws Throwable {
+ ActionCreateArgs createArgs = createRoleOptClientArgs()
+
+ def roleOpts = createArgs.compOptionMap
+
+ def clusterRoleMap = Maps.newHashMap([
+ "master":["cheese":"french"],
+ "worker":["env.CHEESE":"french"]
+ ])
+ SliderUtils.applyCommandLineRoleOptsToRoleMap(clusterRoleMap, roleOpts);
+
+ def masterOpts = clusterRoleMap["master"];
+ assert masterOpts["cheese"] == "swiss"
+
+ def workerOpts = clusterRoleMap["worker"];
+ assert workerOpts["env.CHEESE"] == "stilton"
+ }
+
+ @Test
+ public void testEnvVariableApply() throws Throwable {
+ ActionCreateArgs createArgs = createRoleOptClientArgs()
+
+
+ def roleOpts = createArgs.compOptionMap
+ Map<String, Map<String, String>> clusterRoleMap = Maps.newHashMap([
+ "master": ["cheese": "french"],
+ "worker": ["env.CHEESE": "french"]
+ ])
+ SliderUtils.applyCommandLineRoleOptsToRoleMap(clusterRoleMap, roleOpts);
+
+ def workerOpts = Maps.newHashMap(clusterRoleMap["worker"])
+ assert workerOpts["env.CHEESE"] == "stilton";
+
+ Map<String, String> envmap = SliderUtils.buildEnvMap(workerOpts);
+ assert envmap["CHEESE"] == "stilton";
+
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/GroovyZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/GroovyZKIntegration.groovy
new file mode 100644
index 0000000..49701e3
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/GroovyZKIntegration.groovy
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import org.apache.slider.core.registry.zk.ZKCallback
+import org.apache.zookeeper.WatchedEvent
+
+class GroovyZKIntegration {
+
+/**
+ * Create a ZK watcher callback that forwards the event to the
+ * specific closure
+ * @param closure closure to invoke
+ * @return a callback which can be registered
+ */
+ static ZKCallback watcher(Closure closure) {
+ return new ZKCallback() {
+ @Override
+ void process(WatchedEvent event) {
+ closure(event);
+ }
+ }
+
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
new file mode 100644
index 0000000..7150b3c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.junit.Test
+
+@Slf4j
+class TestClientResourceRegistration {
+
+ /**
+ * Origin of a slider resource -again, internal tracking
+ * rather than something to set by hand.
+ */
+ private final static String KEY_RESOURCE_ORIGIN = "slider.client.resource.origin";
+
+ @Test
+ public void testRegistration() throws Throwable {
+ assert SliderUtils.registerClientResource();
+ }
+
+ @Test
+ public void testLoad() throws Throwable {
+ assert SliderUtils.registerClientResource();
+ Configuration conf = new Configuration(true);
+ assert conf.get(KEY_RESOURCE_ORIGIN) == "test/resources"
+ }
+
+ @Test
+ public void testMergeConfigs() throws Throwable {
+ Configuration conf1 = new Configuration(false)
+ conf1.set("key1", "conf1")
+ conf1.set("key2", "conf1")
+ Configuration conf2 = new Configuration(false)
+ conf1.set("key1", "conf2")
+ conf1.set("key3", "conf2")
+ ConfigHelper.mergeConfigurations(conf1, conf2, "test")
+ log.info(ConfigHelper.dumpConfigToString(conf1))
+
+ assert conf1.get("key1").equals("conf2")
+ assert conf1.get("key2").equals("conf1")
+ assert conf1.get("key3").equals("conf2")
+ }
+
+ /**
+ * This tests the situation where a yarn-config creation forces
+ * a load of the default resources, which would overwrite any other
+ * resources already in the list.
+ * @throws Throwable
+ */
+ @Test
+ public void testLoadRes() throws Throwable {
+ Configuration conf = SliderUtils.loadClientConfigurationResource()
+ assert conf.get(KEY_RESOURCE_ORIGIN) == "test/resources"
+ String hostname = "nosuchhost:0"
+ conf.set(YarnConfiguration.RM_ADDRESS, hostname)
+ YarnConfiguration yc = new YarnConfiguration()
+ ConfigHelper.mergeConfigurations(yc, conf, "slider-client")
+ InetSocketAddress addr = SliderUtils.getRmAddress(yc)
+ assert SliderUtils.isAddressDefined(addr)
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClusterNames.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClusterNames.groovy
new file mode 100644
index 0000000..06ede10
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClusterNames.groovy
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import org.junit.Assert
+import org.junit.Test
+
+@CompileStatic
+class TestClusterNames {
+
+ void assertValidName(String name) {
+ boolean valid = SliderUtils.isClusternameValid(name)
+ Assert.assertTrue("Clustername '$name' mistakenly declared invalid",
+ valid)
+ }
+
+ void assertInvalidName(String name) {
+ boolean valid = SliderUtils.isClusternameValid(name)
+ Assert.assertFalse("Clustername '$name' mistakenly declared valid",
+ valid)
+ }
+
+ void assertInvalid(List<String> names) {
+ names.each { String name -> assertInvalidName(name) }
+ }
+
+ void assertValid(List<String> names) {
+ names.each { String name -> assertValidName(name) }
+ }
+
+ @Test
+ public void testEmptyName() throws Throwable {
+ assertInvalidName('')
+ }
+
+ @Test
+ public void testSpaceName() throws Throwable {
+ assertInvalidName(' ')
+ }
+
+
+ @Test
+ public void testLeadingHyphen() throws Throwable {
+ assertInvalidName('-hyphen')
+ }
+
+ @Test
+ public void testTitleLetters() throws Throwable {
+ assertInvalidName('Title')
+ }
+
+ @Test
+ public void testCapitalLetters() throws Throwable {
+ assertInvalidName('UPPER-CASE-CLUSTER')
+ }
+
+ @Test
+ public void testInnerBraced() throws Throwable {
+ assertInvalidName('a[a')
+ }
+
+ @Test
+ public void testLeadingBrace() throws Throwable {
+ assertInvalidName('[')
+ }
+
+ @Test
+ public void testNonalphaLeadingChars() throws Throwable {
+ assertInvalid([
+ '[a', '#', '@', '=', '*', '.'
+ ])
+ }
+
+ @Test
+ public void testNonalphaInnerChars() throws Throwable {
+ assertInvalid([
+ 'a[a', 'b#', 'c@', 'd=', 'e*', 'f.', 'g ', 'h i'
+ ])
+ }
+
+ @Test
+ public void testClusterValid() throws Throwable {
+ assertValidName('cluster')
+ }
+
+ @Test
+ public void testValidNames() throws Throwable {
+ assertValid([
+ 'cluster',
+ 'cluster1',
+ 'very-very-very-long-cluster-name',
+ 'c1234567890'
+ ])
+
+
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
new file mode 100644
index 0000000..0d21d6c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.test.YarnMiniClusterTestBase
+import org.junit.Test
+
+@Slf4j
+class TestConfigHelperHDFS extends YarnMiniClusterTestBase {
+
+ //diabled for now;
+ @Test
+ public void testConfigHelperHDFS() throws Throwable {
+ YarnConfiguration config = getConfiguration()
+ createMiniHDFSCluster("testConfigHelperHDFS", config)
+
+ Configuration conf= new Configuration(false);
+ conf.set("key","value");
+ URI fsURI = new URI(fsDefaultName)
+ Path root = new Path(fsURI)
+ Path confPath = new Path(root, "conf.xml")
+ HadoopFS dfs = HadoopFS.get(fsURI,config)
+ ConfigHelper.saveConfig(dfs,confPath, conf)
+ //load time
+ Configuration loaded = ConfigHelper.loadConfiguration(dfs,confPath)
+ log.info(ConfigHelper.dumpConfigToString(loaded))
+ assert loaded.get("key") == "value"
+ }
+
+ @Test
+ public void testConfigLoaderIteration() throws Throwable {
+
+ String xml =
+ """<?xml version="1.0" encoding="UTF-8" standalone="no"?><configuration>
+<property><name>key</name><value>value</value><source>programatically</source></property>
+</configuration>
+ """
+ InputStream ins = new ByteArrayInputStream(xml.bytes);
+ Configuration conf = new Configuration(false);
+ conf.addResource(ins);
+ Configuration conf2 = new Configuration(false);
+ for (Map.Entry<String, String> entry : conf) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ conf2.set(key, val, "src")
+ }
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
new file mode 100644
index 0000000..24367a3
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+class TestMiscSliderUtils extends SliderTestBase {
+
+
+ public static final String CLUSTER1 = "cluster1"
+
+ @Test
+ public void testPurgeTempDir() throws Throwable {
+ //SliderUtils. //
+
+ Configuration configuration = new Configuration()
+ HadoopFS fs = HadoopFS.get(new URI("file:///"), configuration)
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration)
+ Path inst = sliderFileSystem.createAppInstanceTempPath(CLUSTER1, "001")
+
+ assert fs.exists(inst)
+ sliderFileSystem.purgeAppInstanceTempFiles(CLUSTER1)
+ assert !fs.exists(inst)
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
new file mode 100644
index 0000000..49bd58e
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import org.junit.Test
+
+@CompileStatic
+class TestPortScan {
+
+ @Test
+ public void testScanPorts() throws Throwable {
+
+ ServerSocket server = new ServerSocket(0)
+
+ try {
+ int serverPort = server.getLocalPort()
+ assert !SliderUtils.isPortAvailable(serverPort)
+ int port = SliderUtils.findFreePort(serverPort, 10)
+ assert port > 0 && serverPort < port
+ } finally {
+ server.close()
+ }
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestSliderFileSystem.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestSliderFileSystem.groovy
new file mode 100644
index 0000000..15d646d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestSliderFileSystem.groovy
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.common.tools
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.fs.Path
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+class TestSliderFileSystem extends SliderTestBase {
+ private static Configuration defaultConfiguration() {
+ new Configuration()
+ }
+
+ private static Configuration createConfigurationWithKV(String key, String value) {
+ def conf = defaultConfiguration()
+ conf.set(key, value)
+ conf
+ }
+
+ @Test
+ public void testSliderBasePathDefaultValue() throws Throwable {
+ Configuration configuration = defaultConfiguration()
+ FileSystem fileSystem = FileSystem.get(configuration)
+
+ def fs2 = new SliderFileSystem(fileSystem, configuration)
+ fs2.baseApplicationPath == new Path(fileSystem.homeDirectory, ".slider")
+ }
+
+ @Test
+ public void testSliderBasePathCustomValue() throws Throwable {
+ Configuration configuration = createConfigurationWithKV(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH, "/slider/cluster")
+ FileSystem fileSystem = FileSystem.get(configuration)
+ def fs2 = new SliderFileSystem(fileSystem, configuration)
+
+ fs2.baseApplicationPath == new Path("/slider/cluster")
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestSliderServiceUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestSliderServiceUtils.groovy
new file mode 100644
index 0000000..b513e94
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestSliderServiceUtils.groovy
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterResponsePBImpl
+import org.apache.slider.core.launch.AMRestartSupport
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+@Slf4j
+class TestSliderServiceUtils extends SliderTestBase {
+
+ @Test
+ public void testRetrieveContainers() throws Throwable {
+ RegisterApplicationMasterResponsePBImpl registration = new RegisterApplicationMasterResponsePBImpl()
+
+ def method = AMRestartSupport.retrieveContainersFromPreviousAttempt(
+ registration)
+ def hasMethod = method != null
+ def containers = AMRestartSupport.retrieveContainersFromPreviousAttempt(
+ registration)
+ def success = containers != null;
+
+ assert (hasMethod == success)
+ log.info("AM container recovery support=$hasMethod")
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestSliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestSliderTestUtils.groovy
new file mode 100644
index 0000000..621af27
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestSliderTestUtils.groovy
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.test.SliderTestUtils
+import org.junit.Test
+import org.junit.internal.AssumptionViolatedException
+
+class TestSliderTestUtils extends SliderTestUtils {
+
+
+ @Test
+ public void testAssumeTrue() throws Throwable {
+
+ try {
+ assume(true, "true")
+ } catch (AssumptionViolatedException e) {
+ throw new Exception(e)
+ }
+ }
+
+ @Test
+ public void testAssumeFalse() throws Throwable {
+
+ try {
+ assume(false, "false")
+ fail("expected an exception")
+ } catch (AssumptionViolatedException ignored) {
+ //expected
+ }
+ }
+
+ @Test
+ public void testAssumeBoolOptionSetInConf() throws Throwable {
+ Configuration conf = new Configuration(false)
+ conf.set("key", "true")
+ try {
+ assumeBoolOption(conf, "key", false)
+ } catch (AssumptionViolatedException e) {
+ throw new Exception(e)
+ }
+ }
+
+ @Test
+ public void testAssumeBoolOptionUnsetInConf() throws Throwable {
+ Configuration conf = new Configuration(false)
+ try {
+ assumeBoolOption(conf, "key", true)
+ } catch (AssumptionViolatedException e) {
+ throw new Exception(e)
+ }
+ }
+
+
+ @Test
+ public void testAssumeBoolOptionFalseInConf() throws Throwable {
+ Configuration conf = new Configuration(false)
+ conf.set("key", "false")
+ try {
+ assumeBoolOption(conf, "key", true)
+ fail("expected an exception")
+ } catch (AssumptionViolatedException ignored) {
+ //expected
+ }
+ }
+
+ @Test
+ public void testAssumeBoolOptionFalseUnsetInConf() throws Throwable {
+ Configuration conf = new Configuration(false)
+ try {
+ assumeBoolOption(conf, "key", false)
+ fail("expected an exception")
+ } catch (AssumptionViolatedException ignored) {
+ //expected
+ }
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
new file mode 100644
index 0000000..3770656
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.core.registry.zk.ZKIntegration
+import org.apache.slider.test.KeysForTests
+import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.apache.zookeeper.CreateMode
+import org.apache.zookeeper.ZooDefs
+import org.apache.zookeeper.data.Stat
+import org.junit.Before
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+
+class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTests {
+
+ @Before
+ void createCluster() {
+ Configuration conf = getConfiguration()
+ createMicroZKCluster(conf)
+ }
+
+ @Test
+ public void testIntegrationCreate() throws Throwable {
+ assertHasZKCluster()
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "cluster1", true, false, 5000)
+ String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ Stat stat = zki.stat(userPath)
+ assert stat != null
+ log.info("User path $userPath has stat $stat")
+ }
+
+ @Test
+ public void testListUserClustersWithoutAnyClusters() throws Throwable {
+ assertHasZKCluster()
+
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
+ String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ List<String> clusters = zki.clusters
+ assert clusters.empty
+ }
+
+ @Test
+ public void testListUserClustersWithOneCluster() throws Throwable {
+ assertHasZKCluster()
+
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
+ String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ String fullPath = zki.createPath(userPath, "/cluster-",
+ ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.EPHEMERAL_SEQUENTIAL)
+ log.info("Ephemeral path $fullPath")
+ List<String> clusters = zki.clusters
+ assert clusters.size() == 1
+ assert fullPath.endsWith(clusters[0])
+ }
+
+ @Test
+ public void testListUserClustersWithTwoCluster() throws Throwable {
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
+ String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ String c1 = createEphemeralChild(zki, userPath)
+ log.info("Ephemeral path $c1")
+ String c2 = createEphemeralChild(zki, userPath)
+ log.info("Ephemeral path $c2")
+ List<String> clusters = zki.clusters
+ assert clusters.size() == 2
+ assert (c1.endsWith(clusters[0]) && c1.endsWith(clusters[1])) ||
+ (c1.endsWith(clusters[1]) && c2.endsWith(clusters[0]))
+ }
+
+ public String createEphemeralChild(ZKIntegration zki, String userPath) {
+ return zki.createPath(userPath, "/cluster-",
+ ZooDefs.Ids.OPEN_ACL_UNSAFE,
+ CreateMode.EPHEMERAL_SEQUENTIAL)
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/core/conf/ExampleConfResources.groovy b/slider-core/src/test/groovy/org/apache/slider/core/conf/ExampleConfResources.groovy
new file mode 100644
index 0000000..b132aba
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/core/conf/ExampleConfResources.groovy
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf
+
+import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.providers.slideram.SliderAMClientProvider
+
+/*
+ names of the example configs
+ */
+
+class ExampleConfResources {
+
+ static final String overridden = "overridden.json"
+ static final String overriddenRes = "overridden-resolved.json"
+ static final String internal = "internal.json"
+ static final String internalRes = "internal-resolved.json"
+ static final String app_configuration = "app_configuration.json"
+ final
+ static String app_configurationRes = "app_configuration-resolved.json"
+ static final String resources = "resources.json"
+ static final String empty = "empty.json"
+
+ static final String PACKAGE = "/org/apache/slider/core/conf/examples/"
+
+
+ static
+ final String[] all_examples = [overridden, overriddenRes, internal, internalRes,
+ app_configuration, app_configurationRes, resources, empty];
+
+ static final List<String> all_example_resources = [];
+ static {
+ all_examples.each { all_example_resources << (PACKAGE + it) }
+
+ all_example_resources <<
+ SliderAMClientProvider.RESOURCES_JSON <<
+ SliderAMClientProvider.INTERNAL_JSON <<
+ SliderAMClientProvider.APPCONF_JSON
+
+ }
+
+ /**
+ * Build up an aggregate conf by loading in the details of the individual resources
+ * and then aggregating them
+ * @return a new instance
+ */
+ static AggregateConf loadExampleAggregateResource() {
+ JsonSerDeser<ConfTree> confTreeJsonSerDeser =
+ new JsonSerDeser<ConfTree>(ConfTree)
+ ConfTree internal = confTreeJsonSerDeser.fromResource(PACKAGE + internal)
+ ConfTree app_conf = confTreeJsonSerDeser.fromResource(PACKAGE + app_configuration)
+ ConfTree resources = confTreeJsonSerDeser.fromResource(PACKAGE + resources)
+ AggregateConf aggregateConf = new AggregateConf(
+ resources,
+ app_conf,
+ internal)
+ return aggregateConf;
+ }
+
+ static ConfTree loadResource(String name) {
+ JsonSerDeser<ConfTree> confTreeJsonSerDeser =
+ new JsonSerDeser<ConfTree>(ConfTree)
+ return confTreeJsonSerDeser.fromResource(PACKAGE + name)
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeLoadExamples.groovy b/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeLoadExamples.groovy
new file mode 100644
index 0000000..c69994d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeLoadExamples.groovy
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf
+
+import org.apache.slider.core.persist.JsonSerDeser
+import org.junit.Assert
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+
+
+/**
+ * Test
+ */
+@RunWith(value = Parameterized.class)
+class TestConfTreeLoadExamples extends Assert {
+
+ String resource;
+
+ static final JsonSerDeser<ConfTree> confTreeJsonSerDeser =
+ new JsonSerDeser<ConfTree>(ConfTree)
+
+ TestConfTreeLoadExamples(String resource) {
+ this.resource = resource
+ }
+
+ @Parameterized.Parameters
+ public static filenames() {
+ return ExampleConfResources.all_example_resources.collect { [it] as String[] }
+ }
+
+ @Test
+ public void testLoadResource() throws Throwable {
+ def confTree = confTreeJsonSerDeser.fromResource(resource)
+ ConfTreeOperations ops = new ConfTreeOperations(confTree)
+ ops.resolve()
+ ops.validate()
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy b/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy
new file mode 100644
index 0000000..b655be8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.conf
+
+import groovy.util.logging.Slf4j
+import org.junit.Assert
+import org.junit.Test
+
+import static org.apache.slider.core.conf.ExampleConfResources.overridden
+
+/**
+ * Test
+ */
+@Slf4j
+class TestConfTreeResolve extends Assert {
+ @Test
+ public void testOverride() throws Throwable {
+
+ def orig = ExampleConfResources.loadResource(overridden)
+
+ ConfTreeOperations origOperations = new ConfTreeOperations(orig)
+ origOperations.validate()
+
+
+ def global = origOperations.globalOptions
+ assert global["g1"] == "a"
+ assert global["g2"] == "b"
+
+ def simple = origOperations.getMandatoryComponent("simple")
+ assert simple.size() == 0
+
+ def master = origOperations.getMandatoryComponent("master")
+ assert master["name"] == "m"
+ assert master["g1"] == "overridden"
+
+ def worker = origOperations.getMandatoryComponent("worker")
+ log.info("worker = $worker")
+ assert worker.size() == 3
+
+ assert worker["name"] == "worker"
+ assert worker["g1"] == "overridden-by-worker"
+ assert worker["g2"] == null
+ assert worker["timeout"] == "1000"
+
+ // here is the resolution
+ origOperations.resolve()
+
+ global = origOperations.globalOptions
+ log.info("global = $global")
+ assert global["g1"] == "a"
+ assert global["g2"] == "b"
+
+ simple = origOperations.getMandatoryComponent("simple")
+ assert simple.size() == 2
+ simple.getMandatoryOption("g1")
+ assert simple["g1"]
+
+
+ master = origOperations.getMandatoryComponent("master")
+ log.info("master = $master")
+ assert master.size() == 3
+ assert master["name"] == "m"
+ assert master["g1"] == "overridden"
+ assert master["g2"] == "b"
+
+ worker = origOperations.getMandatoryComponent("worker")
+ log.info("worker = $worker")
+ assert worker.size() == 4
+
+ assert worker["name"] == "worker"
+ assert worker["g1"] == "overridden-by-worker"
+ assert worker["g2"] == "b"
+ assert worker["timeout"] == "1000"
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/core/persist/TestConfPersisterLocksHDFS.groovy b/slider-core/src/test/groovy/org/apache/slider/core/persist/TestConfPersisterLocksHDFS.groovy
new file mode 100644
index 0000000..1c013c4
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/core/persist/TestConfPersisterLocksHDFS.groovy
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.hdfs.MiniDFSCluster
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.tools.CoreFileSystem
+import org.apache.slider.test.YarnMiniClusterTestBase
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class TestConfPersisterLocksHDFS extends YarnMiniClusterTestBase {
+ static MiniDFSCluster hdfs
+ static private YarnConfiguration conf = new YarnConfiguration()
+ static CoreFileSystem coreFileSystem
+ static URI fsURI
+ static HadoopFS dfsClient
+
+ TestConfPersisterLocksHDFS() {
+
+ }
+
+ @BeforeClass
+ public static void createCluster() {
+ hdfs = buildMiniHDFSCluster(
+ "TestConfPersister",
+ conf)
+
+ fsURI = new URI(buildFsDefaultName(hdfs))
+ dfsClient = HadoopFS.get(fsURI, conf);
+ coreFileSystem = new CoreFileSystem(dfsClient, conf)
+ }
+
+ @AfterClass
+ public static void destroyCluster() {
+ hdfs?.shutdown()
+ hdfs = null;
+ }
+
+ /**
+ * Create the persister. This also creates the destination directory
+ * @param name name of cluster
+ * @return a conf persister
+ */
+ public ConfPersister createPersister(String name) {
+ def path = coreFileSystem.buildClusterDirPath(name);
+ ConfPersister persister = new ConfPersister(
+ coreFileSystem,
+ path)
+ coreFileSystem.getFileSystem().mkdirs(path)
+ return persister
+ }
+
+ @Test
+ public void testReleaseNonexistentWritelock() throws Exception {
+
+ ConfPersister persister = createPersister("testReleaseNonexistentWritelock")
+ assert !persister.releaseWritelock();
+ }
+
+
+ @Test
+ public void testAcqRelWriteLock() throws Throwable {
+ ConfPersister persister = createPersister("testAcqRelWriteLock")
+ persister.acquireWritelock();
+ assert persister.releaseWritelock();
+ assert !persister.releaseWritelock()
+ }
+
+ @Test
+ public void testSecondWriteLockAcqFails() throws Throwable {
+ ConfPersister persister = createPersister("testSecondWriteLockAcqFails")
+ persister.acquireWritelock();
+ try {
+ persister.acquireWritelock();
+ fail "write lock acquired twice"
+ } catch (LockAcquireFailedException lafe) {
+ //expected
+ assert lafe.path.toString().endsWith(Filenames.WRITELOCK)
+ }
+ assert persister.releaseWritelock();
+
+ //now we can ask for it
+ persister.acquireWritelock();
+ }
+
+ @Test
+ public void testReleaseNonexistentReadlockOwner() throws Exception {
+ ConfPersister persister = createPersister("testReleaseNonexistentReadlock")
+ assert !persister.releaseReadlock(true);
+ }
+
+ @Test
+ public void testReleaseNonexistentReadlock() throws Exception {
+ ConfPersister persister = createPersister("testReleaseNonexistentReadlock")
+ assert !persister.releaseReadlock(false)
+ }
+
+ @Test
+ public void testAcqRelReadlock() throws Exception {
+ ConfPersister persister = createPersister("testAcqRelReadlock")
+ assert persister.acquireReadLock();
+ assert persister.readLockExists();
+
+ assert !persister.releaseReadlock(false);
+ assert persister.readLockExists();
+ assert persister.releaseReadlock(true);
+ }
+
+ @Test
+ public void testAcqAcqRelReadlock() throws Exception {
+ ConfPersister persister = createPersister("testAcqRelReadlock")
+ assert persister.acquireReadLock();
+ assert persister.readLockExists();
+ assert !persister.acquireReadLock();
+ assert persister.readLockExists();
+
+ assert !persister.releaseReadlock(false);
+ assert persister.readLockExists();
+ assert persister.releaseReadlock(true);
+ assert !persister.readLockExists();
+ }
+
+ @Test
+ public void testAcqAcqRelReadlockOtherOrderOfRelease() throws Exception {
+ ConfPersister persister = createPersister("testAcqRelReadlock")
+ assert persister.acquireReadLock();
+ assert persister.readLockExists();
+ assert !persister.acquireReadLock();
+ assert persister.readLockExists();
+
+ assert persister.releaseReadlock(true);
+ assert !persister.readLockExists();
+ assert !persister.releaseReadlock(false)
+
+ }
+
+
+ @Test
+ public void testNoReadlockWhenWriteHeld() throws Throwable {
+ ConfPersister persister = createPersister("testNoReadlockWhenWriteHeld")
+ persister.acquireWritelock();
+ try {
+ persister.acquireReadLock();
+ fail "read lock acquired"
+ } catch (LockAcquireFailedException lafe) {
+ //expected
+ assertWritelockBlocked(lafe)
+ }
+ assert persister.releaseWritelock();
+ assert !persister.writelockExists();
+
+ //now we can ask for it
+ persister.acquireReadLock();
+ }
+
+ public void assertWritelockBlocked(LockAcquireFailedException lafe) {
+ assert lafe.path.toString().endsWith(Filenames.WRITELOCK)
+ }
+
+ public void assertReadlockBlocked(LockAcquireFailedException lafe) {
+ assert lafe.path.toString().endsWith(Filenames.READLOCK)
+ }
+
+ @Test
+ public void testNoWritelockWhenReadHeld() throws Throwable {
+ ConfPersister persister = createPersister("testNoWritelockWhenReadHeld")
+ assert persister.acquireReadLock();
+ try {
+ persister.acquireWritelock();
+ fail "write lock acquired"
+ } catch (LockAcquireFailedException lafe) {
+ //expected
+ assertReadlockBlocked(lafe)
+ }
+ assert persister.releaseReadlock(true);
+
+ //now we can ask for it
+ persister.acquireWritelock();
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/core/persist/TestConfPersisterReadWrite.groovy b/slider-core/src/test/groovy/org/apache/slider/core/persist/TestConfPersisterReadWrite.groovy
new file mode 100644
index 0000000..389c7f6
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/core/persist/TestConfPersisterReadWrite.groovy
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.tools.CoreFileSystem
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.core.conf.ExampleConfResources
+import org.apache.slider.test.YarnMiniClusterTestBase
+import org.junit.BeforeClass
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class TestConfPersisterReadWrite extends YarnMiniClusterTestBase {
+ static private YarnConfiguration conf = new YarnConfiguration()
+ static CoreFileSystem coreFileSystem
+ static URI fsURI
+ static HadoopFS dfsClient
+ static final JsonSerDeser<ConfTree> confTreeJsonSerDeser =
+ new JsonSerDeser<ConfTree>(ConfTree)
+ AggregateConf aggregateConf = ExampleConfResources.loadExampleAggregateResource()
+
+
+ TestConfPersisterReadWrite() {
+
+ }
+
+ @BeforeClass
+ public static void createCluster() {
+ fsURI = new URI(buildFsDefaultName(null))
+ dfsClient = HadoopFS.get(fsURI, conf);
+ coreFileSystem = new CoreFileSystem(dfsClient, conf)
+ }
+
+ /**
+ * Create the persister. This also creates the destination directory
+ * @param name name of cluster
+ * @return a conf persister
+ */
+ public ConfPersister createPersister(String name) {
+ def path = coreFileSystem.buildClusterDirPath(name);
+ ConfPersister persister = new ConfPersister(
+ coreFileSystem,
+ path)
+ coreFileSystem.getFileSystem().mkdirs(path)
+ return persister
+ }
+
+ @Test
+ public void testSaveLoadEmptyConf() throws Throwable {
+ AggregateConf aggregateConf = new AggregateConf()
+
+ def persister = createPersister("testSaveLoad")
+ persister.save(aggregateConf, null)
+ AggregateConf loaded = new AggregateConf()
+ persister.load(loaded)
+ loaded.validate()
+ }
+
+
+ @Test
+ public void testSaveLoadTestConf() throws Throwable {
+ def persister = createPersister("testSaveLoadTestConf")
+ persister.save(aggregateConf, null)
+ AggregateConf loaded = new AggregateConf()
+ persister.load(loaded)
+ loaded.validate()
+ }
+
+
+
+ @Test
+ public void testSaveLoadTestConfResolveAndCheck() throws Throwable {
+ def appConfOperations = aggregateConf.getAppConfOperations()
+ appConfOperations.getMandatoryComponent("master")["PATH"]="."
+ def persister = createPersister("testSaveLoadTestConf")
+ persister.save(aggregateConf, null)
+ AggregateConf loaded = new AggregateConf()
+ persister.load(loaded)
+ loaded.validate()
+ loaded.resolve();
+ def resources = loaded.getResourceOperations()
+ def master = resources.getMandatoryComponent("master")
+ assert master["yarn.memory"] == "1024"
+
+ def appConfOperations2 = loaded.getAppConfOperations()
+ assert appConfOperations2.getMandatoryComponent("master")["PATH"] == "."
+
+ }
+
+ @Test
+ public void testSaveFailsIfWritelocked() throws Throwable {
+ def persister = createPersister("testSaveFailsIfWritelocked")
+ persister.releaseWritelock()
+ persister.acquireWritelock()
+ try {
+ expectSaveToFailOnLock(persister, aggregateConf)
+ } finally {
+ persister.releaseWritelock()
+ }
+ }
+
+ @Test
+ public void testSaveFailsIfReadlocked() throws Throwable {
+ def persister = createPersister("testSaveFailsIfReadlocked")
+ persister.releaseWritelock()
+ persister.acquireReadLock()
+ try {
+ expectSaveToFailOnLock(persister, aggregateConf)
+ } finally {
+ persister.releaseReadlock(true)
+ }
+ }
+
+ @Test
+ public void testLoadFailsIfWritelocked() throws Throwable {
+ def persister = createPersister("testLoadFailsIfWritelocked")
+ persister.acquireWritelock()
+ try {
+ expectLoadToFailOnLock(persister, aggregateConf)
+ } finally {
+ persister.releaseWritelock()
+ }
+ }
+
+ @Test
+ public void testLoadFailsIfDestDoesNotExist() throws Throwable {
+ def persister = createPersister("testLoadFailsIfDestDoesNotExist")
+ try {
+ persister.load(aggregateConf)
+ fail "expected save to fail to find a file"
+ } catch (FileNotFoundException e) {
+ //expected
+ }
+ }
+
+ @Test
+ public void testLoadSucceedsIfReadlocked() throws Throwable {
+ def persister = createPersister("testLoadSucceedsIfReadlocked")
+ persister.releaseReadlock(true)
+ try {
+ persister.save(aggregateConf, null)
+ persister.acquireReadLock()
+ AggregateConf loaded = new AggregateConf()
+ persister.load(loaded)
+ loaded.validate()
+ loaded.resolve()
+ } finally {
+ persister.releaseReadlock(true)
+ }
+ }
+
+ public void expectSaveToFailOnLock(
+ ConfPersister persister,
+ AggregateConf aggregateConf) {
+ try {
+ persister.save(aggregateConf, null)
+ fail "expected save to fail to get a lock"
+ } catch (LockAcquireFailedException lafe) {
+ //expected
+ }
+ }
+
+
+ public void expectLoadToFailOnLock(
+ ConfPersister persister,
+ AggregateConf aggregateConf) {
+ try {
+ persister.load(aggregateConf)
+ fail "expected save to fail to get a lock"
+ } catch (LockAcquireFailedException lafe) {
+ //expected
+ }
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/TestProviderFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/TestProviderFactory.groovy
new file mode 100644
index 0000000..5a6ebb0
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/TestProviderFactory.groovy
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.agent.AgentKeys
+import org.apache.slider.providers.agent.AgentProviderFactory
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestProviderFactory {
+
+
+ @Test
+ public void testLoadAgentProvider() throws Throwable {
+ SliderProviderFactory factory = SliderProviderFactory.createSliderProviderFactory(AgentKeys.PROVIDER_AGENT);
+ assert factory instanceof AgentProviderFactory
+ }
+
+ @Test
+ public void testCreateClientProvider() throws Throwable {
+ SliderProviderFactory factory = SliderProviderFactory.createSliderProviderFactory(
+ AgentKeys.PROVIDER_AGENT);
+ assert null != factory.createClientProvider();
+ }
+
+ @Test
+ public void testCreateHBaseProvider() throws Throwable {
+ SliderProviderFactory factory = SliderProviderFactory.createSliderProviderFactory(
+ AgentKeys.PROVIDER_AGENT);
+ assert null != factory.createServerProvider();
+ }
+
+ @Test
+ public void testCreateProviderByClassname() throws Throwable {
+ SliderProviderFactory factory = SliderProviderFactory.createSliderProviderFactory(
+ AgentProviderFactory.CLASSNAME);
+ assert null != factory.createServerProvider();
+ assert factory instanceof AgentProviderFactory
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/TestProviderUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/TestProviderUtils.groovy
new file mode 100644
index 0000000..5133f07
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/TestProviderUtils.groovy
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.test.SliderTestBase
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+
+class TestProviderUtils extends SliderTestBase {
+
+ ProviderUtils providerUtils = new ProviderUtils(log)
+
+ File archive = new File("target/testProviderUtils/tar")
+ File hbase = new File(archive,"hbase-0.96.0")
+ File bin = new File(hbase,"bin")
+ File script = new File(bin, "hbase")
+
+ @Before
+ public void setup() {
+ archive.mkdirs()
+ bin.mkdirs()
+ script.withPrintWriter { PrintWriter out ->
+ out.println("Hello, world")
+ }
+ }
+
+ @Test
+ public void testScriptExists() throws Throwable {
+ assert script.exists()
+ assert script.file
+ }
+ @Test
+ public void testFullSearch() throws Throwable {
+ File sh = providerUtils.findBinScriptInExpandedArchive(archive,"bin","hbase")
+ assert sh == script
+ }
+
+
+ @Test
+ public void testFailScriptMissing() throws Throwable {
+
+ File sh = providerUtils.findBinScriptInExpandedArchive(archive,"bin","hbase")
+ assert sh == script
+ }
+
+ @Test
+ public void testAdditionalArgs() {
+ final String extraArgs = "--address 0.0.0.0";
+ Map<String,String> roleOptions = [ (RoleKeys.ROLE_NAME):"foo",
+ (RoleKeys.ROLE_ADDITIONAL_ARGS):(extraArgs)
+ ];
+
+ String actualExtraArgs = ProviderUtils.getAdditionalArgs(roleOptions);
+
+ assert extraArgs == actualExtraArgs;
+ }
+
+ @Test
+ public void testUndefinedAdditionalArgs() {
+ Map<String,String> roleOptions = [ (RoleKeys.ROLE_NAME):"foo",
+ ("newkey"):"1",
+ ];
+
+ String actualExtraArgs = ProviderUtils.getAdditionalArgs(roleOptions);
+
+ assert "" == actualExtraArgs;
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
new file mode 100644
index 0000000..b6d0c6c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.test.YarnZKMiniClusterTestBase
+
+import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
+import static org.apache.slider.providers.agent.AgentKeys.CONF_RESOURCE
+
+/**
+ * test base for all agent clusters
+ */
+@CompileStatic
+@Slf4j
+public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
+
+ public static
+ final int AGENT_CLUSTER_STARTUP_TIME = 1000 * DEFAULT_AGENT_LAUNCH_TIME_SECONDS
+
+ /**
+ * The time to sleep before trying to talk to the HBase Master and
+ * expect meaningful results.
+ */
+ public static
+ final int AGENT_CLUSTER_STARTUP_TO_LIVE_TIME = AGENT_CLUSTER_STARTUP_TIME
+ public static final int AGENT_GO_LIVE_TIME = 60000
+
+
+ @Override
+ public String getTestConfigurationPath() {
+ return "src/main/resources/" + CONF_RESOURCE;
+ }
+
+ @Override
+ void setup() {
+ super.setup()
+ YarnConfiguration conf = testConfiguration
+ checkTestAssumptions(conf)
+ }
+
+ @Override
+ public String getArchiveKey() {
+ return KEY_TEST_AGENT_TAR
+ }
+
+ /**
+ * Get the key for the application
+ * @return
+ */
+ @Override
+ public String getApplicationHomeKey() {
+ return KEY_TEST_AGENT_HOME
+ }
+
+ /**
+ * Assume that HBase home is defined. This does not check that the
+ * path is valid -that is expected to be a failure on tests that require
+ * HBase home to be set.
+ */
+
+ public void checkTestAssumptions(YarnConfiguration conf) {
+ assumeBoolOption(SLIDER_CONFIG, KEY_TEST_AGENT_ENABLED, true)
+// assumeArchiveDefined();
+ assumeApplicationHome();
+ }
+
+ /**
+ * Create an agent cluster
+ * @param clustername
+ * @param roles
+ * @param extraArgs
+ * @param deleteExistingData
+ * @param blockUntilRunning
+ * @return the cluster launcher
+ */
+ public ServiceLauncher<SliderClient> buildAgentCluster(
+ String clustername,
+ Map<String, Integer> roles,
+ List<String> extraArgs,
+ boolean deleteExistingData,
+ boolean create,
+ boolean blockUntilRunning) {
+
+
+ YarnConfiguration conf = testConfiguration
+
+ def clusterOps = [
+ :
+ ]
+
+ return createOrBuildCluster(
+ create ? SliderActions.ACTION_CREATE : SliderActions.ACTION_BUILD,
+ clustername,
+ roles,
+ extraArgs,
+ deleteExistingData,
+ create && blockUntilRunning,
+ clusterOps)
+ }
+
+ public String getApplicationHome() {
+ return "/"
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestUtils.groovy
new file mode 100644
index 0000000..5888557
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestUtils.groovy
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent
+
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.config.ClientConfig
+import com.sun.jersey.api.client.config.DefaultClientConfig
+import com.sun.jersey.api.json.JSONConfiguration
+import org.apache.slider.server.appmaster.web.rest.agent.Register
+import org.codehaus.jettison.json.JSONException
+import org.codehaus.jettison.json.JSONObject
+
+class AgentTestUtils {
+
+ public static Client createTestClient() {
+ ClientConfig clientConfig = new DefaultClientConfig();
+ clientConfig.getFeatures().put(
+ JSONConfiguration.FEATURE_POJO_MAPPING,
+ Boolean.TRUE);
+ return Client.create(clientConfig);
+ }
+
+
+ public static Register createDummyJSONRegister() throws JSONException {
+ Register register = new Register();
+ register.setResponseId(-1);
+ register.setTimestamp(System.currentTimeMillis());
+ register.setHostname("dummyHost");
+ return register;
+ }
+
+ public static JSONObject createDummyHeartBeat() throws JSONException {
+ JSONObject json = new JSONObject();
+ json.put("responseId", -1);
+ json.put("timestamp", System.currentTimeMillis());
+ json.put("hostname", "dummyHost");
+ return json;
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
new file mode 100644
index 0000000..48cb2dd
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent
+
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.WebResource
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.server.appmaster.web.SliderAMWebApp
+import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse
+import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus
+import org.junit.Test
+
+import javax.ws.rs.core.MediaType
+
+import static org.apache.slider.common.params.Arguments.ARG_OPTION
+import static org.apache.slider.providers.agent.AgentKeys.*
+import static org.apache.slider.providers.agent.AgentTestUtils.createDummyJSONRegister
+import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient
+import static org.apache.slider.test.SliderTestUtils.log
+
+@CompileStatic
+@Slf4j
+class TestAgentAMManagementWS extends AgentTestBase {
+
+ public static final String MANAGEMENT_URI = SliderAMWebApp.BASE_PATH +"/ws/v1/slider/mgmt/";
+ public static final String AGENT_URI = "ws/v1/slider/agents/";
+
+ @Test
+ public void testAgentAMManagementWS() throws Throwable {
+ def clustername = "test_agentammanagementws"
+ createMiniCluster(
+ clustername,
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+ Map<String, Integer> roles = [:]
+ File slider_core = new File(new File(".").absoluteFile, "src/test/python");
+ String app_def = "appdef_1.tar"
+ File app_def_path = new File(slider_core, app_def)
+ String agt_ver = "version"
+ File agt_ver_path = new File(slider_core, agt_ver)
+ String agt_conf = "agent.ini"
+ File agt_conf_path = new File(slider_core, agt_conf)
+ assert app_def_path.exists()
+ assert agt_ver_path.exists()
+ assert agt_conf_path.exists()
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ roles,
+ [
+ ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
+ ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ ],
+ true, true,
+ true)
+ SliderClient sliderClient = launcher.service
+ def report = waitForClusterLive(sliderClient)
+ def trackingUrl = report.trackingUrl
+ log.info("tracking URL is $trackingUrl")
+ def agent_url = trackingUrl + AGENT_URI
+
+
+ def status = dumpClusterStatus(sliderClient, "agent AM")
+ def liveURL = status.getInfo(StatusKeys.INFO_AM_WEB_URL)
+ if (liveURL) {
+ agent_url = liveURL + AGENT_URI
+ }
+
+ log.info("Agent is $agent_url")
+ log.info("stacks is ${liveURL}stacks")
+ log.info("conf is ${liveURL}conf")
+
+
+ def sleeptime = 60
+ log.info "sleeping for $sleeptime seconds"
+ Thread.sleep(sleeptime * 1000)
+
+
+ String page = fetchWebPageWithoutError(agent_url);
+ log.info(page);
+
+ //WS get
+ Client client = createTestClient();
+
+
+ WebResource webResource = client.resource(agent_url + "test/register");
+ RegistrationResponse response = webResource.type(MediaType.APPLICATION_JSON)
+ .post(
+ RegistrationResponse.class,
+ createDummyJSONRegister());
+
+ //TODO: assert failure as actual agent is not started. This test only starts the AM.
+ assert RegistrationStatus.FAILED == response.getResponseStatus();
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
new file mode 100644
index 0000000..908a5ff
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+import static org.apache.slider.common.params.Arguments.*
+import static org.apache.slider.providers.agent.AgentKeys.*
+
+/**
+ * Tests an echo command
+ */
+@CompileStatic
+@Slf4j
+class TestAgentEcho extends AgentTestBase {
+
+
+ @Override
+ void checkTestAssumptions(YarnConfiguration conf) {
+
+ }
+
+ @Test
+ public void testEchoOperation() throws Throwable {
+ def clustername = "test_agent_echo"
+ createMiniCluster(
+ clustername,
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ File slider_core = new File(new File(".").absoluteFile, "src/test/python");
+ String echo_py = "echo.py"
+ File echo_py_path = new File(slider_core, echo_py)
+ String app_def = "appdef_1.tar"
+ File app_def_path = new File(slider_core, app_def)
+ String agt_ver = "version"
+ File agt_ver_path = new File(slider_core, agt_ver)
+ String agt_conf = "agent.ini"
+ File agt_conf_path = new File(slider_core, agt_conf)
+ assert echo_py_path.exists()
+ assert app_def_path.exists()
+ assert agt_ver_path.exists()
+ assert agt_conf_path.exists()
+
+ def role = "echo"
+ Map<String, Integer> roles = [
+ (role): 1,
+ ];
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ roles,
+ [
+ ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
+ ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
+ ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
+ ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
+ ],
+ true, true,
+ true)
+ SliderClient sliderClient = launcher.service
+
+ waitForRoleCount(sliderClient, roles, AGENT_CLUSTER_STARTUP_TIME)
+ //sleep a bit
+ sleep(20000)
+ //expect the role count to be the same
+ waitForRoleCount(sliderClient, roles, 1000)
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
new file mode 100644
index 0000000..f1c1493
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -0,0 +1,443 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.exceptions.BadConfigException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.ConfPersister
+import org.junit.Test
+
+import static org.apache.slider.common.params.Arguments.*
+import static org.apache.slider.providers.agent.AgentKeys.*
+
+@CompileStatic
+@Slf4j
+class TestBuildBasicAgent extends AgentTestBase {
+ static String TEST_FILES = "./src/test/resources/org/apache/slider/providers/agent/tests/"
+
+ @Override
+ void checkTestAssumptions(YarnConfiguration conf) {
+
+ }
+
+ private static class TestResources {
+ static File slider_core = new File(new File(".").absoluteFile, "src/test/python");
+ static String app_def = "appdef_1.tar"
+ static File app_def_path = new File(slider_core, app_def)
+ static String agt_conf = "agent.ini"
+ static File agt_conf_path = new File(slider_core, agt_conf)
+
+ static public File getAppDef() {
+ return app_def_path;
+ }
+
+ static public File getAgentConf() {
+ return agt_conf_path;
+ }
+
+ static public File getAgentImg() {
+ return app_def_path;
+ }
+ }
+
+ @Test
+ public void testBuildMultipleRoles() throws Throwable {
+
+ def clustername = "test_build_basic_agent"
+ createMiniCluster(
+ clustername,
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+ buildAgentCluster("test_build_basic_agent_node_only",
+ [(ROLE_NODE): 5],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_OPTION, SCRIPT_PATH, "agent/scripts/agent.py",
+ ARG_COMP_OPT, ROLE_NODE, SCRIPT_PATH, "agent/scripts/agent.py",
+ ARG_RES_COMP_OPT, ROLE_NODE, ResourceKeys.COMPONENT_PRIORITY, "1",
+ ],
+ true, false,
+ false)
+
+ def master = "hbase-master"
+ def rs = "hbase-rs"
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ [
+ (ROLE_NODE): 5,
+ (master): 1,
+ (rs): 5
+ ],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_OPTION, PACKAGE_PATH, ".",
+ ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_COMP_OPT, master, SCRIPT_PATH, "agent/scripts/agent.py",
+ ARG_COMP_OPT, rs, SCRIPT_PATH, "agent/scripts/agent.py",
+ ARG_RES_COMP_OPT, master, ResourceKeys.COMPONENT_PRIORITY, "2",
+ ARG_RES_COMP_OPT, rs, ResourceKeys.COMPONENT_PRIORITY, "3",
+ ARG_COMP_OPT, master, SERVICE_NAME, "HBASE",
+ ARG_COMP_OPT, rs, SERVICE_NAME, "HBASE",
+ ARG_COMP_OPT, master, AgentKeys.APP_HOME, "/share/hbase/hbase-0.96.1-hadoop2",
+ ARG_COMP_OPT, rs, AgentKeys.APP_HOME, "/share/hbase/hbase-0.96.1-hadoop2",
+ ARG_COMP_OPT, ROLE_NODE, SCRIPT_PATH, "agent/scripts/agent.py",
+ ARG_RES_COMP_OPT, ROLE_NODE, ResourceKeys.COMPONENT_PRIORITY, "1",
+ ],
+ true, false,
+ false)
+ def instanceD = launcher.service.loadPersistedClusterDescription(
+ clustername)
+ dumpClusterDescription("$clustername:", instanceD)
+ def resource = instanceD.getResourceOperations()
+
+
+ def agentComponent = resource.getMandatoryComponent(ROLE_NODE)
+ agentComponent.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY)
+
+ def masterC = resource.getMandatoryComponent(master)
+ assert "2" == masterC.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY)
+
+ def rscomponent = resource.getMandatoryComponent(rs)
+ assert "5" == rscomponent.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES)
+
+ // now create an instance with no role priority for the newnode role
+ try {
+ def name2 = clustername + "-2"
+ buildAgentCluster(name2,
+ [
+ (ROLE_NODE): 5,
+ "role3": 1,
+ "newnode": 5
+ ],
+ [
+ ARG_COMP_OPT, ROLE_NODE, SERVICE_NAME, "HBASE",
+ ARG_COMP_OPT, "role3", SERVICE_NAME, "HBASE",
+ ARG_COMP_OPT, "newnode", SERVICE_NAME, "HBASE",
+ ARG_RES_COMP_OPT, "role3", ResourceKeys.COMPONENT_PRIORITY, "2",
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(name2, "no priority for one role")
+ } catch (BadConfigException expected) {
+ }
+
+ //duplicate priorities
+ try {
+ def name3 = clustername + "-3"
+ buildAgentCluster(name3,
+ [
+ (ROLE_NODE): 5,
+ (master): 1,
+ (rs): 5
+ ],
+ [
+ ARG_RES_COMP_OPT, master, ResourceKeys.COMPONENT_PRIORITY, "2",
+ ARG_RES_COMP_OPT, rs, ResourceKeys.COMPONENT_PRIORITY, "2"
+ ],
+
+ true, false,
+ false)
+ failWithBuildSucceeding(name3, "duplicate priorities")
+ } catch (BadConfigException expected) {
+ }
+
+
+
+ def cluster4 = clustername + "-4"
+
+ def jvmopts = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
+ buildAgentCluster(cluster4,
+ [
+ (master): 1,
+ (rs): 5
+ ],
+ [
+ ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_PACKAGE, ".",
+ ARG_COMP_OPT, SliderKeys.COMPONENT_AM, RoleKeys.JVM_OPTS, jvmopts,
+ ARG_COMP_OPT, master, RoleKeys.JVM_OPTS, jvmopts,
+ ARG_COMP_OPT, rs, RoleKeys.JVM_OPTS, jvmopts,
+ ARG_RES_COMP_OPT, master, ResourceKeys.COMPONENT_PRIORITY, "2",
+ ARG_RES_COMP_OPT, rs, ResourceKeys.COMPONENT_PRIORITY, "3",
+ ],
+
+ true, false,
+ false)
+
+ //now we want to look at the value
+ AggregateConf instanceDefinition = loadInstanceDefinition(cluster4)
+ def opt = instanceDefinition.getAppConfOperations().getComponentOpt(
+ SliderKeys.COMPONENT_AM,
+ RoleKeys.JVM_OPTS,
+ "")
+
+ assert jvmopts == opt
+
+ // now create an instance with no component options, hence no
+ // entry in the app config
+ def name5 = clustername + "-5"
+ buildAgentCluster(name5,
+ [
+ "role": 1,
+ ],
+ [
+ ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_PACKAGE, ".",
+ ARG_RES_COMP_OPT, "role", ResourceKeys.COMPONENT_PRIORITY, "3",
+ ],
+ true, false,
+ false)
+ }
+
+ public AggregateConf loadInstanceDefinition(String name) {
+ def cluster4
+ def sliderFS = createSliderFileSystem()
+ def dirPath = sliderFS.buildClusterDirPath(name)
+ ConfPersister persister = new ConfPersister(sliderFS, dirPath)
+ AggregateConf instanceDefinition = new AggregateConf();
+ persister.load(instanceDefinition)
+ return instanceDefinition
+ }
+
+ @Test
+ public void testBadAgentArgs() throws Throwable {
+ def clustername = "test_bad_template_args"
+ createMiniCluster(
+ clustername,
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ try {
+ def badArgs1 = "test_bad_agent_args-1"
+ buildAgentCluster(clustername,
+ [:],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_RESOURCES, TEST_FILES + "good/resources.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(badArgs1, "missing package home or image path")
+ } catch (BadConfigException expected) {
+
+ }
+
+ try {
+ def badArgs1 = "test_bad_agent_args-2"
+ buildAgentCluster(clustername,
+ [:],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_IMAGE, "file://" + TestResources.getAgentImg().absolutePath + ".badfile",
+ ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_RESOURCES, TEST_FILES + "good/resources.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(badArgs1, "bad image path")
+ } catch (BadConfigException expected) {
+ }
+
+ try {
+ def badArgs1 = "test_bad_agent_args-3"
+ buildAgentCluster(clustername,
+ [:],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_RESOURCES, TEST_FILES + "good/resources.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(badArgs1, "bad app def file")
+ } catch (BadConfigException expected) {
+ }
+
+ try {
+ def badArgs1 = "test_bad_agent_args-5"
+ buildAgentCluster(clustername,
+ [:],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
+ ARG_RESOURCES, TEST_FILES + "good/resources.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(badArgs1, "bad agent conf file")
+ } catch (BadConfigException expected) {
+ }
+ }
+
+ @Test
+ public void testTemplateArgs() throws Throwable {
+
+
+ def clustername = "test_build_template_args"
+ createMiniCluster(
+ clustername,
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+ buildAgentCluster("test_build_template_args_good",
+ [:],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_PACKAGE, ".",
+ ARG_RESOURCES, TEST_FILES + "good/resources.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
+ ],
+ true, false,
+ false)
+ }
+
+
+ @Test
+ public void testBadTemplates() throws Throwable {
+
+
+ def clustername = "test_bad_template_args"
+ createMiniCluster(
+ clustername,
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ try {
+
+
+ def badArgs1 = "test_build_template_args_bad-1"
+ buildAgentCluster(badArgs1,
+ [:],
+ [
+
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_RESOURCES, TEST_FILES + "bad/appconf-1.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(badArgs1, "bad resource template")
+ } catch (BadConfigException expected) {
+ }
+
+ try {
+
+ def bad2 = "test_build_template_args_bad-2"
+ buildAgentCluster(bad2,
+ [:],
+ [
+
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_TEMPLATE, TEST_FILES + "bad/appconf-1.json",
+ ],
+ true, false,
+ false)
+
+ failWithBuildSucceeding(bad2, "a bad app conf")
+ } catch (BadConfigException expected) {
+ }
+
+ try {
+
+ def bad3 = "test_build_template_args_bad-3"
+ buildAgentCluster(bad3,
+ [:],
+ [
+
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_TEMPLATE, "unknown.json",
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(bad3, "missing template file")
+ } catch (BadConfigException expected) {
+ }
+
+
+ try {
+
+ def bad4 = "test_build_template_args_bad-4"
+ buildAgentCluster(bad4,
+ [:],
+ [
+
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_TEMPLATE, TEST_FILES + "bad/appconf-2.json",
+ ],
+ true, false,
+ false)
+
+ failWithBuildSucceeding(bad4, "Unparseable JSON")
+ } catch (BadConfigException expected) {
+ }
+
+ }
+
+ public void failWithBuildSucceeding(String name, String reason) {
+ def badArgs1
+ AggregateConf instanceDefinition = loadInstanceDefinition(name)
+ log.error(
+ "Build operation should have failed from $reason : \n$instanceDefinition")
+ fail("Build operation should have failed from $reason")
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy
new file mode 100644
index 0000000..3af4a06
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.registry.curator
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.server.services.curator.CuratorHelper
+import org.apache.slider.server.services.curator.RegistryBinderService
+import org.apache.slider.server.services.curator.RegistryNaming
+import org.apache.slider.test.MicroZKCluster
+import org.apache.slider.test.SliderTestUtils
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+class TestLocalRegistry {
+ MicroZKCluster miniZK
+ RegistryBinderService<ServiceInstanceData> registryBinder
+
+ @Before
+ void setup() {
+ miniZK = new MicroZKCluster()
+ miniZK.createCluster()
+
+ registryBinder = createRegistry()
+ }
+
+ def RegistryBinderService<ServiceInstanceData> createRegistry(
+ ) {
+ def conf = new YarnConfiguration()
+ CuratorHelper curatorHelper =
+ new CuratorHelper(conf, miniZK.zkBindingString);
+
+ def registry
+ registry = curatorHelper.createRegistryBinderService("/services");
+ registry.init(conf)
+ registry.start()
+ return registry
+ }
+
+ @After
+ void teardown() {
+ registryBinder?.stop()
+ miniZK?.close()
+ }
+
+
+ @Test
+ public void testRegisterAndQuery() throws Throwable {
+ registryBinder.register(SliderKeys.APP_TYPE, "instance3",
+ new URL("http", "localhost", 80, "/"),
+ null)
+ def instance = registryBinder.queryForInstance(
+ SliderKeys.APP_TYPE,
+ "instance3")
+ assert instance != null
+ }
+
+ @Test
+ public void testRegisterAndList() throws Throwable {
+ registryBinder.register(SliderKeys.APP_TYPE, "instance3",
+ new URL("http", "localhost", 80, "/"),
+ null)
+ registryBinder.register(SliderKeys.APP_TYPE, "instance2",
+ new URL("http", "localhost", 8090, "/"),
+ null)
+ def instances = registryBinder.instanceIDs(SliderKeys.APP_TYPE)
+ assert instances.size() ==2
+ def instance = registryBinder.queryForInstance(
+ SliderKeys.APP_TYPE,
+ "instance3")
+ assert instance != null
+ }
+
+ @Test
+ public void testMultipleRegistryBinders() throws Throwable {
+ registryBinder.register(SliderKeys.APP_TYPE, "instance3",
+ new URL("http", "localhost", 80, "/"),
+ null)
+ registryBinder.register(SliderKeys.APP_TYPE, "instance2",
+ new URL("http", "localhost", 8090, "/"),
+ null)
+ RegistryBinderService<ServiceInstanceData> registry2 = createRegistry()
+ RegistryBinderService<ServiceInstanceData> registry3 = createRegistry()
+ try {
+ def instances = registry3.instanceIDs(SliderKeys.APP_TYPE)
+ assert instances.size() == 2
+ def instance = registryBinder.queryForInstance(
+ SliderKeys.APP_TYPE,
+ "instance3")
+ assert instance.id == "instance3"
+ assert instance.name == SliderKeys.APP_TYPE
+ } finally {
+ registry3.stop()
+ registry2.stop()
+
+ }
+ }
+
+ @Test
+ public void testNamingPolicy() throws Throwable {
+
+ String hobbitName = RegistryNaming.createRegistryName("hobbiton",
+ "bilbo",
+ SliderKeys.APP_TYPE);
+ String hobbitId =
+ RegistryNaming.createUniqueInstanceId(
+ "hobbiton",
+ "bilbo",
+ SliderKeys.APP_TYPE,
+ 1);
+ String mordorName = RegistryNaming.createRegistryName("mordor",
+ "bilbo",
+ SliderKeys.APP_TYPE);
+ String mordorId =
+ RegistryNaming.createUniqueInstanceId(
+ "mordor",
+ "bilbo",
+ SliderKeys.APP_TYPE,
+ 1);
+
+ // service have same name
+ assert hobbitName == mordorName;
+ assert mordorId != hobbitId;
+ registryBinder.register(mordorName, mordorId,
+ new URL("http", "localhost", 8090, "/"),
+ new ServiceInstanceData())
+ registryBinder.register(hobbitName, hobbitId,
+ new URL("http", "localhost", 80, "/mordor"),
+ new ServiceInstanceData())
+ def mordorInstance = registryBinder.queryForInstance(
+ mordorName,
+ mordorId)
+ assert mordorInstance.port == 8090
+
+ def instances = registryBinder.listInstances(SliderKeys.APP_TYPE);
+ SliderTestUtils.dumpRegistryInstances(instances)
+ assert instances.size() == 2
+
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
new file mode 100644
index 0000000..1a7ee5a
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.registry.curator
+
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.ClientResponse
+import com.sun.jersey.api.client.UniformInterfaceException
+import com.sun.jersey.api.client.WebResource
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.curator.x.discovery.ServiceType
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.providers.agent.AgentTestBase
+import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.apache.slider.server.services.curator.CuratorServiceInstance
+import org.apache.slider.server.services.curator.CuratorServiceInstances
+import org.apache.slider.server.services.curator.RegistryNaming
+import org.junit.Test
+
+import javax.ws.rs.core.MediaType
+
+import static org.apache.slider.common.params.Arguments.ARG_OPTION
+import static org.apache.slider.providers.agent.AgentKeys.*
+import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient
+
+@CompileStatic
+@Slf4j
+class TestRegistryRestResources extends AgentTestBase {
+
+ public static final String REGISTRY_URI = RestPaths.SLIDER_PATH_REGISTRY;
+
+
+ private String id(String instanceName) {
+
+ RegistryNaming.createUniqueInstanceId(
+ instanceName,
+ UserGroupInformation.getCurrentUser().getUserName(),
+ SliderKeys.APP_TYPE,
+ 1);
+ }
+
+
+ @Test
+ public void testRestURIs() throws Throwable {
+ def clustername = "test_registryws"
+ createMiniCluster(
+ clustername,
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+ Map<String, Integer> roles = [:]
+ File slider_core = new File(new File(".").absoluteFile, "src/test/python");
+ String app_def = "appdef_1.tar"
+ File app_def_path = new File(slider_core, app_def)
+ String agt_ver = "version"
+ File agt_ver_path = new File(slider_core, agt_ver)
+ String agt_conf = "agent.ini"
+ File agt_conf_path = new File(slider_core, agt_conf)
+ assert app_def_path.exists()
+ assert agt_ver_path.exists()
+ assert agt_conf_path.exists()
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ roles,
+ [
+ ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + app_def_path.absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + agt_conf_path.absolutePath,
+ ARG_OPTION, AGENT_VERSION, "file://" + agt_ver_path.absolutePath,
+ ],
+ true, true,
+ true)
+ SliderClient sliderClient = launcher.service
+ def report = waitForClusterLive(sliderClient)
+ def trackingUrl = report.trackingUrl
+ log.info("tracking URL is $trackingUrl")
+ def registry_url = appendToURL(trackingUrl, REGISTRY_URI)
+
+
+ def status = dumpClusterStatus(sliderClient, "agent AM")
+ def liveURL = status.getInfo(StatusKeys.INFO_AM_WEB_URL)
+ if (liveURL) {
+ registry_url = appendToURL(liveURL, REGISTRY_URI)
+
+ }
+
+ log.info("Registry is $registry_url")
+ log.info("stacks is ${liveURL}stacks")
+ log.info("conf is ${liveURL}conf")
+
+
+ //WS get
+ Client client = createTestClient();
+
+ // test the exposed WADL link
+ WebResource webResource = client.resource(registry_url);
+ ClientResponse response = webResource.type(MediaType.APPLICATION_XML)
+ .get(ClientResponse.class);
+ assert response.status == 200
+ assert response.getType() == (new MediaType("application", "vnd.sun.wadl+xml"))
+
+ // test the available GET URIs
+ webResource = client.resource(
+ appendToURL(registry_url, RestPaths.REGISTRY_SERVICE));
+
+ response = webResource.type(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ def responseStr = response.getEntity(String.class)
+ log.info("response is " + responseStr)
+
+ "{\"names\":[\"${SliderKeys.APP_TYPE}\"]}".equals(responseStr)
+
+ webResource = client.resource(
+ appendToURL(registry_url,
+ "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}"));
+ CuratorServiceInstances<ServiceInstanceData> services = webResource.type(MediaType.APPLICATION_JSON)
+ .get(CuratorServiceInstances.class);
+ assert services.services.size() == 1
+ CuratorServiceInstance<ServiceInstanceData> service = services.services.get(0)
+ validateService(service)
+
+ webResource = client.resource(
+ appendToURL(registry_url,
+ "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}/"+id("test_registryws")));
+ service = webResource.type(MediaType.APPLICATION_JSON)
+ .get(CuratorServiceInstance.class);
+ validateService(service)
+
+ webResource = client.resource(
+ appendToURL(
+ registry_url, "${RestPaths.REGISTRY_ANYSERVICE}/${SliderKeys.APP_TYPE}"));
+ service = webResource.type(MediaType.APPLICATION_JSON)
+ .get(CuratorServiceInstance.class);
+ validateService(service)
+
+ // some negative tests...
+ webResource = client.resource(
+ appendToURL(registry_url, "${RestPaths.REGISTRY_SERVICE}/dummy"))
+ services = webResource.type(MediaType.APPLICATION_JSON)
+ .get(CuratorServiceInstances.class);
+ assert services.services.size() == 0
+
+ try {
+ webResource = client.resource(appendToURL(registry_url,
+ "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}/test_registryws-99"));
+
+ service = webResource.type(MediaType.APPLICATION_JSON)
+ .get(CuratorServiceInstance.class);
+ fail("should throw an exception for a 404 response....")
+ } catch (UniformInterfaceException e) {
+ assert e.response.getStatus() == 404
+ }
+
+ try {
+ webResource = client.resource(
+ appendToURL(registry_url, "${RestPaths.REGISTRY_ANYSERVICE}/dummy"));
+
+ service = webResource.type(MediaType.APPLICATION_JSON)
+ .get(CuratorServiceInstance.class);
+ fail("should throw an exception for a 404 response....")
+ } catch (UniformInterfaceException e) {
+ assert e.response.getStatus() == 404
+ }
+ }
+
+ private void validateService(CuratorServiceInstance service) {
+ assert service.name.equals(SliderKeys.APP_TYPE)
+ assert service.serviceType == ServiceType.DYNAMIC
+ assert service.id.contains("test_registryws")
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy
new file mode 100644
index 0000000..47924d2
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.registry.curator
+
+import groovy.util.logging.Slf4j
+import org.apache.curator.x.discovery.ServiceInstance
+import org.apache.curator.x.discovery.ServiceInstanceBuilder
+import org.apache.curator.x.discovery.UriSpec
+import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.server.services.curator.CuratorServiceInstance
+import org.junit.Test
+
+/**
+ * Here to identify why curator was complaining about deserializing
+ * its own published artifacts.
+ *
+ * ... its a Jackson version thing
+ */
+@Slf4j
+class TestServiceInstanceSerDeser {
+
+ JsonSerDeser<ServiceInstance> serializer = new JsonSerDeser<>(
+ ServiceInstance.class)
+ JsonSerDeser<CuratorServiceInstance> deser = new JsonSerDeser<>(
+ CuratorServiceInstance.class)
+
+ @Test
+ public void testDefault() throws Throwable {
+ def builder = ServiceInstance.builder()
+ builder.name("defined")
+ buildAndRoundTrip("testDefault", builder)
+ }
+
+ @Test
+ public void testEmpty() throws Throwable {
+ def builder = ServiceInstance.builder()
+ builder.address(null).id("").name("").port(0).uriSpec(null)
+
+ buildAndRoundTrip("testEmpty", builder)
+ }
+
+ @Test
+ public void testFilled() throws Throwable {
+ def builder = ServiceInstance.builder()
+
+ builder.id("service").name("name").port(0)
+ .uriSpec(new UriSpec("http:{}:{}"))
+
+
+ buildAndRoundTrip("testFilled", builder)
+ }
+
+ @Test
+ public void testPayload() throws Throwable {
+ def builder = ServiceInstance.builder()
+ builder.address(null).id("testPayload").name("").port(0).uriSpec(null)
+
+ ServiceInstanceData data = new ServiceInstanceData()
+ data.name = "testPayload"
+ data.externalView.documentsURL = "http://documents"
+ builder.payload(data)
+
+ def instance = buildAndRoundTrip("", builder)
+
+ def payload = instance.payload as ServiceInstanceData
+ log.info("payload = $payload")
+ assert payload.externalView.documentsURL == "http://documents"
+ }
+
+ @Test
+ public void testHackedDeser() throws Throwable {
+ def builder = ServiceInstance.builder()
+ builder.address("localhost")
+ builder.id("service").name("name").port(8080)
+ .uriSpec(new UriSpec("http:{}:{}"))
+ .sslPort(443)
+ ServiceInstanceData data = new ServiceInstanceData()
+ data.externalView.documentsURL = "http://documents"
+
+ builder.payload(data)
+ log.info("Test: testHackedDeser")
+ String json = serialize(builder)
+ CuratorServiceInstance<ServiceInstanceData> curatorInstance = deser.fromJson(json)
+ log.info("resolved =$curatorInstance")
+ def payload = curatorInstance.payload
+ log.info("payload = $payload")
+ assert payload.externalView.documentsURL == "http://documents"
+
+ }
+
+ def buildAndRoundTrip(String testname, ServiceInstanceBuilder builder) {
+ log.info("Test: $testname")
+ String json = serialize(builder)
+
+ return serializer.fromJson(json)
+ }
+
+ def String serialize(ServiceInstanceBuilder builder) {
+ ServiceInstance<ServiceInstanceData> instance = builder.build()
+ def json = serializer.toJson(instance)
+ log.info(json)
+ return json
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy
new file mode 100644
index 0000000..9c17763
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestAppStateContainerFailure extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateContainerFailure"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(8000, 4)
+ }
+
+ @Test
+ public void testShortLivedFail() throws Throwable {
+
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == 1
+
+ RoleInstance instance = instances[0]
+ long created = instance.createTime
+ long started = instance.startTime
+ assert created > 0
+ assert started >= created
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ assert appState.isShortLived(instance)
+ AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid, 1))
+ assert result.roleInstance != null
+ assert result.containerFailed
+ RoleStatus status = role0Status
+ assert status.failed == 1
+ assert status.startFailed == 1
+
+ //view the world
+ appState.getRoleHistory().dump();
+ List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
+ assert queue.size() == 0
+
+ }
+
+ @Test
+ public void testLongLivedFail() throws Throwable {
+
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == 1
+
+ RoleInstance instance = instances[0]
+ instance.startTime = System.currentTimeMillis() - 60 * 60 * 1000;
+ assert !appState.isShortLived(instance)
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ AppState.NodeCompletionResult result = appState.onCompletedNode(
+ containerStatus(cid, 1))
+ assert result.roleInstance != null
+ assert result.containerFailed
+ RoleStatus status = role0Status
+ assert status.failed == 1
+ assert status.startFailed == 0
+
+ //view the world
+ appState.getRoleHistory().dump();
+ List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
+ assert queue.size() == 1
+
+ }
+
+ @Test
+ public void testNodeStartFailure() throws Throwable {
+
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndSubmitNodes()
+ assert instances.size() == 1
+
+ RoleInstance instance = instances[0]
+
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops"))
+ RoleStatus status = role0Status
+ assert status.failed == 1
+ assert status.startFailed == 1
+
+
+ RoleHistory history = appState.roleHistory
+ history.dump();
+ List<NodeInstance> queue = history.cloneAvailableList(0)
+ assert queue.size() == 0
+
+ NodeInstance ni = history.getOrCreateNodeInstance(instance.container)
+ NodeEntry re = ni.get(0)
+ assert re.failed == 1
+ assert re.startFailed == 1
+ }
+
+ @Test
+ public void testRecurrentStartupFailure() throws Throwable {
+
+ role0Status.desired = 1
+ try {
+ for (int i = 0; i< 100; i++) {
+ List<RoleInstance> instances = createAndSubmitNodes()
+ assert instances.size() == 1
+
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ log.info("$i instance $instances[0] $cid")
+ assert cid
+ appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops"))
+ AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid))
+ assert result.containerFailed
+ }
+ fail("Cluster did not fail from too many startup failures")
+ } catch (TriggerClusterTeardownException teardown) {
+ log.info("Exception $teardown.exitCode : $teardown")
+ }
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
new file mode 100644
index 0000000..6e387d8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestAppStateDynamicRoles extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateDynamicRoles"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4)
+ }
+
+ @Override
+ void initApp() {
+ super.initApp()
+ appState = new AppState(new MockRecordFactory())
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+
+ def instance = factory.newInstanceDefinition(0,0,0)
+
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "4",
+ ]
+
+ instance.resourceOperations.components["dynamic"]= opts
+
+
+ appState.buildInstance(
+ instance,
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null, null)
+ }
+
+ @Test
+ public void testAllocateReleaseRealloc() throws Throwable {
+
+ List<RoleInstance> instances = createAndStartNodes()
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ appState.getRoleHistory().dump();
+
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
new file mode 100644
index 0000000..190e927
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestAppStateRebuildOnAMRestart extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateRebuildOnAMRestart"
+ }
+
+ @Test
+ public void testRebuild() throws Throwable {
+
+ int r0 = 1
+ int r1 = 2
+ int r2 = 3
+ role0Status.desired = r0
+ role1Status.desired = r1
+ role2Status.desired = r2
+ List<RoleInstance> instances = createAndStartNodes()
+
+ int clusterSize = r0 + r1 + r2
+ assert instances.size() == clusterSize
+
+ //clone the list
+ List<RoleInstance> cloned = [];
+ List<Container> containers = []
+ instances.each { RoleInstance elt ->
+ cloned.add(elt.clone() as RoleInstance)
+ containers.add(elt.container)
+ }
+ NodeMap nodemap = appState.roleHistory.cloneNodemap()
+
+ // now destroy the app state
+ appState = new AppState(new MockRecordFactory())
+
+ //and rebuild
+ appState.buildInstance(
+ factory.newInstanceDefinition(r0, r1, r2),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ containers, null)
+
+ assert appState.getStartedCountainerCount() == clusterSize
+
+ appState.getRoleHistory().dump();
+
+ //check that the app state direct structures match
+ List<RoleInstance> r0live = appState.enumLiveNodesInRole(ROLE0)
+ List<RoleInstance> r1live = appState.enumLiveNodesInRole(ROLE1)
+ List<RoleInstance> r2live = appState.enumLiveNodesInRole(ROLE2)
+
+ assert r0 == r0live.size()
+ assert r1 == r1live.size()
+ assert r2 == r2live.size()
+
+ //now examine the role history
+ NodeMap newNodemap = appState.roleHistory.cloneNodemap()
+
+ for (NodeInstance nodeInstance : newNodemap.values()) {
+ String hostname = nodeInstance.hostname
+ NodeInstance orig = nodemap[hostname]
+ assertNotNull("Null entry in original nodemap for " + hostname, orig)
+
+ for (int i = 0; i < ROLE_COUNT; i++) {
+
+ assert (nodeInstance.getActiveRoleInstances(i) ==
+ orig.getActiveRoleInstances(i))
+ NodeEntry origRE = orig.getOrCreate(i)
+ NodeEntry newRE = nodeInstance.getOrCreate(i)
+ assert origRE.live == newRE.live
+ assert newRE.starting == 0
+ }
+ }
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 0
+
+ def status = appState.getClusterStatus()
+ // verify the AM restart container count was set
+ String restarted = status.getInfo(
+ StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ assert restarted != null;
+ //and that the count == 1 master + the region servers
+ assert Integer.parseInt(restarted) == containers.size()
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy
new file mode 100644
index 0000000..fba1ea0
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
+
+/**
+ * Test that the app state lets you ask for nodes, get a specific host,
+ * release it and then get that one back again.
+ */
+@CompileStatic
+@Slf4j
+class TestAppStateRolePlacement extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateRolePlacement"
+ }
+
+
+ @Test
+ public void testAllocateReleaseRealloc() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ AMRMClient.ContainerRequest request = operation.request
+ Container allocated = engine.allocateContainer(request)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> operations = []
+ appState.onContainersAllocated([(Container)allocated], assignments, operations)
+ assert operations.size() == 0
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container container = assigned.container
+ assert container.id == allocated.id
+ int roleId = assigned.role.priority
+ assert roleId == extractRole(request.priority)
+ assert assigned.role.name == ROLE0
+ String containerHostname = RoleHistoryUtils.hostnameOf(container);
+ RoleInstance ri = roleInstance(assigned)
+ //tell the app it arrived
+ appState.containerStartSubmitted(container, ri);
+ assert appState.onNodeManagerContainerStarted(container.id)
+ assert role0Status.started == 1
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 0
+
+ //now it is surplus
+ role0Status.desired = 0
+ ops = appState.reviewRequestAndReleaseNodes()
+ ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0]
+
+ assert release.containerId == container.id
+ engine.execute(ops)
+ assert appState.onCompletedNode(containerStatus(container)).roleInstance
+
+ //view the world
+ appState.getRoleHistory().dump();
+
+ //now ask for a new one
+ role0Status.desired = 1
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ operation = (ContainerRequestOperation) ops[0]
+ AMRMClient.ContainerRequest request2 = operation.request
+ assert request2 != null
+ assert request2.nodes[0] == containerHostname
+ engine.execute(ops)
+
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy
new file mode 100644
index 0000000..f087a30
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestAppStateRoleRelease extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateRolePlacement"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4)
+ }
+
+ @Test
+ public void testAllocateReleaseRealloc() throws Throwable {
+ /**
+ * Allocate to all nodes
+ */
+ role0Status.desired = 6
+ role1Status.desired = 5
+ role2Status.desired = 4
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == 15
+
+ //now it is surplus
+ role0Status.desired = 0
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+
+ List<ContainerId> released = []
+ engine.execute(ops, released)
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+ released.each { ContainerId cid ->
+ assert appState.onCompletedNode(containerStatus(cid)).roleInstance
+ assert ids.contains(cid)
+ }
+
+ //view the world
+ appState.getRoleHistory().dump();
+
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy
new file mode 100644
index 0000000..a0b1100
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.ContainerRequestOperation
+import org.junit.Test
+
+/**
+ * Test the container resource allocation logic
+ */
+@CompileStatic
+@Slf4j
+class TestContainerResourceAllocations extends BaseMockAppStateTest {
+
+ @Override
+ String getTestName() {
+ "TestContainerResourceAllocations"
+ }
+
+ @Test
+ public void testNormalAllocations() throws Throwable {
+ ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
+ ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
+
+ cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY, 512)
+ cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
+ appState.updateResourceDefinitions(clusterSpec)
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ Resource requirements = operation.request.capability
+ assert requirements.memory == 512
+ assert requirements.virtualCores == 2
+ }
+
+ @Test
+ public void testMaxMemAllocations() throws Throwable {
+ ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
+ ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
+
+ cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
+ ResourceKeys.YARN_RESOURCE_MAX)
+ cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
+ appState.updateResourceDefinitions(clusterSpec)
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ Resource requirements = operation.request.capability
+ assert requirements.memory == RM_MAX_RAM
+ assert requirements.virtualCores == 2
+ }
+
+ @Test
+ public void testMaxCoreAllocations() throws Throwable {
+ ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
+ ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
+ cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
+ 512)
+ cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES,
+ ResourceKeys.YARN_RESOURCE_MAX)
+ appState.updateResourceDefinitions(clusterSpec)
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ Resource requirements = operation.request.capability
+ assert requirements.memory == 512
+ assert requirements.virtualCores == RM_MAX_CORES
+ }
+
+ @Test
+ public void testMaxDefaultAllocations() throws Throwable {
+
+ ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
+ appState.updateResourceDefinitions(clusterSpec)
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ Resource requirements = operation.request.capability
+ assert requirements.memory == ResourceKeys.DEF_YARN_MEMORY
+ assert requirements.virtualCores == ResourceKeys.DEF_YARN_CORES
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
new file mode 100644
index 0000000..1693365
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.core.exceptions.BadConfigException
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.avro.RoleHistoryWriter
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestFlexDynamicRoles extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateDynamicRoles"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4)
+ }
+
+ @Override
+ void initApp() {
+ super.initApp()
+ appState = new AppState(new MockRecordFactory())
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+
+ def instance = factory.newInstanceDefinition(0, 0, 0)
+
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "6",
+ ]
+
+ instance.resourceOperations.components["dynamic"] = opts
+
+
+ appState.buildInstance(instance,
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null, null)
+ }
+
+
+ private ConfTreeOperations init() {
+ createAndStartNodes();
+ def resources = appState.instanceDefinition.resources;
+ return new ConfTreeOperations(resources)
+ }
+
+ @Test
+ public void testDynamicFlexAddRole() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "7",
+ ]
+
+ cd.components["role4"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ dumpClusterDescription("updated CD", appState.getClusterStatus())
+ appState.lookupRoleStatus(7)
+ appState.lookupRoleStatus(6)
+ //gaps are still there
+ try {
+ assert null == appState.lookupRoleStatus(5)
+ } catch (RuntimeException expected) {
+ }
+ }
+
+ @Test
+ public void testDynamicFlexAddRoleConflictingPriority() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "6",
+ ]
+
+ cd.components["role4"] = opts
+ try {
+ appState.updateResourceDefinitions(cd.confTree);
+ dumpClusterDescription("updated CD", appState.getClusterStatus())
+ fail("Expected an exception")
+ } catch (BadConfigException expected) {
+ }
+ }
+
+ @Test
+ public void testDynamicFlexDropRole() throws Throwable {
+ def cd = init()
+ cd.components.remove("dynamic");
+ appState.updateResourceDefinitions(cd.confTree);
+
+ def getCD = appState.getClusterStatus()
+ dumpClusterDescription("updated CD", getCD)
+ //status is retained for future
+ appState.lookupRoleStatus(6)
+ }
+
+
+ @Test
+ public void testHistorySaveFlexLoad() throws Throwable {
+ def cd = init()
+ def roleHistory = appState.roleHistory
+ Path history = roleHistory.saveHistory(0x0001)
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "7",
+ ]
+
+ cd.components["role4"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ historyWriter.read(fs, history, appState.roleHistory)
+ }
+
+ @Test
+ public void testHistoryFlexSaveLoad() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "7",
+ ]
+
+ cd.components["role4"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ def roleHistory = appState.roleHistory
+ Path history = roleHistory.saveHistory(0x0002)
+ //now reset the app state
+ def historyWorkDir2 = new File("target/history" + testName + "-0002")
+ def historyPath2 = new Path(historyWorkDir2.toURI())
+ appState = new AppState(new MockRecordFactory())
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+ appState.buildInstance(
+ factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath2,
+ null, null)
+ historyWriter.read(fs, history, appState.roleHistory)
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
new file mode 100644
index 0000000..7f92f9c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
+import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
+
+@Slf4j
+class TestMockRMOperations extends BaseMockAppStateTest implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockRMOperations"
+ }
+
+ @Test
+ public void testPriorityOnly() throws Throwable {
+ assert 5 == buildPriority(5, false)
+ }
+
+ @Test
+ public void testPriorityRoundTrip() throws Throwable {
+ assert 5 == extractRole(buildPriority(5, false))
+ }
+
+ @Test
+ public void testPriorityRoundTripWithRequest() throws Throwable {
+ int priority = buildPriority(5, false)
+ assert 5 == extractRole(priority)
+ }
+
+ @Test
+ public void testMockAddOp() throws Throwable {
+ role0Status.desired = 1
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ int priority = operation.request.priority.priority
+ assert extractRole(priority) == MockFactory.PROVIDER_ROLE0.id
+ RMOperationHandler handler = new MockRMOperationHandler()
+ handler.execute(ops)
+
+ //tell the container its been allocated
+ AbstractRMOperation op = handler.operations[0]
+ assert op instanceof ContainerRequestOperation
+ }
+
+ @Test
+ public void testAllocateReleaseOp() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ AMRMClient.ContainerRequest request = operation.request
+ Container cont = engine.allocateContainer(request)
+ List<Container> allocated = [cont]
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> operations = []
+ appState.onContainersAllocated(allocated, assignments, operations)
+ assert operations.size() == 0
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container target = assigned.container
+ assert target.id == cont.id
+ int roleId = assigned.role.priority
+ assert roleId == extractRole(request.priority)
+ assert assigned.role.name == ROLE0
+ RoleInstance ri = roleInstance(assigned)
+ //tell the app it arrived
+ appState.containerStartSubmitted(target, ri);
+ appState.innerOnNodeManagerContainerStarted(target.id)
+ assert role0Status.started == 1
+
+ //now release it by changing the role status
+ role0Status.desired = 0
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+
+ assert ops[0] instanceof ContainerReleaseOperation
+ ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0]
+ assert release.containerId == cont.id
+ }
+
+ @Test
+ public void testComplexAllocation() throws Throwable {
+ role0Status.desired = 1
+ role1Status.desired = 3
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ List<Container> allocations = engine.execute(ops)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> releases = []
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert releases.size() == 0
+ assert assignments.size() == 4
+ assignments.each { ContainerAssignment assigned ->
+ Container target = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+ appState.containerStartSubmitted(target, ri);
+ }
+ //insert some async operation here
+ assignments.each { ContainerAssignment assigned ->
+ Container target = assigned.container
+ appState.innerOnNodeManagerContainerStarted(target.id)
+ }
+ assert engine.containerCount() == 4;
+ role1Status.desired = 0
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 3
+ allocations = engine.execute(ops)
+ assert engine.containerCount() == 1;
+
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert assignments.empty
+ assert releases.empty
+ }
+
+ @Test
+ public void testDoubleNodeManagerStartEvent() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ List<Container> allocations = engine.execute(ops)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> releases = []
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container target = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+ appState.containerStartSubmitted(target, ri);
+ RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
+ assert ri2 == ri
+ //try a second time, expect an error
+ try {
+ appState.innerOnNodeManagerContainerStarted(target.id)
+ fail("Expected an exception")
+ } catch (RuntimeException expected) {
+ // expected
+ }
+ //and non-faulter should not downgrade to a null
+ log.warn("Ignore any exception/stack trace that appears below")
+ log.warn("===============================================================")
+ RoleInstance ri3 = appState.onNodeManagerContainerStarted(target.id)
+ log.warn("===============================================================")
+ log.warn("Ignore any exception/stack trace that appeared above")
+ assert ri3 == null
+ }
+
+ @Test
+ public void testFlexDuringLaunchPhase() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ List<Container> allocations = engine.execute(
+ ops)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> releases = []
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container target = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.empty
+
+ //now this is the start point.
+ appState.containerStartSubmitted(target, ri);
+
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.empty
+
+ RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
+ }
+
+ @Test
+ public void testFlexBeforeAllocationPhase() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert !ops.empty
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert ops2.empty
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
new file mode 100644
index 0000000..dab03f5
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.ProviderRole
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.appmaster.state.RoleStatus
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * Testing finding nodes for new instances.
+ * These tests validate the (currently) suboptimal
+ * behavior of not listing any known nodes when there
+ * are none in the available list -even if there are nodes
+ * known to be not running live instances in the cluster.
+ */
+@Slf4j
+@CompileStatic
+class TestFindNodesForNewInstances extends BaseMockAppStateTest {
+
+ @Override
+ String getTestName() {
+ return "TestFindNodesForNewInstances"
+ }
+
+ NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1)
+ NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
+ NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+
+ String roleName = "test"
+ RoleStatus roleStat = new RoleStatus(new ProviderRole(roleName, 0))
+ RoleStatus roleStat2 = new RoleStatus(new ProviderRole(roleName, 2))
+
+ @Before
+ public void setupNodeMap() {
+ roleHistory.insert(nodes)
+ roleHistory.buildAvailableNodeLists();
+ }
+
+
+ public List<NodeInstance> findNodes(int count, RoleStatus roleStatus = roleStat) {
+ List < NodeInstance > found = [];
+ for (int i = 0; i < count; i++) {
+ NodeInstance f = roleHistory.findNodeForNewInstance(roleStatus)
+ if (f) {
+ found << f
+ };
+ }
+ return found
+ }
+
+ @Test
+ public void testFind1NodeR0() throws Throwable {
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ log.info("found: $found")
+ assert [age3Active0].contains(found)
+ }
+
+ @Test
+ public void testFind2NodeR0() throws Throwable {
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ log.info("found: $found")
+ assert [age2Active0, age3Active0].contains(found)
+ NodeInstance found2 = roleHistory.findNodeForNewInstance(roleStat)
+ log.info("found: $found2")
+ assert [age2Active0, age3Active0].contains(found2)
+ assert found != found2;
+ }
+ @Test
+ public void testFind3NodeR0ReturnsNull() throws Throwable {
+ assert 2== findNodes(2).size()
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ assert found == null;
+ }
+
+ @Test
+ public void testFindNodesOneEntry() throws Throwable {
+ List<NodeInstance> nodes = findNodes(4, roleStat2)
+ assert 0 == nodes.size()
+ }
+
+ @Test
+ public void testFindNodesIndependent() throws Throwable {
+ assert 2 == findNodes(2).size()
+ roleHistory.dump()
+ assert 0 == findNodes(3, roleStat2).size()
+ }
+
+ @Test
+ public void testFindNodesFallsBackWhenUsed() throws Throwable {
+ // mark age2 and active 0 as busy, expect a null back
+ age2Active0.get(0).onStartCompleted()
+ assert age2Active0.getActiveRoleInstances(0) != 0
+ age3Active0.get(0).onStartCompleted()
+ assert age3Active0.getActiveRoleInstances(0) != 0
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ log.info(found ?.toFullString())
+ assert found == null
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy
new file mode 100644
index 0000000..92915dd
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.junit.Before
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+class TestFindNodesForRelease extends BaseMockAppStateTest {
+
+
+ @Override
+ String getTestName() {
+ return "TestFindNodesForRelease"
+ }
+ NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(2, 2, 0, 0)
+ NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
+ NodeMap nodeMap = new NodeMap(MockFactory.ROLE_COUNT);
+
+
+ @Before
+ public void setupNodeMap() {
+ nodeMap.insert(nodes)
+ }
+
+ private void assertReleased(
+ int count,
+ List<NodeInstance> expected,
+ int role = 0) {
+ List<NodeInstance> released = nodeMap.findNodesForRelease(role, count)
+ assertListEquals(released, expected)
+ }
+ private void assertReleased(
+ List<NodeInstance> expected,
+ int role = 0) {
+ List<NodeInstance> released = nodeMap.findNodesForRelease(role, expected.size())
+ assertListEquals(released, expected)
+ }
+
+ @Test
+ public void testListActiveNodes() throws Throwable {
+ assertListEquals(nodeMap.listActiveNodes(0),
+ [age1Active4,age2Active2, age4Active1])
+ }
+
+ @Test
+ public void testReleaseMinus1() throws Throwable {
+ try {
+ nodeMap.findNodesForRelease(0, -1)
+ fail("Expected an exception")
+ } catch (IllegalArgumentException e) {
+ }
+ }
+ @Test
+ public void testReleaseO() throws Throwable {
+ assertReleased(0, [])
+ }
+
+ @Test
+ public void testRelease1() throws Throwable {
+ assertReleased(1, [age1Active4])
+ }
+
+ @Test
+ public void testRelease2() throws Throwable {
+ assertReleased(2, [age1Active4, age1Active4])
+ }
+
+ @Test
+ public void testRelease3() throws Throwable {
+ assertReleased(3, [age1Active4, age1Active4, age1Active4 ])
+ }
+
+ @Test
+ public void testRelease4() throws Throwable {
+ assertReleased(4, [age1Active4, age1Active4, age1Active4 , age2Active2])
+ }
+
+ @Test
+ public void testRelease5() throws Throwable {
+ assertReleased([age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1])
+ }
+
+ @Test
+ public void testRelease6() throws Throwable {
+ assertReleased(
+ [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, age1Active4])
+ }
+
+ @Test
+ public void testRelease7() throws Throwable {
+ assertReleased(
+ [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
+ age1Active4, age2Active2])
+ }
+
+ @Test
+ public void testRelease8() throws Throwable {
+ assertReleased(8,
+ [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
+ age1Active4, age2Active2])
+ }
+
+ @Test
+ public void testPurgeInactiveTime3() throws Throwable {
+ assert nodeMap.purgeUnusedEntries(3) == 0;
+ }
+
+ @Test
+ public void testPurgeInactiveTime4() throws Throwable {
+ assert nodeMap.purgeUnusedEntries(4) == 1;
+ }
+ @Test
+ public void testPurgeInactiveTime5() throws Throwable {
+ assert nodeMap.purgeUnusedEntries(5) == 1;
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy
new file mode 100644
index 0000000..b646661
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.FSDataOutputStream
+import org.apache.hadoop.fs.Path
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.avro.RoleHistoryWriter
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+class TestHistoryRW extends BaseMockAppStateTest {
+
+ static long time = System.currentTimeMillis();
+
+ @Override
+ String getTestName() {
+ return "TestHistoryRW"
+ }
+
+ @Test
+ public void testWriteReadEmpty() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ Path history = roleHistory.saveHistory(time++)
+ assert fs.isFile(history)
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ historyWriter.read(fs, history, roleHistory)
+ }
+
+ @Test
+ public void testWriteReadData() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert !roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path history = roleHistory.saveHistory(time++)
+ assert fs.isFile(history)
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+
+ assert 0 < historyWriter.read(fs, history, rh2)
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+ NodeEntry ne2 = ni2.get(0)
+ assert ne2 !=null
+ assert ne2.lastUsed == ne1.lastUsed
+ }
+
+ @Test
+ public void testWriteReadActiveData() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ String addr2 = "rack1server5"
+ NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry orig1 = localhost.getOrCreate(0)
+ orig1.lastUsed = 0x10
+ NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2)
+ NodeEntry orig2 = rack1server5.getOrCreate(1)
+ orig2.live = 3
+ assert !orig2.available
+ NodeEntry orig3 = localhost.getOrCreate(1)
+ orig3.lastUsed = 0x20
+ orig3.live = 1
+ assert !orig3.available
+ orig3.release()
+ assert orig3.available
+ roleHistory.dump()
+
+ long savetime = 0x0001000
+ Path history = roleHistory.saveHistory(savetime)
+ assert fs.isFile(history)
+ describe("Loaded")
+ log.info("testWriteReadActiveData in $history")
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+
+ assert 3 == historyWriter.read(fs, history, rh2)
+ rh2.dump()
+
+ assert rh2.clusterSize == 2;
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+ NodeEntry loadedNE = ni2.get(0)
+ assert loadedNE.lastUsed == orig1.lastUsed
+ NodeInstance ni2b = rh2.getExistingNodeInstance(addr2)
+ assert ni2b != null
+ NodeEntry loadedNE2 = ni2b.get(1)
+ assert loadedNE2 != null
+ assert loadedNE2.lastUsed == savetime
+ assert rh2.thawedDataTime == savetime
+
+ // now thaw it
+ rh2.buildAvailableNodeLists();
+ describe("thawing")
+ rh2.dump();
+ List<NodeInstance> available0 = rh2.cloneAvailableList(0)
+ assert available0.size() == 1
+
+ NodeInstance entry = available0.get(0)
+ assert entry.hostname == "localhost"
+ assert entry == localhost
+ List<NodeInstance> available1 = rh2.cloneAvailableList(1)
+ assert available1.size() == 2
+ //and verify that even if last used was set, the save time is picked up
+ assert entry.get(1).lastUsed == roleHistory.saveTime
+
+ }
+
+ @Test
+ public void testWriteThaw() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert !roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path history = roleHistory.saveHistory(time++)
+ long savetime =roleHistory.saveTime;
+ assert fs.isFile(history)
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ assert rh2.onStart(fs, historyPath)
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+ NodeEntry ne2 = ni2.get(0)
+ assert ne2 != null
+ assert ne2.lastUsed == ne1.lastUsed
+ assert rh2.thawedDataTime == savetime
+ }
+
+
+ @Test
+ public void testPurgeOlderEntries() throws Throwable {
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ time = 1;
+ Path file1 = touch(historyWriter, time++)
+ Path file2 = touch(historyWriter, time++)
+ Path file3 = touch(historyWriter, time++)
+ Path file4 = touch(historyWriter, time++)
+ Path file5 = touch(historyWriter, time++)
+ Path file6 = touch(historyWriter, time++)
+
+ assert historyWriter.purgeOlderHistoryEntries(fs, file1) == 0
+ assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 1
+ assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 0
+ assert historyWriter.purgeOlderHistoryEntries(fs, file5) == 3
+ assert historyWriter.purgeOlderHistoryEntries(fs, file6) == 1
+ try {
+ // make an impossible assertion that will fail if the method
+ // actually completes
+ assert -1 == historyWriter.purgeOlderHistoryEntries(fs, file1)
+ } catch (FileNotFoundException ignored) {
+ // expected
+ }
+
+ }
+
+ public Path touch(RoleHistoryWriter historyWriter, long time){
+ Path path = historyWriter.createHistoryFilename(historyPath, time);
+ FSDataOutputStream out = fs.create(path);
+ out.close()
+ return path
+ }
+
+ @Test
+ public void testSkipEmptyFileOnRead() throws Throwable {
+ describe "verify that empty histories are skipped on read; old histories purged"
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ time = 0
+ Path oldhistory = roleHistory.saveHistory(time++)
+
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path goodhistory = roleHistory.saveHistory(time++)
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ Path touched = touch(historyWriter, time++)
+
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ assert rh2.onStart(fs, historyPath)
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+
+ //and assert the older file got purged
+ assert !fs.exists(oldhistory)
+ assert fs.exists(goodhistory)
+ assert fs.exists(touched )
+ }
+
+ @Test
+ public void testSkipBrokenFileOnRead() throws Throwable {
+ describe "verify that empty histories are skipped on read; old histories purged"
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ time = 0
+ Path oldhistory = roleHistory.saveHistory(time++)
+
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path goodhistory = roleHistory.saveHistory(time++)
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ Path badfile = historyWriter.createHistoryFilename(historyPath, time++)
+ FSDataOutputStream out = fs.create(badfile)
+ out.writeBytes("{broken:true}")
+ out.close()
+
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ describe("IGNORE STACK TRACE BELOW")
+
+ assert rh2.onStart(fs, historyPath)
+
+ describe( "IGNORE STACK TRACE ABOVE")
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+
+ //and assert the older file got purged
+ assert !fs.exists(oldhistory)
+ assert fs.exists(goodhistory)
+ assert fs.exists(badfile )
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy
new file mode 100644
index 0000000..6ec046c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.avro.RoleHistoryWriter
+import org.junit.Test
+
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+@Slf4j
+class TestHistoryRWOrdering extends BaseMockAppStateTest {
+
+ def paths = pathlist(
+ [
+ "hdfs://localhost/history-0406c.json",
+ "hdfs://localhost/history-5fffa.json",
+ "hdfs://localhost/history-0001a.json",
+ "hdfs://localhost/history-0001f.json",
+ ]
+ )
+ Path h_0406c = paths[0]
+ Path h_5fffa = paths[1]
+ Path h_0001a = paths[3]
+
+
+ List<Path> pathlist(List<String> pathnames) {
+ def result = []
+ pathnames.each { result << new Path(new URI(it as String)) }
+ result
+ }
+
+ @Override
+ String getTestName() {
+ return "TestHistoryRWOrdering"
+ }
+
+
+ /**
+ * This tests regexp pattern matching. It uses the current time so isn't
+ * repeatable -but it does test a wider range of values in the process
+ * @throws Throwable
+ */
+ @Test
+ public void testPatternRoundTrip() throws Throwable {
+ describe "test pattern matching of names"
+ long value=System.currentTimeMillis()
+ String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,value)
+ String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN
+ Pattern pattern = Pattern.compile(matchpattern)
+ Matcher matcher = pattern.matcher(name);
+ if (!matcher.find()) {
+ throw new Exception("No match for pattern $matchpattern in $name")
+ }
+ }
+
+
+ @Test
+ public void testWriteSequenceReadData() throws Throwable {
+ describe "test that if multiple entries are written, the newest is picked up"
+ long time = System.currentTimeMillis();
+
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert !roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path history1 = roleHistory.saveHistory(time++)
+ Path history2 = roleHistory.saveHistory(time++)
+ Path history3 = roleHistory.saveHistory(time++)
+
+ //inject a later file with a different name
+ sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello, world")
+
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+
+ List<Path> entries = historyWriter.findAllHistoryEntries(
+ fs,
+ historyPath,
+ false)
+ assert entries.size() == 3
+ assert entries[0] == history3
+ assert entries[1] == history2
+ assert entries[2] == history1
+ }
+
+ @Test
+ public void testPathStructure() throws Throwable {
+ assert h_5fffa.getName() == "history-5fffa.json"
+ }
+
+ @Test
+ public void testPathnameComparator() throws Throwable {
+
+ def newerName = new RoleHistoryWriter.NewerFilesFirst()
+
+ log.info("$h_5fffa name is ${h_5fffa.getName()}")
+ log.info("$h_0406c name is ${h_0406c.getName()}")
+ assert newerName.compare(h_5fffa, h_5fffa) == 0
+ assert newerName.compare(h_5fffa, h_0406c) < 0
+ assert newerName.compare(h_5fffa, h_0001a) < 0
+ assert newerName.compare(h_0001a, h_5fffa) > 0
+
+ }
+
+ @Test
+ public void testPathSort() throws Throwable {
+ def paths2 = new ArrayList(paths)
+ RoleHistoryWriter.sortHistoryPaths(paths2)
+ assertListEquals(paths2,
+ [
+ paths[1],
+ paths[0],
+ paths[3],
+ paths[2]
+ ])
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy
new file mode 100644
index 0000000..77119d5
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.junit.Test
+
+/**
+ * Unit test to verify the comparators sort as expected
+ */
+class TestNIComparators extends BaseMockAppStateTest {
+
+ NodeInstance age1Active4 = nodeInstance(1000, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(1001, 2, 0, 0)
+ NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
+
+ @Override
+ String getTestName() {
+ return "TestNIComparators"
+ }
+
+ @Test
+ public void testNewerThan() throws Throwable {
+
+ Collections.sort(nodes, new NodeInstance.newerThan(0))
+ assertListEquals(nodes,
+ [age4Active1, age3Active0, age2Active2, age1Active4])
+ }
+
+ @Test
+ public void testNewerThanNoRole() throws Throwable {
+
+ nodes << empty
+ Collections.sort(nodes, new NodeInstance.newerThan(0))
+ assertListEquals(nodes,
+ [age4Active1, age3Active0, age2Active2, age1Active4, empty])
+ }
+
+ @Test
+ public void testMoreActiveThan() throws Throwable {
+
+ Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
+ assertListEquals(nodes,
+ [age1Active4, age2Active2, age4Active1, age3Active0],)
+ }
+
+ @Test
+ public void testMoreActiveThanEmpty() throws Throwable {
+ nodes << empty
+ Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
+ assertListEquals(nodes,
+ [age1Active4, age2Active2, age4Active1, age3Active0, empty])
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy
new file mode 100644
index 0000000..8d1f4b0
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.OutstandingRequest
+import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
+import org.junit.Test
+
+class TestOutstandingRequestTracker extends BaseMockAppStateTest {
+
+ NodeInstance host1 = new NodeInstance("host1", 3)
+ NodeInstance host2 = new NodeInstance("host2", 3)
+
+ OutstandingRequestTracker tracker = new OutstandingRequestTracker()
+
+ @Override
+ String getTestName() {
+ return "TestOutstandingRequestTracker"
+ }
+
+ @Test
+ public void testAddRetrieveEntry() throws Throwable {
+ OutstandingRequest request = tracker.addRequest(host1, 0)
+ assert tracker.lookup(0, "host1").equals(request)
+ assert tracker.remove(request).equals(request)
+ assert !tracker.lookup(0, "host1")
+ }
+
+ @Test
+ public void testAddCompleteEntry() throws Throwable {
+ tracker.addRequest(host1, 0)
+ tracker.addRequest(host2, 0)
+ tracker.addRequest(host1, 1)
+ assert tracker.onContainerAllocated(1, "host1")
+ assert !tracker.lookup(1, "host1")
+ assert tracker.lookup(0, "host1")
+ }
+
+ @Test
+ public void testCancelEntries() throws Throwable {
+ OutstandingRequest r1 = tracker.addRequest(host1, 0)
+ OutstandingRequest r2 = tracker.addRequest(host2, 0)
+ OutstandingRequest r3 = tracker.addRequest(host1, 1)
+ List<NodeInstance> canceled = tracker.cancelOutstandingRequests(0)
+ assert canceled.size() == 2
+ assert canceled.contains(host1)
+ assert canceled.contains(host2)
+ assert tracker.lookup(1, "host1")
+ assert !tracker.lookup(0, "host1")
+ canceled = tracker.cancelOutstandingRequests(0)
+ assert canceled.size() == 0
+ assert tracker.cancelOutstandingRequests(1).size() == 1
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
new file mode 100644
index 0000000..795b48f
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.Priority
+import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.providers.ProviderRole
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockContainer
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockNodeId
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * Test container events at the role history level -one below
+ * the App State
+ */
+@Slf4j
+@CompileStatic
+class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
+
+ @Override
+ String getTestName() {
+ return "TestRoleHistoryContainerEvents"
+ }
+
+ NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1)
+ NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
+ NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ String roleName = "test"
+
+ List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+
+ Resource resource
+
+ @Before
+ public void setupRH() {
+ roleHistory.onStart(fs, historyPath)
+ roleHistory.insert(nodes)
+ roleHistory.buildAvailableNodeLists();
+ resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES,
+ ResourceKeys.DEF_YARN_MEMORY);
+ }
+
+ @Test
+ public void testFindAndCreate() throws Throwable {
+ int role = 0
+ ProviderRole provRole = new ProviderRole(roleName, role)
+ RoleStatus roleStatus = new RoleStatus(provRole)
+
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestNode(roleStatus, resource);
+
+ List<String> nodes = request.getNodes()
+ assert nodes != null
+ assert nodes.size() == 1
+ String hostname = nodes[0]
+ assert hostname == age3Active0.hostname
+
+ //build a container
+ MockContainer container = factory.newContainer()
+ container.nodeId = new MockNodeId(hostname, 0)
+ container.priority = request.getPriority()
+ roleHistory.onContainerAssigned(container);
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname)
+ NodeEntry roleEntry = allocated.get(role)
+ assert roleEntry.starting == 1
+ assert !roleEntry.available
+ RoleInstance ri = new RoleInstance(container);
+ //start it
+ roleHistory.onContainerStartSubmitted(container, ri)
+ //later, declare that it started
+ roleHistory.onContainerStarted(container)
+ assert roleEntry.starting == 0
+ assert !roleEntry.available
+ assert roleEntry.active == 1
+ assert roleEntry.live == 1
+ }
+
+ @Test
+ public void testCreateAndRelease() throws Throwable {
+ int role = 1
+ ProviderRole provRole = new ProviderRole(roleName, role)
+ RoleStatus roleStatus = new RoleStatus(provRole)
+
+ //verify it is empty
+ assert roleHistory.findNodesForRelease(role, 1).isEmpty()
+
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestNode(roleStatus, resource);
+
+ List<String> nodes = request.getNodes()
+ assert nodes == null
+
+ //pick an idle host
+ String hostname = age3Active0.hostname;
+
+ //build a container
+ MockContainer container = factory.newContainer()
+ container.nodeId = new MockNodeId(hostname, 0)
+ container.priority = request.getPriority()
+ roleHistory.onContainerAssigned(container);
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname)
+ NodeEntry roleEntry = allocated.get(role)
+ assert roleEntry.starting == 1
+ assert !roleEntry.available
+ RoleInstance ri = new RoleInstance(container);
+ //start it
+ roleHistory.onContainerStartSubmitted(container, ri)
+ //later, declare that it started
+ roleHistory.onContainerStarted(container)
+ assert roleEntry.starting == 0
+ assert !roleEntry.available
+ assert roleEntry.active == 1
+ assert roleEntry.live == 1
+
+ // now pick that instance to destroy
+
+ List<NodeInstance> forRelease = roleHistory.findNodesForRelease(role, 1)
+ assert forRelease.size() == 1
+ NodeInstance target = forRelease[0]
+ assert target == allocated
+ roleHistory.onContainerReleaseSubmitted(container);
+ assert roleEntry.releasing == 1
+ assert roleEntry.live == 1
+ assert roleEntry.active == 0
+
+ // release completed
+ roleHistory.onReleaseCompleted(container)
+ assert roleEntry.releasing == 0
+ assert roleEntry.live == 0
+ assert roleEntry.active == 0
+
+ // verify it is empty
+ assert roleHistory.findNodesForRelease(role, 1).isEmpty()
+
+ // ask for a container and expect to get the recently released one
+ AMRMClient.ContainerRequest request2 =
+ roleHistory.requestNode(roleStatus, resource);
+
+ List<String> nodes2 = request2.getNodes()
+ assert nodes2 != null
+ String hostname2 = nodes2[0]
+
+ //pick an idle host
+ assert hostname2 == age3Active0.hostname;
+ }
+
+
+ @Test
+ public void testStartWithoutWarning() throws Throwable {
+ int role = 0
+ //pick an idle host
+ String hostname = age3Active0.hostname;
+ //build a container
+ MockContainer container = factory.newContainer()
+ container.nodeId = new MockNodeId(hostname, 0)
+ container.priority = ContainerPriority.createPriority(0, false)
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname)
+ NodeEntry roleEntry = allocated.get(role)
+
+ RoleInstance ri = new RoleInstance(container);
+ //tell RH that it started
+ roleHistory.onContainerStarted(container)
+ assert roleEntry.starting == 0
+ assert !roleEntry.available
+ assert roleEntry.active == 1
+ assert roleEntry.live == 1
+ }
+
+ @Test
+ public void testStartFailed() throws Throwable {
+ int role = 0
+ ProviderRole provRole = new ProviderRole(roleName, role)
+ RoleStatus roleStatus = new RoleStatus(provRole)
+
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestNode(roleStatus, resource);
+
+ String hostname = request.getNodes()[0]
+ assert hostname == age3Active0.hostname
+
+ //build a container
+ MockContainer container = factory.newContainer()
+ container.nodeId = new MockNodeId(hostname, 0)
+ container.priority = request.getPriority()
+ roleHistory.onContainerAssigned(container);
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname)
+ NodeEntry roleEntry = allocated.get(role)
+ assert roleEntry.starting == 1
+ assert !roleEntry.available
+ RoleInstance ri = new RoleInstance(container);
+ //start it
+ roleHistory.onContainerStartSubmitted(container, ri)
+ //later, declare that it failed on startup
+ assert !roleHistory.onNodeManagerContainerStartFailed(container)
+ assert roleEntry.starting == 0
+ assert roleEntry.startFailed == 1
+ assert roleEntry.failed == 1
+ assert roleEntry.available
+ assert roleEntry.active == 0
+ assert roleEntry.live == 0
+ }
+
+ @Test
+ public void testStartFailedWithoutWarning() throws Throwable {
+ int role = 0
+ ProviderRole provRole = new ProviderRole(roleName, role)
+ RoleStatus roleStatus = new RoleStatus(provRole)
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestNode(roleStatus, resource);
+
+ String hostname = request.getNodes()[0]
+ assert hostname == age3Active0.hostname
+
+ //build a container
+ MockContainer container = factory.newContainer()
+ container.nodeId = new MockNodeId(hostname, 0)
+ container.priority = request.getPriority()
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname)
+ NodeEntry roleEntry = allocated.get(role)
+
+ assert !roleHistory.onNodeManagerContainerStartFailed(container)
+ assert roleEntry.starting == 0
+ assert roleEntry.startFailed == 1
+ assert roleEntry.failed == 1
+ assert roleEntry.available
+ assert roleEntry.active == 0
+ assert roleEntry.live == 0
+ }
+
+ @Test
+ public void testContainerFailed() throws Throwable {
+ describe("fail a container without declaring it as starting")
+
+ int role = 0
+ ProviderRole provRole = new ProviderRole(roleName, role)
+ RoleStatus roleStatus = new RoleStatus(provRole)
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestNode(roleStatus, resource);
+
+ String hostname = request.getNodes()[0]
+ assert hostname == age3Active0.hostname
+
+ //build a container
+ MockContainer container = factory.newContainer()
+ container.nodeId = new MockNodeId(hostname, 0)
+ container.priority = request.getPriority()
+ roleHistory.onContainerAssigned(container);
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname)
+ NodeEntry roleEntry = allocated.get(role)
+ assert roleEntry.starting == 1
+ assert !roleEntry.available
+ RoleInstance ri = new RoleInstance(container);
+ //start it
+ roleHistory.onContainerStartSubmitted(container, ri)
+ roleHistory.onContainerStarted(container)
+
+ //later, declare that it failed
+ roleHistory.onFailedContainer(container, false)
+ assert roleEntry.starting == 0
+ assert roleEntry.available
+ assert roleEntry.active == 0
+ assert roleEntry.live == 0
+ }
+
+ @Test
+ public void testContainerFailedWithoutWarning() throws Throwable {
+ describe( "fail a container without declaring it as starting")
+ int role = 0
+ ProviderRole provRole = new ProviderRole(roleName, role)
+ RoleStatus roleStatus = new RoleStatus(provRole)
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestNode(roleStatus, resource);
+
+ String hostname = request.getNodes()[0]
+ assert hostname == age3Active0.hostname
+
+ //build a container
+ MockContainer container = factory.newContainer()
+ container.nodeId = new MockNodeId(hostname, 0)
+ container.priority = request.getPriority()
+
+
+ NodeMap nodemap = roleHistory.cloneNodemap();
+ NodeInstance allocated = nodemap.get(hostname)
+ NodeEntry roleEntry = allocated.get(role)
+ assert roleEntry.available
+ roleHistory.onFailedContainer(container, false)
+ assert roleEntry.starting == 0
+ assert roleEntry.failed == 1
+ assert roleEntry.available
+ assert roleEntry.active == 0
+ assert roleEntry.live == 0
+ }
+
+ @Test
+ public void testAllocationListPrep() throws Throwable {
+ describe("test prepareAllocationList")
+ int role = 0
+ ProviderRole provRole = new ProviderRole(roleName, role)
+ RoleStatus roleStatus = new RoleStatus(provRole)
+ AMRMClient.ContainerRequest request =
+ roleHistory.requestNode(roleStatus, resource);
+
+ String hostname = request.getNodes()[0]
+ assert hostname == age3Active0.hostname
+
+ MockContainer container1 = factory.newContainer()
+ container1.nodeId = new MockNodeId(hostname, 0)
+ container1.priority = Priority.newInstance(0);
+
+ MockContainer container2 = factory.newContainer()
+ container2.nodeId = new MockNodeId(hostname, 0)
+ container2.priority = Priority.newInstance(1);
+
+ // put containers in List with role == 1 first
+ List<Container> containers = [ (Container) container2, (Container) container1 ]
+ List<Container> sortedContainers = roleHistory.prepareAllocationList(containers)
+
+ // verify that the first container has role == 0 after sorting
+ MockContainer c1 = (MockContainer) sortedContainers[0]
+ assert c1.priority.getPriority() == 0
+ MockContainer c2 = (MockContainer) sortedContainers[1]
+ assert c2.priority.getPriority() == 1
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
new file mode 100644
index 0000000..0a2ba60
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.providers.ProviderRole
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockContainer
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.OutstandingRequest
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.appmaster.state.RoleStatus
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * Test the RH availability list and request tracking: that hosts
+ * get removed and added
+ */
+class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
+
+ String roleName = "test"
+
+ NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1)
+ NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
+ NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ Resource resource = Resource.newInstance(1, 1)
+
+ ProviderRole provRole = new ProviderRole(roleName, 0)
+ RoleStatus roleStatus = new RoleStatus(provRole)
+
+ @Override
+ String getTestName() {
+ return "TestRoleHistoryAvailableList"
+ }
+
+ @Before
+ public void setupNodeMap() {
+ roleHistory.insert(nodes)
+ roleHistory.buildAvailableNodeLists();
+ }
+
+ @Test
+ public void testAvailableListBuiltForRoles() throws Throwable {
+ List<NodeInstance> available0 = roleHistory.cloneAvailableList(0)
+ assertListEquals([age3Active0, age2Active0], available0)
+ }
+
+ @Test
+ public void testRequestedNodeOffList() throws Throwable {
+ List<NodeInstance> available0 = roleHistory.cloneAvailableList(0)
+ NodeInstance ni = roleHistory.findNodeForNewInstance(roleStatus)
+ assert age3Active0 == ni
+ AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
+ roleStatus,
+ resource)
+ List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
+ assertListEquals([age2Active0], a2)
+ }
+
+ @Test
+ public void testFindAndRequestNode() throws Throwable {
+ AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+
+ assert age3Active0.hostname == req.nodes[0]
+ List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
+ assertListEquals([age2Active0], a2)
+ }
+
+ @Test
+ public void testRequestedNodeIntoReqList() throws Throwable {
+ AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ assert requests.size() == 1
+ assert age3Active0.hostname == requests[0].hostname
+ }
+
+ @Test
+ public void testCompletedRequestDropsNode() throws Throwable {
+ AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ assert requests.size() == 1
+ String hostname = requests[0].hostname
+ assert age3Active0.hostname == hostname
+ assert hostname == req.nodes[0]
+ MockContainer container = factory.newContainer(req, hostname)
+ assert roleHistory.onContainerAllocated(container , 2, 1)
+ assert roleHistory.outstandingRequestList.empty
+ }
+
+ @Test
+ public void testTwoRequests() throws Throwable {
+ AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
+ List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ assert requests.size() == 2
+ MockContainer container = factory.newContainer(req, req.nodes[0])
+ assert roleHistory.onContainerAllocated(container , 2, 1)
+ assert roleHistory.outstandingRequestList.size() == 1
+ container = factory.newContainer(req2, req2.nodes[0])
+ assert roleHistory.onContainerAllocated(container, 2, 2)
+ assert roleHistory.outstandingRequestList.empty
+ }
+
+
+ @Test
+ public void testThreeRequestsOneUnsatisified() throws Throwable {
+ AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
+ List<OutstandingRequest> requests = roleHistory.outstandingRequestList
+ assert requests.size() == 2
+ MockContainer container = factory.newContainer(req, req.nodes[0])
+ assert roleHistory.onContainerAllocated(container , 2, 1)
+ assert roleHistory.outstandingRequestList.size() == 1
+
+ container = factory.newContainer(req3, "three")
+ assert !roleHistory.onContainerAllocated(container, 3, 2)
+ assert roleHistory.outstandingRequestList.size() == 1
+
+ // the final allocation will trigger a cleanup
+ container = factory.newContainer(req2, "four")
+ // no node dropped
+ assert !roleHistory.onContainerAllocated(container, 3, 3)
+ // yet the list is now empty
+ assert roleHistory.outstandingRequestList.empty
+
+ // and the remainder goes onto the available list
+ List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
+ assertListEquals([age2Active0], a2)
+
+ }
+
+
+ @Test
+ public void testThreeRequests() throws Throwable {
+ AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
+ assert roleHistory.outstandingRequestList.size() == 2
+ assert req3.nodes == null
+ MockContainer container = factory.newContainer(req, req.nodes[0])
+ assert roleHistory.onContainerAllocated(container , 3, 1)
+ assert roleHistory.outstandingRequestList.size() == 1
+ container = factory.newContainer(req2, req2.nodes[0])
+ assert roleHistory.onContainerAllocated(container, 3, 2)
+ assert roleHistory.outstandingRequestList.empty
+ container = factory.newContainer(req3, "three")
+ assert !roleHistory.onContainerAllocated(container, 3, 3)
+ assert roleHistory.outstandingRequestList.empty
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
new file mode 100644
index 0000000..639c632
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.client.api.AMRMClient
+
+/**
+ * Provides allocation services to a cluster -both random and placed.
+ *
+ * Important: container allocations need an app attempt ID put into the container ID
+ */
+@CompileStatic
+@Slf4j
+class Allocator {
+
+ final MockYarnCluster cluster;
+ /**
+ * Rolling index into the cluster used for the
+ * next "random" assignment
+ */
+ private int rollingIndex = 0;
+
+ Allocator(MockYarnCluster cluster) {
+ this.cluster = cluster
+ }
+
+ /**
+ * Allocate a node using the list of nodes in the container as the
+ * hints.
+ * @param request request
+ * @return the allocated container -or null for none
+ */
+ MockContainer allocate(AMRMClient.ContainerRequest request) {
+ MockYarnCluster.MockYarnClusterNode node = null
+ MockYarnCluster.MockYarnClusterContainer allocated = null
+ if (request.nodes != null) {
+ for (String host : request.nodes) {
+ node = cluster.lookup(host)
+ allocated = node.allocate()
+ if (allocated != null) {
+ break
+ }
+ }
+ }
+
+ if (allocated) {
+ return createContainerRecord(request, allocated, node)
+ } else {
+ if (request.relaxLocality || request.nodes.isEmpty()) {
+ // fallback to anywhere
+ return allocateRandom(request)
+ } else {
+ //no match and locality can't be requested
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Allocate a node without any positioning -use whatever policy this allocator
+ * chooses.
+ * @param request request
+ * @return the allocated container -or null for none
+ */
+ MockContainer allocateRandom(AMRMClient.ContainerRequest request) {
+ int start = rollingIndex;
+ MockYarnCluster.MockYarnClusterNode node = cluster.nodeAt(rollingIndex)
+ MockYarnCluster.MockYarnClusterContainer allocated = node.allocate();
+ // if there is no space, try again -but stop when all the nodes
+ // have failed
+ while (allocated == null && start != nextIndex()) {
+ node = cluster.nodeAt(rollingIndex)
+ allocated = node.allocate();
+ }
+
+ //here the allocation is set, so create the response
+ return createContainerRecord(request, allocated, node)
+ }
+
+ /**
+ * Create a container record -if one was allocated
+ * @param allocated allocation -may be null
+ * @param node node with the container
+ * @return a container record, or null if there was no allocation
+ */
+ public MockContainer createContainerRecord(
+ AMRMClient.ContainerRequest request,
+ MockYarnCluster.MockYarnClusterContainer allocated,
+ MockYarnCluster.MockYarnClusterNode node) {
+ if (allocated == null) {
+ // no space
+ return null;
+ }
+ MockContainer container = new MockContainer()
+ container.id = new MockContainerId(allocated.cid)
+ container.nodeId = node.nodeId
+ container.nodeHttpAddress = node.httpAddress()
+ container.priority = request.priority
+ return container;
+ }
+
+ private int nextIndex() {
+ rollingIndex = (rollingIndex + 1) % cluster.clusterSize;
+ return rollingIndex;
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
new file mode 100644
index 0000000..628c729
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.ContainerState
+import org.apache.hadoop.yarn.api.records.ContainerStatus
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.test.SliderTestBase
+import org.junit.Before
+
+@CompileStatic
+@Slf4j
+abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles {
+ public static final int RM_MAX_RAM = 4096
+ public static final int RM_MAX_CORES = 64
+ MockFactory factory = new MockFactory()
+ AppState appState
+ MockYarnEngine engine
+ protected HadoopFS fs
+ protected SliderFileSystem sliderFileSystem
+ protected File historyWorkDir
+ protected Path historyPath;
+
+ @Override
+ void setup() {
+ super.setup()
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+ fs = HadoopFS.get(new URI("file:///"), conf)
+ sliderFileSystem = new SliderFileSystem(fs, conf)
+ engine = createYarnEngine()
+ }
+
+ /**
+ * Override point: called in setup() to create the YARN engine; can
+ * be changed for different sizes and options
+ * @return
+ */
+ public MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(64, 1)
+ }
+
+ @Before
+ void initApp(){
+
+ String historyDirName = testName;
+
+
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+
+ fs = HadoopFS.get(new URI("file:///"), conf)
+ historyWorkDir = new File("target/history", historyDirName)
+ historyPath = new Path(historyWorkDir.toURI())
+ fs.delete(historyPath, true)
+ appState = new AppState(new MockRecordFactory())
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+ appState.buildInstance(
+ factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null, null)
+ }
+
+ abstract String getTestName();
+
+ public RoleStatus getRole0Status() {
+ return appState.lookupRoleStatus(ROLE0)
+ }
+
+ public RoleStatus getRole1Status() {
+ return appState.lookupRoleStatus(ROLE1)
+ }
+
+ public RoleStatus getRole2Status() {
+ return appState.lookupRoleStatus(ROLE2)
+ }
+
+ /**
+ * Build a role instance from a container assignment
+ * @param assigned
+ * @return
+ */
+ RoleInstance roleInstance(ContainerAssignment assigned) {
+ Container target = assigned.container
+ RoleInstance ri = new RoleInstance(target)
+ ri.roleId = assigned.role.priority
+ ri.role = assigned.role
+ return ri
+ }
+
+
+ public NodeInstance nodeInstance(long age, int live0, int live1=0, int live2=0) {
+ NodeInstance ni = new NodeInstance("age${age}live[${live0},${live1},$live2]",
+ MockFactory.ROLE_COUNT)
+ ni.getOrCreate(0).lastUsed = age
+ ni.getOrCreate(0).live = live0;
+ if (live1 > 0) {
+ ni.getOrCreate(1).live = live1;
+ }
+ if (live2 > 0) {
+ ni.getOrCreate(2).live = live2;
+ }
+ return ni
+ }
+
+ /**
+ * Create a container status event
+ * @param c container
+ * @return a status
+ */
+ ContainerStatus containerStatus(Container c) {
+ return containerStatus(c.id)
+ }
+
+ /**
+ * Create a container status instance for the given ID, declaring
+ * that it was shut down by the application itself
+ * @param cid container Id
+ * @return the instance
+ */
+ public ContainerStatus containerStatus(ContainerId cid) {
+ ContainerStatus status = containerStatus(cid,
+ LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN)
+ return status
+ }
+
+ public ContainerStatus containerStatus(ContainerId cid, int exitCode) {
+ ContainerStatus status = ContainerStatus.newInstance(
+ cid,
+ ContainerState.COMPLETE,
+ "",
+ exitCode)
+ return status
+ }
+
+ /**
+ * Create nodes and bring them to the started state
+ * @return a list of roles
+ */
+ protected List<RoleInstance> createAndStartNodes() {
+ List<RoleInstance> instances = createAndSubmitNodes()
+ for (RoleInstance instance : instances) {
+ assert appState.onNodeManagerContainerStarted(instance.containerId)
+ }
+ return instances
+ }
+
+ /**
+ * Create nodes and submit them
+ * @return a list of roles
+ */
+ public List<RoleInstance> createAndSubmitNodes() {
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ List<Container> allocatedContainers = engine.execute(ops)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> operations = []
+ appState.onContainersAllocated(allocatedContainers, assignments, operations)
+ List<RoleInstance> instances = []
+ for (ContainerAssignment assigned : assignments) {
+ Container container = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+ instances << ri
+ //tell the app it arrived
+ appState.containerStartSubmitted(container, ri);
+ }
+ return instances
+ }
+
+ /**
+ * Extract the list of container IDs from the list of role instances
+ * @param instances instance list
+ * @param role role to look up
+ * @return the list of CIDs
+ */
+ List<ContainerId> extractContainerIds(
+ List<RoleInstance> instances,
+ int role) {
+ List<ContainerId> cids = []
+ instances.each { RoleInstance instance ->
+ if (instance.roleId == role) {
+ cids << instance.id
+ }
+ }
+ return cids
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
new file mode 100644
index 0000000..25c957e
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.slider.server.appmaster.state.AbstractRecordFactory
+import org.apache.slider.server.appmaster.state.AppState
+
+class MockAppState extends AppState {
+
+ public MockAppState(AbstractRecordFactory recordFactory) {
+ super(recordFactory);
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.groovy
new file mode 100644
index 0000000..3cb0a63
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.groovy
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
+import org.apache.hadoop.yarn.api.records.ApplicationId
+
+class MockApplicationAttemptId extends ApplicationAttemptId {
+
+ ApplicationId applicationId
+ int attemptId
+
+ @Override
+ ApplicationId getApplicationId() {
+ return applicationId
+ }
+
+ @Override
+ void setApplicationId(ApplicationId applicationId) {
+ this.applicationId = applicationId
+ }
+
+ @Override
+ int getAttemptId() {
+ return attemptId
+ }
+
+ @Override
+ void setAttemptId(int attemptId) {
+ this.attemptId = attemptId
+ }
+
+ @Override
+ protected void build() {
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationId.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationId.groovy
new file mode 100644
index 0000000..d7a63e1
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockApplicationId.groovy
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.ApplicationId
+
+class MockApplicationId extends ApplicationId {
+
+ private int id;
+ private long clusterTimestamp
+
+ @Override
+ int getId() {
+ return id;
+ }
+
+ @Override
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ @Override
+ long getClusterTimestamp() {
+ return clusterTimestamp
+ }
+
+ @Override
+ public void setClusterTimestamp(long clusterTimestamp) {
+ this.clusterTimestamp = clusterTimestamp
+ }
+
+ @Override
+ public void build() {
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
new file mode 100644
index 0000000..25bee36
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.*
+
+class MockContainer extends Container{
+
+ ContainerId id;
+ NodeId nodeId
+ String nodeHttpAddress;
+ Resource resource
+ Priority priority;
+ Token containerToken
+
+ @Override
+ public int compareTo(Container other) {
+ if (this.getId().compareTo(other.getId()) == 0) {
+ if (this.getNodeId().compareTo(other.getNodeId()) == 0) {
+ return this.getResource().compareTo(other.getResource());
+ } else {
+ return this.getNodeId().compareTo(other.getNodeId());
+ }
+ } else {
+ return this.getId().compareTo(other.getId());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "MockContainer{ id=$id" +
+ ", nodeHttpAddress='$nodeHttpAddress'," +
+ " priority=$priority }"
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerId.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerId.groovy
new file mode 100644
index 0000000..8985f3a
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerId.groovy
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
+import org.apache.hadoop.yarn.api.records.ContainerId
+
+class MockContainerId extends ContainerId implements Cloneable {
+
+ int id;
+ ApplicationAttemptId applicationAttemptId;
+
+ MockContainerId() {
+ }
+
+ MockContainerId(int id) {
+ setId(id);
+ }
+
+ MockContainerId(ContainerId that) {
+ id = that.id
+ applicationAttemptId = that.applicationAttemptId
+ }
+
+
+ int getId() {
+ return id
+ }
+
+ void setId(int id) {
+ this.id = id
+ }
+
+ ApplicationAttemptId getApplicationAttemptId() {
+ return applicationAttemptId
+ }
+
+ void setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) {
+ this.applicationAttemptId = applicationAttemptId
+ }
+
+ @Override
+ public void build() {
+
+ }
+
+ @Override
+ public String toString() {
+ return "mockcontainer_" + id;
+ }
+
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ return super.clone()
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
new file mode 100644
index 0000000..bfb00b0
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.ContainerState
+import org.apache.hadoop.yarn.api.records.ContainerStatus
+
+class MockContainerStatus extends ContainerStatus {
+
+ ContainerId containerId
+ ContainerState state
+ String diagnostics
+ int exitStatus
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
new file mode 100644
index 0000000..52472f0
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import com.google.common.collect.Maps
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
+import org.apache.hadoop.yarn.api.records.ApplicationId
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.providers.ProviderRole
+
+/**
+ * Factory for creating things
+ */
+//@CompileStatic
+@Slf4j
+class MockFactory implements MockRoles {
+
+ public static final ProviderRole PROVIDER_ROLE0 = new ProviderRole(
+ MockRoles.ROLE0,
+ 0)
+ public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole(
+ MockRoles.ROLE1,
+ 1)
+ public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
+ MockRoles.ROLE2,
+ 2)
+ int appIdCount;
+ int attemptIdCount;
+ int containerIdCount;
+
+ ApplicationId appId = newAppId()
+ ApplicationAttemptId attemptId = newApplicationAttemptId(appId)
+
+ /**
+ * List of roles
+ */
+ public static final List<ProviderRole> ROLES = [
+ PROVIDER_ROLE0,
+ PROVIDER_ROLE1,
+ PROVIDER_ROLE2,
+ ]
+
+ public static final int ROLE_COUNT = ROLES.size();
+
+ MockContainerId newContainerId() {
+ newContainerId(attemptId)
+ }
+
+ MockContainerId newContainerId(ApplicationAttemptId attemptId) {
+ MockContainerId cid = new MockContainerId()
+ cid.id = containerIdCount++
+ cid.applicationAttemptId = attemptId;
+ return cid;
+ }
+
+ MockApplicationAttemptId newApplicationAttemptId(ApplicationId appId) {
+ MockApplicationAttemptId id = new MockApplicationAttemptId()
+ id.attemptId = attemptIdCount++;
+ id.applicationId = appId
+ return id;
+ }
+
+ MockApplicationId newAppId() {
+ MockApplicationId id = new MockApplicationId()
+ id.setId(appIdCount++);
+ return id;
+ }
+
+ MockNodeId newNodeId() {
+ MockNodeId nodeId = new MockNodeId()
+ }
+
+ MockContainer newContainer(ContainerId cid) {
+ MockContainer c = new MockContainer()
+ c.id = cid
+ return c
+ }
+
+ MockContainer newContainer() {
+ newContainer(newContainerId())
+ }
+
+ /**
+ * Build a new container using the request to suppy priority and resource
+ * @param req request
+ * @param host hostname to assign to
+ * @return the container
+ */
+ MockContainer newContainer(AMRMClient.ContainerRequest req, String host) {
+ MockContainer container = newContainer(newContainerId())
+ container.resource = req.capability
+ container.priority = req.priority
+ container.nodeId = new MockNodeId(host)
+ return container
+ }
+
+ /**
+ * Create a cluster spec with the given desired role counts
+ * @param r1
+ * @param r2
+ * @param r3
+ * @return
+ */
+ ClusterDescription newClusterSpec(int r1, int r2, int r3) {
+ ClusterDescription cd = new ClusterDescription()
+ cd.roles = newComponentsSection(r1, r2, r3)
+
+ return cd
+ }
+
+ public HashMap<String, LinkedHashMap<String, String>> newComponentsSection(
+ int r1,
+ int r2,
+ int r3) {
+ return Maps.newHashMap([
+ (ROLE0): roleMap(r1),
+ (ROLE1): roleMap(r2),
+ (ROLE2): roleMap(r3),
+ ])
+ }
+
+ /**
+ * Create a cluster spec with the given desired role counts
+ * @param r1
+ * @param r2
+ * @param r3
+ * @return
+ */
+ ConfTree newConfTree(int r1, int r2, int r3) {
+ ConfTree cd = new ConfTree()
+
+ cd.components = newComponentsSection(r1, r2, r3)
+
+ return cd
+ }
+
+ /**
+ * Create a new instance with the given components definined in the
+ * resources section
+ * @param r1
+ * @param r2
+ * @param r3
+ * @return
+ */
+ AggregateConf newInstanceDefinition(int r1, int r2, int r3) {
+ AggregateConf instance = new AggregateConf()
+ instance.setResources(newConfTree(r1, r2, r3))
+ return instance
+ }
+
+
+
+ def roleMap(int count) {
+ return [
+ (ResourceKeys.COMPONENT_INSTANCES):count.toString()
+ ]
+ }
+
+ MockResource newResource() {
+ return new MockResource()
+ }
+
+ MockContainerStatus newContainerStatus() {
+ return new MockContainerStatus()
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java
new file mode 100644
index 0000000..b614834
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFileSystem.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.model.mock;
+
+import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public class MockFileSystem extends FilterFileSystem{
+ @Override
+ public Path resolvePath(Path p) throws IOException {
+ return new Path("hdfs://localhost/", p);
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeId.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeId.groovy
new file mode 100644
index 0000000..65a035d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeId.groovy
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.NodeId
+
+class MockNodeId extends NodeId {
+ String host
+ int port
+
+ MockNodeId() {
+ }
+
+ MockNodeId(String host) {
+ this.host = host
+ }
+
+ MockNodeId(String host, int port) {
+ this.host = host
+ this.port = port
+ }
+
+ String getHost() {
+ return host
+ }
+
+ void setHost(String host) {
+ this.host = host
+ }
+
+ int getPort() {
+ return port
+ }
+
+ void setPort(int port) {
+ this.port = port
+ }
+
+ @Override
+ protected void build() {
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
new file mode 100644
index 0000000..9f541d5
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.service.LifecycleEvent
+import org.apache.hadoop.service.Service.STATE
+import org.apache.hadoop.service.ServiceStateChangeListener
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.conf.MapOperations
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.launch.ContainerLauncher
+import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.providers.ProviderRole
+import org.apache.slider.providers.ProviderService
+import org.apache.slider.server.appmaster.state.StateAccessForProviders
+import org.apache.slider.server.appmaster.web.rest.agent.*
+import org.apache.slider.server.services.curator.RegistryBinderService
+import org.apache.slider.server.services.docstore.utility.EventCallback
+
+class MockProviderService implements ProviderService {
+
+ @Override
+ public String getName() {
+ return null;
+ }
+
+ @Override
+ public List<ProviderRole> getRoles() {
+ return null;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return null;
+ }
+
+ @Override
+ public void validateInstanceDefinition(AggregateConf instanceDefinition) throws SliderException {
+ }
+
+ @Override
+ public void init(Configuration config) {
+ }
+
+ @Override
+ public void start() {
+ }
+
+ @Override
+ public void stop() {
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public void registerServiceListener(ServiceStateChangeListener listener) {
+ }
+
+ @Override
+ public void unregisterServiceListener(ServiceStateChangeListener listener) {
+ }
+
+ @Override
+ public Configuration getConfig() {
+ return null;
+ }
+
+ @Override
+ public STATE getServiceState() {
+ return null;
+ }
+
+ @Override
+ public long getStartTime() {
+ return 0;
+ }
+
+ @Override
+ public boolean isInState(STATE state) {
+ return false;
+ }
+
+ @Override
+ public Throwable getFailureCause() {
+ return null;
+ }
+
+ @Override
+ public STATE getFailureState() {
+ return null;
+ }
+
+ @Override
+ public boolean waitForServiceToStop(long timeout) {
+ return false;
+ }
+
+ @Override
+ public List<LifecycleEvent> getLifecycleHistory() {
+ return null;
+ }
+
+ @Override
+ public Map<String,String> getBlockers() {
+ return null;
+ }
+
+ @Override
+ public int getExitCode() {
+ return 0;
+ }
+
+ @Override
+ public void buildContainerLaunchContext(ContainerLaunchContext ctx, SliderFileSystem sliderFileSystem, Path generatedConfPath, String role,
+ ClusterDescription clusterSpec, Map<String,String> roleOptions) throws IOException, SliderException {
+ }
+
+
+ @Override
+ public boolean exec(
+ AggregateConf instanceDefinition,
+ File confDir,
+ Map<String, String> env,
+ EventCallback execInProgress) throws IOException, SliderException {
+ return false;
+ }
+
+ @Override
+ public boolean isSupportedRole(String role) {
+ return false;
+ }
+
+ @Override
+ public Configuration loadProviderConfigurationInformation(File confDir) throws BadCommandArgumentsException, IOException {
+ return null;
+ }
+
+ @Override
+ public void validateApplicationConfiguration(
+ AggregateConf instanceDefinition,
+ File confDir,
+ boolean secure) throws IOException, SliderException {
+ }
+
+
+ @Override
+ public Map<String,String> buildProviderStatus() {
+ return null;
+ }
+
+ @Override
+ void buildContainerLaunchContext(
+ ContainerLauncher containerLauncher,
+ AggregateConf instanceDefinition,
+ Container container,
+ String role,
+ SliderFileSystem sliderFileSystem,
+ Path generatedConfPath,
+ MapOperations resourceComponent,
+ MapOperations appComponent,
+ Path containerTmpDirPath) throws IOException, SliderException {
+
+ }
+
+ @Override
+ public Map<String, URL> buildMonitorDetails(ClusterDescription clusterSpec) {
+ return null;
+ }
+
+ @Override
+ void bind(
+ StateAccessForProviders stateAccessor,
+ RegistryBinderService<ServiceInstanceData> registry) {
+
+ }
+
+ @Override
+ AgentRestOperations getAgentRestOperations() {
+ return new AgentRestOperations() {
+ @Override
+ public RegistrationResponse handleRegistration(Register registration) {
+ // dummy impl
+ RegistrationResponse response = new RegistrationResponse();
+ response.setResponseStatus(RegistrationStatus.OK);
+ return response;
+ }
+
+ @Override
+ public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
+ // dummy impl
+ return new HeartBeatResponse();
+ }
+ }
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
new file mode 100644
index 0000000..10a7708
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.state.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.RMOperationHandler
+
+@Slf4j
+class MockRMOperationHandler extends RMOperationHandler {
+ public List<AbstractRMOperation> operations = [];
+
+ @Override
+ public void releaseAssignedContainer(ContainerId containerId) {
+ operations.add(new ContainerReleaseOperation(containerId))
+ log.info("Releasing container ID " + containerId.getId())
+ }
+
+ @Override
+ public void addContainerRequest(AMRMClient.ContainerRequest req) {
+ operations.add(new ContainerRequestOperation(req))
+ log.info("Requesting container role #" + req.priority);
+ }
+
+ /**
+ * clear the history
+ */
+ public void clear() {
+ operations.clear()
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
new file mode 100644
index 0000000..f7d353f
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.slider.server.appmaster.state.AbstractRecordFactory
+
+class MockRecordFactory extends AbstractRecordFactory {
+
+ @Override
+ Resource newResource() {
+ return new MockResource()
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy
new file mode 100644
index 0000000..f4c54f3
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.Resource
+
+class MockResource extends Resource {
+ int memory
+ int virtualCores
+
+ MockResource(int memory=0, int vcores=0) {
+ this.memory = memory
+ this.virtualCores = vcores
+ }
+
+ @Override
+ public int compareTo(Resource other) {
+ int diff = this.getMemory() - other.getMemory();
+ if (diff == 0) {
+ diff = this.getVirtualCores() - other.getVirtualCores();
+ }
+ return diff;
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
new file mode 100644
index 0000000..b44482d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
@@ -0,0 +1,27 @@
+package org.apache.slider.server.appmaster.model.mock
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public interface MockRoles {
+
+ String ROLE0 = "role0"
+ String ROLE1 = "role1"
+ String ROLE2 = "role2"
+ int ROLE_COUNT = 3
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockSliderClusterProtocol.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockSliderClusterProtocol.groovy
new file mode 100644
index 0000000..e02b5b5
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockSliderClusterProtocol.groovy
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.ipc.ProtocolSignature
+import org.apache.hadoop.yarn.exceptions.YarnException
+import org.apache.slider.api.SliderClusterProtocol
+import org.apache.slider.api.proto.Messages
+import org.apache.slider.api.proto.Messages.AMSuicideRequestProto
+import org.apache.slider.api.proto.Messages.AMSuicideResponseProto
+import org.apache.slider.api.proto.Messages.EchoRequestProto
+import org.apache.slider.api.proto.Messages.EchoResponseProto
+import org.apache.slider.api.proto.Messages.FlexClusterRequestProto
+import org.apache.slider.api.proto.Messages.FlexClusterResponseProto
+import org.apache.slider.api.proto.Messages.GetClusterNodesRequestProto
+import org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto
+import org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto
+import org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto
+import org.apache.slider.api.proto.Messages.GetNodeRequestProto
+import org.apache.slider.api.proto.Messages.GetNodeResponseProto
+import org.apache.slider.api.proto.Messages.KillContainerRequestProto
+import org.apache.slider.api.proto.Messages.KillContainerResponseProto
+import org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto
+import org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto
+import org.apache.slider.api.proto.Messages.StopClusterRequestProto
+import org.apache.slider.api.proto.Messages.StopClusterResponseProto
+
+/**
+ *
+ */
+class MockSliderClusterProtocol implements SliderClusterProtocol {
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+ return 0;
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol, long clientVersion, int clientMethodsHash) throws IOException {
+ return null;
+ }
+
+ @Override
+ public StopClusterResponseProto stopCluster(StopClusterRequestProto request) throws IOException, YarnException {
+ return null;
+ }
+
+ @Override
+ public FlexClusterResponseProto flexCluster(FlexClusterRequestProto request) throws IOException, YarnException {
+ return null;
+ }
+
+ @Override
+ public GetJSONClusterStatusResponseProto getJSONClusterStatus(GetJSONClusterStatusRequestProto request) throws IOException, YarnException {
+ return null;
+ }
+
+ @Override
+ public ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(ListNodeUUIDsByRoleRequestProto request) throws IOException, YarnException {
+ return null;
+ }
+
+ @Override
+ public GetNodeResponseProto getNode(GetNodeRequestProto request) throws IOException, YarnException {
+ return null;
+ }
+
+ @Override
+ public GetClusterNodesResponseProto getClusterNodes(GetClusterNodesRequestProto request) throws IOException, YarnException {
+ return null;
+ }
+
+ @Override
+ public EchoResponseProto echo(EchoRequestProto request) throws IOException, YarnException {
+ return null;
+ }
+
+ @Override
+ public KillContainerResponseProto killContainer(KillContainerRequestProto request) throws IOException, YarnException {
+ return null;
+ }
+
+ @Override
+ public AMSuicideResponseProto amSuicide(AMSuicideRequestProto request) throws IOException, YarnException {
+ return null;
+ }
+
+ @Override
+ Messages.GetInstanceDefinitionResponseProto getInstanceDefinition(
+ Messages.GetInstanceDefinitionRequestProto request)
+ throws IOException, YarnException {
+ return null
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
new file mode 100644
index 0000000..eac5b6c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.NodeId
+
+/**
+ * Models the cluster itself: a set of mock cluster nodes.
+ *
+ * nodes retain the slot model with a limit of 2^8 slots/host -this
+ * lets us use 24 bits of the container ID for hosts, and so simulate
+ * larger hosts.
+ *
+ * upper 32: index into nodes in the cluster
+ * NodeID hostname is the index in hex format; this is parsed down to the index
+ * to resolve the host
+ *
+ * Important: container IDs will be reused as containers get recycled. This
+ * is not an attempt to realistically mimic a real YARN cluster, just
+ * simulate it enough for Slider to explore node re-use and its handling
+ * of successful and unsuccessful allocations.
+ *
+ * There is little or no checking of valid parameters in here -this is for
+ * test use, not production.
+ */
+@CompileStatic
+@Slf4j
+public class MockYarnCluster {
+
+ final int clusterSize;
+ final int containersPerNode;
+ MockYarnClusterNode[] nodes;
+
+ MockYarnCluster(int clusterSize, int containersPerNode) {
+ this.clusterSize = clusterSize
+ this.containersPerNode = containersPerNode
+ build();
+ }
+
+ /**
+ * Build the cluster.
+ */
+ private void build() {
+ nodes = new MockYarnClusterNode[clusterSize]
+ for (int i = 0; i < clusterSize; i++) {
+ nodes[i] = new MockYarnClusterNode(i, containersPerNode)
+ }
+ }
+
+ MockYarnClusterNode nodeAt(int index) {
+ return nodes[index]
+ }
+
+ MockYarnClusterNode lookup(String hostname) {
+ int index = Integer.valueOf(hostname, 16)
+ return nodeAt(index)
+ }
+
+ MockYarnClusterNode lookup(NodeId nodeId) {
+ return lookup(nodeId.host)
+ }
+
+ MockYarnClusterNode lookupOwner(ContainerId cid) {
+ return nodeAt(extractHost(cid.id))
+ }
+
+ /**
+ * Release a container: return true if it was actually in use
+ * @param cid container ID
+ * @return the container released
+ */
+ MockYarnClusterContainer release(ContainerId cid) {
+ int host = extractHost(cid.id)
+ return nodeAt(host).release(cid.id)
+ }
+
+ int containersInUse() {
+ int count = 0;
+ nodes.each { MockYarnClusterNode it -> count += it.containersInUse() }
+ return count;
+ }
+
+ /**
+ * Containers free
+ * @return
+ */
+ int containersFree() {
+ return totalClusterCapacity() - containersInUse();
+ }
+
+ int totalClusterCapacity() {
+ return clusterSize * containersPerNode;
+ }
+
+ /**
+ * Reset all the containers
+ */
+ public void reset() {
+ nodes.each { MockYarnClusterNode node ->
+ node.reset()
+ }
+ }
+
+ /**
+ * Bulk allocate the specific number of containers on a range of the cluster
+ * @param startNode start of the range
+ * @param endNode end of the range
+ * @param count count
+ * @return the number actually allocated -it will be less the count supplied
+ * if the node was full
+ */
+ public int bulkAllocate(int startNode, int endNode, int count) {
+ int total = 0;
+ for (int i in startNode..endNode) {
+ total += nodeAt(i).bulkAllocate(count).size()
+ }
+ return total;
+ }
+
+
+/**
+ * Model cluster nodes on the simpler "slot" model than the YARN-era
+ * resource allocation model. Why? Makes it easier to implement.
+ *
+ * When a cluster is offline,
+ */
+ public static class MockYarnClusterNode {
+
+ public final int nodeIndex
+ public final String hostname;
+ public final MockNodeId nodeId;
+ public final MockYarnClusterContainer[] containers;
+ private boolean offline;
+
+ public MockYarnClusterNode(int index, int size) {
+ nodeIndex = index;
+ hostname = String.format(Locale.ENGLISH, "%08x", index)
+ nodeId = new MockNodeId(hostname, 0);
+
+ containers = new MockYarnClusterContainer[size];
+ for (int i = 0; i < size; i++) {
+ int cid = makeCid(index, i);
+ MockContainerId mci = new MockContainerId(id: cid)
+ containers[i] = new MockYarnClusterContainer(mci)
+ }
+ }
+
+ public MockYarnClusterContainer lookup(int containerId) {
+ return containers[extractContainer(containerId)]
+ }
+
+ /**
+ * Go offline; release all containers
+ */
+ public void goOffline() {
+ if (!offline) {
+ offline = true;
+ reset()
+ }
+ }
+
+ public void goOnline() {
+ offline = false;
+ }
+
+ /**
+ * allocate a container -if one is available
+ * @return the container or null for none free
+ * -or the cluster node is offline
+ */
+ public MockYarnClusterContainer allocate() {
+ if (!offline) {
+ for (int i = 0; i < containers.size(); i++) {
+ MockYarnClusterContainer c = containers[i]
+ if (!c.busy) {
+ c.busy = true;
+ return c;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Bulk allocate the specific number of containers
+ * @param count count
+ * @return the list actually allocated -it will be less the count supplied
+ * if the node was full
+ */
+ public List<MockYarnClusterContainer> bulkAllocate(int count) {
+ List < MockYarnClusterContainer > result = []
+ for (int i = 0; i < count; i++) {
+ MockYarnClusterContainer allocation = allocate();
+ if (allocation == null) {
+ break;
+ }
+ result << allocation
+ }
+ return result
+ }
+
+
+
+ /**
+ * Release a container
+ * @param cid container ID
+ * @return the container if the container was busy before the release
+ */
+ public MockYarnClusterContainer release(int cid) {
+ MockYarnClusterContainer container = containers[extractContainer(cid)]
+ boolean b = container.busy;
+ container.busy = false;
+ return b? container: null;
+ }
+
+ public String httpAddress() {
+ return "http://$hostname/"
+ }
+
+ /**
+ * Reset all the containers
+ */
+ public void reset() {
+ containers.each { MockYarnClusterContainer cont ->
+ cont.reset()
+ }
+ }
+
+ public int containersInUse() {
+ int c = 0;
+ containers.each { MockYarnClusterContainer cont ->
+ c += cont.busy ? 1 : 0
+ }
+ return c
+ }
+
+ public int containersFree() {
+ return containers.length - containersInUse();
+ }
+ }
+
+ /**
+ * Cluster container
+ */
+ public static class MockYarnClusterContainer {
+ MockContainerId cid;
+ boolean busy;
+
+ MockYarnClusterContainer(MockContainerId cid) {
+ this.cid = cid
+ }
+
+ void reset() {
+ busy = false;
+ }
+ }
+
+ public static int makeCid(int hostIndex, int containerIndex) {
+ return (hostIndex << 8) | containerIndex & 0xff;
+ }
+
+ public static final int extractHost(int cid) {
+ return (cid >>> 8);
+ }
+
+ public static final int extractContainer(int cid) {
+ return (cid & 0xff);
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
new file mode 100644
index 0000000..7ebdf52
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
+import org.apache.hadoop.yarn.api.records.ApplicationId
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.state.ContainerRequestOperation
+
+/**
+ * This is an evolving engine to mock YARN operations
+ */
+@CompileStatic
+@Slf4j
+class MockYarnEngine {
+
+ MockYarnCluster cluster;
+ Allocator allocator;
+ List<ContainerRequestOperation> pending = [];
+
+ ApplicationId appId = new MockApplicationId(
+ id: 0,
+ clusterTimestamp: 0,
+ )
+
+ ApplicationAttemptId attemptId = new MockApplicationAttemptId(
+ applicationId: appId,
+ attemptId: 1,
+ )
+
+ int containerCount() {
+ return cluster.containersInUse();
+ }
+
+ MockYarnEngine(int clusterSize, int containersPerNode) {
+ cluster = new MockYarnCluster(clusterSize, containersPerNode)
+ allocator = new Allocator(cluster)
+ }
+
+/**
+ * Allocate a container from a request. The containerID will be
+ * unique, nodeId and other fields chosen internally with
+ * no such guarantees; resource and priority copied over
+ * @param request request
+ * @return container
+ */
+ Container allocateContainer(AMRMClient.ContainerRequest request) {
+ MockContainer allocated = allocator.allocate(request)
+ if (allocated != null) {
+ MockContainerId id = allocated.id as MockContainerId
+ id.applicationAttemptId = attemptId;
+ }
+ return allocated
+ }
+
+ MockYarnCluster.MockYarnClusterContainer releaseContainer(ContainerId containerId) {
+ return cluster.release(containerId)
+ }
+
+ /**
+ * Process a list of operations -release containers to be released,
+ * allocate those for which there is space (but don't rescan the list after
+ * the scan)
+ * @param ops
+ * @return
+ */
+ List<Container> execute(
+ List<AbstractRMOperation> ops
+ ) {
+ return execute(ops, [])
+ }
+
+ /**
+ * Process a list of operations -release containers to be released,
+ * allocate those for which there is space (but don't rescan the list after
+ * the scan). Unsatisifed entries are appended to the "pending" list
+ * @param ops operations
+ * @return the list of all satisfied operations
+ */
+ List<Container> execute(List<AbstractRMOperation> ops,
+ List<ContainerId> released) {
+ List<Container> allocation = [];
+ ops.each { AbstractRMOperation op ->
+ if (op instanceof ContainerReleaseOperation) {
+ ContainerReleaseOperation cro = (ContainerReleaseOperation) op
+ ContainerId cid = cro.containerId
+ releaseContainer(cid);
+ released.add(cid)
+ } else {
+ ContainerRequestOperation req = (ContainerRequestOperation) op
+ Container container = allocateContainer(req.request)
+ if (container != null) {
+ allocation.add(container)
+ } else {
+ log.debug("Unsatisfied allocation $req")
+ pending.add(req)
+ }
+ }
+ }
+ return allocation
+ }
+
+}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
new file mode 100644
index 0000000..ca3421b
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.view
+
+import com.google.inject.AbstractModule
+import com.google.inject.Guice
+import com.google.inject.Injector
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.SliderClusterProtocol
+import org.apache.slider.providers.ProviderService
+import org.apache.slider.server.appmaster.model.mock.MockAppState
+import org.apache.slider.server.appmaster.model.mock.MockProviderService
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockSliderClusterProtocol
+import org.apache.slider.server.appmaster.state.AbstractRecordFactory
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.web.WebAppApi
+import org.apache.slider.server.appmaster.web.WebAppApiImpl
+import org.junit.Before
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+public class TestClusterSpecificationBlock {
+
+ private ClusterSpecificationBlock clusterSpecBlock;
+
+ @Before
+ public void setup() {
+ SliderClusterProtocol clusterProto = new MockSliderClusterProtocol();
+ AppState appState = new MyAppState(new MockRecordFactory());
+ ProviderService providerService = new MockProviderService();
+
+ WebAppApiImpl inst = new WebAppApiImpl(clusterProto, appState, providerService);
+
+ Injector injector = Guice.createInjector(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(WebAppApi.class).toInstance(inst);
+ }
+ });
+
+ clusterSpecBlock = injector.getInstance(ClusterSpecificationBlock.class);
+ }
+
+ @Test
+ public void testJsonGeneration() {
+ StringWriter sw = new StringWriter(64);
+ PrintWriter pw = new PrintWriter(sw);
+
+ Hamlet hamlet = new Hamlet(pw, 0, false);
+
+ int level = hamlet.nestLevel();
+ clusterSpecBlock.doRender(hamlet);
+
+ assert level == hamlet.nestLevel();
+ }
+
+ private static class MyAppState extends MockAppState {
+ public MyAppState(AbstractRecordFactory recordFactory) {
+ super(recordFactory);
+ this.clusterStatus = new MockClusterDescription();
+ }
+ }
+
+ private static class MockClusterDescription extends ClusterDescription {
+ @Override
+ public String toJsonString() {
+ return "{\"foo\": \"bar\"}";
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy
new file mode 100644
index 0000000..2290dfa
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.view
+
+import com.google.inject.AbstractModule
+import com.google.inject.Guice
+import com.google.inject.Injector
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.Priority
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR
+import org.apache.hadoop.yarn.webapp.hamlet.HamletImpl.EImp
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.api.SliderClusterProtocol
+import org.apache.slider.providers.ProviderService
+import org.apache.slider.server.appmaster.model.mock.*
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.web.WebAppApi
+import org.apache.slider.server.appmaster.web.WebAppApiImpl
+import org.apache.slider.server.appmaster.web.view.ContainerStatsBlock.ClusterNodeNameComparator
+import org.apache.slider.server.appmaster.web.view.ContainerStatsBlock.TableAnchorContent
+import org.apache.slider.server.appmaster.web.view.ContainerStatsBlock.TableContent
+import org.junit.Before
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+public class TestContainerStatsBlock {
+
+ private ContainerStatsBlock statsBlock;
+
+
+ private Container cont1, cont2;
+
+ @Before
+ public void setup() {
+ SliderClusterProtocol clusterProto = new MockSliderClusterProtocol();
+ AppState appState = new MockAppState(new MockRecordFactory());
+ ProviderService providerService = new MockProviderService();
+
+ WebAppApiImpl inst = new WebAppApiImpl(clusterProto, appState, providerService);
+
+ Injector injector = Guice.createInjector(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(WebAppApi.class).toInstance(inst);
+ }
+ });
+
+ statsBlock = injector.getInstance(ContainerStatsBlock.class);
+
+ cont1 = new MockContainer();
+ cont1.id = new MockContainerId();
+ ((MockContainerId) cont1.id).setId(0);
+ cont1.nodeId = new MockNodeId();
+ cont1.priority = Priority.newInstance(1);
+ cont1.resource = new MockResource();
+
+ cont2 = new MockContainer();
+ cont2.id = new MockContainerId();
+ ((MockContainerId) cont2.id).setId(1);
+ cont2.nodeId = new MockNodeId();
+ cont2.priority = Priority.newInstance(1);
+ cont2.resource = new MockResource();
+ }
+
+ @Test
+ public void testGetContainerInstances() {
+ List<RoleInstance> roles = [
+ new RoleInstance(cont1),
+ new RoleInstance(cont2),
+ ];
+ Map<String, RoleInstance> map = statsBlock.getContainerInstances(roles);
+
+ assert 2 == map.size();
+
+ assert map.containsKey("mockcontainer_0");
+ assert map.get("mockcontainer_0").equals(roles[0]);
+
+ assert map.containsKey("mockcontainer_1");
+ assert map.get("mockcontainer_1").equals(roles[1]);
+ }
+
+ @Test
+ public void testGenerateRoleDetailsWithTwoColumns() {
+ StringWriter sw = new StringWriter(64);
+ PrintWriter pw = new PrintWriter(sw);
+
+ Hamlet hamlet = new Hamlet(pw, 0, false);
+
+ // Make a div to put the content into
+ DIV<Hamlet> div = hamlet.div();
+
+ String detailsName = "testing";
+ String selector = "selector";
+ Map<TableContent,String> data = new HashMap<TableContent,String>();
+ data.put(new ContainerStatsBlock.TableContent("Foo"), "bar");
+
+ int levelPrior = hamlet.nestLevel();
+ statsBlock.generateRoleDetails(div, selector, detailsName, data.entrySet());
+
+ // Close out the div we made
+ // DIV<Hamlet>._() will actually invoke the wrong method (creating <p>), explicit
+ // cast to make sure we're closing out the <div>
+ ((EImp) div)._();
+
+ assert levelPrior == hamlet.nestLevel();
+ }
+
+ @Test
+ public void testGenerateRoleDetailsWithOneColumn() {
+ StringWriter sw = new StringWriter(64);
+ PrintWriter pw = new PrintWriter(sw);
+
+ Hamlet hamlet = new Hamlet(pw, 0, false);
+ DIV<Hamlet> div = hamlet.div();
+
+ String detailsName = "testing";
+ String selector = "selector";
+ Map<TableContent,String> data = new HashMap<TableContent,String>();
+ data.put(new ContainerStatsBlock.TableContent("Bar"), null);
+
+ int levelPrior = hamlet.nestLevel();
+ statsBlock.generateRoleDetails(div, selector, detailsName, data.entrySet());
+
+ // Close out the div we made
+ // DIV<Hamlet>._() will actually invoke the wrong method (creating <p>), explicit
+ // cast to make sure we're closing out the <div>
+ ((EImp) div)._();
+
+ assert levelPrior == hamlet.nestLevel();
+ }
+
+ @Test
+ public void testGenerateRoleDetailsWithNoData() {
+ StringWriter sw = new StringWriter(64);
+ PrintWriter pw = new PrintWriter(sw);
+
+ Hamlet hamlet = new Hamlet(pw, 0, false);
+ DIV<Hamlet> div = hamlet.div();
+
+ String detailsName = "testing";
+ String selector = "selector";
+ Map<TableContent,String> data = new HashMap<TableContent,String>();
+
+ int levelPrior = hamlet.nestLevel();
+ statsBlock.generateRoleDetails(div, selector, detailsName, data.entrySet());
+
+ // Close out the div we made
+ // DIV<Hamlet>._() will actually invoke the wrong method (creating <p>), explicit
+ // cast to make sure we're closing out the <div>
+ ((EImp) div)._();
+
+ assert levelPrior == hamlet.nestLevel();
+ }
+
+ @Test
+ public void testClusterNodeNameComparator() {
+ ClusterNode n1 = new ClusterNode(new MockContainerId(1)),
+ n2 = new ClusterNode(new MockContainerId(2)),
+ n3 = new ClusterNode(new MockContainerId(3));
+
+ List<ClusterNode> nodes = new ArrayList<ClusterNode>();
+ nodes.add(n2);
+ nodes.add(n3);
+ nodes.add(n1);
+
+ Collections.sort(nodes, new ClusterNodeNameComparator());
+
+ String prevName = "";
+ for (ClusterNode node : nodes) {
+ assert prevName.compareTo(node.name) <= 0;
+ prevName = node.name;
+ }
+ }
+
+ @Test
+ public void testTableContent() {
+ StringWriter sw = new StringWriter(64);
+ PrintWriter pw = new PrintWriter(sw);
+ TableContent tc = new TableContent("foo");
+
+ Hamlet hamlet = new Hamlet(pw, 0, false);
+ TR<TABLE<Hamlet>> tr = hamlet.table().tr();
+
+ int prevLevel = hamlet.nestLevel();
+ // printCell should not end the tr
+ tc.printCell(tr);
+ tr._();
+ assert prevLevel == hamlet.nestLevel();
+ }
+
+ @Test
+ public void testTableAnchorContent() {
+ StringWriter sw = new StringWriter(64);
+ PrintWriter pw = new PrintWriter(sw);
+ TableContent tc = new TableAnchorContent("foo", "http://bar.com");
+
+ Hamlet hamlet = new Hamlet(pw, 0, false);
+ TR<TABLE<Hamlet>> tr = hamlet.table().tr();
+
+ int prevLevel = hamlet.nestLevel();
+ // printCell should not end the tr
+ tc.printCell(tr);
+ tr._();
+ assert prevLevel == hamlet.nestLevel();
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
new file mode 100644
index 0000000..0311eaa
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.view
+
+import com.google.inject.AbstractModule
+import com.google.inject.Guice
+import com.google.inject.Injector
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.Priority
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet
+import org.apache.slider.api.SliderClusterProtocol
+import org.apache.slider.providers.ProviderService
+import org.apache.slider.server.appmaster.model.mock.*
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.web.WebAppApi
+import org.apache.slider.server.appmaster.web.WebAppApiImpl
+import org.junit.Before
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+public class TestIndexBlock {
+
+ private IndexBlock indexBlock;
+
+ private Container cont1, cont2;
+
+ @Before
+ public void setup() {
+ SliderClusterProtocol clusterProto = new MockSliderClusterProtocol();
+ AppState appState = new MockAppState(new MockRecordFactory());
+ ProviderService providerService = new MockProviderService();
+
+ WebAppApiImpl inst = new WebAppApiImpl(clusterProto, appState, providerService);
+
+ Injector injector = Guice.createInjector(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(WebAppApi.class).toInstance(inst);
+ }
+ });
+
+ indexBlock = injector.getInstance(IndexBlock.class);
+
+ cont1 = new MockContainer();
+ cont1.id = new MockContainerId();
+ ((MockContainerId) cont1.id).setId(0);
+ cont1.nodeId = new MockNodeId();
+ cont1.priority = Priority.newInstance(1);
+ cont1.resource = new MockResource();
+
+ cont2 = new MockContainer();
+ cont2.id = new MockContainerId();
+ ((MockContainerId) cont2.id).setId(1);
+ cont2.nodeId = new MockNodeId();
+ cont2.priority = Priority.newInstance(1);
+ cont2.resource = new MockResource();
+ }
+
+ @Test
+ public void testIndex() {
+ StringWriter sw = new StringWriter(64);
+ PrintWriter pw = new PrintWriter(sw);
+
+ Hamlet hamlet = new Hamlet(pw, 0, false);
+
+ int level = hamlet.nestLevel();
+ indexBlock.doIndex(hamlet, "accumulo");
+
+ assert level == hamlet.nestLevel();
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/MockService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/MockService.groovy
new file mode 100644
index 0000000..50d8560
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/MockService.groovy
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility
+
+import org.apache.hadoop.service.AbstractService
+import org.apache.hadoop.service.ServiceStateException
+import org.apache.slider.core.main.ExitCodeProvider
+
+/**
+ * Little mock service to simulate delays
+ */
+class MockService extends AbstractService implements ExitCodeProvider {
+
+ boolean fail = false;
+ int exitCode;
+ int lifespan = -1;
+
+ MockService() {
+ super("mock")
+ }
+
+ MockService(String name, boolean fail, int lifespan) {
+ super(name)
+ this.fail = fail
+ this.lifespan = lifespan;
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ //act on the lifespan here
+ if (lifespan > 0) {
+ Thread.start {
+ Thread.sleep(lifespan)
+ finish()
+ }
+ } else {
+ if (lifespan == 0) {
+ finish();
+ } else {
+ //continue until told not to
+ }
+ }
+ }
+
+ void finish() {
+ if (fail) {
+ ServiceStateException e = new ServiceStateException("$name failed")
+ noteFailure(e);
+ stop();
+ throw e
+ } else {
+ stop();
+ }
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/TestCompoundService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/TestCompoundService.groovy
new file mode 100644
index 0000000..d9832c2
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/TestCompoundService.groovy
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.service.Service
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.junit.Test
+
+@Slf4j
+class TestCompoundService extends ServiceLauncherBaseTest {
+
+
+ @Test
+ public void testSingleCompound() throws Throwable {
+ CompoundService parent = startService([new MockService()])
+ parent.stop();
+ }
+
+
+ @Test
+ public void testSingleCompoundTerminating() throws Throwable {
+ CompoundService parent = startService([new MockService("1",false,100)])
+ assert waitForParentToStop(parent);
+ }
+
+ public boolean waitForParentToStop(CompoundService parent) {
+ boolean stop = parent.waitForServiceToStop(1000)
+ if (!stop) {
+ log.error("Service failed to stop $parent")
+ logState(parent)
+ }
+ return stop
+ }
+
+
+ @Test
+ public void testSingleCompoundFailing() throws Throwable {
+ CompoundService parent = startService([new MockService("1",true,100)])
+ assert parent.waitForServiceToStop(1000);
+ assert parent.getFailureCause() != null;
+ }
+
+ @Test
+ public void testCompound() throws Throwable {
+ MockService one = new MockService("one", false, 100)
+ MockService two = new MockService("two", false, 100)
+ CompoundService parent = startService([one, two])
+ assert waitForParentToStop(parent);
+ assert one.isInState(Service.STATE.STOPPED)
+ assert two.isInState(Service.STATE.STOPPED)
+ }
+
+ @Test
+ public void testCompoundOneLongLived() throws Throwable {
+ MockService one = new MockService("one", false, 500)
+ MockService two = new MockService("two", false, 100)
+ CompoundService parent = startService([one, two])
+ assert waitForParentToStop(parent);
+ assert one.isInState(Service.STATE.STOPPED)
+ assert two.isInState(Service.STATE.STOPPED)
+ }
+
+
+ @Test
+ public void testNotificationInCompound() throws Throwable {
+ boolean notified = false;
+ EventCallback ecb = new EventCallback() {
+ @Override
+ void eventCallbackEvent() {
+ log.info("EventCallback")
+ notified = true;
+ }
+ }
+ MockService one = new MockService("one", false, 100)
+ EventNotifyingService ens = new EventNotifyingService(ecb, 100);
+ MockService two = new MockService("two", false, 100)
+ CompoundService parent = startService([one, ens, two])
+ assert waitForParentToStop(parent);
+ assert one.isInState(Service.STATE.STOPPED)
+ assert ens.isInState(Service.STATE.STOPPED)
+ assert two.isInState(Service.STATE.STOPPED)
+ assert notified
+ }
+
+ @Test
+ public void testFailingCompound() throws Throwable {
+ MockService one = new MockService("one", true, 100)
+ MockService two = new MockService("two", false, 100)
+ CompoundService parent = startService([one, two])
+ assert waitForParentToStop(parent);
+
+
+
+ assert one.isInState(Service.STATE.STOPPED)
+ assert one.failureCause != null
+ assert two.isInState(Service.STATE.STOPPED)
+ }
+
+
+
+
+ @Test
+ public void testCompoundInCompound() throws Throwable {
+ MockService one = new MockService("one", false, 100)
+ MockService two = new MockService("two", false, 100)
+ CompoundService parent = buildService([one, two])
+ CompoundService outer = startService([parent])
+ assert outer.waitForServiceToStop(1000);
+ assert one.isInState(Service.STATE.STOPPED)
+ assert two.isInState(Service.STATE.STOPPED)
+ }
+
+ public CompoundService startService(List<Service> services) {
+ CompoundService parent = buildService(services)
+ //expect service to start and stay started
+ parent.start();
+ return parent
+ }
+
+ public CompoundService buildService(List<Service> services) {
+ CompoundService parent = new CompoundService("test")
+ services.each { parent.addService(it) }
+ parent.init(new Configuration())
+ return parent
+ }
+
+
+ void logState(Parent p) {
+ logService(p)
+ for (Service s : p.services) {
+ logService(s)
+ }
+ }
+
+ public void logService(Service s) {
+ log.info(s.toString())
+ if (s.getFailureCause()) {
+ log.info("Failed in state ${s.getFailureState()} with $s.failureCause")
+ }
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/TestMockService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/TestMockService.groovy
new file mode 100644
index 0000000..99b6e05
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/TestMockService.groovy
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.service.Service
+import org.apache.hadoop.service.ServiceStateException
+import org.junit.Test
+
+class TestMockService {
+
+ @Test
+ public void testSimpleLifecycle() throws Throwable {
+ MockService s = new MockService("1",false,-1);
+ s.init(new Configuration())
+ s.start();
+ assert s.isInState(Service.STATE.STARTED)
+ }
+
+ @Test
+ public void testSimpleLifecycleWait() throws Throwable {
+ MockService s = new MockService("1",false,-1);
+ s.init(new Configuration())
+ s.start();
+ assert s.isInState(Service.STATE.STARTED)
+ s.stop();
+ s.waitForServiceToStop(0);
+ }
+
+ @Test
+ public void testStoppingService() throws Throwable {
+ MockService s = new MockService("1",false,100);
+ s.init(new Configuration())
+ s.start();
+ Thread.sleep(1000);
+ assert s.isInState(Service.STATE.STOPPED)
+ }
+
+ @Test
+ public void testStoppingWaitService() throws Throwable {
+ MockService s = new MockService("1",false,100);
+ s.init(new Configuration())
+ s.start();
+ s.waitForServiceToStop(0);
+ assert s.isInState(Service.STATE.STOPPED)
+ }
+
+
+
+ @Test
+ public void testFailingService() throws Throwable {
+ MockService s = new MockService("1",true,100);
+ s.init(new Configuration())
+ s.start();
+ s.waitForServiceToStop(0);
+
+ assert s.isInState(Service.STATE.STOPPED)
+ assert s.failureCause != null
+ }
+
+ @Test
+ public void testFailingInStart() throws Throwable {
+ MockService s = new MockService("1",true,0);
+ s.init(new Configuration())
+ try {
+ s.start();
+ //failure, raise a fault with some text
+ assert null == s
+ } catch (ServiceStateException e) {
+ //expected
+ }
+ assert s.isInState(Service.STATE.STOPPED)
+ assert s.failureCause != null
+ s.waitForServiceToStop(0);
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/TestSequenceService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/TestSequenceService.groovy
new file mode 100644
index 0000000..1670dc5
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/services/docstore/utility/TestSequenceService.groovy
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.docstore.utility
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.service.Service
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.junit.Test
+
+@Slf4j
+class TestSequenceService extends ServiceLauncherBaseTest {
+
+
+ @Test
+ public void testSingleSequence() throws Throwable {
+ SequenceService ss = startService([new MockService()])
+ ss.stop();
+ }
+
+ @Test
+ public void testSequence() throws Throwable {
+ MockService one = new MockService("one", false, 100)
+ MockService two = new MockService("two", false, 100)
+ SequenceService ss = startService([one, two])
+ assert ss.waitForServiceToStop(1000);
+ assert one.isInState(Service.STATE.STOPPED)
+ assert two.isInState(Service.STATE.STOPPED)
+ assert ss.previousService == two
+ }
+
+ @Test
+ public void testNotificationInSequence() throws Throwable {
+ boolean notified = false;
+ EventCallback ecb = new EventCallback() {
+ @Override
+ void eventCallbackEvent() {
+ log.info("EventCallback")
+ notified = true;
+ }
+ }
+ MockService one = new MockService("one", false, 100)
+ EventNotifyingService ens = new EventNotifyingService(ecb, 100);
+ MockService two = new MockService("two", false, 100)
+ SequenceService ss = startService([one, ens, two])
+ assert ss.waitForServiceToStop(1000);
+ assert one.isInState(Service.STATE.STOPPED)
+ assert ens.isInState(Service.STATE.STOPPED)
+ assert two.isInState(Service.STATE.STOPPED)
+ assert notified
+ }
+
+ @Test
+ public void testFailingSequence() throws Throwable {
+ MockService one = new MockService("one", true, 100)
+ MockService two = new MockService("two", false, 100)
+ SequenceService ss = startService([one, two])
+ assert ss.waitForServiceToStop(1000);
+ assert one.isInState(Service.STATE.STOPPED)
+ assert two.isInState(Service.STATE.NOTINITED)
+ assert ss.previousService == one
+
+ }
+
+
+
+ @Test
+ public void testFailInStartNext() throws Throwable {
+ MockService one = new MockService("one", false, 100)
+ MockService two = new MockService("two", true, 0)
+ MockService three = new MockService("3", false, 0)
+ SequenceService ss = startService([one, two, three])
+ assert ss.waitForServiceToStop(1000);
+ assert one.isInState(Service.STATE.STOPPED)
+ assert two.isInState(Service.STATE.STOPPED)
+ Throwable failureCause = two.failureCause
+ assert failureCause != null;
+ Throwable masterFailureCause = ss.failureCause
+ assert masterFailureCause != null;
+ assert masterFailureCause == failureCause
+
+ assert three.isInState(Service.STATE.NOTINITED)
+ }
+
+ @Test
+ public void testSequenceInSequence() throws Throwable {
+ MockService one = new MockService("one", false, 100)
+ MockService two = new MockService("two", false, 100)
+ SequenceService ss = buildService([one, two])
+ SequenceService outer = startService([ss])
+
+ assert outer.waitForServiceToStop(1000);
+ assert one.isInState(Service.STATE.STOPPED)
+ assert two.isInState(Service.STATE.STOPPED)
+ }
+
+
+ @Test
+ public void testVarargsCtor() throws Throwable {
+ MockService one = new MockService("one", false, 100)
+ MockService two = new MockService("two", false, 100)
+ SequenceService ss = new SequenceService("test", one, two);
+ ss.init(new Configuration())
+ ss.start();
+ assert ss.waitForServiceToStop(1000);
+ assert one.isInState(Service.STATE.STOPPED)
+ assert two.isInState(Service.STATE.STOPPED)
+
+
+ }
+ public SequenceService startService(List<Service> services) {
+ SequenceService ss = buildService(services)
+ //expect service to start and stay started
+ ss.start();
+ return ss
+ }
+
+ public SequenceService buildService(List<Service> services) {
+ SequenceService ss = new SequenceService("test")
+ services.each { ss.addService(it) }
+ ss.init(new Configuration())
+ return ss
+ }
+
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
new file mode 100644
index 0000000..ddd52eb
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.test
+
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXMLConfKeysForTesting
+
+/**
+ * Keys shared across tests
+ */
+public interface KeysForTests extends SliderKeys, SliderXMLConfKeysForTesting {
+ /**
+ * Username for all clusters, ZK, etc
+ */
+ String USERNAME = "bigdataborat"
+
+ int WAIT_TIME = 120;
+ String WAIT_TIME_ARG = WAIT_TIME.toString()
+
+ String SLIDER_TEST_XML = "slider-test.xml"
+}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
new file mode 100644
index 0000000..d0da914
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.test
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster
+import org.apache.slider.common.tools.SliderUtils
+
+@Slf4j
+@CompileStatic
+class MicroZKCluster implements Closeable {
+
+ public static final String HOSTS = "127.0.0.1"
+ MiniZooKeeperCluster zkCluster
+ File baseDir
+ String zkBindingString
+ Configuration conf
+ int port
+
+ MicroZKCluster() {
+ this(SliderUtils.createConfiguration())
+ }
+
+ MicroZKCluster(Configuration conf) {
+ this.conf = conf
+ }
+
+ void createCluster() {
+ zkCluster = new MiniZooKeeperCluster(conf)
+ baseDir = File.createTempFile("zookeeper", ".dir")
+ baseDir.delete()
+ baseDir.mkdirs()
+ port = zkCluster.startup(baseDir)
+ zkBindingString = HOSTS + ":" + port
+ log.info("Created $this")
+ }
+
+ @Override
+ void close() throws IOException {
+ zkCluster?.shutdown();
+ baseDir?.deleteDir()
+ }
+
+ @Override
+ String toString() {
+ return "Micro ZK cluster as $zkBindingString data=$baseDir"
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
new file mode 100644
index 0000000..d632b25
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.test
+
+import groovy.transform.CompileStatic
+import org.junit.Before
+
+/**
+ * Base class for unit tests as well as ones starting mini clusters
+ * -the foundational code and methods
+ *
+ *
+ */
+
+@CompileStatic
+public abstract class SliderTestBase extends SliderTestUtils {
+
+ @Before
+ public void setup() {
+ //give our thread a name
+ Thread.currentThread().name = "JUnit"
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
new file mode 100644
index 0000000..a64370d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -0,0 +1,720 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.test
+
+import groovy.json.JsonOutput
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.commons.httpclient.HttpClient
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager
+import org.apache.commons.httpclient.methods.GetMethod
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileStatus
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.tools.Duration
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.exceptions.BadClusterStateException
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.exceptions.WaitTimeoutException
+import org.apache.slider.core.main.ServiceLaunchException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.server.services.curator.CuratorServiceInstance
+import org.junit.Assert
+import org.junit.Assume
+
+import static Arguments.ARG_OPTION
+
+/**
+ * Static utils for tests in this package and in other test projects.
+ *
+ * It is designed to work with mini clusters as well as remote ones
+ *
+ * This class is not final and may be extended for test cases.
+ *
+ * Some of these methods are derived from the SwiftUtils and SwiftTestUtils
+ * classes -replicated here so that they are available in Hadoop-2.0 code
+ */
+@Slf4j
+@CompileStatic
+class SliderTestUtils extends Assert {
+
+ public static void describe(String s) {
+ log.info("");
+ log.info("===============================");
+ log.info(s);
+ log.info("===============================");
+ log.info("");
+ }
+
+ public static String prettyPrint(String json) {
+ JsonOutput.prettyPrint(json)
+ }
+
+ public static void skip(String message) {
+ log.warn("Skipping test: " + message)
+ Assume.assumeTrue(message, false);
+ }
+
+ public static void assume(boolean condition, String message) {
+ if (!condition) {
+ log.warn("Skipping test: " + message)
+ Assume.assumeTrue(message, false);
+ }
+ }
+
+
+ public static void assertListEquals(List left, List right) {
+ assert left.size() == right.size();
+ for (int i = 0; i < left.size(); i++) {
+ assert left[0] == right[0]
+ }
+ }
+
+ /**
+ * Assume that a string option is set and not equal to ""
+ * @param conf configuration file
+ * @param key key to look for
+ */
+ public static void assumeStringOptionSet(Configuration conf, String key) {
+ if (!conf.getTrimmed(key)) {
+ skip("Configuration key $key not set")
+ }
+ }
+
+
+ /**
+ * assert that a string option is set and not equal to ""
+ * @param conf configuration file
+ * @param key key to look for
+ */
+ public static void assertStringOptionSet(Configuration conf, String key) {
+ getRequiredConfOption(conf, key)
+ }
+
+
+
+ /**
+ * Assume that a boolean option is set and true.
+ * Unset or false triggers a test skip
+ * @param conf configuration file
+ * @param key key to look for
+ */
+ public static void assumeBoolOptionTrue(Configuration conf, String key) {
+ assumeBoolOption(conf, key, false)
+ }
+
+ /**
+ * Assume that a boolean option is true.
+ * False triggers a test skip
+ * @param conf configuration file
+ * @param key key to look for
+ * @param defval default value if the property is not defined
+ */
+ public static void assumeBoolOption(
+ Configuration conf, String key, boolean defval) {
+ assume(conf.getBoolean(key, defval),
+ "Configuration key $key is false")
+ }
+
+ /**
+ * Get a required config option (trimmed, incidentally).
+ * Test will fail if not set
+ * @param conf configuration
+ * @param key key
+ * @return the string
+ */
+ public static String getRequiredConfOption(Configuration conf, String key) {
+ String val = conf.getTrimmed(key)
+ if (!val) {
+ fail("Missing configuration option $key")
+ }
+ return val;
+ }
+
+ /**
+ * Fails a test because required behavior has not been implemented.
+ */
+ public static void failNotImplemented() {
+ fail("Not implemented")
+ }
+
+ /**
+ * Wait for the cluster live; fail if it isn't within the (standard) timeout
+ * @param sliderClient client
+ * @return the app report of the live cluster
+ */
+ public static ApplicationReport waitForClusterLive(SliderClient sliderClient,int goLiveTime) {
+ ApplicationReport report = sliderClient.monitorAppToRunning(
+ new Duration(goLiveTime));
+ assertNotNull(
+ "Cluster did not go live in the time $goLiveTime",
+ report);
+ return report;
+ }
+
+ protected static String[] toArray(List<Object> args) {
+ String[] converted = new String[args.size()];
+ for (int i = 0; i < args.size(); i++) {
+ def elt = args.get(i)
+ assert args.get(i) != null
+ converted[i] = elt.toString();
+ }
+ return converted;
+ }
+
+ public static void waitWhileClusterLive(SliderClient client, int timeout) {
+ Duration duration = new Duration(timeout);
+ duration.start()
+ while (client.actionExists(client.deployedClusterName, true) &&
+ !duration.limitExceeded) {
+ sleep(1000);
+ }
+ if (duration.limitExceeded) {
+ fail("Cluster ${client.deployedClusterName} still live after $timeout ms")
+ }
+ }
+
+ public static void waitUntilClusterLive(SliderClient client, int timeout) {
+ Duration duration = new Duration(timeout);
+ duration.start()
+ while (!client.actionExists(client.deployedClusterName, true) &&
+ !duration.limitExceeded) {
+ sleep(1000);
+ }
+ if (duration.limitExceeded) {
+ fail("Cluster ${client.deployedClusterName} not live after $timeout ms")
+ }
+ }
+
+ /**
+ * Spin waiting for the Slider role count to match expected
+ * @param client client
+ * @param role role to look for
+ * @param desiredCount RS count
+ * @param timeout timeout
+ */
+ public static ClusterDescription waitForRoleCount(
+ SliderClient client,
+ String role,
+ int desiredCount,
+ int timeout) {
+ return waitForRoleCount(client, [(role): desiredCount], timeout)
+ }
+
+ /**
+ * Spin waiting for the Slider role count to match expected
+ * @param client client
+ * @param roles map of roles to look for
+ * @param desiredCount RS count
+ * @param timeout timeout
+ */
+ public static ClusterDescription waitForRoleCount(
+ SliderClient client,
+ Map<String, Integer> roles,
+ int timeout,
+ String operation = "startup") {
+ String clustername = client.deployedClusterName;
+ ClusterDescription status = null
+ Duration duration = new Duration(timeout);
+ duration.start()
+ boolean roleCountFound = false;
+ while (!roleCountFound) {
+ StringBuilder details = new StringBuilder()
+
+ boolean timedOut = duration.limitExceeded
+ try {
+ status = client.getClusterDescription(clustername)
+ roleCountFound = true;
+ for (Map.Entry<String, Integer> entry : roles.entrySet()) {
+ String role = entry.key
+ int desiredCount = entry.value
+ List<String> instances = status.instances[role]
+ int instanceCount = instances != null ? instances.size() : 0;
+ if (instanceCount != desiredCount) {
+ roleCountFound = false;
+ }
+ details.append("[$role]: desired: $desiredCount; actual: $instanceCount ")
+ }
+ if (roleCountFound) {
+ //successful
+ log.info("$operation: role count as desired: $details")
+ break;
+ }
+ } catch (BadClusterStateException e) {
+ // cluster not live yet; ignore or rethrow
+ if (timedOut) {
+ throw e;
+ }
+ details.append(e.toString());
+ }
+ if (timedOut) {
+ duration.finish();
+ describe("$operation: role count not met after $duration: $details")
+ log.info(prettyPrint(status.toJsonString()))
+ fail(
+ "$operation: role counts not met after $duration: $details in \n$status ")
+ }
+ log.debug("Waiting: " + details)
+ Thread.sleep(1000)
+ }
+ return status
+ }
+
+ /**
+ * Wait for the hbase master to be live (or past it in the lifecycle)
+ * @param clustername cluster
+ * @param spintime time to wait
+ * @return true if the cluster came out of the sleep time live
+ * @throws IOException
+ * @throws SliderException
+ */
+ public static boolean spinForClusterStartup(SliderClient client, long spintime,
+ String role)
+ throws WaitTimeoutException, IOException, SliderException {
+ int state = client.waitForRoleInstanceLive(role, spintime);
+ return state == ClusterDescription.STATE_LIVE;
+ }
+
+ public static ClusterDescription dumpClusterStatus(SliderClient client, String text) {
+ ClusterDescription status = client.clusterDescription;
+ dumpClusterDescription(text, status)
+ return status;
+ }
+
+ public static List<ClusterNode> listNodesInRole(SliderClient client, String role) {
+ return client.listClusterNodesInRole(role)
+ }
+
+ public static void dumpClusterDescription(String text, ClusterDescription status) {
+ describe(text)
+ log.info(prettyPrint(status.toJsonString()))
+ }
+
+
+ public static void dumpClusterDescription(String text, AggregateConf status) {
+ describe(text)
+ log.info(status.toString())
+ }
+
+
+ /**
+ * Fetch the current site config from the Slider AM, from the
+ * <code>clientProperties</code> field of the ClusterDescription
+ * @param client client
+ * @param clustername name of the cluster
+ * @return the site config
+ */
+ public static Configuration fetchClientSiteConfig(SliderClient client) {
+ ClusterDescription status = client.clusterDescription;
+ Configuration siteConf = new Configuration(false)
+ status.clientProperties.each { String key, String val ->
+ siteConf.set(key, val, "slider cluster");
+ }
+ return siteConf;
+ }
+
+ /**
+ * Fetch a web page
+ * @param url URL
+ * @return the response body
+ */
+
+ public static String GET(URL url) {
+ return fetchWebPageWithoutError(url.toString())
+ }
+
+ public static String GET(URL url, String path) {
+ return GET(url.toString(), path)
+ }
+
+ public static String GET(String base, String path) {
+ String s = appendToURL(base, path)
+ return GET(s)
+
+ }
+
+ def static String GET(String s) {
+ return fetchWebPageWithoutError(s)
+ }
+
+ def static String appendToURL(String base, String path) {
+ StringBuilder fullpath = new StringBuilder(base)
+ if (!base.endsWith("/")) {
+ fullpath.append("/")
+ }
+ if (path.startsWith("/")) {
+ fullpath.append(path.substring(1))
+ } else {
+ fullpath.append(path)
+ }
+
+ def s = fullpath.toString()
+ return s
+ }
+
+ /**
+ * Fetch a web page
+ * @param url URL
+ * @return the response body
+ */
+
+ public static String fetchWebPage(String url) {
+ log.info("GET $url")
+ def httpclient = new HttpClient(new MultiThreadedHttpConnectionManager());
+ httpclient.httpConnectionManager.params.connectionTimeout = 10000;
+ GetMethod get = new GetMethod(url);
+
+ get.followRedirects = true;
+ int resultCode
+ try {
+ resultCode = httpclient.executeMethod(get);
+ if (resultCode!=200) {
+ log.warn("Result code of $resultCode")
+ }
+ } catch (IOException e) {
+ log.error("Failed on $url: $e",e)
+ throw e;
+ }
+ String body = get.responseBodyAsString;
+ return body;
+ }
+
+ /**
+ * Fetches a web page asserting that the response code is between 200 and 400.
+ * Will error on 400 and 500 series response codes and let 200 and 300 through.
+ * @param url
+ * @return
+ */
+ public static String fetchWebPageWithoutError(String url) {
+ assert null != url
+
+ log.info("Fetching HTTP content at " + url);
+
+ def client = new HttpClient(new MultiThreadedHttpConnectionManager());
+ client.httpConnectionManager.params.connectionTimeout = 10000;
+ GetMethod get = new GetMethod(url);
+
+ get.followRedirects = true;
+ int resultCode = client.executeMethod(get);
+
+ def body = get.responseBodyAsString
+ if (!(resultCode >= 200 && resultCode < 400)) {
+ def message = "Request to $url failed with exit code $resultCode, body length ${body?.length()}:\n$body"
+ log.error(message)
+ fail(message)
+ }
+ return body;
+ }
+
+ /**
+ * Assert that a service operation succeeded
+ * @param service service
+ */
+ public static void assertSucceeded(ServiceLauncher service) {
+ assert 0 == service.serviceExitCode;
+ }
+
+ /**
+ * Make an assertion about the exit code of an exception
+ * @param ex exception
+ * @param exitCode exit code
+ * @param text error text to look for in the exception
+ */
+ static void assertExceptionDetails(
+ ServiceLaunchException ex,
+ int exitCode,
+ String text = "") {
+ if (exitCode != ex.exitCode) {
+ log.warn(
+ "Wrong exit code, expected $exitCode but got $ex.exitCode in $ex",
+ ex)
+ assert exitCode == ex.exitCode
+ }
+ if (text) {
+ if (!(ex.toString().contains(text))) {
+ log.warn("String match failed in $ex", ex)
+ assert ex.toString().contains(text);
+ }
+ }
+ }
+
+ /**
+ * Launch the slider client with the specific args; no validation
+ * of return code takes place
+ * @param conf configuration
+ * @param args arg list
+ * @return the return code
+ */
+ protected static ServiceLauncher<SliderClient> execSliderCommand(
+ Configuration conf,
+ List args) {
+ ServiceLauncher<SliderClient> serviceLauncher =
+ new ServiceLauncher<SliderClient>(SliderClient.name);
+ serviceLauncher.launchService(conf,
+ toArray(args),
+ false);
+ return serviceLauncher
+ }
+
+ public static ServiceLauncher launch(Class serviceClass,
+ Configuration conf,
+ List<Object> args) throws
+ Throwable {
+ ServiceLauncher serviceLauncher =
+ new ServiceLauncher(serviceClass.name);
+ serviceLauncher.launchService(conf,
+ toArray(args),
+ false);
+ return serviceLauncher;
+ }
+
+ public static void launchExpectingException(Class serviceClass,
+ Configuration conf,
+ String expectedText,
+ List args)
+ throws Throwable {
+ try {
+ ServiceLauncher launch = launch(serviceClass, conf, args);
+ fail("Expected an exception with text containing " + expectedText
+ + " -but the service completed with exit code "
+ + launch.serviceExitCode);
+ } catch (Throwable thrown) {
+ if (!thrown.toString().contains(expectedText)) {
+ //not the right exception -rethrow
+ throw thrown;
+ }
+ }
+ }
+
+
+ public static ServiceLauncher<SliderClient> launchClientAgainstRM(
+ String address,
+ List args,
+ Configuration conf) {
+ assert address != null
+ log.info("Connecting to rm at ${address}")
+ if (!args.contains(Arguments.ARG_MANAGER)) {
+ args += [Arguments.ARG_MANAGER, address]
+ }
+ ServiceLauncher<SliderClient> launcher = execSliderCommand(conf, args)
+ return launcher
+ }
+
+ /**
+ * Add a configuration parameter as a cluster configuration option
+ * @param extraArgs extra arguments
+ * @param conf config
+ * @param option option
+ */
+ public static void addClusterConfigOption(
+ List<String> extraArgs,
+ YarnConfiguration conf,
+ String option) {
+
+ conf.getTrimmed(option);
+ extraArgs << ARG_OPTION << option << getRequiredConfOption(conf, option)
+
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fs filesystem
+ * @param path path of the directory
+ * @throws IOException on File IO problems
+ */
+ public static void assertIsDirectory(HadoopFS fs,
+ Path path) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(path);
+ assertIsDirectory(fileStatus);
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fileStatus stats to check
+ */
+ public static void assertIsDirectory(FileStatus fileStatus) {
+ assertTrue("Should be a dir -but isn't: " + fileStatus,
+ fileStatus.isDirectory());
+ }
+
+ /**
+ * Assert that a path exists -but make no assertions as to the
+ * type of that entry
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathExists(
+ HadoopFS fileSystem,
+ String message,
+ Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ //failure, report it
+ fail(message + ": not found \"" + path + "\" in " + path.getParent() + "-" +
+ ls(fileSystem, path.getParent()));
+ }
+ }
+
+ /**
+ * Assert that a path does not exist
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathDoesNotExist(
+ HadoopFS fileSystem,
+ String message,
+ Path path) throws IOException {
+ try {
+ FileStatus status = fileSystem.getFileStatus(path);
+ // a status back implies there is a file here
+ fail(message + ": unexpectedly found " + path + " as " + status);
+ } catch (FileNotFoundException expected) {
+ //this is expected
+
+ }
+ }
+
+ /**
+ * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
+ * @param fs filesystem
+ * @param dir directory to scan
+ * @param subdir full path to look for
+ * @throws IOException IO probles
+ */
+ public static void assertListStatusFinds(HadoopFS fs,
+ Path dir,
+ Path subdir) throws IOException {
+ FileStatus[] stats = fs.listStatus(dir);
+ boolean found = false;
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.toString()).append('\n');
+ if (stat.getPath().equals(subdir)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + subdir
+ + " not found in directory " + dir + ":" + builder,
+ found);
+ }
+
+ /**
+ * List a a path to string
+ * @param fileSystem filesystem
+ * @param path directory
+ * @return a listing of the filestatuses of elements in the directory, one
+ * to a line, precedeed by the full path of the directory
+ * @throws IOException connectivity problems
+ */
+ public static String ls(HadoopFS fileSystem, Path path)
+ throws
+ IOException {
+ if (path == null) {
+ //surfaces when someone calls getParent() on something at the top of the path
+ return "/";
+ }
+ FileStatus[] stats;
+ String pathtext = "ls " + path;
+ try {
+ stats = fileSystem.listStatus(path);
+ } catch (FileNotFoundException e) {
+ return pathtext + " -file not found";
+ } catch (IOException e) {
+ return pathtext + " -failed: " + e;
+ }
+ return pathtext + fileStatsToString(stats, "\n");
+ }
+
+ /**
+ * Take an array of filestats and convert to a string (prefixed w/ a [01] counter
+ * @param stats array of stats
+ * @param separator separator after every entry
+ * @return a stringified set
+ */
+ public static String fileStatsToString(FileStatus[] stats, String separator) {
+ StringBuilder buf = new StringBuilder(stats.length * 128);
+ for (int i = 0; i < stats.length; i++) {
+ buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
+ }
+ return buf.toString();
+ }
+
+ public static void waitWhileClusterLive(SliderClient sliderClient) {
+ waitWhileClusterLive(sliderClient, 30000)
+ }
+
+ public static void dumpRegistryInstances(
+ List<CuratorServiceInstance<ServiceInstanceData>> instances) {
+ describe "service registry slider instances"
+ JsonSerDeser<ServiceInstanceData> serDeser = new JsonSerDeser<>(
+ ServiceInstanceData)
+
+ instances.each { CuratorServiceInstance<ServiceInstanceData> svc ->
+ ServiceInstanceData payload = svc.payload
+ def json = serDeser.toJson(payload)
+ log.info("service $svc payload=\n$json")
+ }
+ describe "end list service registry slider instances"
+ }
+
+ public static void dumpRegistryInstanceIDs(List<String> instanceIds) {
+ describe "service registry instance IDs"
+ log.info("number of instanceIds: ${instanceIds.size()}")
+ instanceIds.each { String it -> log.info(it) }
+ }
+
+ public static void dumpRegistryNames(Collection<String> names) {
+ describe "service registry names"
+ log.info("number of names: ${names.size()}")
+ names.each { String it -> log.info(it) }
+ }
+
+ /**
+ * Get a time option in seconds if set, otherwise the default value (also in seconds).
+ * This operation picks up the time value as a system property if set -that
+ * value overrides anything in the test file
+ * @param conf
+ * @param key
+ * @param defVal
+ * @return
+ */
+ public static int getTimeOptionMillis(Configuration conf, String key, int defValMillis) {
+ int val = conf.getInt(key, 0)
+ val = Integer.getInteger(key, val)
+ int time = 1000 * val
+ if (time == 0) {
+ time = defValMillis
+ }
+ return time;
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
new file mode 100644
index 0000000..b1111b2
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -0,0 +1,811 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.test
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.commons.logging.Log
+import org.apache.commons.logging.LogFactory
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hdfs.MiniDFSCluster
+import org.apache.hadoop.service.ServiceOperations
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.server.MiniYARNCluster
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionFreezeArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.Duration
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.apache.slider.server.appmaster.SliderAppMaster
+import org.junit.After
+import org.junit.Rule
+import org.junit.rules.Timeout
+
+import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
+import static org.apache.slider.test.KeysForTests.*
+import static org.apache.slider.test.SliderTestUtils.log
+
+/**
+ * Base class for mini cluster tests -creates a field for the
+ * mini yarn cluster
+ */
+@CompileStatic
+@Slf4j
+public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
+ /**
+ * Mini YARN cluster only
+ */
+ public static final int CLUSTER_GO_LIVE_TIME = 3 * 60 * 1000
+ public static final int CLUSTER_STOP_TIME = 1 * 60 * 1000
+
+ public static final int SIGTERM = -15
+ public static final int SIGKILL = -9
+ public static final int SIGSTOP = -17
+ public static
+ final String NO_ARCHIVE_DEFINED = "Archive configuration option not set: "
+ /**
+ * RAM for the YARN containers: {@value}
+ */
+ public static final String YRAM = "256"
+
+
+ public static final YarnConfiguration SLIDER_CONFIG = SliderUtils.createConfiguration();
+ static {
+ SLIDER_CONFIG.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, 1)
+ SLIDER_CONFIG.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100)
+ SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false)
+ SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false)
+ SLIDER_CONFIG.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1)
+
+ }
+
+
+ public int thawWaitTime = DEFAULT_THAW_WAIT_TIME_SECONDS * 1000
+ public int freezeWaitTime = DEFAULT_TEST_FREEZE_WAIT_TIME_SECONDS * 1000
+ public int sliderTestTimeout = DEFAULT_TEST_TIMEOUT_SECONDS * 1000
+ public boolean teardownKillall = DEFAULT_TEARDOWN_KILLALL
+
+
+ public boolean accumuloTestsEnabled = true
+ public int accumuloLaunchWaitTime = DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS * 1000
+
+ public boolean hbaseTestsEnabled = true
+ public int hbaseLaunchWaitTime = DEFAULT_HBASE_LAUNCH_TIME_SECONDS * 1000
+
+
+ protected MiniDFSCluster hdfsCluster
+ protected MiniYARNCluster miniCluster
+ protected boolean switchToImageDeploy = false
+ protected boolean imageIsRemote = false
+ protected URI remoteImageURI
+
+ protected List<SliderClient> clustersToTeardown = [];
+
+ /**
+ * This is set in a system property
+ */
+/*
+ @Rule
+ public Timeout testTimeout = new Timeout(1000*
+ Integer.getInteger(KEY_TEST_TIMEOUT, DEFAULT_TEST_TIMEOUT))
+
+*/
+
+ @Rule
+ public Timeout testTimeout = new Timeout(
+ getTimeOptionMillis(testConfiguration,
+ KEY_TEST_TIMEOUT,
+ DEFAULT_TEST_TIMEOUT_SECONDS * 1000)
+ )
+
+ @Override
+ void setup() {
+ super.setup()
+ def testConf = getTestConfiguration();
+ thawWaitTime = getTimeOptionMillis(testConf,
+ KEY_TEST_THAW_WAIT_TIME,
+ thawWaitTime)
+ freezeWaitTime = getTimeOptionMillis(testConf,
+ KEY_TEST_FREEZE_WAIT_TIME,
+ freezeWaitTime)
+ sliderTestTimeout = getTimeOptionMillis(testConf,
+ KEY_TEST_TIMEOUT,
+ sliderTestTimeout)
+ teardownKillall =
+ testConf.getBoolean(KEY_TEST_TEARDOWN_KILLALL,
+ teardownKillall)
+
+ hbaseTestsEnabled =
+ testConf.getBoolean(KEY_TEST_HBASE_ENABLED, hbaseTestsEnabled)
+ hbaseLaunchWaitTime = getTimeOptionMillis(testConf,
+ KEY_TEST_HBASE_LAUNCH_TIME,
+ hbaseLaunchWaitTime)
+
+ accumuloTestsEnabled =
+ testConf.getBoolean(KEY_TEST_ACCUMULO_ENABLED, hbaseTestsEnabled)
+ accumuloLaunchWaitTime = getTimeOptionMillis(testConf,
+ KEY_ACCUMULO_LAUNCH_TIME,
+ accumuloLaunchWaitTime)
+ }
+
+ @After
+ public void teardown() {
+ describe("teardown")
+ stopRunningClusters();
+ stopMiniCluster();
+ }
+
+ protected void addToTeardown(SliderClient client) {
+ clustersToTeardown << client;
+ }
+ protected void addToTeardown(ServiceLauncher<SliderClient> launcher) {
+ SliderClient sliderClient = launcher.service
+ if (sliderClient) addToTeardown(sliderClient)
+ }
+
+
+ protected YarnConfiguration getConfiguration() {
+ return SLIDER_CONFIG;
+ }
+
+ /**
+ * Stop any running cluster that has been added
+ */
+ public void stopRunningClusters() {
+ clustersToTeardown.each { SliderClient client ->
+ try {
+ maybeStopCluster(client, "", "Teardown at end of test case");
+ } catch (Exception e) {
+ log.warn("While stopping cluster " + e, e);
+ }
+ }
+ }
+
+ public void stopMiniCluster() {
+ Log l = LogFactory.getLog(this.getClass())
+ ServiceOperations.stopQuietly(l, miniCluster)
+ hdfsCluster?.shutdown();
+ }
+
+ /**
+ * Create and start a minicluster
+ * @param name cluster/test name
+ * @param conf configuration to use
+ * @param noOfNodeManagers #of NMs
+ * @param numLocalDirs #of local dirs
+ * @param numLogDirs #of log dirs
+ * @param startZK create a ZK micro cluster
+ * @param startHDFS create an HDFS mini cluster
+ */
+ protected void createMiniCluster(String name,
+ YarnConfiguration conf,
+ int noOfNodeManagers,
+ int numLocalDirs,
+ int numLogDirs,
+ boolean startHDFS) {
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER,
+ FifoScheduler.class, ResourceScheduler.class);
+ SliderUtils.patchConfiguration(conf)
+ miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs)
+ miniCluster.init(conf)
+ miniCluster.start();
+ if (startHDFS) {
+ createMiniHDFSCluster(name, conf)
+ }
+ }
+
+ /**
+ * Create a mini HDFS cluster and save it to the hdfsClusterField
+ * @param name
+ * @param conf
+ */
+ public void createMiniHDFSCluster(String name, YarnConfiguration conf) {
+ hdfsCluster = buildMiniHDFSCluster(name, conf)
+ }
+
+ /**
+ * Inner work building the mini dfs cluster
+ * @param name
+ * @param conf
+ * @return
+ */
+ public static MiniDFSCluster buildMiniHDFSCluster(
+ String name,
+ YarnConfiguration conf) {
+ File baseDir = new File("./target/hdfs/$name").absoluteFile;
+ //use file: to rm it recursively
+ FileUtil.fullyDelete(baseDir)
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.absolutePath)
+ MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf)
+
+ def cluster = builder.build()
+ return cluster
+ }
+
+ /**
+ * Launch the client with the specific args against the MiniMR cluster
+ * launcher ie expected to have successfully completed
+ * @param conf configuration
+ * @param args arg list
+ * @return the return code
+ */
+ protected ServiceLauncher<SliderClient> launchClientAgainstMiniMR(Configuration conf,
+ List args) {
+ ServiceLauncher<SliderClient> launcher = launchClientNoExitCodeCheck(conf, args)
+ int exited = launcher.serviceExitCode
+ if (exited != 0) {
+ throw new SliderException(exited, "Launch failed with exit code $exited")
+ }
+ return launcher;
+ }
+
+ /**
+ * Launch the client with the specific args against the MiniMR cluster
+ * without any checks for exit codes
+ * @param conf configuration
+ * @param args arg list
+ * @return the return code
+ */
+ public ServiceLauncher<SliderClient> launchClientNoExitCodeCheck(
+ Configuration conf,
+ List args) {
+ assert miniCluster != null
+ return launchClientAgainstRM(RMAddr, args, conf)
+ }
+
+
+ /**
+ * Kill all Slider Services. That i
+ * @param signal
+ */
+ public void killAM(int signal) {
+ killJavaProcesses(SliderAppMaster.SERVICE_CLASSNAME_SHORT, signal)
+ }
+
+ /**
+ * Kill any java process with the given grep pattern
+ * @param grepString string to grep for
+ */
+ public void killJavaProcesses(String grepString, int signal) {
+
+ GString bashCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal"
+ log.info("Bash command = $bashCommand" )
+ Process bash = ["bash", "-c", bashCommand].execute()
+ bash.waitFor()
+
+ log.info(bash.in.text)
+ log.error(bash.err.text)
+ }
+
+ public void killJavaProcesses(List<String> greps, int signal) {
+ for (String grep : greps) {
+ killJavaProcesses(grep,signal)
+ }
+ }
+
+ /**
+ * List any java process with the given grep pattern
+ * @param grepString string to grep for
+ */
+ public String lsJavaProcesses() {
+ Process bash = ["jps","-v"].execute()
+ bash.waitFor()
+ String out = bash.in.text
+ log.info(out)
+ String err = bash.err.text
+ log.error(err)
+ return out + "\n" + err
+ }
+
+
+ public void killServiceLaunchers(int value) {
+ killAM(value)
+ }
+
+ public YarnConfiguration getTestConfiguration() {
+ YarnConfiguration conf = getConfiguration()
+
+ conf.addResource(SLIDER_TEST_XML)
+ return conf
+ }
+
+ protected String getRMAddr() {
+ assert miniCluster != null
+ String addr = miniCluster.config.get(YarnConfiguration.RM_ADDRESS)
+ assert addr != null;
+ assert addr != "";
+ return addr
+ }
+
+ /**
+ * return the default filesystem, which is HDFS if the miniDFS cluster is
+ * up, file:// if not
+ * @return a filesystem string to pass down
+ */
+ protected String getFsDefaultName() {
+ return buildFsDefaultName(hdfsCluster)
+ }
+
+ public static String buildFsDefaultName(MiniDFSCluster miniDFSCluster) {
+ if (miniDFSCluster) {
+ return "hdfs://localhost:${miniDFSCluster.nameNodePort}/"
+ } else {
+ return "file:///"
+ }
+ }
+
+ protected String getWaitTimeArg() {
+ return WAIT_TIME_ARG;
+ }
+
+ protected int getWaitTimeMillis(Configuration conf) {
+
+ return WAIT_TIME * 1000;
+ }
+
+ /**
+ * Create a cluster
+ * @param clustername cluster name
+ * @param roles map of rolename to count
+ * @param extraArgs list of extra args to add to the creation command
+ * @param deleteExistingData should the data of any existing cluster
+ * of this name be deleted
+ * @param blockUntilRunning block until the AM is running
+ * @param clusterOps map of key=value cluster options to set with the --option arg
+ * @return launcher which will have executed the command.
+ */
+ public ServiceLauncher<SliderClient> createCluster(
+ String clustername,
+ Map<String, Integer> roles,
+ List<String> extraArgs,
+ boolean deleteExistingData,
+ boolean blockUntilRunning,
+ Map<String, String> clusterOps) {
+ createOrBuildCluster(
+ SliderActions.ACTION_CREATE,
+ clustername,
+ roles,
+ extraArgs,
+ deleteExistingData,
+ blockUntilRunning,
+ clusterOps)
+ }
+
+ /**
+ * Create or build a cluster (the action is set by the first verb)
+ * @param action operation to invoke: SliderActions.ACTION_CREATE or SliderActions.ACTION_BUILD
+ * @param clustername cluster name
+ * @param roles map of rolename to count
+ * @param extraArgs list of extra args to add to the creation command
+ * @param deleteExistingData should the data of any existing cluster
+ * of this name be deleted
+ * @param blockUntilRunning block until the AM is running
+ * @param clusterOps map of key=value cluster options to set with the --option arg
+ * @return launcher which will have executed the command.
+ */
+ public ServiceLauncher<SliderClient> createOrBuildCluster(String action, String clustername, Map<String, Integer> roles, List<String> extraArgs, boolean deleteExistingData, boolean blockUntilRunning, Map<String, String> clusterOps) {
+ assert clustername != null
+ assert miniCluster != null
+ if (deleteExistingData) {
+ HadoopFS dfs = HadoopFS.get(new URI(fsDefaultName), miniCluster.config)
+ Path clusterDir = new SliderFileSystem(dfs, miniCluster.config).buildClusterDirPath(clustername)
+ log.info("deleting customer data at $clusterDir")
+ //this is a safety check to stop us doing something stupid like deleting /
+ assert clusterDir.toString().contains("/.slider/")
+ dfs.delete(clusterDir, true)
+ }
+
+
+ List<String> componentList = [];
+ roles.each { String role, Integer val ->
+ log.info("Component $role := $val")
+ componentList << Arguments.ARG_COMPONENT << role << Integer.toString(val)
+ }
+
+ List<String> argsList = [
+ action, clustername,
+ Arguments.ARG_MANAGER, RMAddr,
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ Arguments.ARG_DEBUG,
+ Arguments.ARG_CONFDIR, confDir
+ ]
+ if (blockUntilRunning) {
+ argsList << Arguments.ARG_WAIT << WAIT_TIME_ARG
+ }
+
+ argsList += extraCLIArgs
+ argsList += componentList;
+ argsList += imageCommands
+
+ //now inject any cluster options
+ clusterOps.each { String opt, String val ->
+ argsList << Arguments.ARG_OPTION << opt << val;
+ }
+
+ if (extraArgs != null) {
+ argsList += extraArgs;
+ }
+ ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ argsList
+ )
+ assert launcher.serviceExitCode == 0
+ SliderClient client = launcher.service
+ if (blockUntilRunning) {
+ client.monitorAppToRunning(new Duration(CLUSTER_GO_LIVE_TIME))
+ }
+ return launcher;
+ }
+
+ /**
+ * Add arguments to launch Slider with.
+ *
+ * Extra arguments are added after standard arguments and before roles.
+ *
+ * @return additional arguments to launch Slider with
+ */
+ protected List<String> getExtraCLIArgs() {
+ []
+ }
+
+ public String getConfDir() {
+ return resourceConfDirURI
+ }
+
+ /**
+ * Get the key for the application
+ * @return
+ */
+ public String getApplicationHomeKey() {
+ failNotImplemented()
+ null
+ }
+
+ /**
+ * Get the archive path -which defaults to the local one
+ * @return
+ */
+ public String getArchivePath() {
+ return localArchive
+ }
+ /**
+ * Get the local archive -the one defined in the test configuration
+ * @return a possibly null/empty string
+ */
+ public final String getLocalArchive() {
+ return testConfiguration.getTrimmed(archiveKey)
+ }
+ /**
+ * Get the key for archives in tests
+ * @return
+ */
+ public String getArchiveKey() {
+ failNotImplemented()
+ null
+ }
+
+ /**
+ * Merge a k-v pair into a simple k=v string; simple utility
+ * @param key key
+ * @param val value
+ * @return the string to use after a -D option
+ */
+ public String define(String key, String val) {
+ return "$key=$val"
+ }
+
+ public void assumeTestEnabled(boolean flag) {
+ assume(flag, "test disabled")
+ }
+
+ public void assumeArchiveDefined() {
+ String archive = archivePath
+ boolean defined = archive != null && archive != ""
+ if (!defined) {
+ log.warn(NO_ARCHIVE_DEFINED + archiveKey);
+ }
+ assume(defined,NO_ARCHIVE_DEFINED + archiveKey)
+ }
+
+ /**
+ * Assume that application home is defined. This does not check that the
+ * path is valid -that is expected to be a failure on tests that require
+ * application home to be set.
+ */
+ public void assumeApplicationHome() {
+ assume(applicationHome != null && applicationHome != "",
+ "Application home dir option not set " + applicationHomeKey)
+ }
+
+
+ public String getApplicationHome() {
+ return testConfiguration.getTrimmed(applicationHomeKey)
+ }
+
+ public List<String> getImageCommands() {
+ if (switchToImageDeploy) {
+ // its an image that had better be defined
+ assert archivePath
+ if (!imageIsRemote) {
+ // its not remote, so assert it exists
+ File f = new File(archivePath)
+ assert f.exists()
+ return [Arguments.ARG_IMAGE, f.toURI().toString()]
+ } else {
+ assert remoteImageURI
+
+ // if it is remote, then its whatever the archivePath property refers to
+ return [Arguments.ARG_IMAGE, remoteImageURI.toString()];
+ }
+ } else {
+ assert applicationHome
+ assert new File(applicationHome).exists();
+ return [Arguments.ARG_APP_HOME, applicationHome]
+ }
+ }
+
+ /**
+ * Start a cluster that has already been defined
+ * @param clustername cluster name
+ * @param extraArgs list of extra args to add to the creation command
+ * @param blockUntilRunning block until the AM is running
+ * @return launcher which will have executed the command.
+ */
+ public ServiceLauncher<SliderClient> thawCluster(String clustername, List<String> extraArgs, boolean blockUntilRunning) {
+ assert clustername != null
+ assert miniCluster != null
+
+ List<String> argsList = [
+ SliderActions.ACTION_THAW, clustername,
+ Arguments.ARG_MANAGER, RMAddr,
+ Arguments.ARG_WAIT, WAIT_TIME_ARG,
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ ]
+ argsList += extraCLIArgs
+
+ if (extraArgs != null) {
+ argsList += extraArgs;
+ }
+ ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ argsList
+ )
+ assert launcher.serviceExitCode == 0
+ SliderClient client = (SliderClient) launcher.service
+ if (blockUntilRunning) {
+ client.monitorAppToRunning(new Duration(CLUSTER_GO_LIVE_TIME))
+ }
+ return launcher;
+ }
+
+
+ /**
+ * Get the resource configuration dir in the source tree
+ * @return
+ */
+ public File getResourceConfDir() {
+ File f = new File(testConfigurationPath).absoluteFile;
+ if (!f.exists()) {
+ throw new FileNotFoundException("Resource configuration directory $f not found")
+ }
+ return f;
+ }
+
+ public String getTestConfigurationPath() {
+ failNotImplemented()
+ null;
+ }
+
+ /**
+ get a URI string to the resource conf dir that is suitable for passing down
+ to the AM -and works even when the default FS is hdfs
+ */
+ public String getResourceConfDirURI() {;
+ return resourceConfDir.absoluteFile.toURI().toString()
+ }
+
+
+ public void logReport(ApplicationReport report) {
+ log.info(SliderUtils.reportToString(report));
+ }
+
+
+ public void logApplications(List<ApplicationReport> apps) {
+ apps.each { ApplicationReport r -> logReport(r) }
+ }
+
+ /**
+ * Wait for the cluster live; fail if it isn't within the (standard) timeout
+ * @param client client
+ * @return the app report of the live cluster
+ */
+ public ApplicationReport waitForClusterLive(SliderClient client) {
+ return waitForClusterLive(client, CLUSTER_GO_LIVE_TIME)
+ }
+
+ /**
+ * force kill the application after waiting for
+ * it to shut down cleanly
+ * @param client client to talk to
+ */
+ public ApplicationReport waitForAppToFinish(SliderClient client) {
+
+ int waitTime = getWaitTimeMillis(client.config)
+ return waitForAppToFinish(client, waitTime)
+ }
+
+ public static ApplicationReport waitForAppToFinish(
+ SliderClient client,
+ int waitTime) {
+ ApplicationReport report = client.monitorAppToState(
+ YarnApplicationState.FINISHED,
+ new Duration(waitTime));
+ if (report == null) {
+ log.info("Forcibly killing application")
+ dumpClusterStatus(client, "final application status")
+ //list all the nodes' details
+ List<ClusterNode> nodes = listNodesInRole(client, "")
+ if (nodes.empty) {
+ log.info("No live nodes")
+ }
+ nodes.each { ClusterNode node -> log.info(node.toString())}
+ client.forceKillApplication("timed out waiting for application to complete");
+ }
+ return report;
+ }
+
+ /**
+ * stop the cluster via the stop action -and wait for {@link #CLUSTER_STOP_TIME}
+ * for the cluster to stop. If it doesn't
+ * @param sliderClient client
+ * @param clustername cluster
+ * @return the exit code
+ */
+ public int clusterActionFreeze(SliderClient sliderClient, String clustername,
+ String message = "action freeze") {
+ log.info("Freezing cluster $clustername: $message")
+ ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
+ freezeArgs.waittime = CLUSTER_STOP_TIME
+ freezeArgs.message = message
+ int exitCode = sliderClient.actionFreeze(clustername,
+ freezeArgs);
+ if (exitCode != 0) {
+ log.warn("Cluster freeze failed with error code $exitCode")
+ }
+ return exitCode
+ }
+
+ /**
+ * Teardown-time cluster termination; will stop the cluster iff the client
+ * is not null
+ * @param sliderClient client
+ * @param clustername name of cluster to teardown
+ * @return
+ */
+ public int maybeStopCluster(
+ SliderClient sliderClient,
+ String clustername,
+ String message) {
+ if (sliderClient != null) {
+ if (!clustername) {
+ clustername = sliderClient.deployedClusterName;
+ }
+ //only stop a cluster that exists
+ if (clustername) {
+ return clusterActionFreeze(sliderClient, clustername, message);
+ }
+ }
+ return 0;
+ }
+
+
+ String roleMapToString(Map<String,Integer> roles) {
+ StringBuilder builder = new StringBuilder()
+ roles.each { String name, int value ->
+ builder.append("$name->$value ")
+ }
+ return builder.toString()
+ }
+
+ /**
+ * Turn on test runs against a copy of the archive that is
+ * uploaded to HDFS -this method copies up the
+ * archive then switches the tests into archive mode
+ */
+ public void enableTestRunAgainstUploadedArchive() {
+ Path remotePath = copyLocalArchiveToHDFS(localArchive)
+ // image mode
+ switchToRemoteImageDeploy(remotePath);
+ }
+
+ /**
+ * Switch to deploying a remote image
+ * @param remotePath the remote path to use
+ */
+ public void switchToRemoteImageDeploy(Path remotePath) {
+ switchToImageDeploy = true
+ imageIsRemote = true
+ remoteImageURI = remotePath.toUri()
+ }
+
+ /**
+ * Copy a local archive to HDFS
+ * @param localArchive local archive
+ * @return the path of the uploaded image
+ */
+ public Path copyLocalArchiveToHDFS(String localArchive) {
+ assert localArchive
+ File localArchiveFile = new File(localArchive)
+ assert localArchiveFile.exists()
+ assert hdfsCluster
+ Path remoteUnresolvedArchive = new Path(localArchiveFile.name)
+ assert FileUtil.copy(
+ localArchiveFile,
+ hdfsCluster.fileSystem,
+ remoteUnresolvedArchive,
+ false,
+ testConfiguration)
+ Path remotePath = hdfsCluster.fileSystem.resolvePath(
+ remoteUnresolvedArchive)
+ return remotePath
+ }
+
+ /**
+ * Assert that an operation failed because a cluster is in use
+ * @param e exception
+ */
+ public static void assertFailureClusterInUse(SliderException e) {
+ assertExceptionDetails(e,
+ SliderExitCodes.EXIT_APPLICATION_IN_USE,
+ ErrorStrings.E_CLUSTER_RUNNING)
+ }
+
+ /**
+ * Create a SliderFileSystem instance bonded to the running FS.
+ * The YARN cluster must be up and running already
+ * @return
+ */
+ public SliderFileSystem createSliderFileSystem() {
+ HadoopFS dfs = HadoopFS.get(new URI(fsDefaultName), configuration)
+ SliderFileSystem hfs = new SliderFileSystem(dfs, configuration)
+ return hfs
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
new file mode 100644
index 0000000..dcc07c6
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.test
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.core.registry.zk.BlockingZKWatcher
+import org.apache.slider.core.registry.zk.ZKIntegration
+
+import java.util.concurrent.atomic.AtomicBoolean
+
+import static org.apache.slider.test.KeysForTests.USERNAME
+
+/**
+ * Base class for mini cluster tests that use Zookeeper
+ */
+@CompileStatic
+@Slf4j
+public abstract class YarnZKMiniClusterTestBase extends YarnMiniClusterTestBase {
+
+ protected MicroZKCluster microZKCluster
+
+ public void stopMiniCluster() {
+ super.stopMiniCluster()
+ microZKCluster?.close()
+ }
+
+ public ZKIntegration createZKIntegrationInstance(String zkQuorum,
+ String clusterName,
+ boolean createClusterPath,
+ boolean canBeReadOnly,
+ int timeout) {
+
+ BlockingZKWatcher watcher = new BlockingZKWatcher();
+ ZKIntegration zki = ZKIntegration.newInstance(zkQuorum,
+ USERNAME,
+ clusterName,
+ createClusterPath,
+ canBeReadOnly, watcher)
+ zki.init()
+ //here the callback may or may not have occurred.
+ //optionally wait for it
+ if (timeout > 0) {
+ watcher.waitForZKConnection(timeout)
+ }
+ //if we get here, the binding worked
+ log.info("Connected: ${zki}")
+ return zki
+ }
+
+ /**
+ * Wait for a flag to go true
+ * @param connectedFlag
+ */
+ public void waitForZKConnection(AtomicBoolean connectedFlag, int timeout) {
+ synchronized (connectedFlag) {
+ if (!connectedFlag.get()) {
+ log.info("waiting for ZK event")
+ //wait a bit
+ connectedFlag.wait(timeout)
+ }
+ }
+ assert connectedFlag.get()
+ }
+
+ /**
+ * Create and start a minicluster
+ * @param name cluster/test name
+ * @param conf configuration to use
+ * @param noOfNodeManagers #of NMs
+ * @param numLocalDirs #of local dirs
+ * @param numLogDirs #of log dirs
+ * @param startZK create a ZK micro cluster
+ * @param startHDFS create an HDFS mini cluster
+ */
+ protected void createMiniCluster(String name,
+ YarnConfiguration conf,
+ int noOfNodeManagers,
+ int numLocalDirs,
+ int numLogDirs,
+ boolean startZK,
+ boolean startHDFS) {
+ createMiniCluster(name, conf, noOfNodeManagers, numLocalDirs, numLogDirs,
+ startHDFS)
+
+ if (startZK) {
+ createMicroZKCluster(conf)
+ }
+ }
+
+ /**
+ * Create and start a minicluster
+ * @param name cluster/test name
+ * @param conf configuration to use
+ * @param noOfNodeManagers #of NMs
+ * @param startZK create a ZK micro cluster
+ */
+ protected void createMiniCluster(String name,
+ YarnConfiguration conf,
+ int noOfNodeManagers,
+ boolean startZK) {
+ createMiniCluster(name, conf, noOfNodeManagers, 1, 1, startZK, false)
+ }
+
+ public void createMicroZKCluster(Configuration conf) {
+ microZKCluster = new MicroZKCluster(new Configuration(conf))
+ microZKCluster.createCluster();
+ }
+
+ void assertHasZKCluster() {
+ assert microZKCluster != null
+ }
+
+ protected String getZKBinding() {
+ if (!microZKCluster) {
+ return "localhost:1"
+ } else {
+ return microZKCluster.zkBindingString
+ }
+ }
+
+ protected int getZKPort() {
+ return microZKCluster ? microZKCluster.port : 2181;
+ }
+
+ protected String getZKHosts() {
+ return MicroZKCluster.HOSTS;
+ }
+
+ /**
+ * CLI args include all the ZK bindings needed
+ * @return
+ */
+ protected List<String> getExtraCLIArgs() {
+ [
+ "-D", define(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, ZKBinding)
+ ]
+ }
+}
diff --git a/slider-core/src/test/java/org/apache/slider/core/main/ServiceLauncherBaseTest.java b/slider-core/src/test/java/org/apache/slider/core/main/ServiceLauncherBaseTest.java
new file mode 100644
index 0000000..4002e04
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/core/main/ServiceLauncherBaseTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.main;
+
+import org.apache.slider.test.SliderTestBase;
+
+/**
+ * Base class for tests that use the service launcher
+ */
+public class ServiceLauncherBaseTest extends SliderTestBase {
+
+
+}
+
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/AgentProviderServiceTest.java b/slider-core/src/test/java/org/apache/slider/providers/agent/AgentProviderServiceTest.java
new file mode 100644
index 0000000..bbc7115
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/AgentProviderServiceTest.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.server.appmaster.model.mock.MockContainerId;
+import org.apache.slider.server.appmaster.model.mock.MockFileSystem;
+import org.apache.slider.server.appmaster.model.mock.MockNodeId;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat;
+import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse;
+import org.apache.slider.server.appmaster.web.rest.agent.Register;
+import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
+import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.eq;
+
+
+public class AgentProviderServiceTest {
+ protected static final Logger log =
+ LoggerFactory.getLogger(AgentProviderServiceTest.class);
+
+ @Test
+ public void testRegistration() throws IOException {
+
+ ConfTree tree = new ConfTree();
+ tree.global.put(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
+
+ AgentProviderService aps = new AgentProviderService();
+ ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
+ AggregateConf instanceDefinition = new AggregateConf();
+
+ instanceDefinition.setInternal(tree);
+ instanceDefinition.setAppConf(tree);
+ instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, ".");
+ instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, ".");
+ instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");
+
+ Container container = createNiceMock(Container.class);
+ String role = "HBASE_MASTER";
+ SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
+ Path generatedConfPath = new Path(".", "test");
+ MapOperations resourceComponent = new MapOperations();
+ MapOperations appComponent = new MapOperations();
+ Path containerTmpDirPath = new Path(".", "test");
+ FileSystem mockFs = new MockFileSystem();
+ expect(sliderFileSystem.getFileSystem())
+ .andReturn(new FilterFileSystem(mockFs)).anyTimes();
+ expect(sliderFileSystem.createAmResource(anyObject(Path.class),
+ anyObject(LocalResourceType.class)))
+ .andReturn(createNiceMock(LocalResource.class)).anyTimes();
+ expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes();
+ expect(container.getNodeId()).andReturn(new MockNodeId("localhost")).anyTimes();
+ StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
+
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doReturn(access).when(mockAps).getStateAccessor();
+
+ try {
+ doNothing().when(mockAps).addInstallCommand(
+ eq("HBASE_MASTER"),
+ any(HeartBeatResponse.class),
+ eq("scripts/hbase_master.py"));
+ } catch (SliderException e) {
+ }
+
+ expect(access.isApplicationLive()).andReturn(true).anyTimes();
+ ClusterDescription desc = new ClusterDescription();
+ desc.setInfo(StatusKeys.INFO_AM_HOSTNAME, "host1");
+ desc.setInfo(StatusKeys.INFO_AM_WEB_PORT, "8088");
+ desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE");
+ desc.getOrAddRole("HBASE_MASTER").put(AgentKeys.COMPONENT_SCRIPT, "scripts/hbase_master.py");
+ expect(access.getClusterStatus()).andReturn(desc).anyTimes();
+
+ AggregateConf aggConf = new AggregateConf();
+ ConfTreeOperations treeOps = aggConf.getAppConfOperations();
+ treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0");
+ expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf);
+ replay(access, ctx, container, sliderFileSystem);
+
+ try {
+ mockAps.buildContainerLaunchContext(null,
+ instanceDefinition,
+ container,
+ role,
+ sliderFileSystem,
+ generatedConfPath,
+ resourceComponent,
+ appComponent,
+ containerTmpDirPath);
+ } catch (SliderException he) {
+ log.warn(he.getMessage());
+ } catch (IOException ioe) {
+ log.warn(ioe.getMessage());
+ }
+
+ Register reg = new Register();
+ reg.setResponseId(0);
+ reg.setHostname("mockcontainer_1___HBASE_MASTER");
+ RegistrationResponse resp = mockAps.handleRegistration(reg);
+ TestCase.assertEquals(0, resp.getResponseId());
+ TestCase.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
+
+ HeartBeat hb = new HeartBeat();
+ hb.setResponseId(1);
+ hb.setHostname("mockcontainer_1___HBASE_MASTER");
+ HeartBeatResponse hbr = mockAps.handleHeartBeat(hb);
+ TestCase.assertEquals(2, hbr.getResponseId());
+ }
+
+ @Test
+ public void testRoleHostMapping() throws Exception {
+ AgentProviderService aps = new AgentProviderService();
+ aps.setRoleHostMapping("FIRST_ROLE", "FIRST_HOST");
+ aps.setRoleHostMapping("SECOND_ROLE", "SECOND_HOST");
+ aps.setRoleHostMapping("SECOND_ROLE", "THIRD_HOST");
+ Map<String,String> tokens = new HashMap<String, String>();
+ aps.addRoleRelatedTokens(tokens);
+ TestCase.assertEquals(2, tokens.size());
+ TestCase.assertEquals("FIRST_HOST", tokens.get("${FIRST_ROLE_HOST}"));
+ TestCase.assertEquals("SECOND_HOST,THIRD_HOST", tokens.get("${SECOND_ROLE_HOST}"));
+ aps.close();
+ }
+}
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/ComponentInstanceStateTest.java b/slider-core/src/test/java/org/apache/slider/providers/agent/ComponentInstanceStateTest.java
new file mode 100644
index 0000000..06657e8
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/ComponentInstanceStateTest.java
@@ -0,0 +1,287 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import junit.framework.TestCase;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ComponentInstanceStateTest {
+ protected static final Logger log =
+ LoggerFactory.getLogger(ComponentInstanceStateTest.class);
+ private State[] states = new State[]{
+ State.INIT, State.INSTALLING, State.INSTALLED,
+ State.STARTING, State.STARTED, State.INSTALL_FAILED};
+
+ @Test
+ public void testValidateSupportedCommands() {
+ Command[] commands = new Command[]{
+ Command.INSTALL, Command.NOP, Command.START,
+ Command.NOP, Command.NOP, Command.INSTALL};
+
+ for (int index = 0; index < states.length; index++) {
+ TestCase.assertEquals(commands[index], states[index].getSupportedCommand());
+ }
+ }
+
+ @Test
+ public void testGetNextStateBasedOnResult() {
+ TestCase.assertEquals(State.INSTALLING, State.INSTALLING.getNextState(CommandResult.IN_PROGRESS));
+ TestCase.assertEquals(State.STARTING, State.STARTING.getNextState(CommandResult.IN_PROGRESS));
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INIT, CommandResult.IN_PROGRESS);
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INSTALLED, CommandResult.IN_PROGRESS);
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.STARTED, CommandResult.IN_PROGRESS);
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INSTALL_FAILED, CommandResult.IN_PROGRESS);
+
+ TestCase.assertEquals(State.INSTALLED, State.INSTALLING.getNextState(CommandResult.COMPLETED));
+ TestCase.assertEquals(State.STARTED, State.STARTING.getNextState(CommandResult.COMPLETED));
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INIT, CommandResult.COMPLETED);
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INSTALLED, CommandResult.COMPLETED);
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.STARTED, CommandResult.COMPLETED);
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INSTALL_FAILED, CommandResult.COMPLETED);
+
+ TestCase.assertEquals(State.INSTALL_FAILED, State.INSTALLING.getNextState(CommandResult.FAILED));
+ TestCase.assertEquals(State.INSTALLED, State.STARTING.getNextState(CommandResult.FAILED));
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INIT, CommandResult.FAILED);
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INSTALLED, CommandResult.FAILED);
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.STARTED, CommandResult.FAILED);
+ expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INSTALL_FAILED, CommandResult.FAILED);
+ }
+
+ @Test
+ public void testGetNextStateBasedOnCommand() {
+ for (int index = 0; index < states.length; index++) {
+ TestCase.assertEquals(states[index], states[index].getNextState(Command.NOP));
+ }
+
+ TestCase.assertEquals(State.INSTALLING, State.INIT.getNextState(Command.INSTALL));
+ TestCase.assertEquals(State.INSTALLING, State.INSTALL_FAILED.getNextState(Command.INSTALL));
+ expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.INSTALLED, Command.INSTALL);
+ expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.STARTING, Command.INSTALL);
+ expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.STARTED, Command.INSTALL);
+
+ TestCase.assertEquals(State.STARTING, State.INSTALLED.getNextState(Command.START));
+ expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.INIT, Command.START);
+ expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.INSTALL_FAILED, Command.START);
+ expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.STARTING, Command.START);
+ expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.INSTALLING, Command.START);
+ expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.STARTED, Command.START);
+ }
+
+ @Test
+ public void validateStateTransitionNormal() {
+ ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", "CID_001", "AID_001");
+ TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ TestCase.assertEquals(true, componentInstanceState.hasPendingCommand());
+ TestCase.assertEquals(Command.INSTALL, componentInstanceState.getNextCommand());
+ TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.IN_PROGRESS, Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+ TestCase.assertEquals(Command.START, componentInstanceState.getNextCommand());
+ componentInstanceState.commandIssued(Command.START);
+ TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.IN_PROGRESS, Command.START);
+ TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.START);
+ TestCase.assertEquals(State.STARTED, componentInstanceState.getState());
+ }
+
+ @Test
+ public void validateStateTransitionScenario2() {
+ ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", "CID_001", "AID_001");
+ TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ TestCase.assertEquals(true, componentInstanceState.hasPendingCommand());
+ TestCase.assertEquals(Command.INSTALL, componentInstanceState.getNextCommand());
+ TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
+ TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+ TestCase.assertEquals(Command.START, componentInstanceState.getNextCommand());
+
+ componentInstanceState.commandIssued(Command.START);
+ TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.START);
+ TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.START);
+ componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.START);
+ TestCase.assertEquals(State.STARTED, componentInstanceState.getState());
+ }
+
+ @Test
+ public void tolerateMaxFailures() {
+ ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", "CID_001", "AID_001");
+ TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ TestCase.assertEquals(true, componentInstanceState.hasPendingCommand());
+ TestCase.assertEquals(Command.INSTALL, componentInstanceState.getNextCommand());
+ TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
+ TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
+ TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
+ TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+
+ try {
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.fail("Command should not be allowed.");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+
+ @Test
+ public void tolerateFewFailureThenReset() {
+ ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", "CID_001", "AID_001");
+ TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ TestCase.assertEquals(true, componentInstanceState.hasPendingCommand());
+ TestCase.assertEquals(Command.INSTALL, componentInstanceState.getNextCommand());
+ TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
+ TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
+ TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.INSTALL);
+ TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.START);
+ TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.START);
+ TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.START);
+ TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.START);
+ TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+
+ componentInstanceState.commandIssued(Command.START);
+ TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.START);
+ TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+
+ try {
+ componentInstanceState.commandIssued(Command.START);
+ TestCase.fail("Command should not be allowed.");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+
+ @Test
+ public void testBadTransitions() {
+ ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", "CID_001", "AID_001");
+
+ try {
+ componentInstanceState.commandIssued(Command.START);
+ TestCase.fail("Command should not be allowed.");
+ } catch (IllegalArgumentException e) {
+ }
+
+ componentInstanceState.commandIssued(Command.INSTALL);
+ try {
+ componentInstanceState.commandIssued(Command.START);
+ TestCase.fail("Command should not be allowed.");
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.START);
+ TestCase.fail("Command should not be allowed.");
+ } catch (IllegalStateException e) {
+ }
+
+ try {
+ componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.START);
+ TestCase.fail("Command should not be allowed.");
+ } catch (IllegalStateException e) {
+ }
+
+ try {
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.fail("Command should not be allowed.");
+ } catch (IllegalArgumentException e) {
+ }
+
+ componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.INSTALL);
+
+ try {
+ componentInstanceState.commandIssued(Command.INSTALL);
+ TestCase.fail("Command should not be allowed.");
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ componentInstanceState.commandIssued(Command.NOP);
+ TestCase.fail("Command should not be allowed.");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+
+ private <T extends Throwable> void expectExceptionOnGetNextForResult(
+ Class<T> expected, State state, CommandResult result) {
+ try {
+ state.getNextState(result);
+ TestCase.fail("Must fail");
+ } catch (Exception e) {
+ if (!expected.isInstance(e)) {
+ TestCase.fail("Unexpected exception " + e.getClass());
+ }
+ }
+ }
+
+ private <T extends Throwable> void expectExceptionOnGetNextForCommand(
+ Class<T> expected, State state, Command command) {
+ try {
+ state.getNextState(command);
+ TestCase.fail("Must fail");
+ } catch (Exception e) {
+ if (!expected.isInstance(e)) {
+ TestCase.fail("Unexpected exception " + e.getClass());
+ }
+ }
+ }
+}
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
new file mode 100644
index 0000000..ac1bd81
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class MetainfoParserTest {
+ protected static final Logger log =
+ LoggerFactory.getLogger(MetainfoParserTest.class);
+ public static final String METAINFO_XML =
+ "/org/apache/slider/providers/agent/application/metadata/metainfo.xml";
+
+ @Test
+ public void testParse() throws IOException {
+
+ InputStream resStream = this.getClass().getResourceAsStream(
+ METAINFO_XML);
+ MetainfoParser parser = new MetainfoParser();
+ Metainfo metainfo = parser.parse(resStream);
+ assert metainfo != null;
+ assert metainfo.services.size() == 1;
+ Service service = metainfo.getServices().get(0);
+ assert "STORM".equals(service.getName());
+ assert 5 == service.getComponents().size();
+ OSPackage pkg = service.getOSSpecifics().get(0).getPackages().get(0);
+ assert "tarball".equals(pkg.getType());
+ assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName());
+ boolean found = false;
+ for (Component comp : service.getComponents()) {
+ if (comp != null && comp.getName().equals("NIMBUS")) {
+ found = true;
+ }
+ }
+ assert found;
+ }
+}
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/TestSliderAmFilter.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/TestSliderAmFilter.java
new file mode 100644
index 0000000..bdc11df
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/TestSliderAmFilter.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web;
+
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
+import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter;
+import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper;
+import org.glassfish.grizzly.servlet.HttpServletResponseImpl;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
+
+/**
+ * Test AmIpFilter. Requests to a no declared hosts should has way through
+ * proxy. Another requests can be filtered with (without) user name.
+ *
+ */
+public class TestSliderAmFilter {
+
+ private String proxyHost = "localhost";
+ private String proxyUri = "http://bogus";
+ private String doFilterRequest;
+ private AmIpServletRequestWrapper servletWrapper;
+ private String wsContextRoot = "/ws";
+
+ private class TestAmIpFilter extends AmIpFilter {
+
+ private Set<String> proxyAddresses = null;
+
+ protected Set<String> getProxyAddresses() {
+ if (proxyAddresses == null) {
+ proxyAddresses = new HashSet<String>();
+ }
+ proxyAddresses.add(proxyHost);
+ return proxyAddresses;
+ }
+ }
+
+ private static class DummyFilterConfig implements FilterConfig {
+ final Map<String, String> map;
+
+ DummyFilterConfig(Map<String, String> map) {
+ this.map = map;
+ }
+
+ @Override
+ public String getFilterName() {
+ return "dummy";
+ }
+
+ @Override
+ public String getInitParameter(String arg0) {
+ return map.get(arg0);
+ }
+
+ @Override
+ public Enumeration<String> getInitParameterNames() {
+ return Collections.enumeration(map.keySet());
+ }
+
+ @Override
+ public ServletContext getServletContext() {
+ return null;
+ }
+ }
+
+ /**
+ * Test AmIpFilter
+ */
+ @Test(timeout = 1000)
+ public void testFilter() throws Exception {
+ Map<String, String> params = new HashMap<String, String>();
+ params.put(SliderAmIpFilter.PROXY_HOST, proxyHost);
+ params.put(SliderAmIpFilter.PROXY_URI_BASE, proxyUri);
+ params.put(SliderAmIpFilter.WS_CONTEXT_ROOT, wsContextRoot);
+ FilterConfig config = new DummyFilterConfig(params);
+
+ // dummy filter
+ FilterChain chain = new FilterChain() {
+ @Override
+ public void doFilter(ServletRequest servletRequest,
+ ServletResponse servletResponse) throws IOException, ServletException {
+ doFilterRequest = servletRequest.getClass().getName();
+ if (servletRequest instanceof AmIpServletRequestWrapper) {
+ servletWrapper = (AmIpServletRequestWrapper) servletRequest;
+
+ }
+ }
+ };
+ SliderAmIpFilter testFilter = new SliderAmIpFilter();
+ testFilter.init(config);
+
+ HttpServletResponseForTest response = new HttpServletResponseForTest();
+ // Test request should implements HttpServletRequest
+
+ ServletRequest failRequest = Mockito.mock(ServletRequest.class);
+ try {
+ testFilter.doFilter(failRequest, response, chain);
+ fail();
+ } catch (ServletException e) {
+ assertEquals("This filter only works for HTTP/HTTPS", e.getMessage());
+ }
+
+ // request with HttpServletRequest
+ HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+ Mockito.when(request.getRemoteAddr()).thenReturn("redirect");
+ Mockito.when(request.getRequestURI()).thenReturn("/ws/v1/slider/agent");
+ testFilter.doFilter(request, response, chain);
+ // address "redirect" is not in host list
+ assertEquals(0, response.getRedirect().length());
+ Mockito.when(request.getRequestURI()).thenReturn("/redirect");
+ testFilter.doFilter(request, response, chain);
+ assertEquals("http://bogus/redirect", response.getRedirect());
+ // "127.0.0.1" contains in host list. Without cookie
+ Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1");
+ testFilter.doFilter(request, response, chain);
+
+ assertTrue(doFilterRequest
+ .contains("javax.servlet.http.HttpServletRequest"));
+ // cookie added
+ Cookie[] cookies = new Cookie[1];
+ cookies[0] = new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user");
+
+ Mockito.when(request.getCookies()).thenReturn(cookies);
+ testFilter.doFilter(request, response, chain);
+
+ assertEquals(
+ "org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper",
+ doFilterRequest);
+ // request contains principal from cookie
+ assertEquals("user", servletWrapper.getUserPrincipal().getName());
+ assertEquals("user", servletWrapper.getRemoteUser());
+ assertFalse(servletWrapper.isUserInRole(""));
+
+ }
+
+ private class HttpServletResponseForTest extends HttpServletResponseImpl {
+ String redirectLocation = "";
+
+ public String getRedirect() {
+ return redirectLocation;
+ }
+
+ @Override
+ public void sendRedirect(String location) throws IOException {
+ redirectLocation = location;
+ }
+
+ @Override
+ public String encodeRedirectURL(String url) {
+ return url;
+ }
+
+ }
+
+}
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
new file mode 100644
index 0000000..f6155c9
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -0,0 +1,228 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockProviderService;
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory;
+import org.apache.slider.server.appmaster.model.mock.MockSliderClusterProtocol;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.WebAppApiImpl;
+import org.apache.slider.server.appmaster.web.rest.AMWebServices;
+import org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.Path;
+import javax.ws.rs.core.MediaType;
+import java.io.File;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestAMAgentWebServices extends JerseyTest {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAMAgentWebServices.class);
+
+ public static final int RM_MAX_RAM = 4096;
+ public static final int RM_MAX_CORES = 64;
+ public static final String AGENT_URL =
+ "http://localhost:9998/slideram/ws/v1/slider/agents/";
+
+ static MockFactory factory = new MockFactory();
+ private static Configuration conf = new Configuration();
+ private static WebAppApi slider;
+
+ private static Injector injector = createInjector();
+ private static FileSystem fs;
+
+ public static class GuiceServletConfig extends GuiceServletContextListener {
+
+ public GuiceServletConfig() {
+ super();
+ }
+
+ @Override
+ protected Injector getInjector() {
+ return injector;
+ }
+ }
+
+// @Path("/ws/v1/slider/agent")
+ @Path("/ws/v1/slider")
+ public static class MockAMWebServices extends AMWebServices {
+
+ @Inject
+ public MockAMWebServices(WebAppApi slider) {
+ super(slider);
+ }
+
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ injector = createInjector();
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ fs = FileSystem.get(new URI("file:///"), conf);
+ }
+
+ private static Injector createInjector() {
+ return Guice.createInjector(new ServletModule() {
+ @Override
+ protected void configureServlets() {
+
+ AppState appState = null;
+ try {
+ fs = FileSystem.get(new URI("file:///"), conf);
+ File
+ historyWorkDir =
+ new File("target/history", "TestAMAgentWebServices");
+ org.apache.hadoop.fs.Path
+ historyPath =
+ new org.apache.hadoop.fs.Path(historyWorkDir.toURI());
+ fs.delete(historyPath, true);
+ appState = new AppState(new MockRecordFactory());
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
+ appState.buildInstance(
+ factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null, null);
+ } catch (Exception e) {
+ log.error("Failed to set up app {}", e);
+ }
+ slider = new WebAppApiImpl(new MockSliderClusterProtocol(), appState,
+ new MockProviderService());
+
+ bind(SliderJacksonJaxbJsonProvider.class);
+ bind(GenericExceptionHandler.class);
+ bind(MockAMWebServices.class);
+ bind(WebAppApi.class).toInstance(slider);
+ bind(Configuration.class).toInstance(conf);
+
+ Map<String, String> params = new HashMap<String, String>();
+ addLoggingFilter(params);
+ serve("/*").with(GuiceContainer.class, params);
+ }
+ });
+ }
+
+ private static void addLoggingFilter(Map<String, String> params) {
+ params.put("com.sun.jersey.spi.container.ContainerRequestFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
+ params.put("com.sun.jersey.spi.container.ContainerResponseFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
+ }
+
+ public TestAMAgentWebServices() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.appmaster.web")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .initParam("com.sun.jersey.api.json.POJOMappingFeature", "true")
+ .contextPath("slideram").servletPath("/").build());
+ }
+
+ @Test
+ public void testRegistration() throws JSONException, Exception {
+ RegistrationResponse response;
+ Client client = createTestClient();
+ WebResource webResource = client.resource(AGENT_URL + "test/register");
+ response = webResource.type(MediaType.APPLICATION_JSON)
+ .post(RegistrationResponse.class, createDummyJSONRegister());
+ Assert.assertEquals(RegistrationStatus.OK, response.getResponseStatus());
+ }
+
+ protected Client createTestClient() {
+ ClientConfig clientConfig = new DefaultClientConfig();
+ clientConfig.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE);
+ return Client.create(clientConfig);
+ }
+
+ @Test
+ public void testHeartbeat() throws JSONException, Exception {
+ HeartBeatResponse response;
+ Client client = createTestClient();
+ WebResource webResource = client.resource(AGENT_URL + "test/heartbeat");
+ response = webResource.type(MediaType.APPLICATION_JSON)
+ .post(HeartBeatResponse.class, createDummyHeartBeat());
+ assertEquals(response.getResponseId(), 0L);
+ }
+
+ @Test
+ public void testHeadURL() throws JSONException, Exception {
+ Client client = createTestClient();
+ WebResource webResource = client.resource(AGENT_URL);
+ ClientResponse response = webResource.type(MediaType.APPLICATION_JSON)
+ .head();
+ assertEquals(200, response.getStatus());
+ }
+
+ @Test
+ public void testSleepForAWhile() throws Throwable {
+ log.info("Agent is running at {}", AGENT_URL);
+ Thread.sleep(60 * 1000);
+ }
+
+ private Register createDummyJSONRegister() throws JSONException {
+ Register register = new Register();
+ register.setResponseId(-1);
+ register.setTimestamp(System.currentTimeMillis());
+ register.setHostname("dummyHost");
+ return register;
+ }
+
+ private JSONObject createDummyHeartBeat() throws JSONException {
+ JSONObject json = new JSONObject();
+ json.put("responseId", -1);
+ json.put("timestamp", System.currentTimeMillis());
+ json.put("hostname", "dummyHost");
+ return json;
+ }
+
+}
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
new file mode 100644
index 0000000..a3256ca
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -0,0 +1,282 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.management;
+
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.persist.JsonSerDeser;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockProviderService;
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory;
+import org.apache.slider.server.appmaster.model.mock.MockSliderClusterProtocol;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.WebAppApiImpl;
+import org.apache.slider.server.appmaster.web.rest.AMWebServices;
+import org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.Path;
+import javax.ws.rs.core.MediaType;
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class TestAMManagementWebServices extends JerseyTest {
+
+ public static final int RM_MAX_RAM = 4096;
+ public static final int RM_MAX_CORES = 64;
+ public static final String EXAMPLES =
+ "/org/apache/slider/core/conf/examples/";
+ static MockFactory factory = new MockFactory();
+ private static Configuration conf = new Configuration();
+ private static WebAppApi slider;
+
+ private static Injector injector = createInjector();
+ private static FileSystem fs;
+
+ public static class GuiceServletConfig extends GuiceServletContextListener {
+
+ public GuiceServletConfig() {
+ super();
+ }
+
+ @Override
+ protected Injector getInjector() {
+ return injector;
+ }
+ }
+
+ @Path("/ws/v1/slider")
+ public static class MockSliderAMWebServices extends AMWebServices {
+
+ @Inject
+ public MockSliderAMWebServices(WebAppApi slider) {
+ super(slider);
+ }
+
+ @Override
+ public ManagementResource getManagementResource() {
+ return new MockManagementResource(slider);
+ }
+
+ }
+
+ public static class MockManagementResource extends ManagementResource {
+ public MockManagementResource(WebAppApi slider) {
+ super(slider);
+ }
+
+ protected AggregateConf getAggregateConf() {
+ JsonSerDeser<ConfTree> confTreeJsonSerDeser =
+ new JsonSerDeser<ConfTree>(ConfTree.class);
+ ConfTree internal = null;
+ ConfTree app_conf = null;
+ ConfTree resources = null;
+ try {
+ internal =
+ confTreeJsonSerDeser.fromResource(
+ EXAMPLES +"internal.json");
+ app_conf =
+ confTreeJsonSerDeser.fromResource(
+ EXAMPLES + "app_configuration.json");
+ resources =
+ confTreeJsonSerDeser.fromResource(
+ EXAMPLES + "resources.json");
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ AggregateConf aggregateConf = new AggregateConf(
+ resources,
+ app_conf,
+ internal);
+ aggregateConf.setName("test");
+ return aggregateConf;
+ }
+
+ }
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ injector = createInjector();
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ fs = FileSystem.get(new URI("file:///"), conf);
+ }
+
+ private static Injector createInjector() {
+ return Guice.createInjector(new ServletModule() {
+ @Override
+ protected void configureServlets() {
+
+ AppState appState = null;
+ try {
+ fs = FileSystem.get(new URI("file:///"), conf);
+ File
+ historyWorkDir =
+ new File("target/history", "TestAMManagementWebServices");
+ org.apache.hadoop.fs.Path
+ historyPath =
+ new org.apache.hadoop.fs.Path(historyWorkDir.toURI());
+ fs.delete(historyPath, true);
+ appState = new AppState(new MockRecordFactory());
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
+ appState.buildInstance(
+ factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null, null);
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ } catch (URISyntaxException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ } catch (BadClusterStateException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ } catch (BadConfigException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ slider = new WebAppApiImpl(new MockSliderClusterProtocol(), appState,
+ new MockProviderService());
+
+ bind(SliderJacksonJaxbJsonProvider.class);
+ bind(MockSliderAMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ bind(WebAppApi.class).toInstance(slider);
+ bind(Configuration.class).toInstance(conf);
+
+ serve("/*").with(GuiceContainer.class);
+ }
+ });
+ }
+
+ public TestAMManagementWebServices() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.appmaster.web")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("slideram").servletPath("/").build());
+ }
+
+ @Test
+ public void testAppResource() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("slider").path("mgmt").path("app")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(200, response.getStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 4, json.length());
+ assertEquals("wrong href",
+ "http://localhost:9998/slideram/ws/v1/slider/mgmt/app",
+ json.getString("href"));
+ assertNotNull("no resources", json.getJSONObject("resources"));
+ assertNotNull("no internal", json.getJSONObject("internal"));
+ assertNotNull("no appConf", json.getJSONObject("appConf"));
+ }
+
+ @Test
+ public void testAppInternal() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse
+ response =
+ r.path("ws").path("v1").path("slider").path("mgmt").path("app").path("configurations").path(
+ "internal")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(200, response.getStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 4, json.length());
+ assertEquals("wrong href",
+ "http://localhost:9998/slideram/ws/v1/slider/mgmt/app/configurations/internal",
+ json.getString("href"));
+ assertEquals("wrong description",
+ "Internal configuration DO NOT EDIT",
+ json.getJSONObject("metadata").getString("description"));
+ }
+
+ @Test
+ public void testAppResources() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse
+ response =
+ r.path("ws").path("v1").path("slider").path("mgmt").path("app").path("configurations").path(
+ "resources")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+
+ assertEquals(200, response.getStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 4, json.length());
+ assertEquals("wrong href",
+ "http://localhost:9998/slideram/ws/v1/slider/mgmt/app/configurations/resources",
+ json.getString("href"));
+ json = json.getJSONObject("components");
+ assertNotNull("no components", json);
+ assertEquals("incorrect number of components", 2, json.length());
+ assertNotNull("wrong component", json.getJSONObject("worker"));
+ }
+
+ @Test
+ public void testAppAppConf() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse
+ response =
+ r.path("ws").path("v1").path("slider").path("mgmt").path("app").path("configurations").path(
+ "appConf")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(200, response.getStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 4, json.length());
+ assertEquals("wrong href",
+ "http://localhost:9998/slideram/ws/v1/slider/mgmt/app/configurations/appConf",
+ json.getString("href"));
+ json = json.getJSONObject("components");
+ assertNotNull("no components", json);
+ assertEquals("incorrect number of components", 2, json.length());
+ assertNotNull("wrong component", json.getJSONObject("worker"));
+ }
+}
diff --git a/slider-core/src/test/java/org/apache/slider/server/servicemonitor/PortProbeTest.java b/slider-core/src/test/java/org/apache/slider/server/servicemonitor/PortProbeTest.java
new file mode 100644
index 0000000..426c207
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/server/servicemonitor/PortProbeTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+public class PortProbeTest extends Assert {
+ /**
+ * Assert that a port probe failed if the port is closed
+ * @throws Throwable
+ */
+ @Test
+ public void testPortProbeFailsClosedPort() throws Throwable {
+ PortProbe probe = new PortProbe("127.0.0.1", 65500, 100, "", new Configuration());
+ probe.init();
+ ProbeStatus status = probe.ping(true);
+ assertFalse("Expected a failure but got successful result: " + status,
+ status.isSuccess());
+ }
+}
diff --git a/slider-core/src/test/python/agent.ini b/slider-core/src/test/python/agent.ini
new file mode 100644
index 0000000..7b11312
--- /dev/null
+++ b/slider-core/src/test/python/agent.ini
@@ -0,0 +1 @@
+[agent]
diff --git a/slider-core/src/test/python/agent.py b/slider-core/src/test/python/agent.py
new file mode 100644
index 0000000..4be2cd9
--- /dev/null
+++ b/slider-core/src/test/python/agent.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import sys
+import datetime
+import time
+import argparse
+import os
+
+# A representative Agent code for the embedded agent
+def main():
+ print "Executing echo"
+ print 'Argument List: {}'.format(str(sys.argv))
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--log', dest='log_folder', help='log destination')
+ parser.add_argument('--config', dest='conf_folder', help='conf folder')
+ args = parser.parse_args()
+ if args.log_folder:
+ log_file_name = "echo" + str(datetime.datetime.now()) + ".log"
+ log_file_path = os.path.join(args.log_folder, log_file_name)
+ logging.basicConfig(filename=log_file_path, level=logging.DEBUG)
+ print log_file_path
+ logging.debug('Starting echo script ...')
+
+ logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
+ logging.info("Argument List: %s", str(sys.argv))
+ time.sleep(30)
+
+
+if __name__ == "__main__":
+ main()
+ sys.exit(0)
diff --git a/slider-core/src/test/python/agent/main.py b/slider-core/src/test/python/agent/main.py
new file mode 100755
index 0000000..fd8b262
--- /dev/null
+++ b/slider-core/src/test/python/agent/main.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import sys
+import datetime
+import time
+import argparse
+import os
+
+
+def main():
+ print "Executing echo"
+ print 'Argument List: {}'.format(str(sys.argv))
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--log', dest='log_folder', help='log destination')
+ parser.add_argument('--config', dest='conf_folder', help='conf folder')
+ parser.add_argument('--command', dest='command', help='command to execute')
+ parser.add_argument('--label', dest='label', help='label')
+ parser.add_argument('--host', dest='host', help='port')
+ parser.add_argument('--port', dest='port', help='host')
+ args = parser.parse_args()
+ if args.log_folder:
+ log_file_name = "echo" + str(datetime.datetime.now()) + ".log"
+ log_file_path = os.path.join(args.log_folder, log_file_name)
+ logging.basicConfig(filename=log_file_path, level=logging.DEBUG)
+ print log_file_path
+ logging.debug('Starting echo script ...')
+
+ logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
+ logging.info("Argument List: %s", str(sys.argv))
+ time.sleep(30)
+
+
+if __name__ == "__main__":
+ main()
+ sys.exit(0)
diff --git a/slider-core/src/test/python/appdef_1.tar b/slider-core/src/test/python/appdef_1.tar
new file mode 100644
index 0000000..9340ea3
--- /dev/null
+++ b/slider-core/src/test/python/appdef_1.tar
Binary files differ
diff --git a/slider-core/src/test/python/echo.py b/slider-core/src/test/python/echo.py
new file mode 100644
index 0000000..2bcab20
--- /dev/null
+++ b/slider-core/src/test/python/echo.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import sys
+import datetime
+import time
+import argparse
+import os
+
+
+def main():
+ print "Executing echo"
+ print 'Argument List: {}'.format(str(sys.argv))
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--log', dest='log_folder', help='log destination')
+ parser.add_argument('--config', dest='conf_folder', help='conf folder')
+ parser.add_argument('--command', dest='command', help='command to execute')
+ args = parser.parse_args()
+ if args.log_folder:
+ log_file_name = "echo" + str(datetime.datetime.now()) + ".log"
+ log_file_path = os.path.join(args.log_folder, log_file_name)
+ logging.basicConfig(filename=log_file_path, level=logging.DEBUG)
+ print log_file_path
+ logging.debug('Starting echo script ...')
+
+ logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
+ logging.info("Argument List: %s", str(sys.argv))
+ time.sleep(30)
+
+
+if __name__ == "__main__":
+ main()
+ sys.exit(0)
diff --git a/slider-core/src/test/python/version b/slider-core/src/test/python/version
new file mode 100644
index 0000000..bc80560
--- /dev/null
+++ b/slider-core/src/test/python/version
@@ -0,0 +1 @@
+1.5.0
diff --git a/slider-core/src/test/resources/example-slider-test.xml b/slider-core/src/test/resources/example-slider-test.xml
new file mode 100644
index 0000000..084557f
--- /dev/null
+++ b/slider-core/src/test/resources/example-slider-test.xml
@@ -0,0 +1,117 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!-- Slider test options -->
+
+ <!--
+ This is an example of a slider XML test file; it propagates
+ to the dependent modules so can be a single place to keep
+ test information
+ -->
+<configuration>
+
+ <property>
+ <name>slider.test.timeout.seconds</name>
+ <description></description>
+ <value>600</value>
+ </property>
+
+ <property>
+ <name>slider.test.teardown.killall</name>
+ <description>Kill all hbase/accumulo, etc processes on test teardown</description>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>slider.test.thaw.wait.seconds</name>
+ <description>Time to wait for a thaw to work</description>
+ <value>60</value>
+ </property>
+
+ <property>
+ <name>slider.test.freeze.wait.seconds</name>
+ <description>Time to wait for a freeze to work</description>
+ <value>60</value>
+ </property>
+
+
+ <!-- Properties for the slider-hbase-provider only -not HBase-under-agent- -->
+ <property>
+ <name>slider.test.hbase.enabled</name>
+ <description>Flag to enable/disable HBase tests</description>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.launch.wait.seconds</name>
+ <description>Time to wait for the HBase application to be live</description>
+ <value>180</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.home</name>
+ <value>/home/slider/Projects/hbase/hbase-assembly/target/hbase-0.98.1</value>
+ <description>HBASE Home</description>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <value>/home/slider/Projects/hbase/hbase-assembly/target/hbase-0.98.1-bin.tar.gz</value>
+ <description>HBASE archive URI</description>
+ </property>
+
+ <!-- Properties for the slider-accumulo-provider only -not HBase-under-agent- -->
+
+ <property>
+ <name>slider.test.accumulo.enabled</name>
+ <description>Flag to enable/disable Accumulo tests</description>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.launch.wait.seconds</name>
+ <description>Time to wait for the accumulo application to be live</description>
+ <value>180</value>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.home</name>
+ <value>/home/slider/accumulo</value>
+ <description>Accumulo Home</description>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.tar</name>
+ <value>/home/slider/Projects/accumulo/accumulo-1.5.1-bin.tar</value>
+ <description>Accumulo archive URI</description>
+ </property>
+
+ <property>
+ <name>zk.home</name>
+ <value>/home/slider/zookeeper</value>
+ <description>Zookeeper home dir on target systems</description>
+ </property>
+
+ <property>
+ <name>hadoop.home</name>
+ <value>/home/slider/hadoop/</value>
+ <description>Hadoop home dir on target systems</description>
+ </property>
+
+</configuration>
\ No newline at end of file
diff --git a/slider-core/src/test/resources/log4j-disabled.properties b/slider-core/src/test/resources/log4j-disabled.properties
new file mode 100644
index 0000000..b819673
--- /dev/null
+++ b/slider-core/src/test/resources/log4j-disabled.properties
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{3} (%F:%M(%L)) - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=%c{1}: %m%n
+log4j.logger.org.apache.slider.server.appmaster.SliderAppMaster.master=INFO,subprocess
+
+
+#at debug this provides details on what is going on
+log4j.logger.org.apache.slider=DEBUG
+#log4j.logger.org.apache.slider.exec.RunLongLivedApp=ERROR
+
+
+log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+#crank back on some noise
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.hadoop.yarn.client.RMProxy=WARN
+
+# for test runs we don't care about native code
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+# HDFS is noise on tets
+log4j.logger.org.apache.hadoop.hdfs.server.datanode=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.namenode=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+log4j.logger.org.apache.zookeeper=WARN
diff --git a/slider-core/src/test/resources/log4j.properties b/slider-core/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a552a55
--- /dev/null
+++ b/slider-core/src/test/resources/log4j.properties
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+log4j.logger.org.apache.slider=DEBUG
+log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+
+
+
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
+log4j.logger.org.apache.hadoop.metrics2=ERROR
+log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
+log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN
+log4j.logger.org.apache.hadoop.security.token.delegation=WARN
+log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration-resolved.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration-resolved.json
new file mode 100644
index 0000000..5b90ba9
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration-resolved.json
@@ -0,0 +1,42 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+ "components": {
+
+ "worker": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "512M"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json
new file mode 100644
index 0000000..489acda
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json
@@ -0,0 +1,25 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+ "components": {
+
+ "worker": {
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "jvm.heapsize": "512M"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/empty.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/empty.json
new file mode 100644
index 0000000..5c05163
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/empty.json
@@ -0,0 +1,8 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ },
+ "components": {
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json
new file mode 100644
index 0000000..592b4dc
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json
@@ -0,0 +1,24 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "Internal configuration DO NOT EDIT"
+ },
+ "global": {
+ "application.name": "small_cluster",
+ "application.type": "hbase",
+ "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar"
+ },
+ "components": {
+
+ "diagnostics": {
+ "application.name": "small_cluster",
+ "application.type": "hbase",
+ "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar",
+ "create.hadoop.deployed.info": "(release-2.3.0) @dfe463",
+ "create.hadoop.build.info": "2.3.0",
+ "create.time.millis": "1393512091276",
+ "create.time": "27 Feb 2014 14:41:31 GMT"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
new file mode 100644
index 0000000..8617d1f
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
@@ -0,0 +1,21 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "Internal configuration DO NOT EDIT"
+ },
+ "global": {
+ "application.name": "small_cluster",
+ "application.type": "hbase",
+ "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar"
+ },
+ "components": {
+
+ "diagnostics": {
+ "create.hadoop.deployed.info": "(release-2.3.0) @dfe463",
+ "create.hadoop.build.info": "2.3.0",
+ "create.time.millis": "1393512091276",
+ "create.time": "27 Feb 2014 14:41:31 GMT"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/overridden-resolved.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/overridden-resolved.json
new file mode 100644
index 0000000..edd941f
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/overridden-resolved.json
@@ -0,0 +1,25 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "components": {
+ "simple": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "master": {
+ "name": "m",
+ "g1": "overridden",
+ "g2": "b"
+ },
+ "worker": {
+ "name": "worker",
+ "g1": "overridden-by-worker",
+ "g2": "b",
+ "timeout": "1000"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/overridden.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/overridden.json
new file mode 100644
index 0000000..8237ad4
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/overridden.json
@@ -0,0 +1,23 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "components": {
+ "simple": {
+ },
+ "master": {
+ "name": "m",
+ "g1": "overridden"
+
+ },
+ "worker": {
+ "name": "worker",
+ "g1": "overridden-by-worker",
+ "timeout": "1000"
+
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json
new file mode 100644
index 0000000..06c3b54
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json
@@ -0,0 +1,25 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "example of a resources file"
+ },
+
+ "global": {
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ },
+
+ "components": {
+ "master": {
+ "instances": "1",
+ "yarn.vcores": "1",
+ "yarn.memory": "1024"
+ },
+ "worker": {
+ "instances":"5",
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
new file mode 100644
index 0000000..2fcf4cd
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>STORM</name>
+ <comment>Apache Hadoop Stream processing framework</comment>
+ <version>0.9.1.2.1</version>
+ <components>
+
+ <component>
+ <name>NIMBUS</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/nimbus.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_REST_API</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/rest_api.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>SUPERVISOR</name>
+ <category>SLAVE</category>
+ <commandScript>
+ <script>scripts/supervisor.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_UI_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/ui_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>DRPC_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/drpc_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <configuration-dependencies>
+ <config-type>storm-site</config-type>
+ <config-type>global</config-type>
+ </configuration-dependencies>
+ </service>
+ </services>
+</metainfo>
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/appconf-1.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/appconf-1.json
new file mode 100644
index 0000000..43a69ee
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/appconf-1.json
@@ -0,0 +1,11 @@
+{
+ "schema": "http://example.org/specification/v3.0.0",
+
+ "global": {
+ },
+ "components": {
+ "node": {
+ "agent.script":"agent/scripts/agent.py"
+ }
+ }
+}
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/appconf-2.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/appconf-2.json
new file mode 100644
index 0000000..27220d3
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/appconf-2.json
@@ -0,0 +1,3 @@
+THIS IS NOT WELL FORMED JSON
+{
+ []
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/appconf.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/appconf.json
new file mode 100644
index 0000000..9c3bd1e
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/appconf.json
@@ -0,0 +1,11 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ },
+ "components": {
+ "node": {
+ "agent.script":"agent/scripts/agent.py"
+ }
+ }
+}
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json
new file mode 100644
index 0000000..a36a1fa
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json
@@ -0,0 +1,13 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ },
+ "components": {
+ "node": {
+ "yarn.memory": "256",
+ "component.instances": "5",
+ "role.priority":"1"
+ }
+ }
+}
diff --git a/slider-core/src/test/resources/slider-client.xml b/slider-core/src/test/resources/slider-client.xml
new file mode 100644
index 0000000..40f4883
--- /dev/null
+++ b/slider-core/src/test/resources/slider-client.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>slider.client.resource.origin</name>
+ <value>test/resources</value>
+ <comment>Used in TestClientResourceRegistration -do not change</comment>
+ </property>
+</configuration>
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
new file mode 100644
index 0000000..e720100
--- /dev/null
+++ b/slider-funtest/pom.xml
@@ -0,0 +1,257 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-funtest</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ <name>Slider Functional Tests</name>
+ <packaging>jar</packaging>
+ <description>
+ Functional testing for slider: testing is always fun
+ </description>
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+
+ <!-- resources are filtered for dynamic updates. This gets build info in-->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
+ <plugins>
+
+ <!--read in a build.properties file if defined-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <version>${maven.properties.version}</version>
+ <executions>
+ <execution>
+ <phase>initialize</phase>
+ <goals>
+ <goal>read-project-properties</goal>
+ </goals>
+ <configuration>
+ <quiet>true</quiet>
+ <files>
+ <file>build.properties</file>
+ <file>../build.properties</file>
+ </files>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler-plugin.version}</version>
+ <configuration>
+ <compilerId>groovy-eclipse-compiler</compilerId>
+ <!-- set verbose to be true if you want lots of uninteresting messages -->
+ <!-- <verbose>true</verbose> -->
+ <source>${project.java.src.version}</source>
+ <target>${project.java.src.version}</target>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>${groovy-eclipse-compiler.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-batch</artifactId>
+ <version>${groovy-eclipse-batch.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+
+ <!-- test -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ <configuration>
+ <!--mvn process fork options-->
+ <reuseForks>${test.reuseForks}</reuseForks>
+ <forkMode>${test.forkMode}</forkMode>
+ <forkCount>1</forkCount>
+ <forkedProcessTimeoutInSeconds>${test.forkedProcessTimeoutInSeconds}
+ </forkedProcessTimeoutInSeconds>
+ <threadCount>1</threadCount>
+ <argLine>${test.argLine}</argLine>
+ <failIfNoTests>${test.failIfNoTests}</failIfNoTests>
+
+ <trimStackTrace>false</trimStackTrace>
+ <redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
+ <systemPropertyVariables>
+ <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
+ <java.awt.headless>true</java.awt.headless>
+ <java.security.krb5.realm>${slider.test.java.security.krb5.realm}</java.security.krb5.realm>
+ <java.security.krb5.kdc>${slider.test.java.security.krb5.kdc}</java.security.krb5.kdc>
+ <!-- this property must be supplied-->
+ <slider.conf.dir>${slider.conf.dir}</slider.conf.dir>
+ <slider.bin.dir>../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/Test*.java</include>
+ </includes>
+ <excludes>
+ <exclude>**/Test*$*.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/httpfs-signature.secret</exclude>
+ <exclude>**/regionservers</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <reporting>
+ <plugins>
+
+
+
+ </plugins>
+ </reporting>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-assembly</artifactId>
+ <version>${project.version}</version>
+ <classifier>all</classifier>
+ <type>tar.gz</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>itest-common</artifactId>
+ </dependency>
+
+
+ </dependencies>
+
+
+</project>
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy
new file mode 100644
index 0000000..fba7461
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/abstracttests/AbstractTestBuildSetup.groovy
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.abstracttests
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.funtest.framework.ConfLoader
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.test.SliderTestUtils
+import org.junit.Test
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+
+/**
+ * Simple tests to verify that the build has been set up: if these
+ * fail then the arguments to the test run are incomplete.
+ *
+ * This deliberately doesn't depend on SliderCommandTestBase,
+ * so that individual tests fail with more diagnostics
+ * than the @BeforeClass failing
+ */
+@CompileStatic
+@Slf4j
+abstract class AbstractTestBuildSetup extends SliderTestUtils implements FuntestProperties {
+
+
+ String getRequiredSysprop(String name) {
+ String val = System.getProperty(name)
+ if (!val) {
+ fail("System property not set : $name")
+
+ }
+ return val;
+ }
+
+ public String getSliderConfDirProp() {
+ return getRequiredSysprop(SLIDER_CONF_DIR_PROP)
+ }
+
+ public File getSliderConfDirectory() {
+ return getCanonicalFile(sliderConfDirProp)
+ }
+
+ File getCanonicalFile(String dir) {
+ assert dir
+ return new File(dir).canonicalFile
+ }
+
+ File getBinDirectory() {
+ return getCanonicalFile(getBinDirProp())
+ }
+
+ public String getBinDirProp() {
+ return getRequiredSysprop(SLIDER_BIN_DIR_PROP)
+ }
+
+ public File getConfXML() {
+ new File(sliderConfDirectory, CLIENT_CONFIG_FILENAME).canonicalFile
+ }
+
+ public File getSliderScript() {
+ new File(binDirectory, "bin/" + SCRIPT_NAME).canonicalFile
+ }
+
+ /**
+ * Load the client XML file
+ * @return
+ */
+ public Configuration loadSliderConf() {
+ Configuration conf = (new ConfLoader()).loadSliderConf(confXML)
+ return conf
+ }
+
+ @Test
+ public void testConfDirSet() throws Throwable {
+ assert getSliderConfDirProp()
+ log.info("Slider Configuration directory $sliderConfDirProp")
+ }
+
+ @Test
+ public void testConfDirExists() throws Throwable {
+ assert sliderConfDirectory.exists()
+ }
+
+
+ @Test
+ public void testConfDirHasClientXML() throws Throwable {
+ File clientXMLFile = confXML
+ assert clientXMLFile.exists()
+ clientXMLFile.toString()
+ }
+
+
+ @Test
+ public void testBinDirExists() throws Throwable {
+ log.info("binaries dir = $binDirectory")
+ }
+
+ @Test
+ public void testBinScriptExists() throws Throwable {
+ assert sliderScript.exists()
+ }
+
+ @Test
+ public void testConfLoad() throws Throwable {
+ Configuration conf = loadSliderConf()
+ }
+
+ @Test
+ public void testConfHasFileURL() throws Throwable {
+ Configuration conf = loadSliderConf()
+ assert conf.get(KEY_TEST_CONF_XML)
+ String confXml = conf.get(KEY_TEST_CONF_XML)
+ URL confURL = new URL(confXml)
+ log.info("$KEY_TEST_CONF_XML = $confXML -as URL: $confURL")
+ Path path = new Path(confURL.toURI())
+
+ def fs = HadoopFS.get(path.toUri(), conf)
+ assert fs.exists(path)
+
+ }
+
+ @Test
+ public void testConfHasDefaultFS() throws Throwable {
+ Configuration conf = loadSliderConf()
+ assumeBoolOption(conf, KEY_SLIDER_FUNTESTS_ENABLED, true)
+ String fs = conf.get("fs.defaultFS")
+ log.info("Test Filesystem $fs")
+ assert fs != null
+ }
+
+
+ @Test
+ public void testConfHasRM() throws Throwable {
+
+ Configuration conf = loadSliderConf()
+ assumeBoolOption(conf, KEY_SLIDER_FUNTESTS_ENABLED, true)
+ String val = conf.get(YarnConfiguration.RM_ADDRESS)
+ log.info("$YarnConfiguration.RM_ADDRESS = $val")
+ assert val != YarnConfiguration.DEFAULT_RM_ADDRESS
+ }
+
+ @Test
+ public void testSecuritySettingsValid() throws Throwable {
+ Configuration conf = loadSliderConf();
+ if (SliderUtils.maybeInitSecurity(conf)) {
+ log.info("Security enabled")
+ SliderUtils.forceLogin()
+ }
+ log.info("Login User = ${UserGroupInformation.loginUser}")
+ }
+
+
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/categories/FunctionalTests.java b/slider-funtest/src/main/groovy/org/apache/slider/funtest/categories/FunctionalTests.java
new file mode 100644
index 0000000..6516707
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/categories/FunctionalTests.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.categories;
+
+/**
+ * Marker interface for JUnit tests in functional category
+ */
+public interface FunctionalTests {
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
new file mode 100644
index 0000000..615cbde
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import groovy.transform.CompileStatic
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.util.ExitUtil
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.test.SliderTestUtils
+import org.junit.Before
+import org.junit.BeforeClass
+import org.junit.Rule
+import org.junit.rules.Timeout
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import static org.apache.slider.common.SliderExitCodes.*
+import static FuntestProperties.*
+import static Arguments.*
+import static org.apache.slider.common.params.SliderActions.*
+import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
+
+@CompileStatic
+abstract class CommandTestBase extends SliderTestUtils {
+ private static final Logger log =
+ LoggerFactory.getLogger(CommandTestBase.class);
+
+ public static final String SLIDER_CONF_DIR = sysprop(SLIDER_CONF_DIR_PROP)
+ public static final String SLIDER_BIN_DIR = sysprop(SLIDER_BIN_DIR_PROP)
+ public static final File SLIDER_BIN_DIRECTORY = new File(
+ SLIDER_BIN_DIR).canonicalFile
+ public static final File SLIDER_SCRIPT = new File(
+ SLIDER_BIN_DIRECTORY,
+ "bin/slider").canonicalFile
+ public static final File SLIDER_CONF_DIRECTORY = new File(SLIDER_CONF_DIR).canonicalFile
+ public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
+ CLIENT_CONFIG_FILENAME).canonicalFile
+
+ public static final YarnConfiguration SLIDER_CONFIG
+ public static final int THAW_WAIT_TIME
+ public static final int FREEZE_WAIT_TIME
+
+ public static final int ACCUMULO_LAUNCH_WAIT_TIME
+ public static final int SLIDER_TEST_TIMEOUT
+ public static final boolean ACCUMULO_TESTS_ENABLED
+
+ public static final boolean FUNTESTS_ENABLED
+ public static final boolean AGENTTESTS_ENABLED
+
+
+ static {
+ SLIDER_CONFIG = new ConfLoader().loadSliderConf(SLIDER_CONF_XML);
+ THAW_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
+ KEY_TEST_THAW_WAIT_TIME,
+ 1000 * DEFAULT_THAW_WAIT_TIME_SECONDS)
+ FREEZE_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
+ KEY_TEST_FREEZE_WAIT_TIME,
+ 1000 * DEFAULT_TEST_FREEZE_WAIT_TIME_SECONDS)
+ SLIDER_TEST_TIMEOUT = getTimeOptionMillis(SLIDER_CONFIG,
+ KEY_TEST_TIMEOUT,
+ 1000 * DEFAULT_TEST_TIMEOUT_SECONDS)
+ ACCUMULO_LAUNCH_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
+ KEY_ACCUMULO_LAUNCH_TIME,
+ 1000 * DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS)
+ FUNTESTS_ENABLED =
+ SLIDER_CONFIG.getBoolean(KEY_SLIDER_FUNTESTS_ENABLED, true)
+ ACCUMULO_TESTS_ENABLED =
+ SLIDER_CONFIG.getBoolean(KEY_TEST_ACCUMULO_ENABLED, false)
+
+ AGENTTESTS_ENABLED =
+ SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, true)
+ }
+
+ @Rule
+ public final Timeout testTimeout = new Timeout(SLIDER_TEST_TIMEOUT);
+
+
+ @BeforeClass
+ public static void setupTestBase() {
+ Configuration conf = loadSliderConf();
+ if (SliderUtils.maybeInitSecurity(conf)) {
+ log.debug("Security enabled")
+ SliderUtils.forceLogin()
+ } else {
+ log.info "Security off, making cluster dirs broadly accessible"
+ }
+ SliderShell.confDir = SLIDER_CONF_DIRECTORY
+ SliderShell.script = SLIDER_SCRIPT
+ log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " +
+ "and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
+
+ // now patch the settings with the path of the conf direcotry
+
+ }
+
+ /**
+ * give our thread a name
+ */
+ @Before
+ public void nameThread() {
+ Thread.currentThread().name = "JUnit"
+ }
+
+ /**
+ * Add a jar to the slider classpath
+ * @param clazz
+ */
+ public static void addExtraJar(Class clazz) {
+ def jar = SliderUtils.findContainingJarOrFail(clazz)
+
+ def path = jar.absolutePath
+ if (!SliderShell.slider_classpath_extra.contains(path)) {
+ SliderShell.slider_classpath_extra << path
+ }
+ }
+
+ public static String sysprop(String key) {
+ def property = System.getProperty(key)
+ if (!property) {
+ throw new RuntimeException("Undefined property $key")
+ }
+ return property
+ }
+
+ /**
+ * Exec any slider command
+ * @param conf
+ * @param commands
+ * @return the shell
+ */
+ public static SliderShell slider(List<String> commands) {
+ SliderShell shell = new SliderShell(commands)
+ shell.execute()
+ return shell
+ }
+
+ /**
+ * Execute an operation, state the expected error code
+ * @param exitCode exit code
+ * @param commands commands
+ * @return
+ */
+ public static SliderShell slider(int exitCode, List<String> commands) {
+ return SliderShell.run(commands, exitCode)
+ }
+
+ /**
+ * Load the client XML file
+ * @return
+ */
+ public static Configuration loadSliderConf() {
+ Configuration conf = (new ConfLoader()).loadSliderConf(SLIDER_CONF_XML)
+ return conf
+ }
+
+ public static HadoopFS getClusterFS() {
+ return HadoopFS.get(SLIDER_CONFIG)
+ }
+
+ static SliderShell destroy(String name) {
+ slider([
+ ACTION_DESTROY, name
+ ])
+ }
+
+ static SliderShell destroy(int result, String name) {
+ slider(result, [
+ ACTION_DESTROY, name
+ ])
+ }
+
+ static SliderShell exists(String name, boolean live = true) {
+
+ List<String> args = [
+ ACTION_EXISTS, name
+ ]
+ if (live) {
+ args << Arguments.ARG_LIVE
+ }
+ slider(args)
+ }
+
+ static SliderShell exists(int result, String name, boolean live = true) {
+ List<String> args = [
+ ACTION_EXISTS, name
+ ]
+ if (live) {
+ args << ARG_LIVE
+ }
+ slider(result, args)
+ }
+
+ static SliderShell freeze(String name) {
+ slider([
+ ACTION_FREEZE, name
+ ])
+ }
+
+ static SliderShell getConf(String name) {
+ slider([
+ ACTION_GETCONF, name
+ ])
+ }
+
+ static SliderShell getConf(int result, String name) {
+ slider(result,
+ [
+ ACTION_GETCONF, name
+ ])
+ }
+
+ static SliderShell killContainer(String name, String containerID) {
+ slider(0,
+ [
+ ACTION_KILL_CONTAINER,
+ name,
+ containerID
+ ])
+ }
+
+ static SliderShell freezeForce(String name) {
+ slider([
+ ACTION_FREEZE, ARG_FORCE, name
+ ])
+ }
+
+ static SliderShell list(String name) {
+ List<String> cmd = [
+ ACTION_LIST
+ ]
+ if (name != null) {
+ cmd << name
+ }
+ slider(cmd)
+ }
+
+ static SliderShell list(int result, String name) {
+ List<String> cmd = [
+ ACTION_LIST
+ ]
+ if (name != null) {
+ cmd << name
+ }
+ slider(result, cmd)
+ }
+
+ static SliderShell status(String name) {
+ slider([
+ ACTION_STATUS, name
+ ])
+ }
+
+ static SliderShell status(int result, String name) {
+ slider(result,
+ [
+ ACTION_STATUS, name
+ ])
+ }
+
+ static SliderShell thaw(String name) {
+ slider([
+ ACTION_THAW, name
+ ])
+ }
+
+ static SliderShell thaw(int result, String name) {
+ slider(result,
+ [
+ ACTION_THAW, name
+ ])
+ }
+
+ /**
+ * Ensure that a cluster has been destroyed
+ * @param name
+ */
+ static void ensureClusterDestroyed(String name) {
+ def froze = freezeForce(name)
+
+ def result = froze.ret
+ if (result != 0 && result != EXIT_UNKNOWN_INSTANCE) {
+ froze.assertExitCode(0)
+ }
+ destroy(0, name)
+ }
+
+ /**
+ * If the functional tests are enabled, set up the cluster
+ *
+ * @param cluster
+ */
+ static void setupCluster(String cluster) {
+ if (FUNTESTS_ENABLED) {
+ ensureClusterDestroyed(cluster)
+ }
+ }
+
+ /**
+ * Teardown operation -freezes cluster, and may destroy it
+ * though for testing it is best if it is retained
+ * @param name cluster name
+ */
+ static void teardown(String name) {
+ if (FUNTESTS_ENABLED) {
+ freezeForce(name)
+ }
+ }
+
+ /**
+ * Assert the exit code is that the cluster is unknown
+ * @param shell shell
+ */
+ public static void assertSuccess(SliderShell shell) {
+ assertExitCode(shell, 0)
+ }
+ /**
+ * Assert the exit code is that the cluster is unknown
+ * @param shell shell
+ */
+ public static void assertUnknownCluster(SliderShell shell) {
+ assertExitCode(shell, EXIT_UNKNOWN_INSTANCE)
+ }
+
+ /**
+ * Assert a shell exited with a given error code
+ * if not the output is printed and an assertion is raised
+ * @param shell shell
+ * @param errorCode expected error code
+ */
+ public static void assertExitCode(SliderShell shell, int errorCode) {
+ shell.assertExitCode(errorCode)
+ }
+
+ /**
+ * Create a connection to the cluster by execing the status command
+ *
+ * @param clustername
+ * @return
+ */
+ SliderClient bondToCluster(Configuration conf, String clustername) {
+
+ String address = getRequiredConfOption(conf, YarnConfiguration.RM_ADDRESS)
+
+ ServiceLauncher<SliderClient> launcher = launchClientAgainstRM(
+ address,
+ ["exists", clustername],
+ conf)
+
+ int exitCode = launcher.serviceExitCode
+ if (exitCode) {
+ throw new ExitUtil.ExitException(exitCode, "exit code = $exitCode")
+ }
+ SliderClient sliderClient = launcher.service
+ sliderClient.deployedClusterName = clustername
+ return sliderClient;
+ }
+
+ /**
+ * Create or build a slider cluster (the action is set by the first verb)
+ * @param action operation to invoke: ACTION_CREATE or ACTION_BUILD
+ * @param clustername cluster name
+ * @param roles map of rolename to count
+ * @param extraArgs list of extra args to add to the creation command
+ * @param deleteExistingData should the data of any existing cluster
+ * of this name be deleted
+ * @param blockUntilRunning block until the AM is running
+ * @param clusterOps map of key=value cluster options to set with the --option arg
+ * @return shell which will have executed the command.
+ */
+ public SliderShell createOrBuildSliderCluster(
+ String action,
+ String clustername,
+ Map<String, Integer> roles,
+ List<String> extraArgs,
+ boolean blockUntilRunning,
+ Map<String, String> clusterOps) {
+ assert action != null
+ assert clustername != null
+
+
+
+ List<String> roleList = [];
+ roles.each { String role, Integer val ->
+ log.info("Role $role := $val")
+ roleList << ARG_COMPONENT << role << Integer.toString(val)
+ }
+
+ List<String> argsList = [action, clustername]
+
+ argsList << ARG_ZKHOSTS <<
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM)
+
+
+ if (blockUntilRunning) {
+ argsList << ARG_WAIT << Integer.toString(THAW_WAIT_TIME)
+ }
+
+ argsList += roleList;
+
+ //now inject any cluster options
+ clusterOps.each { String opt, String val ->
+ argsList << ARG_OPTION << opt.toString() << val.toString();
+ }
+
+ if (extraArgs != null) {
+ argsList += extraArgs;
+ }
+ slider(0, argsList)
+ }
+
+ /**
+ * Create a slider cluster
+ * @param clustername cluster name
+ * @param roles map of rolename to count
+ * @param extraArgs list of extra args to add to the creation command
+ * @param blockUntilRunning block until the AM is running
+ * @param clusterOps map of key=value cluster options to set with the --option arg
+ * @return launcher which will have executed the command.
+ */
+ public SliderShell createSliderApplication(
+ String clustername,
+ Map<String, Integer> roles,
+ List<String> extraArgs,
+ boolean blockUntilRunning,
+ Map<String, String> clusterOps) {
+ return createOrBuildSliderCluster(
+ ACTION_CREATE,
+ clustername,
+ roles,
+ extraArgs,
+ blockUntilRunning,
+ clusterOps)
+ }
+
+ public Path buildClusterPath(String clustername) {
+ return new Path(clusterFS.homeDirectory, "${SliderKeys.SLIDER_BASE_DIRECTORY}/cluster/${clustername}")
+ }
+
+
+ public ClusterDescription killAmAndWaitForRestart(
+ SliderClient sliderClient, String cluster) {
+
+ assert cluster
+ slider(0, [
+ ACTION_AM_SUICIDE, cluster,
+ ARG_EXITCODE, "1",
+ ARG_WAIT, "1000",
+ ARG_MESSAGE, "suicide"
+ ])
+
+
+
+ def sleeptime = SLIDER_CONFIG.getInt( KEY_AM_RESTART_SLEEP_TIME,
+ DEFAULT_AM_RESTART_SLEEP_TIME)
+ sleep(sleeptime)
+ ClusterDescription status
+
+ try {
+ // am should have restarted it by now
+ // cluster is live
+ exists(0, cluster, true)
+
+ status = sliderClient.clusterDescription
+ } catch (SliderException e) {
+ if (e.exitCode == EXIT_BAD_STATE) {
+ log.error(
+ "Property $YarnConfiguration.RM_AM_MAX_ATTEMPTS may be too low")
+ }
+ throw e;
+ }
+ return status
+ }
+
+ /**
+ * if tests are not enabled: skip them
+ */
+ public static void assumeFunctionalTestsEnabled() {
+ assume(FUNTESTS_ENABLED, "Functional tests disabled")
+ }
+
+ public static void assumeAccumuloTestsEnabled() {
+ assumeFunctionalTestsEnabled()
+ assume(ACCUMULO_TESTS_ENABLED, "Accumulo tests disabled")
+ }
+
+ public void assumeAgentTestsEnabled() {
+ assumeFunctionalTestsEnabled()
+ assume(AGENTTESTS_ENABLED, "Agent tests disabled")
+ }
+
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy
new file mode 100644
index 0000000..8ef06cd
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/ConfLoader.groovy
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+
+public class ConfLoader {
+
+ YarnConfiguration loadSliderConf(File confFile) {
+ URI confURI = confFile.toURI();
+ YarnConfiguration conf = new YarnConfiguration()
+ def confXmlUrl = confURI.toURL()
+ conf.addResource(confXmlUrl)
+ conf.set(FuntestProperties.KEY_TEST_CONF_XML, confXmlUrl.toString())
+ conf.set(FuntestProperties.KEY_TEST_CONF_DIR, CommandTestBase.SLIDER_CONF_DIRECTORY.absolutePath)
+ return conf
+ }
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
new file mode 100644
index 0000000..7157b37
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -0,0 +1,63 @@
+package org.apache.slider.funtest.framework
+
+import groovy.transform.CompileStatic
+import org.apache.slider.common.SliderXMLConfKeysForTesting
+import org.apache.slider.common.SliderKeys
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Properties unique to the functional tests
+ */
+@CompileStatic
+public interface FuntestProperties extends SliderXMLConfKeysForTesting {
+
+ /**
+ * Maven Property of location of slider conf dir: {@value}
+ */
+ String SLIDER_CONF_DIR_PROP = "slider.conf.dir"
+
+ /**
+ * Maven Property of location of slider binary image dir: {@value}
+ */
+ String SLIDER_BIN_DIR_PROP = "slider.bin.dir"
+
+ String KEY_SLIDER_TEST_NUM_WORKERS = "slider.test.cluster.size"
+ int DEFAULT_SLIDER_NUM_WORKERS = 1
+
+ String KEY_SLIDER_TEST_ZK_HOSTS = "slider.test.zkhosts";
+ String DEFAULT_SLIDER_ZK_HOSTS = "localhost:2181";
+
+ /**
+ * Time to sleep waiting for the AM to come back up
+ */
+ String KEY_AM_RESTART_SLEEP_TIME = "slider.test.am.restart.time"
+ int DEFAULT_AM_RESTART_SLEEP_TIME = 30000
+
+ String KEY_SLIDER_FUNTESTS_ENABLED = "slider.funtest.enabled"
+
+ String CLIENT_CONFIG_FILENAME = SliderKeys.CLIENT_RESOURCE
+
+ String ENV_CONF_DIR = "SLIDER_CONF_DIR"
+ String ENV_SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
+
+ String SCRIPT_NAME = "slider"
+ static final String KEY_TEST_CONF_XML = "slider.test.conf.xml"
+ static final String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/PortAssignments.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/PortAssignments.groovy
new file mode 100644
index 0000000..8c40523
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/PortAssignments.groovy
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+/**
+ * Here is where all port assignments should be booked, to ensure that no
+ * other test suite is using the same ports
+ */
+interface PortAssignments {
+
+ int _testHBaseCreateCluster = 7000;
+ int _testHBaseCreateCluster2 = 7001;
+ int _testAccumuloCI = 62424;
+}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
new file mode 100644
index 0000000..318ea47
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import groovy.util.logging.Slf4j
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.common.tools.SliderUtils
+
+@Slf4j
+
+class SliderShell extends Shell {
+
+
+ public static final String BASH = '/bin/bash -s'
+
+ /**
+ * Configuration directory, shared across all instances. Not marked as volatile,
+ * assumed set up during @BeforeClass
+ */
+ public static File confDir;
+
+ public static File script;
+
+ public static final List<String> slider_classpath_extra = []
+
+ final String command
+
+ /**
+ * Build the command
+ * @param commands
+ */
+ SliderShell(List<String> commands) {
+ super(BASH)
+ assert confDir != null;
+ assert script != null;
+ command = script.absolutePath + " " + commands.join(" ")
+ }
+
+ /**
+ * Exec the command
+ * @return the script exit code
+ */
+ int execute() {
+ String confDirCmd = env(FuntestProperties.ENV_CONF_DIR, confDir)
+ log.info(command)
+ List<String> commandLine = [
+ confDirCmd,
+ ]
+ if (!slider_classpath_extra.isEmpty()) {
+ commandLine << env(FuntestProperties.ENV_SLIDER_CLASSPATH_EXTRA,
+ SliderUtils.join(slider_classpath_extra, ":", false))
+ }
+ commandLine << command
+ String script = commandLine.join("\n")
+ log.debug(script)
+ super.exec(script);
+ signCorrectReturnCode()
+ return ret;
+ }
+
+ String env(String var, Object val) {
+ return "export " + var + "=${val.toString()};"
+ }
+
+ /**
+ * Fix up the return code so that a value of 255 is mapped back to -1
+ * @return twos complement return code from an unsigned byte
+ */
+ int signCorrectReturnCode() {
+ ret = signCorrect(ret)
+ }
+
+ int execute(int expectedExitCode) {
+ execute()
+ return assertExitCode(expectedExitCode)
+ }
+
+ /**
+ * Exec any slider command
+ * @param conf
+ * @param commands
+ * @return the shell
+ */
+ public static SliderShell run(List<String> commands, int exitCode) {
+ SliderShell shell = new SliderShell(commands)
+ shell.execute(exitCode);
+ return shell
+ }
+
+ public static int signCorrect(int u) {
+ return (u << 24) >> 24;
+ }
+
+ @Override
+ public String toString() {
+ return ret + " =>" + command
+ }
+
+ public void dump() {
+ log.error(toString())
+ log.error("return code = $ret")
+ if (out.size() != 0) {
+ log.info("\n<stdout>\n${out.join('\n')}\n</stdout>");
+ }
+ if (err.size() != 0) {
+ log.error("\n<stderr>\n${err.join('\n')}\n</stderr>");
+ }
+ }
+ /**
+ * Assert a shell exited with a given error code
+ * if not the output is printed and an assertion is raised
+ * @param shell shell
+ * @param errorCode expected error code
+ */
+ public int assertExitCode(int errorCode) {
+ return assertExitCode(this, errorCode)
+ }
+
+ /**
+ * Assert a shell exited with a given error code
+ * if not the output is printed and an assertion is raised
+ * @param shell shell
+ * @param errorCode expected error code
+ * @throws SliderException if the exit code is wrong (the value in the exception
+ * is the exit code received)
+ */
+ public static int assertExitCode(SliderShell shell, int errorCode) throws
+ SliderException {
+ assert shell != null
+ if (shell.ret != errorCode) {
+ shell.dump()
+ throw new SliderException(shell.ret,"Expected exit code %d - actual=%d", errorCode, shell.ret)
+ }
+ return errorCode
+ }
+}
diff --git a/slider-funtest/src/main/java/org/apache/slider/funtest/StubToForceGroovySrcToCompile.java b/slider-funtest/src/main/java/org/apache/slider/funtest/StubToForceGroovySrcToCompile.java
new file mode 100644
index 0000000..4007c0e
--- /dev/null
+++ b/slider-funtest/src/main/java/org/apache/slider/funtest/StubToForceGroovySrcToCompile.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest;
+
+class StubToForceGroovySrcToCompile {
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestBuildSetup.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestBuildSetup.groovy
new file mode 100644
index 0000000..b6f9a12
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestBuildSetup.groovy
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.basic
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup
+
+/**
+ * Simple tests to verify that the build has been set up: if these
+ * fail then the arguments to the test run are incomplete.
+ *
+ * This deliberately doesn't depend on CommandTestBase,
+ * so that individual tests fail with more diagnostics
+ * than the @BeforeClass failing
+ */
+@CompileStatic
+@Slf4j
+class TestBuildSetup extends AbstractTestBuildSetup {
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestSignCorrection.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestSignCorrection.groovy
new file mode 100644
index 0000000..7feb11d
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestSignCorrection.groovy
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.basic
+
+import org.junit.Test
+
+import static org.apache.slider.funtest.framework.SliderShell.signCorrect
+
+/**
+ * This just verifies the two's complement sign correction that will
+ * be applied after the return code is picked up from the shell
+ */
+class TestSignCorrection {
+
+ @Test
+ public void test255ToMinus1() throws Throwable {
+ assert -1 == signCorrect(255)
+ }
+ @Test
+ public void test74To74() throws Throwable {
+ assert 74 == signCorrect(74)
+ }
+ @Test
+ public void test1To1() throws Throwable {
+ assert 1 == signCorrect(1)
+ }
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/TestListCommand.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/TestListCommand.groovy
new file mode 100644
index 0000000..ce7b497
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/TestListCommand.groovy
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.commands
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.junit.BeforeClass
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class TestListCommand extends CommandTestBase {
+
+ @BeforeClass
+ public static void prepareCluster() {
+ assumeFunctionalTestsEnabled();
+ }
+
+ @Test
+ public void testListAll() throws Throwable {
+ assertSuccess(list(null))
+ }
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/TestSimpleCommands.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/TestSimpleCommands.groovy
new file mode 100644
index 0000000..2d00130
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/TestSimpleCommands.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.commands
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.SliderShell
+import org.apache.slider.common.params.SliderActions
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class TestSimpleCommands extends CommandTestBase {
+
+ @Test
+ public void testVersion() throws Throwable {
+ Shell shell = slider([SliderActions.ACTION_VERSION])
+ assertSuccess(shell)
+ }
+
+ @Test
+ public void testUsage() throws Throwable {
+ SliderShell shell = slider(0, [SliderActions.ACTION_USAGE])
+ assertSuccess(shell)
+ }
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/TestUnknownClusterOperations.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/TestUnknownClusterOperations.groovy
new file mode 100644
index 0000000..7791c3c
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/TestUnknownClusterOperations.groovy
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.commands
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.funtest.categories.FunctionalTests
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.BeforeClass
+import org.junit.Test
+
+/**
+ * Test the return code from ops against unknown clusters are what we expect
+ */
+@CompileStatic
+@Slf4j
+@org.junit.experimental.categories.Category(FunctionalTests)
+public class TestUnknownClusterOperations extends CommandTestBase {
+
+ public static final String UNKNOWN = "unknown_cluster"
+
+ @BeforeClass
+ public static void prepareCluster() {
+ assumeFunctionalTestsEnabled();
+ }
+
+ @Test
+ public void testFreezeUnknownCluster() throws Throwable {
+ SliderShell shell = freeze(UNKNOWN)
+ assertUnknownCluster(shell)
+ }
+
+ @Test
+ public void testFreezeUnknownClusterWithMessage() throws Throwable {
+ slider(SliderExitCodes.EXIT_UNKNOWN_INSTANCE,
+ [
+ SliderActions.ACTION_FREEZE, UNKNOWN,
+ Arguments.ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ Arguments.ARG_MESSAGE, "testFreezeUnknownClusterWithMessage"
+ ])
+ }
+
+ @Test
+ public void testFreezeForceUnknownCluster() throws Throwable {
+ SliderShell shell = freezeForce(UNKNOWN)
+ assertUnknownCluster(shell)
+ }
+
+ @Test
+ public void testDestroyUnknownCluster() throws Throwable {
+ SliderShell shell = destroy(UNKNOWN)
+ assertSuccess(shell)
+ }
+
+ @Test
+ public void testListUnknownCluster() throws Throwable {
+ assertUnknownCluster(list(UNKNOWN))
+ }
+
+ @Test
+ public void testExistsUnknownCluster() throws Throwable {
+ assertUnknownCluster(exists(UNKNOWN, false))
+ }
+
+ @Test
+ public void testExistsLiveUnknownCluster() throws Throwable {
+ assertUnknownCluster(exists(UNKNOWN, true))
+ }
+
+ @Test
+ public void testThawUnknownCluster() throws Throwable {
+ assertUnknownCluster(thaw(UNKNOWN))
+ }
+
+ @Test
+ public void testStatusUnknownCluster() throws Throwable {
+ assertUnknownCluster(status(UNKNOWN))
+ }
+
+ @Test
+ public void testGetConfUnknownCluster() throws Throwable {
+ assertUnknownCluster(getConf(UNKNOWN))
+ }
+
+}
diff --git a/slider-funtest/src/test/java/org/apache/slider/funtest/StubToForceGroovyTestsToCompile.java b/slider-funtest/src/test/java/org/apache/slider/funtest/StubToForceGroovyTestsToCompile.java
new file mode 100644
index 0000000..0f98fc0
--- /dev/null
+++ b/slider-funtest/src/test/java/org/apache/slider/funtest/StubToForceGroovyTestsToCompile.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest;
+
+public class StubToForceGroovyTestsToCompile {
+}
diff --git a/slider-funtest/src/test/manual/python/SliderTester.py b/slider-funtest/src/test/manual/python/SliderTester.py
new file mode 100644
index 0000000..40bdf2b
--- /dev/null
+++ b/slider-funtest/src/test/manual/python/SliderTester.py
@@ -0,0 +1,360 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import logging
+import logging.handlers
+from optparse import OptionParser
+import sys
+import os
+import ConfigParser
+import errno
+import shutil
+import shlex
+import subprocess
+import time
+import fnmatch
+from SliderTesterConfig import SliderTesterConfig
+
+logger = logging.getLogger()
+formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s"
+
+
+class SliderTester:
+ SLIDER_CREATE_CMD_FORMAT = '{0} create {1} --image {2}{3} --option agent.conf {2}{4} --template {5} --resources {6} --option application.def {2}{7}'
+ HDFS_LS = "hdfs dfs -ls"
+ HDFS_CHOWN = "hdfs dfs -chown"
+ HDFS_MKDIR = "hdfs dfs -mkdir"
+ HDFS_RMR = "hdfs dfs -rm -r"
+
+ def __init__(self, config):
+ self.config = config
+ self.slider_user = self.config.get(SliderTesterConfig.TEST_SECTION, SliderTesterConfig.APP_USER)
+ self.hdfs_user = self.config.get(SliderTesterConfig.TEST_SECTION, SliderTesterConfig.HDFS_ROOT_USER)
+ self.jdk_path = self.config.get(SliderTesterConfig.SLIDER_SECTION, SliderTesterConfig.JDK_PATH)
+
+ def run_slider_command(self, cmd, user=None):
+ if self.jdk_path and len(self.jdk_path) > 0:
+ cmd = "PATH=$PATH:{0}&&{1}".format(self.jdk_path, cmd)
+
+ return self.run_os_command(cmd, user)
+
+ def run_os_command(self, cmd, user=None):
+ FORMAT_WITH_SU = 'su -l {0} -c "{1}"'
+ if user:
+ cmd = FORMAT_WITH_SU.format(user, cmd)
+
+ logger.info('Executing command: ' + str(cmd))
+ if type(cmd) == str:
+ cmd = shlex.split(cmd)
+
+ process = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ (stdoutdata, stderrdata) = process.communicate()
+
+ logger.debug("Out: " + stdoutdata)
+ logger.debug("Err: " + stderrdata)
+ logger.debug("Process ret code: " + str(process.returncode))
+ return process.returncode, stdoutdata, stderrdata
+
+ def ensure_path_exists(self, path):
+ try:
+ os.makedirs(path)
+ except OSError as exception:
+ if exception.errno != errno.EEXIST:
+ raise
+ pass
+ pass
+
+ def ensure_ownership(self, path, user):
+ CMD = "chown -R {0} {1}"
+ self.run_os_command(CMD.format(user, path))
+ pass
+
+ def deploy_and_validate_slider(self):
+ UNTAR = "tar -xvf {0} -C {1}"
+ self.slider_package = self.config.get(SliderTesterConfig.SLIDER_SECTION, SliderTesterConfig.PACKAGE)
+ if not os.path.isfile(self.slider_package):
+ raise Exception("Invalid Slider package: " + self.slider_package)
+
+ self.app_package = self.config.get(SliderTesterConfig.APP_SECTION, SliderTesterConfig.PACKAGE)
+ if not os.path.isfile(self.app_package):
+ raise Exception("Invalid App package: " + self.app_package)
+
+ self.test_root = self.config.get(SliderTesterConfig.TEST_SECTION, SliderTesterConfig.TEST_ROOT)
+ if os.path.exists(self.test_root):
+ if os.path.isfile(self.test_root):
+ os.remove(self.test_root)
+ else:
+ shutil.rmtree(self.test_root)
+ pass
+
+ self.slider_root = os.path.join(self.test_root, "slider")
+ self.ensure_path_exists(self.slider_root)
+ self.ensure_ownership(self.slider_root, self.slider_user)
+
+ self.run_os_command(UNTAR.format(self.slider_package, self.slider_root), self.slider_user)
+
+ self.app_root = os.path.join(self.test_root, "expandedapp")
+ self.ensure_path_exists(self.app_root)
+ self.ensure_ownership(self.app_root, self.slider_user)
+
+ self.run_os_command(UNTAR.format(self.app_package, self.app_root), self.slider_user)
+
+ self.app_resources = self.findfile("resources.json", self.app_root)
+ self.app_conf = self.findfile("appConfig.json", self.app_root)
+
+ self.slider_exec = self.findfile("slider", self.slider_root)
+
+ (retcode, out, err) = self.run_slider_command(" ".join([self.slider_exec, "version"]), self.slider_user)
+ if retcode != 0:
+ raise Exception("Could not execute version check using " + self.slider_exec)
+
+ if not "Compiled against Hadoop" in out:
+ raise Exception("Could not execute version check using " + self.slider_exec)
+
+ self.agent_conf = self.findfile("agent.ini", self.slider_root)
+ self.agent_tarball = self.findfile("slider-agent*.tar.gz", self.slider_root)
+
+ pass
+
+ def configure_slider_client(self):
+ slider_conf_file = self.findfile("slider-client.xml", self.slider_root)
+
+ data = None
+ with open(slider_conf_file, 'r') as orig:
+ data = orig.readlines()
+
+ # add/replace a set of well known configs
+
+ name_value_to_add = {
+ SliderTesterConfig.SLIDER_ZK_QUORUM:
+ self.config.get(SliderTesterConfig.CLUSTER_SECTION, SliderTesterConfig.SLIDER_ZK_QUORUM),
+ SliderTesterConfig.YARN_RESOURCEMANAGER_ADDRESS:
+ self.config.get(SliderTesterConfig.CLUSTER_SECTION, SliderTesterConfig.YARN_RESOURCEMANAGER_ADDRESS),
+ SliderTesterConfig.YARN_RESOURCEMANAGER_SCHEDULER_ADDRESS:
+ self.config.get(SliderTesterConfig.CLUSTER_SECTION, SliderTesterConfig.YARN_RESOURCEMANAGER_SCHEDULER_ADDRESS),
+ SliderTesterConfig.FS_DEFAULTFS:
+ self.config.get(SliderTesterConfig.CLUSTER_SECTION, SliderTesterConfig.FS_DEFAULTFS),
+ SliderTesterConfig.YARN_APP_CP:
+ self.config.get(SliderTesterConfig.CLUSTER_SECTION, SliderTesterConfig.YARN_APP_CP),
+ }
+ output = []
+ for line in data:
+ output.append(line)
+ if "<configuration>" in line:
+ output.append(os.linesep)
+ logger.info("Adding additional configuations ...")
+ for (k, v) in name_value_to_add.items():
+ output.append(" <property>" + os.linesep)
+ output.append(" <name>{0}</name>".format(k) + os.linesep)
+ output.append(" <value>{0}</value>".format(v) + os.linesep)
+ output.append(" </property>" + os.linesep)
+ output.append(os.linesep)
+ pass
+ pass
+
+ with open(slider_conf_file, 'w') as modified:
+ modified.writelines(output)
+ pass
+
+ def create_hdfs_folders(self):
+ self.hdfs_work_root = self.config.get(SliderTesterConfig.TEST_SECTION, SliderTesterConfig.HDFS_ROOT_DIR)
+ (ret, out, err) = self.run_os_command(" ".join([SliderTester.HDFS_LS, self.hdfs_work_root]), self.hdfs_user)
+ if ret == 0:
+ self.run_os_command(" ".join([SliderTester.HDFS_RMR, self.hdfs_work_root]), self.hdfs_user)
+
+ self.run_os_command(" ".join([SliderTester.HDFS_MKDIR, self.hdfs_work_root]), self.hdfs_user)
+ self.run_os_command(" ".join([SliderTester.HDFS_CHOWN, self.slider_user, self.hdfs_work_root]), self.hdfs_user)
+
+ self.hdfs_user_root = os.path.join(
+ self.config.get(SliderTesterConfig.TEST_SECTION, SliderTesterConfig.HDFS_USER_DIR), self.slider_user)
+
+ (ret, out, err) = self.run_os_command(" ".join([SliderTester.HDFS_LS, self.hdfs_user_root]), self.hdfs_user)
+ if ret != 0:
+ self.run_os_command(" ".join([SliderTester.HDFS_MKDIR, self.hdfs_user_root]), self.hdfs_user)
+ self.run_os_command(" ".join([SliderTester.HDFS_CHOWN, self.slider_user, self.hdfs_user_root]), self.hdfs_user)
+
+ self.hdfs_agent_root = os.path.join(self.hdfs_work_root, "agent")
+ self.hdfs_agent_conf_root = os.path.join(self.hdfs_agent_root, "conf")
+ self.run_os_command(" ".join([SliderTester.HDFS_MKDIR, self.hdfs_agent_root]), self.slider_user)
+ self.run_os_command(" ".join([SliderTester.HDFS_MKDIR, self.hdfs_agent_conf_root]), self.slider_user)
+
+ self.run_os_command("hdfs dfs -copyFromLocal " + self.agent_tarball + " " + self.hdfs_agent_root, self.slider_user)
+ self.run_os_command("hdfs dfs -copyFromLocal " + self.agent_conf + " " + self.hdfs_agent_conf_root,
+ self.slider_user)
+
+ self.run_os_command("hdfs dfs -copyFromLocal " + self.app_package + " " + self.hdfs_work_root, self.slider_user)
+
+ self.agent_tarball_hdfs = os.path.join(self.hdfs_agent_root, os.path.basename(self.agent_tarball))
+ self.agent_conf_hdfs = os.path.join(self.hdfs_agent_conf_root, os.path.basename(self.agent_conf))
+ self.app_package_hdfs = os.path.join(self.hdfs_work_root, os.path.basename(self.app_package))
+ self.cluster_name = self.config.get(SliderTesterConfig.TEST_SECTION, SliderTesterConfig.CLUSTER_NAME)
+
+ self.cluster_location = os.path.join(self.hdfs_user_root, ".slider/cluster", self.cluster_name)
+ self.run_os_command(" ".join([SliderTester.HDFS_RMR, self.cluster_location]), self.hdfs_user)
+
+ pass
+
+ def create_cluster(self):
+ self.fsdefault = self.config.get(SliderTesterConfig.CLUSTER_SECTION, SliderTesterConfig.FS_DEFAULTFS)
+
+ cmd = SliderTester.SLIDER_CREATE_CMD_FORMAT.format(self.slider_exec,
+ self.cluster_name, self.fsdefault, self.agent_tarball_hdfs,
+ self.agent_conf_hdfs, self.app_conf, self.app_resources,
+ self.app_package_hdfs)
+
+ (retcode, out, err) = self.run_slider_command(cmd, self.slider_user)
+ if retcode != 0:
+ raise Exception("Could not create cluster. Out: " + out + " Err: " + err)
+ pass
+
+ def verify_cluster(self):
+ (retcode, out, err) = self.run_os_command(" ".join([SliderTester.HDFS_LS, self.cluster_location]), self.slider_user)
+ if retcode != 0:
+ raise Exception("Could not verify cluster. Out: " + out + " Err: " + err)
+ pass
+
+ def clean_up(self):
+ (retcode, out, err) = self.run_slider_command(" ".join([self.slider_exec, "freeze", self.cluster_name]),
+ self.slider_user)
+ if retcode != 0:
+ raise Exception("Could not clean cluster. Out: " + out + " Err: " + err)
+ pass
+
+
+ def findfile(self, filepattern, path):
+ matches = []
+ for root, dirnames, filenames in os.walk(path):
+ for filename in fnmatch.filter(filenames, filepattern):
+ matches.append(os.path.join(root, filename))
+ if len(matches) > 0:
+ return matches[0]
+ else:
+ return None
+ pass
+
+
+ def do_test_setup(self):
+ self.cluster_type = self.config.get(SliderTesterConfig.TEST_SECTION, SliderTesterConfig.CLUSTER_TYPE)
+ if self.cluster_type.lower() == "habse":
+ self.hbase_setup()
+ return
+
+ if self.cluster_type.lower() == "storm":
+ self.storm_setup()
+ return
+
+ if self.cluster_type.lower() == "storm":
+ self.storm_setup()
+ return
+ pass
+
+
+ def hbase_setup(self):
+ (ret, out, err) = self.run_os_command(" ".join([SliderTester.HDFS_LS, "/app"]), self.hdfs_user)
+ if ret != 0:
+ self.run_os_command(" ".join([SliderTester.HDFS_MKDIR, "/app"]), self.hdfs_user)
+ pass
+
+ (ret, out, err) = self.run_os_command(" ".join([SliderTester.HDFS_LS, "/app/habse"]), self.hdfs_user)
+ if ret == 0:
+ self.run_os_command(" ".join([SliderTester.HDFS_RMR, "/app/hbase"]), self.hdfs_user)
+
+ self.run_os_command(" ".join([SliderTester.HDFS_MKDIR, "/app/hbase"]), self.hdfs_user)
+ self.run_os_command(" ".join([SliderTester.HDFS_CHOWN, self.slider_user, "/app/hbase"]), self.hdfs_user)
+ pass
+
+ pass
+
+
+def resolve_config(configFile):
+ try:
+ config = SliderTesterConfig.config
+ if os.path.exists(configFile):
+ config.read(configFile)
+ SliderTesterConfig.setConfig(config)
+ else:
+ raise Exception("No config found, use default")
+
+ except Exception, err:
+ logger.warn(err)
+ return config
+
+
+def main():
+ parser = OptionParser()
+ parser.add_option("-c", "--config", dest="config", help="SliderTester config file location", default=None)
+ parser.add_option("-o", "--out", dest="outputfile", default="/tmp/slider-test.log", help="log file to store results.",
+ metavar="FILE")
+ (options, args) = parser.parse_args()
+
+ if os.path.isfile(options.outputfile):
+ os.remove(options.outputfile)
+
+ print "Logging at " + options.outputfile
+
+ global logger
+ logger = logging.getLogger('HostCleanup')
+ handler = logging.FileHandler(options.outputfile)
+ formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+ logging.basicConfig(level=logging.DEBUG)
+
+ if options.config:
+ resolve_config(options.config)
+
+ s = SliderTester(SliderTesterConfig().getConfig())
+
+ s.deploy_and_validate_slider()
+
+ s.configure_slider_client()
+
+ s.create_hdfs_folders()
+
+ s.do_test_setup()
+
+ s.create_cluster()
+
+ time.sleep(120)
+ s.verify_cluster()
+
+ s.clean_up();
+
+ # Copy slider and app package to TEST_ROOT/packages
+ # Expand app package at TEST_ROOT/app
+ # Expand slider package at TEST_ROOT/slider
+ ## set jdk path
+ ## check slider version
+
+ # Create HDFS folders and ensure permission
+ # Populate HDFS folders
+ # Finalize resources and appConf json files
+ # Call create
+ # Validate existence of the app
+ # Call freeze
+
+
+if __name__ == "__main__":
+ main()
+
diff --git a/slider-funtest/src/test/manual/python/SliderTesterConfig.py b/slider-funtest/src/test/manual/python/SliderTesterConfig.py
new file mode 100644
index 0000000..d01e4b3
--- /dev/null
+++ b/slider-funtest/src/test/manual/python/SliderTesterConfig.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import ConfigParser
+import StringIO
+import os
+
+config = ConfigParser.RawConfigParser()
+content = """
+
+[slider]
+package=/pkgs/slider-0.22.0-all.tar.gz
+jdk.path=/usr/jdk64/jdk1.7.0_45/bin
+
+[app]
+package=/pkgs/hbase_v096.tar
+
+[cluster]
+yarn.application.classpath=/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
+slider.zookeeper.quorum=c6401.ambari.apache.org:2181
+yarn.resourcemanager.address=c6401.ambari.apache.org:8050
+yarn.resourcemanager.scheduler.address=c6401.ambari.apache.org:8030
+fs.defaultFS=hdfs://c6401.ambari.apache.org:8020
+
+[test]
+app.user=yarn
+hdfs.root.user=hdfs
+hdfs.root.dir=/slidertst
+hdfs.user.dir=/user
+test.root=/test
+cluster.name=tst1
+cluster.type=hbase
+
+[agent]
+"""
+s = StringIO.StringIO(content)
+config.readfp(s)
+
+
+class SliderTesterConfig:
+
+ SLIDER_SECTION = "slider"
+ APP_SECTION = "app"
+ CLUSTER_SECTION = "cluster"
+ TEST_SECTION = "test"
+ AGENT_SECTION = "agent"
+
+ PACKAGE = "package"
+ SLIDER_ZK_QUORUM = "slider.zookeeper.quorum"
+ YARN_RESOURCEMANAGER_ADDRESS = "yarn.resourcemanager.address"
+ YARN_RESOURCEMANAGER_SCHEDULER_ADDRESS = "yarn.resourcemanager.scheduler.address"
+ FS_DEFAULTFS = "fs.defaultFS"
+ YARN_APP_CP="yarn.application.classpath"
+
+ APP_USER = "app.user"
+ HDFS_ROOT_USER = "hdfs.root.user"
+ HDFS_ROOT_DIR = "hdfs.root.dir"
+ HDFS_USER_DIR = "hdfs.user.dir"
+ TEST_ROOT = "test.root"
+ CLUSTER_NAME = "cluster.name"
+ JDK_PATH="jdk.path"
+ CLUSTER_TYPE="cluster.type"
+
+ def getConfig(self):
+ global config
+ return config
+
+
+def setConfig(customConfig):
+ global config
+ config = customConfig
+
+
+def main():
+ print config
+
+
+if __name__ == "__main__":
+ main()
+
diff --git a/slider-funtest/src/test/manual/testing.md b/slider-funtest/src/test/manual/testing.md
new file mode 100644
index 0000000..018d785
--- /dev/null
+++ b/slider-funtest/src/test/manual/testing.md
@@ -0,0 +1,51 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Manual Testing
+
+Manual testing invloves using Slider package and an AppPackage to perform basic cluster functionalities such as create/destroy, flex up/down, and freeze/thaw. A python helper script is provided that can be used to automatically test and app package.
+
+## SliderTester.py
+Details to be added.
+
+## SliderTester.ini
+The various config parameters are:
+
+* **slider**
+ * *package:* location of the slider package
+ * *jdk.path:* jdk path on the test hosts
+
+* **app**
+ * *package:* location of the app package
+
+* **cluster**
+ * *yarn.application.classpath:* yarn application classpaths
+ * *slider.zookeeper.quorum:* the ZK quorum hosts
+ * *yarn.resourcemanager.address:*
+ * *yarn.resourcemanager.scheduler.address:*
+ * *fs.defaultFS:* e.g. hdfs://NN_HOST:8020
+
+* **test**
+ * *app.user:* user to use for app creation
+ * *hdfs.root.user:* hdfs root user
+ * *hdfs.root.dir:* HDFS root, default /slidertst
+ * *hdfs.user.dir:* HDFS user dir, default /user
+ * *test.root:* local test root folder, default /test
+ * *cluster.name:* name of the test cluster, default tst1
+ * *cluster.type:* cluster type to build and test, e.g. hbase,storm,accumulo
+
+* **agent**
diff --git a/slider-providers/accumulo/accumulo-funtests/pom.xml b/slider-providers/accumulo/accumulo-funtests/pom.xml
new file mode 100644
index 0000000..b0bc17b
--- /dev/null
+++ b/slider-providers/accumulo/accumulo-funtests/pom.xml
@@ -0,0 +1,236 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>accumulo-funtests</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ <name>Slider Accumulo Provider Functional Tests</name>
+ <packaging>jar</packaging>
+ <description>
+ Functional tests for the accumulo provider
+
+ </description>
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ <relativePath>../../../</relativePath>
+ </parent>
+
+ <build>
+
+ <plugins>
+
+ <!--read in a build.properties file if defined-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <version>${maven.properties.version}</version>
+ <executions>
+ <execution>
+ <phase>initialize</phase>
+ <goals>
+ <goal>read-project-properties</goal>
+ </goals>
+ <configuration>
+ <quiet>true</quiet>
+ <files>
+ <file>build.properties</file>
+ <file>../../../build.properties</file>
+ </files>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler-plugin.version}</version>
+ <configuration>
+ <compilerId>groovy-eclipse-compiler</compilerId>
+ <!-- set verbose to be true if you want lots of uninteresting messages -->
+ <!-- <verbose>true</verbose> -->
+ <source>${project.java.src.version}</source>
+ <target>${project.java.src.version}</target>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>${groovy-eclipse-compiler.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-batch</artifactId>
+ <version>${groovy-eclipse-batch.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+
+ <!-- functional test -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ <configuration>
+ <!--mvn process fork options-->
+ <reuseForks>${test.reuseForks}</reuseForks>
+ <forkMode>${test.forkMode}</forkMode>
+ <forkCount>1</forkCount>
+ <forkedProcessTimeoutInSeconds>${test.forkedProcessTimeoutInSeconds}
+ </forkedProcessTimeoutInSeconds>
+ <threadCount>1</threadCount>
+ <argLine>${test.argLine}</argLine>
+ <failIfNoTests>${test.failIfNoTests}</failIfNoTests>
+
+ <trimStackTrace>false</trimStackTrace>
+ <redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
+ <systemPropertyVariables>
+ <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
+ <java.awt.headless>true</java.awt.headless>
+ <java.security.krb5.realm>${slider.test.java.security.krb5.realm}</java.security.krb5.realm>
+ <java.security.krb5.kdc>${slider.test.java.security.krb5.kdc}</java.security.krb5.kdc>
+ <!-- this property must be supplied-->
+ <slider.conf.dir>${slider.conf.dir}</slider.conf.dir>
+ <slider.bin.dir>../../../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/Test*.java</include>
+ </includes>
+ <excludes>
+ <exclude>**/Test*$*.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <reporting>
+ <plugins>
+
+ </plugins>
+ </reporting>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-accumulo-provider</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+<!--
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-accumulo-provider</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+-->
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-assembly</artifactId>
+ <version>${project.version}</version>
+ <classifier>all</classifier>
+ <type>tar.gz</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-funtest</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-trace</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+
+</project>
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy
new file mode 100644
index 0000000..1a97635
--- /dev/null
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.funtest
+
+import static SliderXMLConfKeysForTesting.KEY_TEST_ACCUMULO_APPCONF
+import static SliderXMLConfKeysForTesting.KEY_TEST_ACCUMULO_TAR
+import static org.apache.slider.api.ResourceKeys.YARN_MEMORY
+import static org.apache.slider.providers.accumulo.AccumuloKeys.*
+import static org.apache.slider.common.params.Arguments.ARG_PROVIDER
+import static org.apache.slider.common.params.Arguments.ARG_RES_COMP_OPT
+
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat
+import org.apache.accumulo.fate.ZooStore
+import org.apache.accumulo.trace.instrument.Tracer
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.LocalFileSystem
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.mapreduce.filecache.DistributedCache
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.SliderXMLConfKeysForTesting
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.SliderShell
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.thrift.TException
+import org.junit.Before
+
+/**
+ * Anything specific to accumulo tests
+ */
+abstract class AccumuloCommandTestBase extends CommandTestBase {
+
+ @Before
+ public void verifyPreconditions() {
+
+ //if tests are not enabled: skip tests
+ assumeAccumuloTestsEnabled()
+ // but if they are -fail if the values are missing
+ getRequiredConfOption(SLIDER_CONFIG, OPTION_ZK_HOME)
+ getRequiredConfOption(SLIDER_CONFIG, OPTION_HADOOP_HOME)
+ }
+
+ /**
+ * Create an accumulo cluster
+ *
+ * @param clustername
+ * @param roles
+ * @param argsList
+ * @param blockUntilRunning
+ * @param containerMemory
+ * @return
+ */
+ public SliderShell createAccumuloCluster(String clustername,
+ Map<String, Integer> roles,
+ List<String> argsList,
+ boolean blockUntilRunning,
+ Map<String, String> clusterOps,
+ String containerMemory,
+ String password) {
+ argsList << ARG_PROVIDER << PROVIDER_ACCUMULO;
+
+
+ YarnConfiguration conf = SLIDER_CONFIG
+ clusterOps[OPTION_ZK_HOME] = getRequiredConfOption(
+ SLIDER_CONFIG, OPTION_ZK_HOME)
+ clusterOps[OPTION_HADOOP_HOME] = getRequiredConfOption(
+ SLIDER_CONFIG,
+ OPTION_HADOOP_HOME)
+ argsList << Arguments.ARG_IMAGE <<
+ getRequiredConfOption(SLIDER_CONFIG, KEY_TEST_ACCUMULO_TAR)
+
+ argsList << Arguments.ARG_CONFDIR <<
+ getRequiredConfOption(SLIDER_CONFIG, KEY_TEST_ACCUMULO_APPCONF)
+
+ argsList << Arguments.ARG_OPTION << AccumuloKeys.OPTION_ACCUMULO_PASSWORD << password
+
+ argsList << ARG_RES_COMP_OPT << ROLE_MASTER <<
+ YARN_MEMORY << containerMemory
+ argsList << ARG_RES_COMP_OPT << ROLE_TABLET <<
+ YARN_MEMORY << containerMemory
+ argsList << ARG_RES_COMP_OPT << ROLE_MONITOR <<
+ YARN_MEMORY << containerMemory
+ argsList << ARG_RES_COMP_OPT << ROLE_GARBAGE_COLLECTOR <<
+ YARN_MEMORY << containerMemory
+
+ return createSliderApplication(clustername,
+ roles,
+ argsList,
+ blockUntilRunning,
+ clusterOps)
+ }
+
+ public boolean loadClassesForMapReduce(Configuration conf) {
+ String[] neededClasses = [AccumuloInputFormat.class.getName(), TException.class.getName(), ZooStore.class.getName(), Tracer.class.getName()]
+ String[] neededJars = ["accumulo-core.jar", "libthrift.jar", "accumulo-fate.jar", "accumulo-trace.jar"]
+
+ LocalFileSystem localfs = new LocalFileSystem();
+ localfs.initialize(new URI("file:///"), conf);
+ ArrayList<Path> jarsToLoad = new ArrayList<Path>();
+
+ ClassLoader loader = AccumuloCommandTestBase.class.getClassLoader();
+ boolean missingJar = false
+ try {
+ for (String className : neededClasses) {
+ className = className.replace('.', '/') + ".class"
+ URL url = loader.getResource(className)
+ log.debug("For $className found $url")
+ String path = url.getPath();
+ int separator = path.indexOf('!')
+ if (-1 == separator) {
+ log.info("Could not interpret $path to find a valid path to a jar")
+ missingJar = true;
+ break;
+ }
+ path = path.substring(0, separator)
+ Path jarPath = new Path(path);
+ if (!localfs.exists(jarPath)) {
+ log.info("Could not find $jarPath")
+ missingJar = true
+ jarsToLoad.clear();
+ break
+ } else {
+ jarsToLoad.add(jarPath);
+ }
+ }
+ } catch (Exception e) {
+ log.warn("Got exception trying to parse jars from maven repository", e)
+ missingJar = true
+ }
+
+ if (missingJar) {
+ String accumuloHome = conf.get(SliderXMLConfKeysForTesting.KEY_TEST_ACCUMULO_HOME)
+ if (null == accumuloHome) {
+ log.info(SliderXMLConfKeysForTesting.KEY_TEST_ACCUMULO_HOME + " is not defined in Slider configuration. Cannot load jars from local Accumulo installation")
+ } else {
+ Path p = new Path(accumuloHome + "/lib")
+ if (localfs.exists(p)) {
+ log.info("Found lib directory in local accumulo home: $p")
+ for (String neededJar : neededJars) {
+ Path jarPath = new Path(p, neededJar);
+ if (!localfs.exists(jarPath)) {
+ log.info("Could not find " + jarPath)
+ missingJar = true
+ jarsToLoad.clear();
+ break
+ } else {
+ jarsToLoad.add(jarPath);
+ }
+ }
+ }
+ }
+ }
+
+ if (!missingJar) {
+ for (Path neededJar : jarsToLoad) {
+ log.info("Adding to mapreduce classpath: $neededJar")
+ DistributedCache.addArchiveToClassPath(neededJar, conf, localfs)
+ }
+ return true
+ } else {
+ log.info("Falling back to local mapreduce because the necessary Accumulo classes couldn't be loaded")
+ }
+
+ return false
+ }
+
+ public void tryToLoadMapredSite(Configuration conf) {
+ String hadoopHome = conf.get(AccumuloKeys.OPTION_HADOOP_HOME)
+
+ // Add mapred-site.xml if we can find it
+ if (null == hadoopHome) {
+ log.info(AccumuloKeys.OPTION_HADOOP_HOME + " was not defined in Slider configuration. Running job in local mode");
+ } else {
+ LocalFileSystem localfs = new LocalFileSystem();
+ localfs.initialize(new URI("file:///"), conf);
+
+ // If we found the necessary jars, make sure we throw mapred-site.xml on the classpath
+ // too so that we avoid local mode
+ Path p = new Path(hadoopHome + "/etc/hadoop/mapred-site.xml");
+ if (localfs.exists(p)) {
+ log.info("Loaded mapred-site.xml from " + p);
+ conf.addResource(p);
+ } else {
+ log.info("Failed to load mapred-site.xml as it doesn't exist at " + p);
+ }
+ }
+ }
+}
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloBuildSetup.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloBuildSetup.groovy
new file mode 100644
index 0000000..61366da
--- /dev/null
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloBuildSetup.groovy
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup
+import org.junit.Test
+
+class TestAccumuloBuildSetup extends AbstractTestBuildSetup {
+
+ @Test
+ public void testAccumuloBuildsHavePathsDefined() throws Throwable {
+ Configuration conf = loadSliderConf();
+ assumeBoolOption(conf, KEY_SLIDER_FUNTESTS_ENABLED, true)
+
+ assumeBoolOption(conf, KEY_TEST_ACCUMULO_ENABLED, true)
+
+ assertStringOptionSet(conf, KEY_TEST_ACCUMULO_APPCONF)
+ assertStringOptionSet(conf, KEY_TEST_ACCUMULO_TAR)
+ }
+
+}
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloCI.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloCI.groovy
new file mode 100644
index 0000000..5573dd4
--- /dev/null
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloCI.groovy
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.accumulo.funtest
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+
+import org.apache.accumulo.core.client.Connector
+import org.apache.accumulo.core.client.ZooKeeperInstance
+import org.apache.accumulo.core.client.security.tokens.PasswordToken
+import org.apache.accumulo.test.continuous.ContinuousIngest
+import org.apache.accumulo.test.continuous.ContinuousVerify
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.io.Text
+import org.apache.hadoop.util.ToolRunner
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.PortAssignments
+
+/**
+ *
+ */
+@CompileStatic
+@Slf4j
+class TestAccumuloCI extends TestFunctionalAccumuloCluster {
+
+ @Override
+ String getClusterName() {
+ return "test_accumulo_ci"
+ }
+
+ @Override
+ public int getNumTservers() {
+ return 2;
+ }
+
+ @Override
+ public int getMonitorPort() {
+ return PortAssignments._testAccumuloCI;
+ }
+
+ @Override
+ void clusterLoadOperations(
+ String clustername,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+ assert clustername
+
+ String currentUser = System.getProperty("user.name");
+ String zookeepers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+ FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
+ ZooKeeperInstance inst = new ZooKeeperInstance(currentUser + "-" + clustername, zookeepers)
+ PasswordToken passwd = new PasswordToken(getPassword())
+ Connector conn = inst.getConnector("root", new PasswordToken(getPassword()))
+
+ // Create the test table with some split points
+ String tableName = "testAccumuloCi";
+ conn.tableOperations().create(tableName)
+ TreeSet<Text> splits = new TreeSet<Text>()
+ splits.add(new Text("2"))
+ splits.add(new Text("5"))
+ splits.add(new Text("7"))
+ conn.tableOperations().addSplits(tableName, splits)
+
+ // Write 15M records per tserver -- should take a few minutes
+ String[] ciOpts = ["-i", inst.getInstanceName(),
+ "-z", zookeepers, "-u", "root",
+ "-p", getPassword(), "--table", tableName,
+ "--num", Integer.toString(1000 * 1000 * 15 * getNumTservers()),
+ "--batchMemory", "100000000",
+ "--batchLatency", "600000",
+ "--batchThreads", "1"]
+
+ ContinuousIngest ci = new ContinuousIngest();
+ ci.main(ciOpts);
+
+ // Create a directory for the verify to write its output to
+ Path verifyOutput = new Path("/user/" + currentUser + "/.slider/cluster/" + clustername + "/verify-output")
+ assert !clusterFS.exists(verifyOutput)
+
+ YarnConfiguration verifyConf = new YarnConfiguration(CommandTestBase.SLIDER_CONFIG);
+
+ // Try to load the necessary classes for the Mappers to find them
+ if (loadClassesForMapReduce(verifyConf)) {
+ // If we found those classes, try to run in distributed mode.
+ tryToLoadMapredSite(verifyConf)
+ }
+
+ // Run ContinuousVerify and ensure that no holes exist
+ ContinuousVerify verify = new ContinuousVerify();
+ String[] verifyOpts = ["-i", inst.getInstanceName(),
+ "-z", zookeepers, "-u", "root",
+ "-p", getPassword(), "--table", tableName,
+ "--output", verifyOutput.toString(), "--maxMappers", Integer.toString(2 * getNumTservers()),
+ "--reducers", Integer.toString(getNumTservers())]
+ assert 0 == ToolRunner.run(verifyConf, verify, verifyOpts)
+ }
+}
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloCluster.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloCluster.groovy
new file mode 100644
index 0000000..06fe21c
--- /dev/null
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloCluster.groovy
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.accumulo.funtest
+
+import static org.apache.slider.providers.accumulo.AccumuloConfigFileOptions.*
+import static org.apache.slider.providers.accumulo.AccumuloKeys.*
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+/**
+ *
+ */
+@CompileStatic
+@Slf4j
+class TestFunctionalAccumuloCluster extends AccumuloCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes {
+
+
+ public String getClusterName() {
+ return "test_functional_accumulo_cluster"
+ }
+
+ public String getPassword() {
+ return "password";
+ }
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(getClusterName())
+ }
+
+ @After
+ public void destroyCluster() {
+ teardown(getClusterName())
+ }
+
+ public int getNumMasters() {
+ return 1
+ }
+
+ public int getNumTservers() {
+ return 1
+ }
+
+ public int getNumMonitors() {
+ return 1
+ }
+
+ public int getNumGarbageCollectors() {
+ return 1
+ }
+
+ public int getNumTracers() {
+ return 0
+ }
+
+ public int getMonitorPort() {
+ return 0
+ }
+
+ @Test
+ public void testAccumuloClusterCreate() throws Throwable {
+
+ describe "Create a working Accumulo cluster"
+
+ def path = buildClusterPath(getClusterName())
+ assert !clusterFS.exists(path)
+
+ Map<String, Integer> roleMap = [
+ (ROLE_MASTER) : getNumMasters(),
+ (ROLE_TABLET) : getNumTservers(),
+ (ROLE_MONITOR): getNumMonitors(),
+ (ROLE_GARBAGE_COLLECTOR): getNumGarbageCollectors(),
+ (ROLE_TRACER) : getNumTracers()
+ ];
+
+ Map<String, String> clusterOps = [:]
+ clusterOps["site." + MONITOR_PORT_CLIENT] = Integer.toString(getMonitorPort())
+
+ List<String> extraArgs = []
+
+ createAccumuloCluster(
+ getClusterName(),
+ roleMap,
+ extraArgs,
+ true,
+ clusterOps,
+ "256",
+ getPassword()
+ )
+
+ //get a slider client against the cluster
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, getClusterName())
+ ClusterDescription cd = sliderClient.clusterDescription
+ assert getClusterName() == cd.name
+
+ log.info("Connected via Client {}", sliderClient.toString())
+
+ //wait for the role counts to be reached
+ waitForRoleCount(sliderClient, roleMap, ACCUMULO_LAUNCH_WAIT_TIME)
+
+ clusterLoadOperations(clusterName, roleMap, cd)
+ }
+
+
+ public String getDescription() {
+ return "Create a working Accumulo cluster $clusterName"
+ }
+
+ /**
+ * Override point for any cluster load operations
+ * @param clientConf
+ * @param numWorkers
+ */
+ public void clusterLoadOperations(
+ String clustername,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+
+ log.info("Client Description = " + cd.toJsonString())
+ }
+
+}
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloM1T1GC1Mon1.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloM1T1GC1Mon1.groovy
new file mode 100644
index 0000000..4f07b40
--- /dev/null
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloM1T1GC1Mon1.groovy
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.funtest
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+
+@CompileStatic
+@Slf4j
+public class TestFunctionalAccumuloM1T1GC1Mon1 extends TestFunctionalAccumuloCluster {
+
+ @Override
+ public String getClusterName() {
+ return "test_functional_accumulo_m1t1gc1mon1";
+ }
+
+ /**
+ * Override point for any cluster load operations
+ * @param clientConf
+ * @param numWorkers
+ */
+ @Override
+ public void clusterLoadOperations(
+ String clustername,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+
+ slider(0, [
+ SliderActions.ACTION_FREEZE,
+ getClusterName(),
+ Arguments.ARG_WAIT,
+ Integer.toString(FREEZE_WAIT_TIME),
+ Arguments.ARG_MESSAGE,
+ "freeze-in-test-AccumuloCluster"
+ ])
+
+ //destroy the cluster. This only works if the permissions allow it
+ destroy(0, getClusterName())
+ }
+}
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/resources/log4j.properties b/slider-providers/accumulo/accumulo-funtests/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a552a55
--- /dev/null
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/resources/log4j.properties
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+log4j.logger.org.apache.slider=DEBUG
+log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+
+
+
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
+log4j.logger.org.apache.hadoop.metrics2=ERROR
+log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
+log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN
+log4j.logger.org.apache.hadoop.security.token.delegation=WARN
+log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
diff --git a/slider-providers/accumulo/slider-accumulo-provider/pom.xml b/slider-providers/accumulo/slider-accumulo-provider/pom.xml
new file mode 100644
index 0000000..e660360
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/pom.xml
@@ -0,0 +1,223 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-accumulo-provider</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ <name>Slider Accumulo Provider</name>
+ <packaging>jar</packaging>
+ <description>
+ Direct provider for slider. This is the original provider from Hoya, moved to one side
+ as it is no longer being actively developed
+
+ </description>
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ <relativePath>../../../</relativePath>
+ </parent>
+
+ <build>
+
+ <!-- resources are filtered for dynamic updates. This gets build info in-->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
+ <plugins>
+
+ <!--read in a build.properties file if defined-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <version>${maven.properties.version}</version>
+ <executions>
+ <execution>
+ <phase>initialize</phase>
+ <goals>
+ <goal>read-project-properties</goal>
+ </goals>
+ <configuration>
+ <quiet>true</quiet>
+ <files>
+ <file>build.properties</file>
+ <file>../../build.properties</file>
+ </files>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler-plugin.version}</version>
+ <configuration>
+ <compilerId>groovy-eclipse-compiler</compilerId>
+ <!-- set verbose to be true if you want lots of uninteresting messages -->
+ <!-- <verbose>true</verbose> -->
+ <source>${project.java.src.version}</source>
+ <target>${project.java.src.version}</target>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>${groovy-eclipse-compiler.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-batch</artifactId>
+ <version>${groovy-eclipse-batch.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+
+ <!-- test -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ <configuration>
+ <!--mvn process fork options-->
+ <reuseForks>${test.reuseForks}</reuseForks>
+ <forkMode>${test.forkMode}</forkMode>
+ <forkCount>1</forkCount>
+ <forkedProcessTimeoutInSeconds>${test.forkedProcessTimeoutInSeconds}
+ </forkedProcessTimeoutInSeconds>
+ <threadCount>1</threadCount>
+ <argLine>${test.argLine}</argLine>
+ <failIfNoTests>${test.failIfNoTests}</failIfNoTests>
+
+ <trimStackTrace>false</trimStackTrace>
+ <redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
+ <systemPropertyVariables>
+ <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
+ <java.awt.headless>true</java.awt.headless>
+ <java.security.krb5.realm>${slider.test.java.security.krb5.realm}</java.security.krb5.realm>
+ <java.security.krb5.kdc>${slider.test.java.security.krb5.kdc}</java.security.krb5.kdc>
+ <!-- this property must be supplied-->
+ <slider.conf.dir>${slider.conf.dir}</slider.conf.dir>
+ <slider.bin.dir>../../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/Test*.java</include>
+ </includes>
+ <excludes>
+ <exclude>**/Test*$*.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+
+ </plugins>
+ </build>
+
+ <reporting>
+ <plugins>
+
+
+
+ </plugins>
+ </reporting>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-trace</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+
+</project>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
new file mode 100644
index 0000000..db99360
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * Client-side accumulo provider
+ */
+public class AccumuloClientProvider extends AbstractClientProvider implements
+ AccumuloKeys {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(AccumuloClientProvider.class);
+ private static final ProviderUtils providerUtils = new ProviderUtils(log);
+ private static final String INSTANCE_RESOURCE_BASE =
+ "/org/apache/slider/providers/accumulo/instance/";
+
+
+ protected AccumuloClientProvider(Configuration conf) {
+ super(conf);
+ }
+
+ public static List<ProviderRole> getProviderRoles() {
+ return AccumuloRoles.ROLES;
+ }
+
+ @Override
+ public String getName() {
+ return PROVIDER_ACCUMULO;
+ }
+
+ @Override
+ public List<ProviderRole> getRoles() {
+ return AccumuloRoles.ROLES;
+ }
+
+
+ @Override
+ public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws
+ SliderException,
+ IOException {
+ String resourceTemplate = INSTANCE_RESOURCE_BASE + "resources.json";
+ String appConfTemplate = INSTANCE_RESOURCE_BASE + "appconf.json";
+ mergeTemplates(aggregateConf, null, resourceTemplate, appConfTemplate);
+ aggregateConf.getAppConfOperations().set(OPTION_ACCUMULO_PASSWORD,
+ createAccumuloPassword());
+ }
+
+
+ public String createAccumuloPassword() {
+ return UUID.randomUUID().toString();
+ }
+
+ public void setDatabasePath(Map<String, String> sitexml, String dataPath) {
+ Path path = new Path(dataPath);
+ URI parentUri = path.toUri();
+ String authority = parentUri.getAuthority();
+ String fspath =
+ parentUri.getScheme() + "://" + (authority == null ? "" : authority) + "/";
+ sitexml.put(AccumuloConfigFileOptions.INSTANCE_DFS_URI, fspath);
+ sitexml.put(AccumuloConfigFileOptions.INSTANCE_DFS_DIR,
+ parentUri.getPath());
+ }
+
+ /**
+ * Build the accumulo-site.xml file
+ * This the configuration used by Accumulo directly
+ * @param instanceDescription this is the cluster specification used to define this
+ * @return a map of the dynamic bindings for this Slider instance
+ */
+ public Map<String, String> buildSiteConfFromInstance(
+ AggregateConf instanceDescription)
+ throws BadConfigException {
+
+
+ ConfTreeOperations appconf =
+ instanceDescription.getAppConfOperations();
+
+ MapOperations globalAppOptions = appconf.getGlobalOptions();
+ MapOperations globalInstanceOptions =
+ instanceDescription.getInternalOperations().getGlobalOptions();
+
+
+ Map<String, String> sitexml = new HashMap<String, String>();
+
+ providerUtils.propagateSiteOptions(globalAppOptions, sitexml);
+
+ propagateClientFSBinding(sitexml);
+ setDatabasePath(sitexml,
+ globalInstanceOptions.getMandatoryOption(OptionKeys.INTERNAL_DATA_DIR_PATH));
+
+
+ String quorum =
+ globalAppOptions.getMandatoryOption(OptionKeys.ZOOKEEPER_QUORUM);
+ sitexml.put(AccumuloConfigFileOptions.ZOOKEEPER_HOST, quorum);
+
+ return sitexml;
+
+ }
+
+
+ public void propagateClientFSBinding(Map<String, String> sitexml) throws
+ BadConfigException {
+ String fsDefaultName =
+ getConf().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY);
+ if (fsDefaultName == null) {
+ throw new BadConfigException("Key not found in conf: {}",
+ CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
+ }
+ sitexml.put(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName);
+ sitexml.put(SliderXmlConfKeys.FS_DEFAULT_NAME_CLASSIC, fsDefaultName);
+ }
+
+ @Override
+ public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
+ String clustername,
+ Configuration configuration,
+ AggregateConf instanceDefinition,
+ Path clusterDirPath,
+ Path generatedConfDirPath,
+ boolean secure) throws
+ SliderException,
+ IOException {
+ super.preflightValidateClusterConfiguration(sliderFileSystem, clustername,
+ configuration,
+ instanceDefinition,
+ clusterDirPath,
+ generatedConfDirPath, secure);
+
+ }
+
+ /**
+ * Add Accumulo and its dependencies (only) to the job configuration.
+ * <p>
+ * This is intended as a low-level API, facilitating code reuse between this
+ * class and its mapred counterpart. It also of use to external tools that
+ * need to build a MapReduce job that interacts with Accumulo but want
+ * fine-grained control over the jars shipped to the cluster.
+ * </p>
+ *
+ * @see org.apache.hadoop.hbase.mapred.TableMapReduceUtil
+ * @see <a href="https://issues.apache.org/;jira/browse/PIG-3285">PIG-3285</a>
+ *
+ * @param providerResources provider resources to add resource to
+ * @param sliderFileSystem filesystem
+ * @param libdir relative directory to place resources
+ * @param tempPath path in the cluster FS for temp files
+ * @throws IOException IO problems
+ * @throws SliderException Slider-specific issues
+ */
+ private void addAccumuloDependencyJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ String libdir,
+ Path tempPath) throws
+ IOException,
+ SliderException {
+ String[] jars =
+ {
+ /* "zookeeper.jar",*/
+ };
+ Class<?>[] classes = {
+ //zk
+/* org.apache.zookeeper.ClientCnxn.class*/
+ };
+
+ ProviderUtils.addDependencyJars(providerResources, sliderFileSystem, tempPath,
+ libdir, jars,
+ classes);
+ }
+
+ @Override
+ public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
+ Configuration serviceConf,
+ AbstractLauncher launcher,
+ AggregateConf instanceDescription,
+ Path snapshotConfDirPath,
+ Path generatedConfDirPath,
+ Configuration clientConfExtras,
+ String libdir,
+ Path tempPath,
+ boolean miniClusterTestRun) throws IOException, SliderException {
+ //load in the template site config
+ log.debug("Loading template configuration from {}", snapshotConfDirPath);
+ Configuration siteConf = ConfigHelper.loadTemplateConfiguration(
+ serviceConf,
+ snapshotConfDirPath,
+ AccumuloKeys.SITE_XML,
+ AccumuloKeys.SITE_XML_RESOURCE);
+
+
+
+ Map<String, LocalResource> providerResources;
+ providerResources = fileSystem.submitDirectory(generatedConfDirPath,
+ SliderKeys.PROPAGATED_CONF_DIR_NAME);
+
+
+ ProviderUtils.addProviderJar(providerResources,
+ this,
+ "slider-accumulo-provider.jar",
+ fileSystem,
+ tempPath,
+ libdir,
+ miniClusterTestRun);
+
+
+ addAccumuloDependencyJars(providerResources, fileSystem, libdir, tempPath);
+ launcher.addLocalResources(providerResources);
+
+ //construct the cluster configuration values
+
+ ConfTreeOperations appconf =
+ instanceDescription.getAppConfOperations();
+
+
+ Map<String, String> clusterConfMap = buildSiteConfFromInstance(
+ instanceDescription);
+
+ //merge them
+ ConfigHelper.addConfigMap(siteConf,
+ clusterConfMap.entrySet(),
+ "Accumulo Provider");
+
+ //now, if there is an extra client conf, merge it in too
+ if (clientConfExtras != null) {
+ ConfigHelper.mergeConfigurations(siteConf, clientConfExtras,
+ "Slider Client");
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("Merged Configuration");
+ ConfigHelper.dumpConf(siteConf);
+ }
+
+ Path sitePath = ConfigHelper.saveConfig(serviceConf,
+ siteConf,
+ generatedConfDirPath,
+ AccumuloKeys.SITE_XML);
+
+ log.debug("Saving the config to {}", sitePath);
+ launcher.submitDirectory(generatedConfDirPath,
+ SliderKeys.PROPAGATED_CONF_DIR_NAME);
+
+ }
+
+ private static Set<String> knownRoleNames = new HashSet<String>();
+ static {
+ knownRoleNames.add(SliderKeys.COMPONENT_AM);
+ for (ProviderRole role : AccumuloRoles.ROLES) {
+ knownRoleNames.add(role.name);
+ }
+ }
+
+ @Override
+ public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ SliderException {
+ super.validateInstanceDefinition(instanceDefinition);
+
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+ Set<String> unknownRoles = resources.getComponentNames();
+ unknownRoles.removeAll(knownRoleNames);
+ if (!unknownRoles.isEmpty()) {
+ throw new BadCommandArgumentsException("There is unknown role: %s",
+ unknownRoles.iterator().next());
+ }
+ providerUtils.validateNodeCount(instanceDefinition,
+ AccumuloKeys.ROLE_TABLET,
+ 1, -1);
+
+
+ providerUtils.validateNodeCount(instanceDefinition,
+ AccumuloKeys.ROLE_MASTER, 1, -1);
+
+ providerUtils.validateNodeCount(instanceDefinition,
+ AccumuloKeys.ROLE_GARBAGE_COLLECTOR,
+ 0, -1);
+
+ providerUtils.validateNodeCount(instanceDefinition,
+ AccumuloKeys.ROLE_MONITOR,
+ 0, -1);
+
+ providerUtils.validateNodeCount(instanceDefinition,
+ AccumuloKeys.ROLE_TRACER , 0, -1);
+
+ MapOperations globalAppConfOptions =
+ instanceDefinition.getAppConfOperations().getGlobalOptions();
+ globalAppConfOptions.verifyOptionSet(AccumuloKeys.OPTION_ZK_HOME);
+ globalAppConfOptions.verifyOptionSet(AccumuloKeys.OPTION_HADOOP_HOME);
+ }
+
+
+ /**
+ * Get the path to the script
+ * @return the script
+ */
+ public static String buildScriptBinPath(AggregateConf instanceDefinition)
+ throws FileNotFoundException {
+ return providerUtils.buildPathToScript(instanceDefinition, "bin", "accumulo");
+ }
+
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloConfigFileOptions.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloConfigFileOptions.java
new file mode 100644
index 0000000..c6eec9a
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloConfigFileOptions.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo;
+
+/**
+ * Mappings of config params to env variables for
+ * custom -site.xml files to pick up
+ */
+public interface AccumuloConfigFileOptions {
+
+
+ /**
+ * quorum style, comma separated list of hostname:port values
+ */
+ String ZOOKEEPER_HOST = "instance.zookeeper.host";
+
+ /**
+ * URI to the filesystem
+ */
+ String INSTANCE_DFS_URI = "instance.dfs.uri";
+
+ /**
+ * Dir under the DFS URI
+ */
+ String INSTANCE_DFS_DIR = "instance.dfs.dir";
+
+ // String used to restrict access to data in ZK
+ String INSTANCE_SECRET = "instance.secret";
+
+ // IPC port for master
+ String MASTER_PORT_CLIENT = "master.port.client";
+ String MASTER_PORT_CLIENT_DEFAULT = "9999";
+
+ // IPC port for monitor
+ String MONITOR_PORT_CLIENT = "monitor.port.client";
+ String MONITOR_PORT_CLIENT_DEFAULT = "50095";
+ int MONITOR_PORT_CLIENT_INT = Integer.parseInt(MONITOR_PORT_CLIENT_DEFAULT);
+
+ // Log4j forwarding port
+ String MONITOR_LOG4J_PORT = "monitor.port.log4j";
+ String MONITOR_LOG4J_PORT_DEFAULT = "4560";
+ int MONITOR_LOG4J_PORT_INT = Integer.parseInt(MONITOR_LOG4J_PORT_DEFAULT);
+
+ // IPC port for tracer
+ String TRACE_PORT_CLIENT = "trace.port.client";
+ String TRACE_PORT_CLIENT_DEFAULT = "trace.port.client";
+
+ // IPC port for tserver
+ String TSERV_PORT_CLIENT = "tserver.port.client";
+ String TSERV_PORT_CLIENT_DEFAULT = "tserver.port.client";
+
+ // IPC port for gc
+ String GC_PORT_CLIENT = "gc.port.client";
+ String GC_PORT_CLIENT_DEFAULT = "50091";
+ int GC_PORT_CLIENT_INT = Integer.parseInt(GC_PORT_CLIENT_DEFAULT);
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloKeys.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloKeys.java
new file mode 100644
index 0000000..924ec41
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloKeys.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo;
+
+import org.apache.slider.api.StatusKeys;
+
+/**
+ * Any keys related to acculumulo
+ */
+public interface AccumuloKeys {
+ String PROVIDER_ACCUMULO = "accumulo";
+
+ String ROLE_MASTER = "master";
+
+ String ROLE_TABLET = "tserver";
+ String ROLE_GARBAGE_COLLECTOR = "gc";
+ String ROLE_MONITOR = "monitor";
+ String ROLE_TRACER = "tracer";
+
+ String OPTION_ZK_TAR = "zk.image.path";
+ String OPTION_ZK_HOME = "zk.home";
+ String OPTION_HADOOP_HOME = "hadoop.home";
+ String OPTION_ACCUMULO_PASSWORD = "accumulo.password";
+
+ String DEFAULT_MASTER_HEAP = "256";
+ String DEFAULT_MASTER_YARN_RAM = "384";
+ String DEFAULT_MASTER_YARN_VCORES = "1";
+ String DEFAULT_ROLE_YARN_VCORES = "1";
+ String DEFAULT_ROLE_HEAP = DEFAULT_MASTER_HEAP;
+ String DEFAULT_ROLE_YARN_RAM = DEFAULT_MASTER_YARN_RAM;
+
+// org.apache.slider.providers.accumulo.conf
+
+ String VERSION = "version";
+
+ String CREATE_MASTER = ROLE_MASTER;
+ String CREATE_GC = ROLE_GARBAGE_COLLECTOR;
+ String CREATE_TABLET = ROLE_TABLET;
+ String CREATE_MONITOR = ROLE_MONITOR;
+ String CREATE_TRACER = ROLE_TRACER;
+
+
+ String ACTION_START = "start";
+ String ACTION_STOP = "stop";
+
+ /**
+ * Config directory : {@value}
+ */
+ String ARG_CONFIG = "--config";
+ /**
+ * name of the hbase script relative to the hbase root dir: {@value}
+ */
+ String START_SCRIPT = "bin/accumulo";
+
+ /**
+ * name of the site conf to generate : {@value}
+ */
+ String SITE_XML = "accumulo-site.xml";
+
+ /**
+ * Template stored in the slider classpath -to use if there is
+ * no site-specific template
+ * {@value}
+ */
+ String CONF_RESOURCE = "org/apache/slider/providers/accumulo/conf/";
+ String SITE_XML_RESOURCE = CONF_RESOURCE + SITE_XML;
+ String ACCUMULO_HOME = "ACCUMULO_HOME";
+
+ String ACCUMULO_CONF_DIR = "ACCUMULO_CONF_DIR";
+ String ACCUMULO_LOG_DIR = "ACCUMULO_LOG_DIR";
+ String ACCUMULO_GENERAL_OPTS = "ACCUMULO_GENERAL_OPTS";
+ String HADOOP_HOME = "HADOOP_HOME";
+ String ZOOKEEPER_HOME = "ZOOKEEPER_HOME";
+
+ /**
+ * ":"-separated list of extra jars
+ *
+ */
+ String ACCUMULO_XTRAJARS = "ACCUMULO_XTRAJARS";
+ String HADOOP_PREFIX = "HADOOP_PREFIX" ;
+ int INIT_TIMEOUT_DEFAULT = 60000;
+ /**
+ * timeout in millis for init to complete
+ */
+ String OPTION_ACCUMULO_INIT_TIMEOUT = "accumulo.init.timeout";
+
+ /**
+ * @Parameter(names = "--instance-name", description = "the instance name, if not provided, will prompt")
+ */
+ String PARAM_INSTANCE_NAME = "--instance-name";
+ /**
+ * @Parameter(names = "--password", description = "set the password on the command line")
+ */
+ String PARAM_PASSWORD = "--password";
+
+ String MONITOR_PAGE_JSON = "/json";
+ String MONITOR_PAGE_XML = "/xml";
+
+ String ACCUMULO_VERSION_COMMAND = "version";
+
+ String MASTER_ADDRESS = StatusKeys.INFO_MASTER_ADDRESS;
+ String MONITOR_ADDRESS = "monitor.address";
+ String INSTANCE_ID = "instance_id";
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderFactory.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderFactory.java
new file mode 100644
index 0000000..a090cb1
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderFactory.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.SliderProviderFactory;
+import org.apache.slider.providers.ProviderService;
+
+public class AccumuloProviderFactory extends SliderProviderFactory {
+
+ public AccumuloProviderFactory() {
+ }
+
+ public AccumuloProviderFactory(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return new AccumuloClientProvider(getConf());
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return new AccumuloProviderService();
+ }
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
new file mode 100644
index 0000000..355a6d8
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
@@ -0,0 +1,415 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo;
+
+import com.google.common.net.HostAndPort;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.registry.zk.BlockingZKWatcher;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.server.services.docstore.utility.EventCallback;
+import org.apache.slider.server.services.docstore.utility.EventNotifyingService;
+import org.apache.slider.server.services.docstore.utility.ForkedProcessService;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+
+/**
+ * Server-side accumulo provider
+ */
+public class AccumuloProviderService extends AbstractProviderService implements
+ ProviderCore,
+ AccumuloKeys,
+ SliderKeys {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(AccumuloClientProvider.class);
+ private AccumuloClientProvider clientProvider;
+ private static final ProviderUtils providerUtils = new ProviderUtils(log);
+
+ private SliderFileSystem fileSystem = null;
+
+ public AccumuloProviderService() {
+ super("accumulo");
+ }
+
+
+ @Override
+ public List<ProviderRole> getRoles() {
+ return AccumuloRoles.ROLES;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ clientProvider = new AccumuloClientProvider(conf);
+ }
+
+ @Override
+ public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ SliderException {
+ clientProvider.validateInstanceDefinition(instanceDefinition);
+ }
+
+ @Override
+ public Configuration loadProviderConfigurationInformation(File confDir)
+ throws BadCommandArgumentsException, IOException {
+
+ return loadProviderConfigurationInformation(confDir, SITE_XML);
+ }
+
+ /*
+ ======================================================================
+ Server interface below here
+ ======================================================================
+ */
+
+ @Override
+ public void buildContainerLaunchContext(ContainerLauncher launcher,
+ AggregateConf instanceDefinition,
+ Container container,
+ String role,
+ SliderFileSystem fileSystem,
+ Path generatedConfPath,
+ MapOperations resourceComponent,
+ MapOperations appComponent,
+ Path containerTmpDirPath) throws
+ IOException,
+ SliderException {
+
+ this.fileSystem = fileSystem;
+ this.instanceDefinition = instanceDefinition;
+
+ // Set the environment
+ launcher.putEnv(SliderUtils.buildEnvMap(appComponent));
+
+ Map<String, String> env = SliderUtils.buildEnvMap(appComponent);
+ launcher.setEnv(ACCUMULO_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ ConfTreeOperations appConf =
+ instanceDefinition.getAppConfOperations();
+ String hadoop_home =
+ ApplicationConstants.Environment.HADOOP_COMMON_HOME.$();
+ MapOperations appConfGlobal = appConf.getGlobalOptions();
+ hadoop_home = appConfGlobal.getOption(OPTION_HADOOP_HOME, hadoop_home);
+ launcher.setEnv(HADOOP_HOME, hadoop_home);
+ launcher.setEnv(HADOOP_PREFIX, hadoop_home);
+
+ // By not setting ACCUMULO_HOME, this will cause the Accumulo script to
+ // compute it on its own to an absolute path.
+
+ launcher.setEnv(ACCUMULO_CONF_DIR,
+ ProviderUtils.convertToAppRelativePath(
+ SliderKeys.PROPAGATED_CONF_DIR_NAME));
+ launcher.setEnv(ZOOKEEPER_HOME, appConfGlobal.getMandatoryOption(OPTION_ZK_HOME));
+
+ //local resources
+
+
+ //add the configuration resources
+ launcher.addLocalResources(fileSystem.submitDirectory(
+ generatedConfPath,
+ SliderKeys.PROPAGATED_CONF_DIR_NAME));
+
+ //Add binaries
+ //now add the image if it was set
+ String imageURI = instanceDefinition.getInternalOperations()
+ .get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ fileSystem.maybeAddImagePath(launcher.getLocalResources(), imageURI);
+
+ CommandLineBuilder commandLine = new CommandLineBuilder();
+
+ String heap = "-Xmx" + appComponent.getOption(RoleKeys.JVM_HEAP, DEFAULT_JVM_HEAP);
+ String opt = "ACCUMULO_OTHER_OPTS";
+ if (SliderUtils.isSet(heap)) {
+ if (AccumuloKeys.ROLE_MASTER.equals(role)) {
+ opt = "ACCUMULO_MASTER_OPTS";
+ } else if (AccumuloKeys.ROLE_TABLET.equals(role)) {
+ opt = "ACCUMULO_TSERVER_OPTS";
+ } else if (AccumuloKeys.ROLE_MONITOR.equals(role)) {
+ opt = "ACCUMULO_MONITOR_OPTS";
+ } else if (AccumuloKeys.ROLE_GARBAGE_COLLECTOR.equals(role)) {
+ opt = "ACCUMULO_GC_OPTS";
+ }
+ launcher.setEnv(opt, heap);
+ }
+
+ //this must stay relative if it is an image
+ commandLine.add(providerUtils.buildPathToScript(instanceDefinition,
+ "bin", "accumulo"));
+
+ //role is translated to the accumulo one
+ commandLine.add(AccumuloRoles.serviceForRole(role));
+
+ // Add any role specific arguments to the command line
+ String additionalArgs = ProviderUtils.getAdditionalArgs(appComponent);
+ if (!StringUtils.isBlank(additionalArgs)) {
+ commandLine.add(additionalArgs);
+ }
+
+ commandLine.addOutAndErrFiles(role + "-out.txt", role + "-err.txt");
+
+
+ launcher.addCommand(commandLine.build());
+ }
+
+ public List<String> buildProcessCommandList(AggregateConf instance,
+ File confDir,
+ Map<String, String> env,
+ String... commands) throws
+ IOException,
+ SliderException {
+ env.put(ACCUMULO_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ String hadoop_home = System.getenv(HADOOP_HOME);
+ MapOperations globalOptions =
+ instance.getAppConfOperations().getGlobalOptions();
+ hadoop_home = globalOptions.getOption(OPTION_HADOOP_HOME, hadoop_home);
+ if (hadoop_home == null) {
+ throw new BadConfigException(
+ "Undefined env variable/config option: " + HADOOP_HOME);
+ }
+ ProviderUtils.validatePathReferencesLocalDir("HADOOP_HOME", hadoop_home);
+ env.put(HADOOP_HOME, hadoop_home);
+ env.put(HADOOP_PREFIX, hadoop_home);
+ //buildup accumulo home env variable to be absolute or relative
+ String accumulo_home = providerUtils.buildPathToHomeDir(instance,
+ "bin", "accumulo");
+ File image = new File(accumulo_home);
+ String accumuloPath = image.getAbsolutePath();
+ env.put(ACCUMULO_HOME, accumuloPath);
+ ProviderUtils.validatePathReferencesLocalDir("ACCUMULO_HOME", accumuloPath);
+ env.put(ACCUMULO_CONF_DIR, confDir.getAbsolutePath());
+ String zkHome = globalOptions.getMandatoryOption(OPTION_ZK_HOME);
+ ProviderUtils.validatePathReferencesLocalDir("ZOOKEEPER_HOME", zkHome);
+
+ env.put(ZOOKEEPER_HOME, zkHome);
+
+
+ String accumuloScript = AccumuloClientProvider.buildScriptBinPath(instance);
+ List<String> launchSequence = new ArrayList<String>(8);
+ launchSequence.add(0, accumuloScript);
+ Collections.addAll(launchSequence, commands);
+ return launchSequence;
+ }
+
+ /**
+ * Accumulo startup is a bit more complex than HBase, as it needs
+ * to pre-initialize the data directory.
+ *
+ * This is done by running an init operation before starting the
+ * real master. If the init fails, that is reported to the AM, which
+ * then fails the application.
+ * If the init succeeds, the next service in the queue is started -
+ * a composite service that starts the Accumulo Master and, in parallel,
+ * sends a delayed event to the AM
+ *
+ * @param instanceDefinition component description
+ * @param confDir local dir with the config
+ * @param env environment variables above those generated by
+ * @param execInProgress callback for the event notification
+ * @throws IOException IO problems
+ * @throws SliderException anything internal
+ */
+ @Override
+ public boolean exec(AggregateConf instanceDefinition,
+ File confDir,
+ Map<String, String> env,
+ EventCallback execInProgress) throws
+ IOException,
+ SliderException {
+
+
+ //now pull in these files and do a bit of last-minute validation
+ File siteXML = new File(confDir, SITE_XML);
+ Configuration accumuloSite = ConfigHelper.loadConfFromFile(
+ siteXML);
+ String zkQuorum =
+ accumuloSite.get(AccumuloConfigFileOptions.ZOOKEEPER_HOST);
+ if (zkQuorum == null) {
+ throw new BadConfigException("Accumulo site.xml %s does not contain %s",
+ siteXML,
+ AccumuloConfigFileOptions.ZOOKEEPER_HOST);
+ } else {
+ log.info("ZK Quorum is {}", zkQuorum);
+ }
+ //now test this
+ int timeout = 5000;
+ try {
+ verifyZookeeperLive(zkQuorum, timeout);
+ log.info("Zookeeper is live");
+ } catch (KeeperException e) {
+ throw new BadClusterStateException("Failed to connect to Zookeeper at %s after %d seconds",
+ zkQuorum, timeout);
+ } catch (InterruptedException e) {
+ throw new BadClusterStateException(
+ "Interrupted while trying to connect to Zookeeper at %s",
+ zkQuorum);
+ }
+ boolean inited = isInited(instanceDefinition);
+ if (inited) {
+ // cluster is inited, so don't run anything
+ return false;
+ }
+ List<String> commands;
+
+ log.info("Initializing accumulo datastore {}");
+ ConfTreeOperations appConfOperations =
+ instanceDefinition.getAppConfOperations();
+
+ ConfTreeOperations internalOperations =
+ instanceDefinition.getInternalOperations();
+ ConfTreeOperations resourceOperations =
+ instanceDefinition.getResourceOperations();
+ String accumuloInstanceName = internalOperations.get(OptionKeys.APPLICATION_NAME);
+ commands = buildProcessCommandList(instanceDefinition, confDir, env,
+ "init",
+ PARAM_INSTANCE_NAME,
+ providerUtils.getUserName() + "-" + accumuloInstanceName,
+ PARAM_PASSWORD,
+ appConfOperations.getGlobalOptions().getMandatoryOption(
+ OPTION_ACCUMULO_PASSWORD),
+ "--clear-instance-name");
+
+
+ ForkedProcessService accumulo =
+ queueCommand(getName(), env, commands);
+ //add a timeout to this process
+ accumulo.setTimeout(
+ appConfOperations.getGlobalOptions().getOptionInt(
+ OPTION_ACCUMULO_INIT_TIMEOUT,
+ INIT_TIMEOUT_DEFAULT), 1);
+
+ //callback to AM to trigger cluster review is set up to happen after
+ //the init/verify action has succeeded
+ EventNotifyingService notifier = new EventNotifyingService(execInProgress,
+ internalOperations.getGlobalOptions().getOptionInt(
+ OptionKeys.INTERNAL_CONTAINER_STARTUP_DELAY,
+ OptionKeys.DEFAULT_CONTAINER_STARTUP_DELAY));
+ // register the service for lifecycle management;
+ // this service is started after the accumulo process completes
+ addService(notifier);
+
+ // now trigger the command sequence
+ maybeStartCommandSequence();
+ return true;
+ }
+
+ /**
+ * probe to see if accumulo has already been installed.
+ * @param cd cluster description
+ * @return true if the relevant data directory looks inited
+ * @throws IOException IO problems
+ */
+ private boolean isInited(AggregateConf cd) throws
+ IOException,
+ BadConfigException {
+ String dataDir = cd.getInternalOperations()
+ .getGlobalOptions()
+ .getMandatoryOption(
+ OptionKeys.INTERNAL_DATA_DIR_PATH);
+ Path accumuloInited = new Path(dataDir, INSTANCE_ID);
+ FileSystem fs2 = FileSystem.get(accumuloInited.toUri(), getConf());
+ return fs2.exists(accumuloInited);
+ }
+
+
+
+ private void verifyZookeeperLive(String zkQuorum, int timeout) throws
+ IOException,
+ KeeperException,
+ InterruptedException {
+
+ BlockingZKWatcher watcher = new BlockingZKWatcher();
+ ZooKeeper zookeeper = new ZooKeeper(zkQuorum, 10000, watcher, true);
+ zookeeper.getChildren("/", watcher);
+
+ watcher.waitForZKConnection(timeout);
+
+ }
+
+ @Override
+ public Map<String, String> buildProviderStatus() {
+
+ Map<String,String> status = new HashMap<String, String>();
+
+
+ return status;
+ }
+
+
+ /* non-javadoc
+ * @see org.apache.slider.providers.ProviderService#buildMonitorDetails()
+ */
+ @Override
+ public TreeMap<String,URL> buildMonitorDetails(ClusterDescription clusterDesc) {
+ TreeMap<String,URL> map = new TreeMap<String,URL>();
+
+ map.put("Active Accumulo Master (RPC): " + getInfoAvoidingNull(clusterDesc, AccumuloKeys.MASTER_ADDRESS), null);
+
+ String monitorKey = "Active Accumulo Monitor: ";
+ String monitorAddr = getInfoAvoidingNull(clusterDesc, AccumuloKeys.MONITOR_ADDRESS);
+ if (!StringUtils.isBlank(monitorAddr)) {
+ try {
+ HostAndPort hostPort = HostAndPort.fromString(monitorAddr);
+ map.put(monitorKey, new URL("http", hostPort.getHostText(), hostPort.getPort(), ""));
+ } catch (Exception e) {
+ log.debug("Caught exception parsing Accumulo monitor URL", e);
+ map.put(monitorKey + "N/A", null);
+ }
+ } else {
+ map.put(monitorKey + "N/A", null);
+ }
+
+ return map;
+ }
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloRoles.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloRoles.java
new file mode 100644
index 0000000..6144f9b
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloRoles.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo;
+
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.providers.ProviderRole;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.slider.providers.accumulo.AccumuloKeys.ROLE_GARBAGE_COLLECTOR;
+import static org.apache.slider.providers.accumulo.AccumuloKeys.ROLE_MASTER;
+import static org.apache.slider.providers.accumulo.AccumuloKeys.ROLE_MONITOR;
+import static org.apache.slider.providers.accumulo.AccumuloKeys.ROLE_TABLET;
+import static org.apache.slider.providers.accumulo.AccumuloKeys.ROLE_TRACER;
+
+public class AccumuloRoles {
+ protected static final Logger log =
+ LoggerFactory.getLogger(AccumuloRoles.class);
+
+ /**
+ * List of roles
+ */
+ public static final List<ProviderRole> ROLES =
+ new ArrayList<ProviderRole>();
+
+ private static int BASE;
+
+ /**
+ * Initialize role list
+ */
+ static {
+ BASE = SliderKeys.ROLE_AM_PRIORITY_INDEX;
+ AccumuloRoles.ROLES.add(new ProviderRole(ROLE_MASTER, BASE + 1));
+ AccumuloRoles.ROLES.add(new ProviderRole(ROLE_TABLET, BASE + 2));
+ AccumuloRoles.ROLES.add(new ProviderRole(ROLE_GARBAGE_COLLECTOR, BASE + 3));
+ AccumuloRoles.ROLES.add(new ProviderRole(ROLE_MONITOR, BASE + 4));
+ AccumuloRoles.ROLES.add(new ProviderRole(ROLE_TRACER, BASE + 5));
+ }
+
+
+ /**
+ * Convert a Slider role into the service/classname passed down
+ * to accumulo, (and implicitly , item to grep and kill when force
+ * killing services at the end of a test run)
+ * @param role role being instantiated
+ * @return first argument to Accumulo
+ */
+ public static String serviceForRole(String role) {
+ for (ProviderRole providerRole : ROLES) {
+ if (providerRole.name.equals(role)) {
+ return role;
+ }
+ }
+ //unknown role
+ log.warn("unknown accumulo role {}", role);
+ return role;
+ }
+
+ public static List<String> roleList() {
+ List<String> l = new ArrayList<String>(ROLES.size());
+ for (ProviderRole providerRole : ROLES) {
+ l.add(providerRole.name);
+ }
+ return l;
+ }
+
+ public static List<String> serviceList() {
+ List<String> l = new ArrayList<String>(ROLES.size());
+ for (ProviderRole providerRole : ROLES) {
+ l.add(serviceForRole(providerRole.name));
+ }
+ return l;
+ }
+
+
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo-env.sh b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo-env.sh
new file mode 100755
index 0000000..b7a11ee
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo-env.sh
@@ -0,0 +1,56 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###
+### Configure these environment variables to point to your local installations.
+###
+### The functional tests require conditional values, so keep this style:
+###
+### test -z "$JAVA_HOME" && export JAVA_HOME=/usr/local/lib/jdk-1.6.0
+###
+###
+### Note that the -Xmx -Xms settings below require substantial free memory:
+### you may want to use smaller values, especially when running everything
+### on a single machine.
+###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
+
+test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
+test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
+test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
+then
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
+fi
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m "
+test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
+test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx64m -Xms64m"
+test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
+test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Dorg.apache.slider.container=${CONTAINER_ID}"
+test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx128m -Xms64m"
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/monitor ; echo localhost ) 2>/dev/null | head -1`
+# what do when the JVM runs out of heap memory
+export ACCUMULO_KILL_CMD='kill -9 %p'
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo-metrics.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo-metrics.xml
new file mode 100644
index 0000000..60f9f8d
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo-metrics.xml
@@ -0,0 +1,60 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ This file follows the conventions for XMLConfiguration files specified in the Apache Commons Configuration 1.5 Library. Changes to this file will be noticed
+ at runtime (see the FileChangedReloadingStrategy class in Commons Configuration).
+-->
+<config>
+<!--
+ Metrics log directory
+-->
+ <logging>
+ <dir>${ACCUMULO_HOME}/metrics</dir>
+ </logging>
+<!--
+ Enable/Disable metrics accumulation on the different servers and their components
+ NOTE: Turning on logging can be expensive because it will use several more file handles and will create a lot of short lived objects.
+-->
+ <master>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </master>
+ <logger>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </logger>
+ <tserver>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ <update>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </update>
+ <scan>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </scan>
+ <minc>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </minc>
+ </tserver>
+ <thrift>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </thrift>
+</config>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo-site.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo-site.xml
new file mode 100644
index 0000000..b2dd1cf
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo-site.xml
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- Put your site-specific accumulo configurations here. The available configuration values along with their defaults are documented in docs/config.html Unless
+ you are simply testing at your workstation, you will most definitely need to change the three entries below. -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The property only needs to be set if upgrading from 1.4 which used to store write-ahead logs on the local
+ filesystem. In 1.5 write-ahead logs are stored in DFS. When 1.5 is started for the first time it will copy any 1.4
+ write ahead logs into DFS. It is possible to specify a comma-separated list of directories.
+ </description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers
+ must know in order to communicate with one another.
+ Change it before initialization. To
+ change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.native.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.token.property.password</name>
+ <!-- change this to the root user's password, and/or change the user below -->
+ <value>secret</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.maven.project.basedir</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <!--
+ Add the following for hadoop-2.0
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+ -->
+ <value>
+ $ACCUMULO_HOME/lib/accumulo-server.jar,
+ $ACCUMULO_HOME/lib/accumulo-core.jar,
+ $ACCUMULO_HOME/lib/accumulo-start.jar,
+ $ACCUMULO_HOME/lib/accumulo-fate.jar,
+ $ACCUMULO_HOME/lib/accumulo-proxy.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.</description>
+ </property>
+</configuration>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo.policy.example b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo.policy.example
new file mode 100644
index 0000000..2964f06
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/accumulo.policy.example
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grant codeBase "file:${java.home}/lib/ext/*" {
+ permission java.security.AllPermission;
+};
+
+// These should all be empty in a fielded system
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/server/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/core/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/start/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/examples/target/classes/" {
+ permission java.security.AllPermission;
+};
+
+grant codebase "file:${hadoop.home.dir}/*" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "shutdownHooks"; // hadoop libs use executables to discover usernames, groups, etc.
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.io.FilePermission "<<ALL FILES>>", "read, execute";
+ permission java.io.FilePermission "/tmp", "write, delete";
+ permission java.io.FilePermission "/tmp/-", "write, delete";
+ permission java.io.FilePermission "/", "write";
+ permission java.net.SocketPermission "*", "connect, resolve";
+ permission java.util.PropertyPermission "java.library.path", "read";
+ permission java.util.PropertyPermission "user.dir", "read";
+ permission java.util.PropertyPermission "org.apache.commons.logging.*", "read";
+ permission java.util.PropertyPermission "entityExpansionLimit", "read";
+ permission java.util.PropertyPermission "maxOccurLimit", "read";
+ permission java.util.PropertyPermission "os.name", "read";
+};
+
+grant codebase "file:${hadoop.home.dir}/lib/*" {
+ // monitor's jetty web service
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ // need to accept web requests, and talk to job tracker, name node, etc.
+ permission java.net.SocketPermission "*", "accept, listen, resolve, connect, resolve";
+ permission java.lang.RuntimePermission "getenv.*";
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.util.PropertyPermission "org.mortbay.*", "read";
+ permission java.util.PropertyPermission "VERBOSE", "read";
+ permission java.util.PropertyPermission "IGNORED", "read";
+ permission java.util.PropertyPermission "ISO_8859_1", "read";
+ permission java.util.PropertyPermission "org.apache.commons.logging.*", "read";
+ permission java.util.PropertyPermission "accumulo.*", "read";
+ permission java.util.PropertyPermission "org.jfree.*", "read";
+ permission java.util.PropertyPermission "elementAttributeLimit", "read";
+ permission java.util.PropertyPermission "entityExpansionLimit", "read";
+ permission java.util.PropertyPermission "maxOccurLimit", "read";
+ // some resources come out of accumulo jars
+ permission java.lang.RuntimePermission "getClassLoader";
+ permission java.io.FilePermission "${org.apache.accumulo.core.home.dir}/lib/*", "read";
+ permission java.io.FilePermission "${org.apache.accumulo.core.home.dir}/src/-", "read";
+ permission java.io.FilePermission "${hadoop.home.dir}/lib/*", "read";
+ // images are cached in /tmp
+ permission java.io.FilePermission "/tmp/*", "read, write";
+ permission java.io.FilePermission "/", "write";
+};
+
+grant codebase "file:${zookeeper.home.dir}/*" {
+ permission java.net.SocketPermission "*", "connect, resolve";
+ permission java.util.PropertyPermission "user.*", "read";
+ permission java.util.PropertyPermission "java.*", "read";
+ permission java.util.PropertyPermission "zookeeper.*", "read";
+ permission java.util.PropertyPermission "jute.*", "read";
+ permission java.util.PropertyPermission "os.*", "read";
+ // accumulo properties read in callbacks
+ permission java.util.PropertyPermission "accumulo.*", "read";
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "exitVM";
+};
+
+grant codebase "file:${org.apache.accumulo.core.home.dir}/lib/ext/*" {
+};
+
+grant codebase "file:${org.apache.accumulo.core.home.dir}/lib/*" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ // logging, configuration and getting user id
+ permission java.io.FilePermission "<<ALL FILES>>", "read, write, execute, delete";
+ permission java.util.PropertyPermission "*", "read, write";
+ permission java.lang.RuntimePermission "getenv.*";
+ permission java.lang.RuntimePermission "getClassLoader";
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.lang.RuntimePermission "accessDeclaredMembers";
+ permission java.lang.RuntimePermission "selectorProvider";
+ permission java.lang.RuntimePermission "accessClassInPackage.*";
+ permission java.lang.RuntimePermission "readFileDescriptor";
+ permission java.lang.RuntimePermission "writeFileDescriptor";
+ permission java.lang.RuntimePermission "modifyThread";
+ permission java.lang.RuntimePermission "modifyThreadGroup";
+ permission java.lang.RuntimePermission "createClassLoader";
+ permission java.lang.RuntimePermission "setContextClassLoader";
+ permission java.lang.RuntimePermission "exitVM";
+ permission java.lang.RuntimePermission "shutdownHooks";
+ permission java.security.SecurityPermission "getPolicy";
+ permission java.security.SecurityPermission "getProperty.*";
+ permission java.security.SecurityPermission "putProviderProperty.*";
+ permission java.security.SecurityPermission "setSystemScope";
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.util.logging.LoggingPermission "control";
+ permission java.net.NetPermission "getProxySelector";
+ permission javax.management.MBeanServerPermission "createMBeanServer";
+ permission javax.management.MBeanTrustPermission "register";
+ permission javax.management.MBeanPermission "*", "registerMBean";
+ permission java.net.SocketPermission "*", "accept, connect, listen, resolve";
+};
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/auditLog.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/auditLog.xml
new file mode 100644
index 0000000..327f181
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/auditLog.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+
+
+ <!-- Write out Audit info to an Audit file -->
+ <appender name="Audit" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.slider.container}.audit"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="DatePattern" value="'.'yyyy-MM-dd"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss,SSS/Z} [%c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+ <logger name="Audit" additivity="false">
+ <appender-ref ref="Audit" />
+ <level value="OFF"/>
+ </logger>
+
+
+
+
+
+</log4j:configuration>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/gc b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/gc
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/gc
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/generic_logger.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/generic_logger.xml
new file mode 100644
index 0000000..19537e9
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/generic_logger.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.slider.container}.debug.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.slider.container}.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Send all logging data to a centralized logger -->
+ <appender name="N1" class="org.apache.log4j.net.SocketAppender">
+ <param name="remoteHost" value="${org.apache.accumulo.core.host.log}"/>
+ <param name="port" value="4560"/>
+ <param name="application" value="${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname}"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- If the centralized logger is down, buffer the log events, but drop them if it stays down -->
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="N1" />
+ </appender>
+
+ <!-- Log accumulo events to the debug, normal and remote logs. -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="ASYNC" />
+ </logger>
+
+ <logger name="org.apache.accumulo.core.file.rfile.bcfile">
+ <level value="INFO"/>
+ </logger>
+
+ <logger name="org.mortbay.log">
+ <level value="WARN"/>
+ </logger>
+
+ <logger name="org.apache.zookeeper">
+ <level value="ERROR"/>
+ </logger>
+
+ <!-- Log non-accumulo events to the debug and normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/log4j.properties b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/log4j.properties
new file mode 100644
index 0000000..a4bcb2e
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/log4j.properties
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# default logging properties:
+# by default, log everything at INFO or higher to the console
+log4j.rootLogger=INFO,A1
+
+# hide Jetty junk
+log4j.logger.org.mortbay.log=WARN,A1
+
+# hide "Got brand-new compresssor" messages
+log4j.logger.org.apache.hadoop.io.compress=WARN,A1
+
+# hide junk from TestRandomDeletes
+log4j.logger.org.apache.accumulo.test.TestRandomDeletes=WARN,A1
+
+# hide junk from VFS
+log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN,A1
+
+# hide almost everything from zookeeper
+log4j.logger.org.apache.zookeeper=ERROR,A1
+
+# hide AUDIT messages in the shell, alternatively you could send them to a different logger
+log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=WARN,A1
+
+# Send most things to the console
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.appender.A1.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/masters b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/masters
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/masters
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/monitor b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/monitor
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/monitor
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/monitor_logger.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/monitor_logger.xml
new file mode 100644
index 0000000..7e42a4f
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/monitor_logger.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.slider.container}.debug.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.slider.container}.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Keep the last few log messages for display to the user -->
+ <appender name="GUI" class="org.apache.accumulo.server.monitor.LogService">
+ <param name="keep" value="40"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- Log accumulo messages to debug, normal and GUI -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="GUI" />
+ </logger>
+
+ <!-- Log non-accumulo messages to debug, normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/slaves b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/slaves
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/slaves
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/tracers b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/tracers
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/conf/tracers
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/appconf.json b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/appconf.json
new file mode 100644
index 0000000..d6ac754
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/appconf.json
@@ -0,0 +1,37 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+
+
+ },
+
+ "global": {
+
+ "site.master.port.client": "0",
+ "site.trace.port.client": "0",
+ "site.tserver.port.client": "0",
+ "site.gc.port.client": "0",
+ "site.monitor.port.log4j": "0",
+ "site.instance.secret": "secret",
+ "site.accumulo.password": "secret",
+
+ "jvm.heapsize": "256M"
+ },
+
+
+ "components": {
+ "master": {
+ },
+ "tserver": {
+ },
+ "monitor": {
+ "role.additional.args": "--address 0.0.0.0"
+ },
+ "tracer": {
+ },
+ "gc": {
+ "role.additional.args":"--address 0.0.0.0"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/resources.json b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/resources.json
new file mode 100644
index 0000000..23f4b75
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/instance/resources.json
@@ -0,0 +1,30 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+
+ },
+
+ "global": {
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ },
+
+ "components": {
+ "master": {
+ "component.instances": "1"
+ },
+ "tserver": {
+ "component.instances": "2"
+ },
+ "monitor": {
+ "component.instances": "0"
+ },
+ "tracer": {
+ "component.instances": "0"
+ },
+ "gc": {
+ "component.instances": "0"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-gc.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-gc.xml
new file mode 100644
index 0000000..92308ad
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-gc.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+ <!--
+ Garbage collector
+ -->
+<configuration>
+ <property>
+ <name>role.name</name>
+ <value>gc</value>
+ </property>
+
+ <property>
+ <name>role.instances</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>role.additional.args</name>
+ <value>--address 0.0.0.0</value>
+ </property>
+
+ <property>
+ <name>yarn.vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.memory</name>
+ <value>1024</value>
+ </property>
+
+ <property>
+ <name>jvm.heapsize</name>
+ <value>512M</value>
+ </property>
+
+</configuration>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-master.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-master.xml
new file mode 100644
index 0000000..9e36eea
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-master.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+ <!--
+ These are the default cluster options for a Slider cluster
+ -->
+<configuration>
+ <property>
+ <name>role.name</name>
+ <value>master</value>
+ </property>
+
+ <property>
+ <name>role.instances</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.memory</name>
+ <value>1024</value>
+ </property>
+
+ <property>
+ <name>jvm.heapsize</name>
+ <value>512M</value>
+ </property>
+
+ <property>
+ <name>app.infoport</name>
+ <value>0</value>
+ </property>
+
+</configuration>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-monitor.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-monitor.xml
new file mode 100644
index 0000000..0d55fa3
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-monitor.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <property>
+ <name>role.name</name>
+ <value>monitor</value>
+ </property>
+
+ <property>
+ <name>role.instances</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>role.additional.args</name>
+ <value>--address 0.0.0.0</value>
+ </property>
+
+ <property>
+ <name>yarn.vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.memory</name>
+ <value>384</value>
+ </property>
+
+ <property>
+ <name>jvm.heapsize</name>
+ <value>256M</value>
+ </property>
+
+</configuration>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-other.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-other.xml
new file mode 100644
index 0000000..54669d1
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-other.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+ <!--
+ Accumulo options for any role that doesn't
+ have an explicit configuration
+ -->
+<configuration>
+ <property>
+ <name>role.name</name>
+ <value>other</value>
+ </property>
+
+ <property>
+ <name>role.instances</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.memory</name>
+ <value>1024</value>
+ </property>
+
+ <property>
+ <name>jvm.heapsize</name>
+ <value>512M</value>
+ </property>
+
+</configuration>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-tablet.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-tablet.xml
new file mode 100644
index 0000000..56ad727
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-tablet.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+ <!--
+ These are the default cluster options for a Slider cluster
+ -->
+<configuration>
+ <property>
+ <name>role.name</name>
+ <value>tablet</value>
+ </property>
+
+ <property>
+ <name>role.instances</name>
+ <value>2</value>
+ </property>
+
+ <property>
+ <name>yarn.vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.memory</name>
+ <value>768</value>
+ </property>
+
+ <property>
+ <name>jvm.heapsize</name>
+ <value>512M</value>
+ </property>
+
+ <property>
+ <name>app.infoport</name>
+ <value>0</value>
+ </property>
+
+</configuration>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-tracer.xml b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-tracer.xml
new file mode 100644
index 0000000..8003c9f
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/resources/org/apache/slider/providers/accumulo/role-accumulo-tracer.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+ <!--
+ Accumulo options for any role that doesn't
+ have an explicit configuration
+ -->
+<configuration>
+ <property>
+ <name>role.name</name>
+ <value>tracer</value>
+ </property>
+
+ <property>
+ <name>role.instances</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.memory</name>
+ <value>384</value>
+ </property>
+
+ <property>
+ <name>jvm.heapsize</name>
+ <value>256M</value>
+ </property>
+
+</configuration>
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
new file mode 100644
index 0000000..53ac7f5
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.accumulo.core.client.ZooKeeperInstance
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.test.YarnZKMiniClusterTestBase
+
+import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
+import static org.apache.slider.providers.accumulo.AccumuloKeys.*
+import static org.apache.slider.common.params.Arguments.ARG_PROVIDER
+import static org.apache.slider.common.params.Arguments.ARG_RES_COMP_OPT
+
+/**
+ * test base for accumulo clusters
+ */
+@CompileStatic
+@Slf4j
+public abstract class AccumuloTestBase extends YarnZKMiniClusterTestBase {
+
+ public static final int ACCUMULO_LAUNCH_WAIT_TIME
+ public static final boolean ACCUMULO_TESTS_ENABLED
+
+
+ public static final int ACCUMULO_CLUSTER_STARTUP_TIME = ACCUMULO_LAUNCH_WAIT_TIME
+ public static final int ACCUMULO_CLUSTER_STOP_TIME = 1 * 60 * 1000
+
+ /**
+ * The time to sleep before trying to talk to the HBase Master and
+ * expect meaningful results.
+ */
+ public static final int ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME = ACCUMULO_CLUSTER_STARTUP_TIME
+ public static final int ACCUMULO_GO_LIVE_TIME = 60000
+
+ @Override
+ public String getTestConfigurationPath() {
+ return "src/main/resources/" + CONF_RESOURCE;
+ }
+
+ @Override
+ void setup() {
+ super.setup()
+ assumeBoolOption(SLIDER_CONFIG, KEY_TEST_ACCUMULO_ENABLED, true)
+ assumeArchiveDefined();
+ assumeApplicationHome();
+ YarnConfiguration conf = testConfiguration
+ assumeOtherSettings(conf)
+ }
+
+ /**
+ * Teardown
+ */
+ @Override
+ void teardown() {
+ super.teardown();
+ if (teardownKillall) {
+ killAllAccumuloProcesses();
+ }
+ }
+
+ void killAllAccumuloProcesses() {
+ killJavaProcesses("org.apache.accumulo.start.Main", SIGKILL)
+ }
+
+ @Override
+ public String getArchiveKey() {
+ return KEY_TEST_ACCUMULO_TAR
+ }
+
+ /**
+ * Get the key for the application
+ * @return
+ */
+ @Override
+ public String getApplicationHomeKey() {
+ return KEY_TEST_ACCUMULO_HOME
+ }
+
+ /**
+ * Assume that HBase home is defined. This does not check that the
+ * path is valid -that is expected to be a failure on tests that require
+ * HBase home to be set.
+ */
+
+ public void assumeOtherSettings(YarnConfiguration conf) {
+ assumeStringOptionSet(conf, OPTION_ZK_HOME)
+ }
+
+ /**
+ * Create a full cluster with a master & the requested no. of region servers
+ * @param clustername cluster name
+ * @param tablets # of nodes
+ * @param extraArgs list of extra args to add to the creation command
+ * @param deleteExistingData should the data of any existing cluster
+ * of this name be deleted
+ * @param blockUntilRunning block until the AM is running
+ * @return launcher which will have executed the command.
+ */
+ public ServiceLauncher<SliderClient> createAccCluster(String clustername, int tablets, List<String> extraArgs, boolean deleteExistingData, boolean blockUntilRunning) {
+ Map<String, Integer> roles = [
+ (ROLE_MASTER): 1,
+ (ROLE_TABLET): tablets,
+ ];
+ return createAccCluster(clustername, roles, extraArgs, deleteExistingData, blockUntilRunning);
+}
+
+ /**
+ * Create an accumulo cluster
+ * @param clustername
+ * @param roles
+ * @param extraArgs
+ * @param deleteExistingData
+ * @param blockUntilRunning
+ * @return the cluster launcher
+ */
+ public ServiceLauncher<SliderClient> createAccCluster(String clustername, Map<String, Integer> roles, List<String> extraArgs, boolean deleteExistingData, boolean blockUntilRunning) {
+ extraArgs << ARG_PROVIDER << PROVIDER_ACCUMULO;
+
+ YarnConfiguration conf = testConfiguration
+
+ def clusterOps = [
+ (OPTION_ZK_HOME): conf.getTrimmed(OPTION_ZK_HOME),
+ (OPTION_HADOOP_HOME): conf.getTrimmed(OPTION_HADOOP_HOME),
+ ("site." + AccumuloConfigFileOptions.MONITOR_PORT_CLIENT): AccumuloConfigFileOptions.MONITOR_PORT_CLIENT_DEFAULT,
+ ("site." + AccumuloConfigFileOptions.MASTER_PORT_CLIENT): AccumuloConfigFileOptions.MASTER_PORT_CLIENT_DEFAULT,
+ ]
+
+
+ extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY << YRAM;
+ extraArgs << ARG_RES_COMP_OPT << ROLE_TABLET << ResourceKeys.YARN_MEMORY << YRAM
+ extraArgs << ARG_RES_COMP_OPT << ROLE_MONITOR << ResourceKeys.YARN_MEMORY << YRAM
+ extraArgs << ARG_RES_COMP_OPT << ROLE_GARBAGE_COLLECTOR << ResourceKeys.YARN_MEMORY << YRAM
+
+ return createCluster(clustername,
+ roles,
+ extraArgs,
+ deleteExistingData,
+ blockUntilRunning,
+ clusterOps)
+ }
+
+ def getAccClusterStatus() {
+ ZooKeeperInstance instance = new ZooKeeperInstance("", "localhost:4");
+ instance.getConnector("user", "pass").instanceOperations().tabletServers;
+ }
+
+
+ public String fetchLocalPage(int port, String page) {
+ String url = "http://localhost:" + port+ page
+ return fetchWebPage(url)
+
+ }
+
+ public ClusterDescription flexAccClusterTestRun(
+ String clustername, List<Map<String, Integer>> plan) {
+ int planCount = plan.size()
+ assert planCount > 0
+ createMiniCluster(clustername, getConfiguration(),
+ 1,
+ true);
+ //now launch the cluster
+ SliderClient sliderClient = null;
+ ServiceLauncher launcher = createAccCluster(clustername,
+ plan[0],
+ [],
+ true,
+ true);
+ sliderClient = (SliderClient) launcher.service;
+ try {
+
+ //verify the #of roles is as expected
+ //get the hbase status
+ waitForRoleCount(sliderClient, plan[0],
+ ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME);
+ sleep(ACCUMULO_GO_LIVE_TIME);
+
+ plan.remove(0)
+
+ ClusterDescription cd = null
+ while (!plan.empty) {
+
+ Map<String, Integer> flexTarget = plan.remove(0)
+ //now flex
+ describe(
+ "Flexing " + roleMapToString(flexTarget));
+ boolean flexed = 0 == sliderClient.flex(clustername,
+ flexTarget
+ );
+ cd = waitForRoleCount(sliderClient, flexTarget,
+ ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME);
+
+ sleep(ACCUMULO_GO_LIVE_TIME);
+
+ }
+
+ return cd;
+
+ } finally {
+ maybeStopCluster(sliderClient, null, "end of flex test run");
+ }
+
+ }
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
new file mode 100644
index 0000000..ad8981f
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccCorrectInstanceName.groovy
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.accumulo.core.client.Connector
+import org.apache.accumulo.core.client.ZooKeeperInstance
+import org.apache.accumulo.core.client.security.tokens.PasswordToken
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.accumulo.AccumuloTestBase
+import org.apache.slider.core.registry.zk.ZKIntegration
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestAccCorrectInstanceName extends AccumuloTestBase {
+
+ @Test
+ public void testAccM1T1GC1Mon1() throws Throwable {
+ String clustername = "test_acc_m1t1gc1mon1"
+ int tablets = 1
+ int monitor = 1
+ int gc = 1
+ createMiniCluster(clustername, getConfiguration(), 1, 1, 1, true, false)
+ describe(" Create an accumulo cluster");
+
+ //make sure that ZK is up and running at the binding string
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
+ log.info("ZK up at $zki");
+ //now launch the cluster
+ Map<String, Integer> roles = [
+ (AccumuloKeys.ROLE_MASTER): 1,
+ (AccumuloKeys.ROLE_TABLET): tablets,
+ (AccumuloKeys.ROLE_MONITOR): monitor,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): gc
+ ];
+ String password = "password"
+ List<String> extraArgs = [Arguments.ARG_OPTION, AccumuloKeys.OPTION_ACCUMULO_PASSWORD, password]
+ ServiceLauncher launcher = createAccCluster(clustername, roles, extraArgs, true, true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+
+ waitWhileClusterLive(sliderClient);
+ assert sliderClient.applicationReport.yarnApplicationState == YarnApplicationState.RUNNING
+ waitForRoleCount(sliderClient, roles, ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME)
+ describe("Cluster status")
+ ClusterDescription status
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+
+ //now give the cluster a bit of time to actually start work
+
+ log.info("Sleeping for a while")
+ sleep(ACCUMULO_GO_LIVE_TIME);
+
+ //verify that all is still there
+ waitForRoleCount(sliderClient, roles, 0)
+
+ // Making the connector validates that the instance name is correct, we have the right zk,
+ // and the proper username and password were created in accumulo
+ ZooKeeperInstance instance = new ZooKeeperInstance(System.getProperty("user.name") + "-" + clustername, zki.getConnectionString());
+ Connector c = instance.getConnector("root", new PasswordToken(password));
+
+ log.info("Finishing")
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+ maybeStopCluster(sliderClient, clustername, "shut down $clustername")
+
+ }
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFlexTablets133Mgr113.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFlexTablets133Mgr113.groovy
new file mode 100644
index 0000000..166c6c4
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFlexTablets133Mgr113.groovy
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.live
+
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.providers.accumulo.AccumuloTestBase
+import org.junit.Test
+
+@Slf4j
+class TestAccFlexTablets133Mgr113 extends AccumuloTestBase {
+
+ @Test
+ public void testAccFlexTablets133Mgr113() throws Throwable {
+ ClusterDescription cd = flexAccClusterTestRun(
+ "test_acc_flex_tablets133mgr113",
+ [
+ [
+ (AccumuloKeys.ROLE_MASTER) : 1,
+ (AccumuloKeys.ROLE_TABLET) : 1,
+ (AccumuloKeys.ROLE_MONITOR): 1,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): 1
+ ],
+ [
+ (AccumuloKeys.ROLE_MASTER) : 1,
+ (AccumuloKeys.ROLE_TABLET) : 3,
+ (AccumuloKeys.ROLE_MONITOR): 1,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): 1
+ ],
+ [
+ (AccumuloKeys.ROLE_MASTER) : 3,
+ (AccumuloKeys.ROLE_TABLET) : 3,
+ (AccumuloKeys.ROLE_MONITOR): 1,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): 1
+ ],
+
+ ]
+ )
+
+ log.info("Final CD \n{}", cd)
+ }
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFlexTablets1to3.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFlexTablets1to3.groovy
new file mode 100644
index 0000000..b084cdc
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFlexTablets1to3.groovy
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.providers.accumulo.AccumuloTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestAccFlexTablets1to3 extends AccumuloTestBase {
+
+ @Test
+ public void testAccFlexTablets1to3() throws Throwable {
+ Map<String, Integer> plan1 = (Map<String, Integer>) [
+ (AccumuloKeys.ROLE_MASTER): 1,
+ (AccumuloKeys.ROLE_TABLET): 1,
+ (AccumuloKeys.ROLE_MONITOR): 1,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): 1]
+
+ Map<String, Integer> plan2 = (Map<String, Integer>) [
+ (AccumuloKeys.ROLE_MASTER): 1,
+ (AccumuloKeys.ROLE_TABLET): 3,
+ (AccumuloKeys.ROLE_MONITOR): 1,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): 1]
+
+ ClusterDescription cd = flexAccClusterTestRun(
+ "test_acc_flex_tablets1to3",
+ [plan1, plan2]
+ )
+
+ log.info("Final CD \n{}", cd)
+ }
+
+
+
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
new file mode 100644
index 0000000..b3eecd2
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccFreezeThaw.groovy
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.accumulo.AccumuloTestBase
+import org.apache.slider.core.registry.zk.ZKIntegration
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestAccFreezeThaw extends AccumuloTestBase {
+
+ @Test
+ public void testAccFreezeThaw() throws Throwable {
+ String clustername = "test_acc_freeze_thaw"
+ int tablets = 1
+ int monitor = 1
+ int gc = 1
+ createMiniCluster(clustername, configuration, 1, 1, 1, true, false)
+ describe(" Create an accumulo cluster");
+
+ //make sure that ZK is up and running at the binding string
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
+ log.info("ZK up at $zki");
+ //now launch the cluster
+ Map<String, Integer> roles = [
+ (AccumuloKeys.ROLE_MASTER): 1,
+ (AccumuloKeys.ROLE_TABLET): tablets,
+ (AccumuloKeys.ROLE_MONITOR): monitor,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): gc
+ ];
+ ServiceLauncher launcher = createAccCluster(clustername, roles, [], true, true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+
+ waitForRoleCount(sliderClient, roles, ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME)
+ //now give the cluster a bit of time to actually start work
+
+ log.info("Sleeping for a while")
+ sleepForAccumuloClusterLive();
+ //verify that all is still there
+ waitForRoleCount(sliderClient, roles, 0, "extended cluster operation")
+
+ String page = fetchLocalPage(AccumuloConfigFileOptions.MONITOR_PORT_CLIENT_INT,
+ AccumuloKeys.MONITOR_PAGE_JSON)
+ log.info(page);
+
+ log.info("Freezing")
+ clusterActionFreeze(sliderClient, clustername, "freeze");
+ waitForAppToFinish(sliderClient)
+
+ //make sure the fetch fails
+ try {
+ page = fetchLocalPage(AccumuloConfigFileOptions.MONITOR_PORT_CLIENT_INT,
+ AccumuloKeys.MONITOR_PAGE_JSON)
+ //this should have failed
+ log.error("The accumulo monitor is still live")
+ log.error(page)
+ } catch (ConnectException expected) {
+ //
+ log.debug("expected exception", expected)
+ }
+ //force kill any accumulo processes
+ killAllAccumuloProcesses()
+
+ sleepForAccumuloClusterLive();
+
+ log.info("Thawing")
+
+ ServiceLauncher launcher2 = thawCluster(clustername, [], true);
+ SliderClient sliderClient2 = (SliderClient) launcher2.service
+ addToTeardown(sliderClient2)
+ waitForRoleCount(sliderClient, roles, ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME, "thawing")
+
+
+ sleepForAccumuloClusterLive();
+ //verify that all is still there
+ waitForRoleCount(sliderClient, roles, 0, "extended cluster operation after thawing")
+ page = fetchLocalPage(
+ AccumuloConfigFileOptions.MONITOR_PORT_CLIENT_INT,
+ AccumuloKeys.MONITOR_PAGE_JSON)
+ log.info(page);
+ }
+
+ public void sleepForAccumuloClusterLive() {
+ sleep(ACCUMULO_GO_LIVE_TIME)
+ }
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
new file mode 100644
index 0000000..7f16a6f
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveHDFSArchive.groovy
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.accumulo.AccumuloTestBase
+import org.apache.slider.core.registry.zk.ZKIntegration
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestAccLiveHDFSArchive extends AccumuloTestBase {
+
+ @Test
+ public void testAccLiveHDFSArchive() throws Throwable {
+ String clustername = "test_acc_live_hdfs_archive"
+ int tablets = 1
+ int monitor = 1
+ int gc = 1
+ createMiniCluster(clustername, configuration, 1, 1, 1, true, true)
+ describe(" Create an accumulo cluster from an archive");
+
+ enableTestRunAgainstUploadedArchive();
+ //make sure that ZK is up and running at the binding string
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
+ log.info("ZK up at $zki");
+ //now launch the cluster
+ Map<String, Integer> roles = [
+ (AccumuloKeys.ROLE_MASTER): 1,
+ (AccumuloKeys.ROLE_TABLET): tablets,
+ (AccumuloKeys.ROLE_MONITOR): monitor,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): gc
+ ];
+ ServiceLauncher launcher = createAccCluster(clustername, roles, [], true, true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+
+ waitWhileClusterLive(sliderClient);
+ assert sliderClient.applicationReport.yarnApplicationState == YarnApplicationState.RUNNING
+ waitForRoleCount(sliderClient, roles, ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME)
+ describe("Cluster status")
+ ClusterDescription status
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+
+ //now give the cluster a bit of time to actually start work
+
+ log.info("Sleeping for a while")
+ sleep(ACCUMULO_GO_LIVE_TIME);
+
+ //verify that all is still there
+ waitForRoleCount(sliderClient, roles, 0)
+
+ String page = fetchLocalPage(AccumuloConfigFileOptions.MONITOR_PORT_CLIENT_INT,
+ AccumuloKeys.MONITOR_PAGE_JSON)
+ log.info(page);
+
+ log.info("Finishing")
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+ maybeStopCluster(sliderClient, clustername, "shut down $clustername")
+
+ }
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
new file mode 100644
index 0000000..e8f6edb
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccLiveLocalArchive.groovy
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.accumulo.AccumuloTestBase
+import org.apache.slider.core.registry.zk.ZKIntegration
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestAccLiveLocalArchive extends AccumuloTestBase {
+
+ @Test
+ public void testAccLiveLocalArchive() throws Throwable {
+ String clustername = "test_acc_live_local_archive"
+ int tablets = 1
+ int monitor = 1
+ int gc = 1
+ createMiniCluster(clustername, getConfiguration(), 1, 1, 1, true, false)
+ describe(" Create an accumulo cluster from an archive");
+
+ //image mode
+ switchToImageDeploy = true
+
+ //make sure that ZK is up and running at the binding string
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
+ log.info("ZK up at $zki");
+ //now launch the cluster
+ Map<String, Integer> roles = [
+ (AccumuloKeys.ROLE_MASTER): 1,
+ (AccumuloKeys.ROLE_TABLET): tablets,
+ (AccumuloKeys.ROLE_MONITOR): monitor,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): gc
+ ];
+ ServiceLauncher launcher = createAccCluster(clustername, roles, [], true, true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+
+ waitWhileClusterLive(sliderClient);
+ assert sliderClient.applicationReport.yarnApplicationState == YarnApplicationState.RUNNING
+ waitForRoleCount(sliderClient, roles, ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME)
+ describe("Cluster status")
+ ClusterDescription status
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+
+ //now give the cluster a bit of time to actually start work
+
+ log.info("Sleeping for a while")
+ sleep(ACCUMULO_GO_LIVE_TIME);
+
+ //verify that all is still there
+ waitForRoleCount(sliderClient, roles, 0)
+
+ String page = fetchLocalPage(AccumuloConfigFileOptions.MONITOR_PORT_CLIENT_INT,
+ AccumuloKeys.MONITOR_PAGE_JSON)
+ log.info(page);
+
+ log.info("Finishing")
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+ maybeStopCluster(sliderClient, clustername, "shut down $clustername")
+
+ }
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
new file mode 100644
index 0000000..0db2602
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM1T1GC1Mon1.groovy
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.accumulo.AccumuloTestBase
+import org.apache.slider.core.registry.zk.ZKIntegration
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestAccM1T1GC1Mon1 extends AccumuloTestBase {
+
+ @Test
+ public void testAccM1T1GC1Mon1() throws Throwable {
+ String clustername = "test_acc_m1t1gc1mon1"
+ int tablets = 1
+ int monitor = 1
+ int gc = 1
+ createMiniCluster(clustername, getConfiguration(), 1, 1, 1, true, false)
+ describe(" Create an accumulo cluster");
+
+ //make sure that ZK is up and running at the binding string
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
+ log.info("ZK up at $zki");
+ //now launch the cluster
+ Map<String, Integer> roles = [
+ (AccumuloKeys.ROLE_MASTER): 1,
+ (AccumuloKeys.ROLE_TABLET): tablets,
+ (AccumuloKeys.ROLE_MONITOR): monitor,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): gc
+ ];
+ ServiceLauncher launcher = createAccCluster(clustername, roles, [], true, true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+
+ waitWhileClusterLive(sliderClient);
+ assert sliderClient.applicationReport.yarnApplicationState == YarnApplicationState.RUNNING
+ waitForRoleCount(sliderClient, roles, ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME)
+ describe("Cluster status")
+ ClusterDescription status
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+
+ //now give the cluster a bit of time to actually start work
+
+ log.info("Sleeping for a while")
+ sleep(ACCUMULO_GO_LIVE_TIME);
+
+ //verify that all is still there
+ waitForRoleCount(sliderClient, roles, 0)
+
+
+ log.info("Finishing")
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+ maybeStopCluster(sliderClient, clustername, "shut down $clustername")
+
+ }
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
new file mode 100644
index 0000000..bf403b8
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccM2T2GC1Mon1.groovy
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.accumulo.AccumuloTestBase
+import org.apache.slider.core.registry.zk.ZKIntegration
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestAccM2T2GC1Mon1 extends AccumuloTestBase {
+
+ @Test
+ public void testAccM1T1GC1Mon1() throws Throwable {
+ String clustername = "test_acc_m2t2gc1mon1"
+ int master = 2
+ int tablets = 2
+ int monitor = 1
+ int gc = 1
+ createMiniCluster(clustername, getConfiguration(), 1, 1, 1, true, false)
+ describe(" Create an accumulo cluster");
+
+ //make sure that ZK is up and running at the binding string
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
+ log.info("ZK up at $zki");
+ //now launch the cluster
+ Map<String, Integer> roles = [
+ (AccumuloKeys.ROLE_MASTER): master,
+ (AccumuloKeys.ROLE_TABLET): tablets,
+ (AccumuloKeys.ROLE_MONITOR): monitor,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): gc
+ ];
+ ServiceLauncher launcher = createAccCluster(clustername, roles, [], true, true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+
+ waitWhileClusterLive(sliderClient);
+ assert sliderClient.applicationReport.yarnApplicationState == YarnApplicationState.RUNNING
+ waitForRoleCount(sliderClient, roles, ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME)
+ describe("Cluster status")
+ ClusterDescription status
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+
+ //now give the cluster a bit of time to actually start work
+
+ log.info("Sleeping for a while")
+ sleep(ACCUMULO_GO_LIVE_TIME);
+
+ //verify that all is still there
+ waitForRoleCount(sliderClient, roles, 0)
+
+ String page = fetchLocalPage(AccumuloConfigFileOptions.MONITOR_PORT_CLIENT_INT,
+ AccumuloKeys.MONITOR_PAGE_JSON)
+ log.info(page);
+
+ log.info("Finishing")
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+ maybeStopCluster(sliderClient, clustername, "shut down $clustername")
+
+ }
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
new file mode 100644
index 0000000..9eb2717
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestAccumuloAMWebApp.groovy
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.accumulo.AccumuloConfigFileOptions
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.server.appmaster.web.SliderAMWebApp
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.accumulo.AccumuloTestBase
+import org.apache.slider.core.registry.zk.ZKIntegration
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestAccumuloAMWebApp extends AccumuloTestBase {
+
+ @Test
+ public void testAccumuloAMWebApp() throws Throwable {
+ String clustername = "test_accumulo_am_webapp"
+ int tablets = 1
+ int monitor = 1
+ int gc = 1
+ createMiniCluster(clustername, getConfiguration(), 1, 1, 1, true, false)
+ describe(" Create an accumulo cluster");
+
+ //make sure that ZK is up and running at the binding string
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
+ log.info("ZK up at $zki");
+ //now launch the cluster
+ Map<String, Integer> roles = [
+ (AccumuloKeys.ROLE_MASTER): 1,
+ (AccumuloKeys.ROLE_TABLET): tablets,
+ (AccumuloKeys.ROLE_MONITOR): monitor,
+ (AccumuloKeys.ROLE_GARBAGE_COLLECTOR): gc
+ ];
+ ServiceLauncher launcher = createAccCluster(clustername, roles, [], true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+
+
+ waitWhileClusterLive(sliderClient);
+ assert sliderClient.applicationReport.yarnApplicationState == YarnApplicationState.RUNNING
+ waitForRoleCount(sliderClient, roles, ACCUMULO_CLUSTER_STARTUP_TO_LIVE_TIME)
+ describe("Cluster status")
+ ClusterDescription status
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+
+ //now give the cluster a bit of time to actually start work
+
+ log.info("Sleeping for a while")
+ sleep(ACCUMULO_GO_LIVE_TIME);
+
+ //verify that all is still there
+ waitForRoleCount(sliderClient, roles, 0)
+
+ String page = fetchLocalPage(AccumuloConfigFileOptions.MONITOR_PORT_CLIENT_INT,
+ AccumuloKeys.MONITOR_PAGE_JSON)
+ log.info(page);
+
+ log.info("Finishing")
+
+ ApplicationReport appReport = sliderClient.getApplicationReport();
+
+ String url = appReport.getTrackingUrl();
+
+ // Should redirect to (or at least serve content from) SliderAMWebApp.BASE_PATH
+ fetchWebPageWithoutError(url);
+
+ // TrackingUrl has a trailing slash on it already for us (which is apparently very important)
+ // let's make sure it's there.
+ if ('/' != url.charAt(url.length() -1)) {
+ url = url + '/';
+ }
+
+ url = url + SliderAMWebApp.BASE_PATH;
+
+ // This should also give us content
+ fetchWebPageWithoutError(url);
+
+ url = url + SliderAMWebApp.CONTAINER_STATS;
+
+ fetchWebPageWithoutError(url);
+
+
+ status = sliderClient.getClusterDescription(clustername)
+ log.info(prettyPrint(status.toJsonString()))
+ maybeStopCluster(sliderClient, clustername, "shut down $clustername")
+
+ }
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
new file mode 100644
index 0000000..d86a158
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/live/TestInvalidMonitorAddress.groovy
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.core.main.ServiceLaunchException
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.providers.accumulo.AccumuloKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.accumulo.AccumuloTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+
+class TestInvalidMonitorAddress extends AccumuloTestBase {
+
+ @Test
+ public void testInvalidMonitorAddress() throws Throwable {
+ String clustername = "test_invalid_monitor_address"
+ createMiniCluster(clustername, configuration, 1, true)
+
+ describe "verify that bad Java heap options are picked up"
+
+ Map<String, Integer> roles = [
+ (AccumuloKeys.ROLE_MASTER): 1,
+ (AccumuloKeys.ROLE_TABLET): 1,
+ (AccumuloKeys.ROLE_MONITOR): 1,
+ ];
+ try {
+ ServiceLauncher launcher = createAccCluster(clustername, roles,
+ [
+ Arguments.ARG_COMP_OPT, AccumuloKeys.ROLE_MONITOR, RoleKeys.ROLE_ADDITIONAL_ARGS, "--address foobar",
+ ],
+ true,
+ true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.clusterDescription
+ dumpClusterDescription("Remote CD", status)
+ } catch (ServiceLaunchException e) {
+ assertExceptionDetails(e, SliderExitCodes.EXIT_YARN_SERVICE_FAILED)
+ }
+
+ }
+
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/java/org/apache/slider/providers/accumulo/Stub.java b/slider-providers/accumulo/slider-accumulo-provider/src/test/java/org/apache/slider/providers/accumulo/Stub.java
new file mode 100644
index 0000000..6b70a81
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/java/org/apache/slider/providers/accumulo/Stub.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo;
+
+public class Stub {
+}
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/resources/log4j.properties b/slider-providers/accumulo/slider-accumulo-provider/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a552a55
--- /dev/null
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/resources/log4j.properties
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+log4j.logger.org.apache.slider=DEBUG
+log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+
+
+
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
+log4j.logger.org.apache.hadoop.metrics2=ERROR
+log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
+log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN
+log4j.logger.org.apache.hadoop.security.token.delegation=WARN
+log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
diff --git a/slider-providers/hbase/hbase-funtests/pom.xml b/slider-providers/hbase/hbase-funtests/pom.xml
new file mode 100644
index 0000000..a2f3082
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/pom.xml
@@ -0,0 +1,248 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hbase-funtests</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ <name>Slider HBase Provider Functional Tests</name>
+ <packaging>jar</packaging>
+ <description>
+ Functional tests for the hbase provider
+
+ </description>
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ <relativePath>../../../</relativePath>
+ </parent>
+
+ <build>
+
+ <plugins>
+
+ <!--read in a build.properties file if defined-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <version>${maven.properties.version}</version>
+ <executions>
+ <execution>
+ <phase>initialize</phase>
+ <goals>
+ <goal>read-project-properties</goal>
+ </goals>
+ <configuration>
+ <quiet>true</quiet>
+ <files>
+ <file>build.properties</file>
+ <file>../../../build.properties</file>
+ </files>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler-plugin.version}</version>
+ <configuration>
+ <compilerId>groovy-eclipse-compiler</compilerId>
+ <!-- set verbose to be true if you want lots of uninteresting messages -->
+ <!-- <verbose>true</verbose> -->
+ <source>${project.java.src.version}</source>
+ <target>${project.java.src.version}</target>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>${groovy-eclipse-compiler.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-batch</artifactId>
+ <version>${groovy-eclipse-batch.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+
+ <!-- functional test -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ <configuration>
+
+ <!--mvn process fork options-->
+ <reuseForks>${test.reuseForks}</reuseForks>
+ <forkMode>${test.forkMode}</forkMode>
+ <forkCount>1</forkCount>
+ <forkedProcessTimeoutInSeconds>${test.forkedProcessTimeoutInSeconds}
+ </forkedProcessTimeoutInSeconds>
+ <threadCount>1</threadCount>
+ <argLine>${test.argLine}</argLine>
+ <failIfNoTests>${test.failIfNoTests}</failIfNoTests>
+
+ <trimStackTrace>false</trimStackTrace>
+ <redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
+ <systemPropertyVariables>
+ <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
+ <java.awt.headless>true</java.awt.headless>
+ <java.security.krb5.realm>${slider.test.java.security.krb5.realm}</java.security.krb5.realm>
+ <java.security.krb5.kdc>${slider.test.java.security.krb5.kdc}</java.security.krb5.kdc>
+ <!-- this property must be supplied-->
+ <slider.conf.dir>${slider.conf.dir}</slider.conf.dir>
+ <slider.bin.dir>../../../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/Test*.java</include>
+ </includes>
+ <excludes>
+ <exclude>**/Test*$*.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <reporting>
+ <plugins>
+
+ </plugins>
+ </reporting>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-hbase-provider</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-hbase-provider</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-assembly</artifactId>
+ <version>${project.version}</version>
+ <classifier>all</classifier>
+ <type>tar.gz</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-funtest</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+
+
+ </dependencies>
+
+
+</project>
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy
new file mode 100644
index 0000000..6e1f91b
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.slider.funtest.categories.FunctionalTests
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.SliderShell
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.providers.hbase.HBaseClientProvider
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.junit.Before
+import org.junit.BeforeClass
+
+import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
+
+/**
+ * Anything specific to HBase tests
+ */
+@org.junit.experimental.categories.Category(FunctionalTests)
+
+abstract class HBaseCommandTestBase extends CommandTestBase {
+ public static final boolean HBASE_TESTS_ENABLED
+ public static final int HBASE_LAUNCH_WAIT_TIME
+
+ static {
+ HBASE_TESTS_ENABLED =
+ SLIDER_CONFIG.getBoolean(KEY_TEST_HBASE_ENABLED, true)
+ HBASE_LAUNCH_WAIT_TIME = SLIDER_CONFIG.getInt(
+ KEY_TEST_HBASE_LAUNCH_TIME,
+ DEFAULT_HBASE_LAUNCH_TIME_SECONDS)
+ }
+
+ @BeforeClass
+ public static void extendClasspath() {
+ addExtraJar(HBaseClientProvider)
+ }
+
+ @Before
+ public void verifyPreconditions() {
+ assumeHBaseTestsEnabled()
+ getRequiredConfOption(SLIDER_CONFIG, KEY_TEST_HBASE_TAR)
+ getRequiredConfOption(SLIDER_CONFIG, KEY_TEST_HBASE_APPCONF)
+ }
+
+
+ public void assumeHBaseTestsEnabled() {
+ assumeFunctionalTestsEnabled()
+ assume(HBASE_TESTS_ENABLED, "HBase tests disabled")
+ }
+
+ /**
+ * Create an hbase cluster -this patches in the relevant attributes
+ * for the HBase cluster
+ * @param name name
+ * @param masters no. of master nodes
+ * @param workers no. of region servers
+ * @param argsList list of arguments
+ * @param clusterOps map of cluster options
+ * @return the role map -for use when waiting for the cluster to go live
+ */
+ public Map<String, Integer> createHBaseCluster(
+ String name,
+ int masters,
+ int workers,
+ List<String> argsList,
+ Map<String, String> clusterOps) {
+ Map<String, Integer> roleMap = [
+ (HBaseKeys.ROLE_MASTER): masters,
+ (HBaseKeys.ROLE_WORKER): workers
+ ]
+
+ argsList << Arguments.ARG_IMAGE <<
+ SLIDER_CONFIG.getTrimmed(KEY_TEST_HBASE_TAR)
+
+ argsList << Arguments.ARG_CONFDIR <<
+ SLIDER_CONFIG.getTrimmed(KEY_TEST_HBASE_APPCONF)
+
+ argsList << Arguments.ARG_PROVIDER << HBaseKeys.PROVIDER_HBASE
+
+ SliderShell shell = createSliderApplication(
+ name,
+ roleMap,
+ argsList,
+ true,
+ clusterOps
+ )
+ return roleMap
+ }
+
+
+ public String getDescription() {
+ return "Create an HBase"
+ }
+
+ public int getWorkerPortAssignment() {
+ return 0
+ }
+
+ public int getMasterPortAssignment() {
+ return 0
+ }
+
+}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterBuildDestroy.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterBuildDestroy.groovy
new file mode 100644
index 0000000..0fe67c3
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterBuildDestroy.groovy
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class TestClusterBuildDestroy extends HBaseCommandTestBase
+ implements FuntestProperties, Arguments {
+
+
+ static String CLUSTER = "test_cluster_build_destroy"
+
+
+ @BeforeClass
+ public static void prepareCluster() {
+ assumeFunctionalTestsEnabled();
+ setupCluster(CLUSTER)
+ }
+
+ @AfterClass
+ public static void destroyCluster() {
+ teardown(CLUSTER)
+ }
+
+ @Test
+ public void testBuildAndDestroyCluster() throws Throwable {
+ def clusterDir = SliderKeys.SLIDER_BASE_DIRECTORY + "/cluster/$CLUSTER"
+ def clusterDirPath = new Path(clusterFS.homeDirectory, clusterDir)
+ clusterFS.delete(clusterDirPath, true)
+ slider(0,
+ [
+ SliderActions.ACTION_BUILD,
+ CLUSTER,
+ ARG_PROVIDER, HBaseKeys.PROVIDER_HBASE,
+ ARG_ZKHOSTS,
+ SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
+ ARG_IMAGE,
+ SLIDER_CONFIG.get(KEY_TEST_HBASE_TAR),
+ ARG_CONFDIR,
+ SLIDER_CONFIG.get(KEY_TEST_HBASE_APPCONF),
+ ARG_COMPONENT, HBaseKeys.ROLE_MASTER, "1",
+ ARG_COMPONENT, HBaseKeys.ROLE_WORKER, "1",
+ ARG_OPTION, "site.hbase.master.info.port", "8180",
+ ])
+
+
+
+ assert clusterFS.exists(clusterDirPath)
+ //cluster exists if you don't want it to be live
+ exists(0, CLUSTER, false)
+ // condition returns false if it is required to be live
+ exists(LauncherExitCodes.EXIT_FALSE, CLUSTER, true)
+ destroy(CLUSTER)
+
+ }
+
+
+}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterLifecycle.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterLifecycle.groovy
new file mode 100644
index 0000000..0308a98
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterLifecycle.groovy
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.client.SliderClient
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class TestClusterLifecycle extends HBaseCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes {
+
+
+ static String CLUSTER = "test_cluster_lifecycle"
+
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(CLUSTER)
+ }
+
+ @After
+ public void destroyCluster() {
+ teardown(CLUSTER)
+ }
+
+ @Test
+ public void testClusterLifecycle() throws Throwable {
+
+ describe "Walk a 0-role cluster through its lifecycle"
+
+
+ def clusterpath = buildClusterPath(CLUSTER)
+ assert !clusterFS.exists(clusterpath)
+
+
+ Map<String, Integer> roleMap = createHBaseCluster(CLUSTER,
+ 0,
+ 0,
+ [],
+ [:])
+
+
+ //at this point the cluster should exist.
+ assertPathExists(clusterFS,"Cluster parent directory does not exist", clusterpath.parent)
+
+ assertPathExists(clusterFS,"Cluster directory does not exist", clusterpath)
+
+ // assert it exists on the command line
+ exists(0, CLUSTER)
+
+ //destroy will fail in use
+
+ destroy(EXIT_APPLICATION_IN_USE, CLUSTER)
+
+ //thaw will fail as cluster is in use
+ thaw(EXIT_APPLICATION_IN_USE, CLUSTER)
+
+ //it's still there
+ exists(0, CLUSTER)
+
+ //listing the cluster will succeed
+ list(0, CLUSTER)
+
+ //simple status
+ status(0, CLUSTER)
+
+ //now status to a temp file
+ File jsonStatus = File.createTempFile("tempfile", ".json")
+ try {
+ slider(0,
+ [
+ SliderActions.ACTION_STATUS, CLUSTER,
+ ARG_OUTPUT, jsonStatus.canonicalPath
+ ])
+
+ assert jsonStatus.exists()
+ ClusterDescription cd = ClusterDescription.fromFile(jsonStatus)
+
+ assert CLUSTER == cd.name
+
+ log.info(cd.toJsonString())
+
+ getConf(0, CLUSTER)
+
+ //get a slider client against the cluster
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
+ ClusterDescription cd2 = sliderClient.getClusterDescription()
+ assert CLUSTER == cd2.name
+
+ log.info("Connected via Client {}", sliderClient.toString())
+
+ //freeze
+ slider(0, [
+ SliderActions.ACTION_FREEZE, CLUSTER,
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "freeze-in-testHBaseCreateCluster"
+ ])
+
+ //cluster exists if you don't want it to be live
+ exists(0, CLUSTER, false)
+ // condition returns false if it is required to be live
+ exists(EXIT_FALSE, CLUSTER, true)
+
+
+ // thaw then freeze the cluster
+
+ slider(0,
+ [
+ SliderActions.ACTION_THAW, CLUSTER,
+ ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
+ ])
+ exists(0, CLUSTER)
+ slider(0, [
+ SliderActions.ACTION_FREEZE, CLUSTER,
+ ARG_FORCE,
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "forced-freeze-in-test"
+ ])
+
+ //cluster is no longer live
+ exists(0, CLUSTER, false)
+
+ // condition returns false if it is required to be live
+ exists(EXIT_FALSE, CLUSTER, true)
+
+ // thaw with a restart count set to enable restart
+
+ describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
+ slider(0,
+ [
+ SliderActions.ACTION_THAW, CLUSTER,
+ ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
+ ARG_DEFINE, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=3"
+ ])
+
+
+
+ ClusterDescription status = killAmAndWaitForRestart(sliderClient, CLUSTER)
+
+ def restarted = status.getInfo(
+ StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ assert restarted != null
+ assert Integer.parseInt(restarted) == 0
+ freeze(CLUSTER)
+
+ destroy(0, CLUSTER)
+
+ //cluster now missing
+ exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+
+ } finally {
+ jsonStatus.delete()
+ }
+
+
+ }
+
+
+}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy
new file mode 100644
index 0000000..abfd262
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.common.tools.ConfigHelper
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.HBaseConfigFileOptions
+import org.apache.slider.providers.hbase.HBaseTestUtils
+import org.apache.zookeeper.*
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_MASTER
+import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_WORKER
+
+@CompileStatic
+@Slf4j
+public class TestFunctionalHBaseCluster extends HBaseCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes {
+
+
+ public static final String HBASE_HEAP = "96m"
+
+ public String getClusterName() {
+ return "test_functional_hbase_cluster"
+ }
+
+ public String getClusterZNode() {
+ return "/yarnapps_slider_yarn_" + getClusterName();
+ }
+
+ @Before
+ public void prepareCluster() {
+
+ String quorumServers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS)
+
+ ZooKeeper monitor = new ZooKeeper(quorumServers,
+ 1000, new Watcher(){
+ @Override
+ public void process(WatchedEvent watchedEvent) {
+ }
+ }, false)
+ try {
+ ZKUtil.deleteRecursive(monitor, getClusterZNode())
+ } catch (KeeperException.NoNodeException ignored) {
+ log.info(getClusterZNode() + " not there")
+ }
+ setupCluster(clusterName)
+ }
+
+ @After
+ public void teardownCluster() {
+ teardown(clusterName)
+ }
+
+ @Test
+ public void testHBaseCreateCluster() throws Throwable {
+
+ describe description
+
+ int numWorkers = SLIDER_CONFIG.getInt(KEY_SLIDER_TEST_NUM_WORKERS,
+ DEFAULT_SLIDER_NUM_WORKERS);
+
+ def clusterpath = buildClusterPath(clusterName)
+ assert !clusterFS.exists(clusterpath)
+ Map<String, Integer> roleMap = createHBaseCluster(
+ clusterName,
+ 1, numWorkers,
+ [
+ ARG_OPTION,
+ HBaseConfigFileOptions.KEY_HBASE_MASTER_INFO_PORT,
+ Integer.toString(masterPortAssignment),
+ ARG_COMP_OPT, ROLE_MASTER, RoleKeys.JVM_HEAP, HBASE_HEAP,
+ ARG_OPTION,
+ HBaseConfigFileOptions.KEY_REGIONSERVER_PORT,
+ Integer.toString(workerPortAssignment),
+ ARG_COMP_OPT, ROLE_WORKER, RoleKeys.JVM_HEAP, HBASE_HEAP,
+ ],
+ [:]
+ )
+
+ //get a slider client against the cluster
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, clusterName)
+ ClusterDescription cd2 = sliderClient.getClusterDescription()
+ assert clusterName == cd2.name
+
+ log.info("Connected via Client {} with {} workers", sliderClient.toString(),
+ numWorkers)
+
+ //wait for the role counts to be reached
+ waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
+
+ Configuration clientConf = HBaseTestUtils.createHBaseConfiguration(sliderClient)
+ HBaseTestUtils.assertHBaseMasterFound(clientConf)
+ HBaseTestUtils.waitForHBaseRegionServerCount(sliderClient, clusterName,
+ numWorkers, HBASE_LAUNCH_WAIT_TIME)
+
+ clusterLoadOperations(clusterName, clientConf, numWorkers, roleMap, cd2)
+ }
+
+
+ public String getDescription() {
+ return "Create a working HBase cluster $clusterName"
+ }
+
+ /**
+ * Override point for any cluster load operations
+ * @param clientConf
+ * @param numWorkers
+ */
+ public void clusterLoadOperations(
+ String clustername,
+ Configuration clientConf,
+ int numWorkers,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+
+ log.info("Client Configuration = " + ConfigHelper.dumpConfigToString(clientConf))
+ }
+
+}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseBuildSetup.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseBuildSetup.groovy
new file mode 100644
index 0000000..c8f3be3
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseBuildSetup.groovy
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup
+import org.junit.Test
+
+class TestHBaseBuildSetup extends AbstractTestBuildSetup {
+
+ @Test
+ public void testHBaseBuildsHavePathsDefined() throws Throwable {
+ Configuration conf = loadSliderConf();
+ assumeBoolOption(conf, KEY_SLIDER_FUNTESTS_ENABLED, true)
+
+ assumeBoolOption(conf, KEY_TEST_HBASE_ENABLED, true)
+
+ assertStringOptionSet(conf, KEY_TEST_HBASE_APPCONF)
+ assertStringOptionSet(conf, KEY_TEST_HBASE_TAR)
+ }
+}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseIntegration.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseIntegration.groovy
new file mode 100644
index 0000000..eab93a5
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseIntegration.groovy
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.IntegrationTestIngest
+import org.apache.hadoop.hbase.IntegrationTestingUtility
+import org.apache.hadoop.util.ToolRunner
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.hbase.HBaseConfigFileOptions;
+
+/* Runs IntegrationTestIngest on cluster
+ *
+ * Note: this test runs for about 20 minutes
+ * please set slider.test.timeout.seconds accordingly
+ */
+class TestHBaseIntegration extends TestFunctionalHBaseCluster {
+
+ @Override
+ String getClusterName() {
+ return "test_hbase_integration"
+ }
+
+ @Override
+ void clusterLoadOperations(
+ String clustername,
+ Configuration clientConf,
+ int numWorkers,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+ String parent = "/yarnapps_slider_yarn_" + clustername
+ clientConf.set(HBaseConfigFileOptions.KEY_ZNODE_PARENT, parent)
+
+ clientConf.set(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER, "true")
+
+ String[] args = []
+ IntegrationTestIngest test = new IntegrationTestIngest();
+ test.setConf(clientConf)
+ int ret = ToolRunner.run(clientConf, test, args);
+ assert ret == 0;
+ }
+
+
+ public int getWorkerPortAssignment() {
+ return 0
+ }
+
+ public int getMasterPortAssignment() {
+ return 0
+ }
+}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseLoad.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseLoad.groovy
new file mode 100644
index 0000000..eb2e03a
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseLoad.groovy
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.util.LoadTestTool
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.hbase.HBaseConfigFileOptions
+
+class TestHBaseLoad extends TestFunctionalHBaseCluster {
+
+ @Override
+ String getClusterName() {
+ return "test_hbase_load"
+ }
+
+ @Override
+ void clusterLoadOperations(
+ String clustername,
+ Configuration clientConf,
+ int numWorkers,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+ assert clustername
+ int numKeys = 4000 * numWorkers
+ String[] args = ["-tn", "test", "-write", "4:100",
+ "-num_keys", numKeys,
+ "-zk", clientConf.get(HBaseConfigFileOptions.KEY_ZOOKEEPER_QUORUM),
+ "-zk_root", clientConf.get(HBaseConfigFileOptions.KEY_ZNODE_PARENT),
+
+ ]
+ LoadTestTool loadTool = new LoadTestTool();
+ loadTool.setConf(clientConf)
+ int ret = loadTool.run(args);
+ assert ret == 0;
+ }
+
+
+ public int getWorkerPortAssignment() {
+ return 0
+ }
+
+ public int getMasterPortAssignment() {
+ return 0
+ }
+}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseNodeFailure.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseNodeFailure.groovy
new file mode 100644
index 0000000..b26ccf8
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseNodeFailure.groovy
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionKillContainerArgs
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.providers.hbase.HBaseTestUtils
+
+class TestHBaseNodeFailure extends TestFunctionalHBaseCluster {
+
+
+ public static final int RESTART_SLEEP_TIME = 5000
+
+ @Override
+ String getClusterName() {
+ return "test_hbase_node_failure"
+ }
+
+ @Override
+ String getDescription() {
+ "Fail containers and verify that the cluster recovers"
+ }
+
+ @Override
+ void clusterLoadOperations(
+ String clustername,
+ Configuration clientConf,
+ int numWorkers,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, clusterName)
+
+
+ killInstanceOfRole(sliderClient, HBaseKeys.ROLE_WORKER)
+ // let it take
+ sleep(RESTART_SLEEP_TIME)
+
+ //wait for the role counts to be reached
+ cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
+ // then expect a restart
+ HBaseTestUtils.waitForHBaseRegionServerCount(
+ sliderClient,
+ clusterName,
+ numWorkers,
+ HBASE_LAUNCH_WAIT_TIME)
+ assert cd.roles[HBaseKeys.ROLE_WORKER][RoleKeys.ROLE_FAILED_INSTANCES] == "1"
+ killInstanceOfRole(sliderClient, HBaseKeys.ROLE_WORKER)
+ // let it take
+ sleep(RESTART_SLEEP_TIME)
+ // then expect a restart
+
+ //wait for the role counts to be reached
+ cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
+
+ HBaseTestUtils.waitForHBaseRegionServerCount(
+ sliderClient,
+ clusterName,
+ numWorkers,
+ HBASE_LAUNCH_WAIT_TIME)
+ assert cd.roles[HBaseKeys.ROLE_WORKER][RoleKeys.ROLE_FAILED_INSTANCES] == "2"
+
+ killInstanceOfRole(sliderClient, HBaseKeys.ROLE_MASTER)
+ // let it take
+ sleep(RESTART_SLEEP_TIME)
+
+ // wait for the role counts to be reached
+ cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
+ HBaseTestUtils.waitForHBaseRegionServerCount(
+ sliderClient,
+ clusterName,
+ numWorkers,
+ HBASE_LAUNCH_WAIT_TIME)
+ assert cd.roles[HBaseKeys.ROLE_MASTER][RoleKeys.ROLE_FAILED_INSTANCES] == "1"
+
+ // now trigger AM failure
+ ClusterDescription status = killAmAndWaitForRestart(sliderClient, clusterName)
+
+ def restarted = status.getInfo(
+ StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ assert restarted != null
+ assert Integer.parseInt(restarted) == 1 + numWorkers
+
+ }
+
+ /**
+ * Kill a random in instance of a role in the cluster
+ * @param sliderClient client
+ * @param role
+ * @return ID of container killed
+ */
+ public String killInstanceOfRole(
+ SliderClient sliderClient, String role) {
+ ClusterDescription cd = sliderClient.getClusterDescription()
+ def instances = cd.instances[role]
+ if (instances == null || instances.size() == 0) {
+ log.info("No instances of role $role to kill")
+ return null;
+ }
+ String id = instances[new Random().nextInt(instances.size())]
+ ActionKillContainerArgs args = new ActionKillContainerArgs()
+ args.id = id
+ sliderClient.actionKillContainer(clusterName, args)
+ return id;
+ }
+
+
+ public int getWorkerPortAssignment() {
+ return 0
+ }
+
+ public int getMasterPortAssignment() {
+ return 0
+ }
+}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestImages.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestImages.groovy
new file mode 100644
index 0000000..128e087
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestImages.groovy
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.junit.Before
+import org.junit.Test
+
+class TestImages extends CommandTestBase implements FuntestProperties {
+
+
+ @Before
+ public void verifyPreconditions() {
+ assumeBoolOption(SLIDER_CONFIG, KEY_SLIDER_FUNTESTS_ENABLED, true)
+ assumeBoolOption(SLIDER_CONFIG, KEY_TEST_HBASE_ENABLED, true)
+ }
+
+ @Test
+ public void testImageExists() throws Throwable {
+
+ Configuration conf = loadSliderConf()
+ String testImage = conf.get(KEY_TEST_HBASE_TAR)
+ assert testImage
+ Path path = new Path(testImage)
+ HadoopFS fs = HadoopFS.get(
+ path.toUri(),
+ conf)
+ assert fs.exists(path)
+ }
+
+ @Test
+ public void testAppConfExists() throws Throwable {
+ Configuration conf = loadSliderConf()
+ String dir = conf.get(KEY_TEST_HBASE_APPCONF)
+
+ assert conf.get(KEY_TEST_HBASE_APPCONF)
+ Path path = new Path(dir)
+ HadoopFS fs = HadoopFS.get(
+ path.toUri(),
+ conf)
+ assert fs.exists(path)
+ }
+
+
+}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/java/org/apache/slider/providers/hbase/funtest/StubCompile.java b/slider-providers/hbase/hbase-funtests/src/test/java/org/apache/slider/providers/hbase/funtest/StubCompile.java
new file mode 100644
index 0000000..d426635
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/java/org/apache/slider/providers/hbase/funtest/StubCompile.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest;
+
+class StubCompile {
+}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/resources/log4j.properties b/slider-providers/hbase/hbase-funtests/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a552a55
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/resources/log4j.properties
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+log4j.logger.org.apache.slider=DEBUG
+log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+
+
+
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
+log4j.logger.org.apache.hadoop.metrics2=ERROR
+log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
+log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN
+log4j.logger.org.apache.hadoop.security.token.delegation=WARN
+log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
diff --git a/slider-providers/hbase/slider-hbase-provider/pom.xml b/slider-providers/hbase/slider-hbase-provider/pom.xml
new file mode 100644
index 0000000..88d1eca
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/pom.xml
@@ -0,0 +1,242 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-hbase-provider</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ <name>Slider HBase Provider</name>
+ <packaging>jar</packaging>
+ <description>
+ Direct provider for slider. This is the original provider from Hoya, moved to one side
+ as it is no longer being actively developed. It contains tests against
+ mini clusters as well as full functional tests
+
+ </description>
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.23.0-SNAPSHOT</version>
+ <relativePath>../../../</relativePath>
+ </parent>
+
+ <build>
+
+ <!-- resources are filtered for dynamic updates. This gets build info in-->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
+ <plugins>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler-plugin.version}</version>
+ <configuration>
+ <compilerId>groovy-eclipse-compiler</compilerId>
+ <!-- set verbose to be true if you want lots of uninteresting messages -->
+ <!-- <verbose>true</verbose> -->
+ <source>${project.java.src.version}</source>
+ <target>${project.java.src.version}</target>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>${groovy-eclipse-compiler.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-batch</artifactId>
+ <version>${groovy-eclipse-batch.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${maven-jar-plugin.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- test -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ <configuration>
+ <!--mvn process fork options-->
+ <reuseForks>${test.reuseForks}</reuseForks>
+ <forkMode>${test.forkMode}</forkMode>
+ <forkCount>1</forkCount>
+ <forkedProcessTimeoutInSeconds>${test.forkedProcessTimeoutInSeconds}</forkedProcessTimeoutInSeconds>
+ <threadCount>1</threadCount>
+ <argLine>${test.argLine}</argLine>
+ <failIfNoTests>${test.failIfNoTests}</failIfNoTests>
+
+ <trimStackTrace>false</trimStackTrace>
+ <redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
+ <systemPropertyVariables>
+ <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
+ <java.awt.headless>true</java.awt.headless>
+ <java.security.krb5.realm>${slider.test.java.security.krb5.realm}</java.security.krb5.realm>
+ <java.security.krb5.kdc>${slider.test.java.security.krb5.kdc}</java.security.krb5.kdc>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/Test*.java</include>
+ </includes>
+ <excludes>
+ <exclude>**/Test*$*.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>src/main/resources/org/apache/slider/providers/hbase/conf/regionservers</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <reporting>
+ <plugins>
+
+
+
+ </plugins>
+ </reporting>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+
+</project>
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
new file mode 100644
index 0000000..7627829
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.registry.zk.ZookeeperUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This class implements the client-side aspects
+ * of an HBase Cluster
+ */
+public class HBaseClientProvider extends AbstractClientProvider implements
+ HBaseKeys, SliderKeys,
+ HBaseConfigFileOptions {
+
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(HBaseClientProvider.class);
+ protected static final String NAME = "hbase";
+ private static final String INSTANCE_RESOURCE_BASE = "/org/apache/slider/providers/hbase/instance/";
+
+
+ protected HBaseClientProvider(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public List<ProviderRole> getRoles() {
+ return HBaseRoles.getRoles();
+ }
+
+
+ @Override
+ public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws
+ SliderException,
+ IOException {
+ String resourceTemplate = INSTANCE_RESOURCE_BASE + "resources.json";
+ String appConfTemplate = INSTANCE_RESOURCE_BASE + "appconf.json";
+ mergeTemplates(aggregateConf, null, resourceTemplate, appConfTemplate);
+ }
+
+ /**
+ * Build the hdfs-site.xml file
+ * This the configuration used by HBase directly
+ * @param instanceDescription this is the cluster specification used to define this
+ * @return a map of the dynamic bindings for this Slider instance
+ */
+ public Map<String, String> buildSiteConfFromInstance(
+ AggregateConf instanceDescription)
+ throws BadConfigException {
+
+
+ ConfTreeOperations appconf =
+ instanceDescription.getAppConfOperations();
+
+ MapOperations globalAppOptions = appconf.getGlobalOptions();
+ MapOperations globalInstanceOptions = instanceDescription.getInternalOperations().getGlobalOptions();
+ MapOperations master = appconf.getMandatoryComponent(HBaseKeys.ROLE_MASTER);
+
+ MapOperations worker = appconf.getMandatoryComponent(HBaseKeys.ROLE_WORKER);
+
+ Map<String, String> sitexml = new HashMap<String, String>();
+
+ //map all cluster-wide site. options
+ providerUtils.propagateSiteOptions(globalAppOptions, sitexml);
+/*
+ //this is where we'd do app-indepdenent keytabs
+
+ String keytab =
+ clusterSpec.getOption(OptionKeys.OPTION_KEYTAB_LOCATION, "");
+
+*/
+
+
+ sitexml.put(KEY_HBASE_ROOTDIR,
+ globalInstanceOptions.getMandatoryOption(
+ OptionKeys.INTERNAL_DATA_DIR_PATH)
+ );
+ providerUtils.propagateOption(globalAppOptions, OptionKeys.ZOOKEEPER_PATH,
+ sitexml, KEY_ZNODE_PARENT);
+ String quorum =
+ globalAppOptions.getMandatoryOption(OptionKeys.ZOOKEEPER_QUORUM);
+ sitexml.put(KEY_ZOOKEEPER_QUORUM, ZookeeperUtils.convertToHostsOnlyList(quorum));
+ //and assume first port works everywhere
+ sitexml.put(KEY_ZOOKEEPER_PORT,
+ Integer.toString(ZookeeperUtils.getFirstPort(quorum, HBASE_ZK_PORT)));
+
+ return sitexml;
+ }
+
+ @Override //Client
+ public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
+ String clustername,
+ Configuration configuration,
+ AggregateConf instanceDefinition,
+ Path clusterDirPath,
+ Path generatedConfDirPath,
+ boolean secure) throws
+ SliderException,
+ IOException {
+ super.preflightValidateClusterConfiguration(sliderFileSystem, clustername,
+ configuration,
+ instanceDefinition,
+ clusterDirPath,
+ generatedConfDirPath, secure);
+
+ Path templatePath = new Path(generatedConfDirPath, HBaseKeys.SITE_XML);
+ //load the HBase site file or fail
+ Configuration siteConf = ConfigHelper.loadConfiguration(sliderFileSystem.getFileSystem(),
+ templatePath);
+
+ //core customizations
+ validateHBaseSiteXML(siteConf, secure, templatePath.toString());
+
+ }
+
+ /**
+ * Validate the hbase-site.xml values
+ * @param siteConf site config
+ * @param secure secure flag
+ * @param origin origin for exceptions
+ * @throws BadConfigException if a config is missing/invalid
+ */
+ public void validateHBaseSiteXML(Configuration siteConf,
+ boolean secure,
+ String origin) throws BadConfigException {
+ try {
+ SliderUtils.verifyOptionSet(siteConf, KEY_HBASE_CLUSTER_DISTRIBUTED,
+ false);
+ SliderUtils.verifyOptionSet(siteConf, KEY_HBASE_ROOTDIR, false);
+ SliderUtils.verifyOptionSet(siteConf, KEY_ZNODE_PARENT, false);
+ SliderUtils.verifyOptionSet(siteConf, KEY_ZOOKEEPER_QUORUM, false);
+ SliderUtils.verifyOptionSet(siteConf, KEY_ZOOKEEPER_PORT, false);
+ int zkPort =
+ siteConf.getInt(HBaseConfigFileOptions.KEY_ZOOKEEPER_PORT, 0);
+ if (zkPort == 0) {
+ throw new BadConfigException(
+ "ZK port property not provided at %s in configuration file %s",
+ HBaseConfigFileOptions.KEY_ZOOKEEPER_PORT,
+ siteConf);
+ }
+
+ if (secure) {
+ //better have the secure cluster definition up and running
+ SliderUtils.verifyOptionSet(siteConf, KEY_MASTER_KERBEROS_PRINCIPAL,
+ false);
+ SliderUtils.verifyOptionSet(siteConf, KEY_MASTER_KERBEROS_KEYTAB,
+ false);
+ SliderUtils.verifyOptionSet(siteConf,
+ KEY_REGIONSERVER_KERBEROS_PRINCIPAL,
+ false);
+ SliderUtils.verifyOptionSet(siteConf,
+ KEY_REGIONSERVER_KERBEROS_KEYTAB, false);
+ }
+ } catch (BadConfigException e) {
+ //bad configuration, dump it
+
+ log.error("Bad site configuration {} : {}", origin, e, e);
+ log.info(ConfigHelper.dumpConfigToString(siteConf));
+ throw e;
+ }
+ }
+
+ private static Set<String> knownRoleNames = new HashSet<String>();
+ static {
+ List<ProviderRole> roles = HBaseRoles.getRoles();
+ knownRoleNames.add(SliderKeys.COMPONENT_AM);
+ for (ProviderRole role : roles) {
+ knownRoleNames.add(role.name);
+ }
+ }
+
+ /**
+ * Validate the instance definition.
+ * @param instanceDefinition instance definition
+ */
+ @Override
+ public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ SliderException {
+ super.validateInstanceDefinition(instanceDefinition);
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+ Set<String> unknownRoles = resources.getComponentNames();
+ unknownRoles.removeAll(knownRoleNames);
+ if (!unknownRoles.isEmpty()) {
+ throw new BadCommandArgumentsException("Unknown component: %s",
+ unknownRoles.iterator().next());
+ }
+ providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_WORKER,
+ 0, -1);
+ providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_MASTER,
+ 0, -1);
+
+ }
+
+ @Override
+ public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
+ Configuration serviceConf,
+ AbstractLauncher launcher,
+ AggregateConf instanceDescription,
+ Path snapshotConfDirPath,
+ Path generatedConfDirPath,
+ Configuration clientConfExtras,
+ String libdir,
+ Path tempPath, boolean miniClusterTestRun) throws
+ IOException,
+ SliderException {
+
+ // add any and all dependency files
+ Map<String, LocalResource> providerResources =
+ new HashMap<String, LocalResource>();
+
+ ProviderUtils.addProviderJar(providerResources,
+ this,
+ "slider-hbase-provider.jar",
+ fileSystem,
+ tempPath,
+ libdir, miniClusterTestRun);
+
+ Class<?>[] classes = {
+ };
+ String[] jars =
+ {
+ };
+ ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath,
+ libdir, jars,
+ classes);
+
+
+ launcher.addLocalResources(providerResources);
+
+ //load in the template site config
+ log.debug("Loading template configuration from {}", snapshotConfDirPath);
+ Configuration siteConf = ConfigHelper.loadTemplateConfiguration(
+ serviceConf,
+ snapshotConfDirPath,
+ HBaseKeys.SITE_XML,
+ HBaseKeys.HBASE_TEMPLATE_RESOURCE);
+
+ if (log.isDebugEnabled()) {
+ log.debug("Configuration came from {}",
+ siteConf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN));
+ ConfigHelper.dumpConf(siteConf);
+ }
+ //construct the cluster configuration values
+
+ ConfTreeOperations appconf =
+ instanceDescription.getAppConfOperations();
+
+
+ Map<String, String> clusterConfMap = buildSiteConfFromInstance(
+ instanceDescription);
+
+ //merge them
+ ConfigHelper.addConfigMap(siteConf,
+ clusterConfMap.entrySet(),
+ "HBase Provider");
+
+ //now, if there is an extra client conf, merge it in too
+ if (clientConfExtras != null) {
+ ConfigHelper.mergeConfigurations(siteConf, clientConfExtras,
+ "Slider Client");
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("Merged Configuration");
+ ConfigHelper.dumpConf(siteConf);
+ }
+
+ Path sitePath = ConfigHelper.saveConfig(serviceConf,
+ siteConf,
+ generatedConfDirPath,
+ HBaseKeys.SITE_XML);
+
+ log.debug("Saving the config to {}", sitePath);
+ launcher.submitDirectory(generatedConfDirPath,
+ SliderKeys.PROPAGATED_CONF_DIR_NAME);
+
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseConfigFileOptions.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseConfigFileOptions.java
new file mode 100644
index 0000000..ac1a6ac
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseConfigFileOptions.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase;
+
+/**
+ * Mappings of config params to env variables for
+ * custom -site.xml files to pick up
+ *
+ * A lot of these come from HConstants -the reason they have been copied
+ * and pasted in here is to remove dependencies on HBase from
+ * the Slider Client and AM.
+ */
+public interface HBaseConfigFileOptions {
+
+ String KEY_HBASE_CLUSTER_DISTRIBUTED = "hbase.cluster.distributed";
+ String KEY_HBASE_ROOTDIR = "hbase.rootdir";
+
+ String KEY_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
+ //HConstants.ZOOKEEPER_QUORUM;
+ String KEY_ZOOKEEPER_PORT = "hbase.zookeeper.property.clientPort";
+ //HConstants.ZOOKEEPER_CLIENT_PORT;
+ String KEY_ZNODE_PARENT = "zookeeper.znode.parent";
+
+
+ int DEFAULT_MASTER_PORT = 60000;
+ int DEFAULT_MASTER_INFO_PORT = 60010;
+
+ String KEY_HBASE_MASTER_PORT = "hbase.master.port";
+ String KEY_HBASE_MASTER_INFO_PORT = "hbase.master.info.port";
+
+ int HBASE_ZK_PORT = 2181; // HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT;
+
+
+ String KEY_REGIONSERVER_PORT = "hbase.regionserver.port";
+ String KEY_REGIONSERVER_INFO_PORT = "hbase.regionserver.info.port";
+
+ /**
+ * needed to relax constraints in negotiations, including tests
+ */
+ String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH =
+ "ipc.client.fallback-to-simple-auth-allowed";
+
+ /*
+ <property>
+ <name>hbase.regionserver.kerberos.principal</name>
+ <value>hbase/_HOST@YOUR-REALM.COM</value>
+ </property>
+ <property>
+ <name>hbase.regionserver.keytab.file</name>
+ <value>/etc/hbase/conf/keytab.krb5</value>
+ </property>
+ <property>
+ <name>hbase.master.kerberos.principal</name>
+ <value>hbase/_HOST@YOUR-REALM.COM</value>
+ </property>
+ <property>
+ <name>hbase.master.keytab.file</name>
+ <value>/etc/hbase/conf/keytab.krb5</value>
+ </property>
+ */
+
+
+ String KEY_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal";
+ String KEY_REGIONSERVER_KERBEROS_KEYTAB = "hbase.regionserver.keytab.file";
+
+ String KEY_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal";
+ String KEY_MASTER_KERBEROS_KEYTAB = "hbase.master.keytab.file";
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
new file mode 100644
index 0000000..ed3cde3
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase;
+
+public interface HBaseKeys {
+
+ /** {@value */
+ String MASTER = "master";
+ String ROLE_WORKER = "worker";
+
+ String ROLE_MASTER = MASTER;
+
+ /** {@value */
+ String REGION_SERVER = "regionserver";
+
+ /**
+ * What is the command for hbase to print a version: {@value}
+ */
+ String COMMAND_VERSION = "version";
+
+ String ACTION_START = "start";
+ String ACTION_STOP = "stop";
+
+ /**
+ * Config directory : {@value}
+ */
+ String ARG_CONFIG = "--config";
+ /**
+ * name of the hbase script relative to the hbase root dir: {@value}
+ */
+ String HBASE_SCRIPT = "hbase";
+
+ /**
+ * name of the site conf to generate : {@value}
+ */
+ String SITE_XML = "hbase-site.xml";
+ /**
+ * Template stored in the slider classpath -to use if there is
+ * no site-specific template
+ * {@value}
+ */
+ String HBASE_CONF_RESOURCE = "org/apache/slider/providers/hbase/conf/";
+ String HBASE_TEMPLATE_RESOURCE = HBASE_CONF_RESOURCE + SITE_XML;
+
+
+ String DEFAULT_HBASE_WORKER_HEAP = "512M";
+ String DEFAULT_HBASE_MASTER_HEAP = "512M";
+ String DEFAULT_HBASE_WORKER_INFOPORT = "0";
+ String DEFAULT_HBASE_MASTER_INFOPORT = "0";
+ String PROVIDER_HBASE = "hbase";
+ String HBASE_LOG_DIR = "HBASE_LOG_DIR";
+
+}
+
+
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderFactory.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderFactory.java
new file mode 100644
index 0000000..aa10cbd
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderFactory.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.SliderProviderFactory;
+import org.apache.slider.providers.ProviderService;
+
+public class HBaseProviderFactory extends SliderProviderFactory {
+
+ public HBaseProviderFactory() {
+ }
+
+ public HBaseProviderFactory(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return new HBaseClientProvider(getConf());
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return new HBaseProviderService();
+ }
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
new file mode 100644
index 0000000..0322844
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.SliderInternalStateException;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
+import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat;
+import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse;
+import org.apache.slider.server.appmaster.web.rest.agent.Register;
+import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
+import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
+import org.apache.slider.server.services.docstore.utility.EventCallback;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * This class implements the server-side aspects
+ * of an HBase Cluster
+ */
+public class HBaseProviderService extends AbstractProviderService implements
+ ProviderCore,
+ HBaseKeys,
+ SliderKeys,
+ AgentRestOperations{
+
+ public static final String ERROR_UNKNOWN_ROLE = "Unknown role ";
+ protected static final Logger log =
+ LoggerFactory.getLogger(HBaseProviderService.class);
+ protected static final String NAME = "hbase";
+ private static final ProviderUtils providerUtils = new ProviderUtils(log);
+ private HBaseClientProvider clientProvider;
+ private Configuration siteConf;
+ private SliderFileSystem sliderFileSystem = null;
+
+ public HBaseProviderService() {
+ super("HBaseProviderService");
+ setAgentRestOperations(this);
+ }
+
+ @Override
+ public List<ProviderRole> getRoles() {
+ return HBaseRoles.getRoles();
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ clientProvider = new HBaseClientProvider(conf);
+ }
+
+ @Override
+ public Configuration loadProviderConfigurationInformation(File confDir)
+ throws BadCommandArgumentsException, IOException {
+
+ return loadProviderConfigurationInformation(confDir, SITE_XML);
+ }
+
+ /**
+ * Validate the cluster specification. This can be invoked on both
+ * server and client
+ * @param instanceDefinition
+ */
+ @Override // Client and Server
+ public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ SliderException {
+ clientProvider.validateInstanceDefinition(instanceDefinition);
+ }
+
+ @Override
+ public void buildContainerLaunchContext(ContainerLauncher launcher,
+ AggregateConf instanceDefinition,
+ Container container,
+ String role,
+ SliderFileSystem sliderFileSystem,
+ Path generatedConfPath,
+ MapOperations resourceComponent,
+ MapOperations appComponent,
+ Path containerTmpDirPath) throws IOException, SliderException {
+
+ this.sliderFileSystem = sliderFileSystem;
+ this.instanceDefinition = instanceDefinition;
+ // Set the environment
+ launcher.putEnv(SliderUtils.buildEnvMap(appComponent));
+
+ launcher.setEnv(HBASE_LOG_DIR, providerUtils.getLogdir());
+
+ launcher.setEnv("PROPAGATED_CONFDIR",
+ ProviderUtils.convertToAppRelativePath(
+ SliderKeys.PROPAGATED_CONF_DIR_NAME) );
+
+
+ //local resources
+
+ //add the configuration resources
+ launcher.addLocalResources(sliderFileSystem.submitDirectory(
+ generatedConfPath,
+ SliderKeys.PROPAGATED_CONF_DIR_NAME));
+ //Add binaries
+ //now add the image if it was set
+ String imageURI = instanceDefinition.getInternalOperations().get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ sliderFileSystem.maybeAddImagePath(launcher.getLocalResources(), imageURI);
+
+ CommandLineBuilder cli = new CommandLineBuilder();
+
+ String heap = appComponent.getOption(RoleKeys.JVM_HEAP, DEFAULT_JVM_HEAP);
+ if (SliderUtils.isSet(heap)) {
+ String adjustedHeap = SliderUtils.translateTrailingHeapUnit(heap);
+ launcher.setEnv("HBASE_HEAPSIZE", adjustedHeap);
+ }
+
+ String gcOpts = appComponent.getOption(RoleKeys.GC_OPTS, DEFAULT_GC_OPTS);
+ if (SliderUtils.isSet(gcOpts)) {
+ launcher.setEnv("SERVER_GC_OPTS", gcOpts);
+ }
+
+ //this must stay relative if it is an image
+ cli.add(providerUtils.buildPathToScript(
+ instanceDefinition,
+ "bin",
+ HBASE_SCRIPT));
+ //config dir is relative to the generated file
+ cli.add(ARG_CONFIG);
+ cli.add("$PROPAGATED_CONFDIR");
+
+ String roleCommand;
+ String logfile;
+ //now look at the role
+ if (ROLE_WORKER.equals(role)) {
+ //role is region server
+ roleCommand = REGION_SERVER;
+ logfile = "/region-server.txt";
+ } else if (ROLE_MASTER.equals(role)) {
+ roleCommand = MASTER;
+
+ logfile ="/master.txt";
+ } else {
+ throw new SliderInternalStateException("Cannot start role %s", role);
+ }
+
+ cli.add(roleCommand);
+ cli.add(ACTION_START);
+ //log details
+ cli.addOutAndErrFiles(logfile, null);
+ launcher.addCommand(cli.build());
+
+ }
+
+ /**
+ * Run this service
+ *
+ *
+ * @param instanceDefinition component description
+ * @param confDir local dir with the config
+ * @param env environment variables above those generated by
+ * @param execInProgress callback for the event notification
+ * @throws IOException IO problems
+ * @throws SliderException anything internal
+ */
+ @Override
+ public boolean exec(AggregateConf instanceDefinition,
+ File confDir,
+ Map<String, String> env,
+ EventCallback execInProgress) throws
+ IOException,
+ SliderException {
+
+ return false;
+ }
+
+
+ /**
+ * This is a validation of the application configuration on the AM.
+ * Here is where things like the existence of keytabs and other
+ * not-seen-client-side properties can be tested, before
+ * the actual process is spawned.
+ * @param instanceDefinition clusterSpecification
+ * @param confDir configuration directory
+ * @param secure flag to indicate that secure mode checks must exist
+ * @throws IOException IO problemsn
+ * @throws SliderException any failure
+ */
+ @Override
+ public void validateApplicationConfiguration(AggregateConf instanceDefinition,
+ File confDir,
+ boolean secure
+ ) throws IOException,
+ SliderException {
+ String siteXMLFilename = SITE_XML;
+ File siteXML = new File(confDir, siteXMLFilename);
+ if (!siteXML.exists()) {
+ throw new BadCommandArgumentsException(
+ "Configuration directory %s doesn't contain %s - listing is %s",
+ confDir, siteXMLFilename, SliderUtils.listDir(confDir));
+ }
+
+ //now read it in
+ siteConf = ConfigHelper.loadConfFromFile(siteXML);
+ //look in the site spec to see that it is OK
+ clientProvider.validateHBaseSiteXML(siteConf, secure, siteXMLFilename);
+
+ if (secure) {
+ //secure mode: take a look at the keytab of master and RS
+ SliderUtils.verifyKeytabExists(siteConf,
+ HBaseConfigFileOptions.KEY_MASTER_KERBEROS_KEYTAB);
+ SliderUtils.verifyKeytabExists(siteConf,
+ HBaseConfigFileOptions.KEY_REGIONSERVER_KERBEROS_KEYTAB);
+
+ }
+ }
+
+
+ /**
+ * Build the provider status, can be empty
+ * @return the provider status - map of entries to add to the info section
+ */
+ public Map<String, String> buildProviderStatus() {
+ Map<String, String> stats = new HashMap<String, String>();
+
+ return stats;
+ }
+
+ /* non-javadoc
+ * @see org.apache.slider.providers.ProviderService#buildMonitorDetails()
+ */
+ @Override
+ public TreeMap<String,URL> buildMonitorDetails(ClusterDescription clusterDesc) {
+ TreeMap<String,URL> map = new TreeMap<String,URL>();
+
+ map.put("Active HBase Master (RPC): " + getInfoAvoidingNull(clusterDesc, StatusKeys.INFO_MASTER_ADDRESS), null);
+
+ return map;
+ }
+
+ @Override
+ public RegistrationResponse handleRegistration(Register registration) {
+ // dummy impl
+ RegistrationResponse response = new RegistrationResponse();
+ response.setResponseStatus(RegistrationStatus.OK);
+ return response;
+ }
+
+ @Override
+ public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
+ // dummy impl
+ long id = heartBeat.getResponseId();
+ HeartBeatResponse response = new HeartBeatResponse();
+ response.setResponseId(id + 1L);
+ return response;
+ }
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
new file mode 100644
index 0000000..552374c
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase;
+
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class HBaseRoles {
+
+ /**
+ * List of roles
+ */
+ protected static final List<ProviderRole> ROLES =
+ new ArrayList<ProviderRole>();
+
+ public static final int KEY_WORKER = SliderKeys.ROLE_AM_PRIORITY_INDEX + 1;
+
+ public static final int KEY_MASTER = SliderKeys.ROLE_AM_PRIORITY_INDEX + 2;
+
+ /**
+ * Initialize role list
+ */
+ static {
+ ROLES.add(new ProviderRole(HBaseKeys.ROLE_WORKER, KEY_WORKER));
+ // Master doesn't need data locality
+ ROLES.add(new ProviderRole(HBaseKeys.ROLE_MASTER, KEY_MASTER,PlacementPolicy.NO_DATA_LOCALITY));
+ }
+
+
+ public static List<ProviderRole> getRoles() {
+ return ROLES;
+ }
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hadoop-metrics.properties b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hadoop-metrics.properties
new file mode 100644
index 0000000..fd37d44
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hadoop-metrics.properties
@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See http://wiki.apache.org/hadoop/GangliaMetrics
+# Make sure you know whether you are using ganglia 3.0 or 3.1.
+# If 3.1, you will have to patch your hadoop instance with HADOOP-4675
+# And, yes, this file is named hadoop-metrics.properties rather than
+# hbase-metrics.properties because we're leveraging the hadoop metrics
+# package and hadoop-metrics.properties is an hardcoded-name, at least
+# for the moment.
+#
+# See also http://hadoop.apache.org/hbase/docs/current/metrics.html
+# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia
+# meta daemon (gmetad) service is running
+
+# Configuration of the "hbase" context for NullContextWithUpdateThread
+# NullContextWithUpdateThread is a null context which has a thread calling
+# periodically when monitoring is started. This keeps the data sampled
+# correctly.
+hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+hbase.period=10
+
+# Configuration of the "hbase" context for file
+# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# hbase.fileName=/tmp/metrics_hbase.log
+
+# HBase-specific configuration to reset long-running stats (e.g. compactions)
+# If this variable is left out, then the default is no expiration.
+hbase.extendedperiod = 3600
+
+# Configuration of the "hbase" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# hbase.period=10
+# hbase.servers=GMETADHOST_IP:8649
+
+# Configuration of the "jvm" context for null
+jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+jvm.period=10
+
+# Configuration of the "jvm" context for file
+# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# jvm.fileName=/tmp/metrics_jvm.log
+
+# Configuration of the "jvm" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# jvm.period=10
+# jvm.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rpc" context for null
+rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+rpc.period=10
+
+# Configuration of the "rpc" context for file
+# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# rpc.fileName=/tmp/metrics_rpc.log
+
+# Configuration of the "rpc" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rpc.period=10
+# rpc.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rest" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rest.period=10
+# rest.servers=GMETADHOST_IP:8649
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-env.sh b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-env.sh
new file mode 100644
index 0000000..57c9c26
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-env.sh
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set environment variables here.
+
+# This script sets variables multiple times over the course of starting an hbase process,
+# so try to keep things idempotent unless you want to take an even deeper look
+# into the startup scripts (bin/hbase, etc.)
+
+# The java implementation to use. Java 1.6 required.
+# export JAVA_HOME=/usr/java/jdk1.6.0/
+
+# Extra Java CLASSPATH elements. Optional.
+# export HBASE_CLASSPATH=
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+
+# Uncomment below to enable java garbage collection logging for the server-side processes
+# this enables basic gc logging for the server processes to the .out file
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M $HBASE_GC_OPTS"
+
+# Uncomment below to enable java garbage collection logging for the client processes in the .out file.
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS).
+# This applies to both the server and client GC options above
+# export HBASE_USE_GC_LOGFILE=true
+
+
+# Uncomment below if you intend to use the EXPERIMENTAL off heap cache.
+# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize="
+# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value.
+
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers
+
+# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default.
+# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored. $HBASE_HOME/logs by default.
+# export HBASE_LOG_DIR=${HBASE_HOME}/logs
+
+# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073"
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+# export HBASE_PID_DIR=/var/hadoop/pids
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+# export HBASE_MANAGES_ZK=true
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-policy.xml b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-policy.xml
new file mode 100644
index 0000000..e45f23c
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-policy.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HRegionInterface protocol implementations (ie.
+ clients talking to HRegionServers)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterInterface protocol implementation (ie.
+ clients talking to HMaster for admin operations).
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterRegionInterface protocol implementations
+ (for HRegionServers communicating with HMaster)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+</configuration>
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-site.xml b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-site.xml
new file mode 100644
index 0000000..2b21659
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-site.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+ <!--
+ This is a template HBase site file that
+ does not include all the properties
+ required for a valid configuration -
+ the remainder are injected during
+ conversion from a template to
+ actual file
+ -->
+<configuration>
+ <property>
+ <name>slider.template.origin</name>
+ <value>resource</value>
+ </property>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hbase.regionserver.info.port.auto</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>./hbase-tmp</value>
+ </property>
+ <property>
+ <name>hbase.regionserver.hlog.tolerable.lowreplication</name>
+ <value>1</value>
+ </property>
+</configuration>
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/log4j.properties b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/log4j.properties
new file mode 100644
index 0000000..3ea8b71
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/log4j.properties
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,DRFA
+hbase.security.logger=INFO,DRFA
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+#
+# Security audit appender
+#
+hbase.security.log.file=SecurityAuth.audit
+log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file}
+log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+log4j.logger.org.apache.zookeeper.ClientCnxn=ERROR
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of .META. messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/regionservers b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/regionservers
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/regionservers
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/appconf.json b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/appconf.json
new file mode 100644
index 0000000..5320354
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/appconf.json
@@ -0,0 +1,29 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+
+
+ },
+
+ "global": {
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.hbase.cluster.distributed": "true",
+ "site.hbase.regionserver.info.port.auto": "true",
+ "site.hbase.tmp.dir": "./hbase-tmp",
+ "site.hbase.regionserver.hlog.tolerable.lowreplication": "1",
+ "site.hbase.master.port": "0",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "256M"
+ },
+
+ "components": {
+ "master": {
+ },
+ "worker": {
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json
new file mode 100644
index 0000000..9af7345
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json
@@ -0,0 +1,24 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+
+ },
+
+ "global": {
+
+ },
+
+ "components": {
+ "master": {
+ "component.instances": "1",
+ "yarn.vcores": "1",
+ "yarn.memory": "1024"
+ },
+ "worker": {
+ "component.instances": "2",
+ "yarn.vcores": "1",
+ "yarn.memory": "768"
+ }
+ }
+}
\ No newline at end of file
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-master.xml b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-master.xml
new file mode 100644
index 0000000..005e56d
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-master.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+ <!--
+ These are the default cluster options for a Slider cluster
+ -->
+<configuration>
+ <property>
+ <name>role.name</name>
+ <value>master</value>
+ </property>
+
+ <property>
+ <name>role.instances</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.memory</name>
+ <value>1024</value>
+ </property>
+
+ <property>
+ <name>jvm.heapsize</name>
+ <value>512M</value>
+ </property>
+
+ <property>
+ <name>app.infoport</name>
+ <value>0</value>
+ </property>
+ <property>
+ <name>env.MALLOC_ARENA_MAX</name>
+ <value>4</value>
+ </property>
+
+</configuration>
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-worker.xml b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-worker.xml
new file mode 100644
index 0000000..39ba99e
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-worker.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+ <!--
+ These are the default cluster options for a Slider cluster
+ -->
+<configuration>
+ <property>
+ <name>role.name</name>
+ <value>worker</value>
+ </property>
+
+ <property>
+ <name>role.instances</name>
+ <value>2</value>
+ </property>
+
+ <property>
+ <name>yarn.vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.memory</name>
+ <value>768</value>
+ </property>
+
+ <property>
+ <name>jvm.heapsize</name>
+ <value>512M</value>
+ </property>
+
+ <property>
+ <name>app.infoport</name>
+ <value>0</value>
+ </property>
+
+ <property>
+ <name>env.MALLOC_ARENA_MAX</name>
+ <value>4</value>
+ </property>
+
+</configuration>
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/HBaseTestUtils.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/HBaseTestUtils.groovy
new file mode 100644
index 0000000..590e5a1
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/HBaseTestUtils.groovy
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.hadoop.hbase.HBaseConfiguration
+import org.apache.hadoop.hbase.HConstants
+import org.apache.hadoop.hbase.ServerName
+import org.apache.hadoop.hbase.client.HBaseAdmin
+import org.apache.hadoop.hbase.client.HConnection
+import org.apache.hadoop.hbase.client.HConnectionManager
+import org.apache.hadoop.hbase.client.RetriesExhaustedException
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.exceptions.WaitTimeoutException
+import org.apache.slider.test.SliderTestUtils
+import org.apache.slider.common.tools.ConfigHelper
+import org.apache.slider.common.tools.Duration
+import org.apache.slider.client.SliderClient
+
+/**
+ * Static HBase test utils
+ */
+@Slf4j
+@CompileStatic
+class HBaseTestUtils extends SliderTestUtils {
+
+ /**
+ * Create an (unshared) HConnection talking to the hbase service that
+ * Slider should be running
+ * @param sliderClient slider client
+ * @param clustername the name of the Slider cluster
+ * @return the connection
+ */
+ public static HConnection createHConnection(Configuration clientConf) {
+ HConnection hbaseConnection = HConnectionManager.createConnection(
+ clientConf);
+ return hbaseConnection
+ }
+
+ /**
+ * get a string representation of an HBase cluster status
+ * @param status cluster status
+ * @return a summary for printing
+ */
+ public static String hbaseStatusToString(ClusterStatus status) {
+ StringBuilder builder = new StringBuilder();
+ builder << "Cluster " << status.clusterId;
+ builder << " @ " << status.master << " version " << status.HBaseVersion;
+ builder << "\nlive [\n"
+ if (!status.servers.empty) {
+ status.servers.each() { ServerName name ->
+ builder << " Server " << name << " :" << status.getLoad(name) << "\n"
+ }
+ } else {
+ }
+ builder << "]\n"
+ if (status.deadServers > 0) {
+ builder << "\n dead servers=${status.deadServers}"
+ }
+ return builder.toString()
+ }
+
+ public static ClusterStatus getHBaseClusterStatus(SliderClient sliderClient) {
+ Configuration clientConf = createHBaseConfiguration(sliderClient)
+ return getHBaseClusterStatus(clientConf)
+ }
+
+ public static ClusterStatus getHBaseClusterStatus(Configuration clientConf) {
+ try {
+ HConnection hbaseConnection1 = createHConnection(clientConf)
+ HConnection hbaseConnection = hbaseConnection1;
+ HBaseAdmin hBaseAdmin = new HBaseAdmin(hbaseConnection);
+ ClusterStatus hBaseClusterStatus = hBaseAdmin.clusterStatus;
+ return hBaseClusterStatus;
+ } catch (NoSuchMethodError e) {
+ throw new Exception("Using an incompatible version of HBase!", e);
+ }
+ }
+
+ /**
+ * Create an HBase config to work with
+ * @param sliderClient slider client
+ * @param clustername cluster
+ * @return an hbase config extended with the custom properties from the
+ * cluster, including the binding to the HBase cluster
+ */
+ public static Configuration createHBaseConfiguration(SliderClient sliderClient) {
+ Configuration siteConf = fetchClientSiteConfig(sliderClient);
+ Configuration conf = HBaseConfiguration.create(siteConf);
+ // patch in some timeouts
+ conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10)
+ conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000)
+
+ //fixed time of 1 s per attempt, not any multiplicative pause
+ conf.setInt(HConstants.HBASE_CLIENT_PAUSE, 100)
+ conf.setInt("hbase.client.retries.longer.multiplier", 1)
+ return conf
+ }
+
+ /**
+ * Ask the AM for the site configuration -then dump it
+ * @param sliderClient
+ * @param clustername
+ */
+ public static void dumpHBaseClientConf(SliderClient sliderClient) {
+ Configuration conf = fetchClientSiteConfig(sliderClient);
+ describe("AM-generated site configuration");
+ ConfigHelper.dumpConf(conf);
+ }
+
+ /**
+ * Create a full HBase configuration by merging the AM data with
+ * the rest of the local settings. This is the config that would
+ * be used by any clients
+ * @param sliderClient slider client
+ * @param clustername name of the cluster
+ */
+ public static void dumpFullHBaseConf(SliderClient sliderClient) {
+ Configuration conf = createHBaseConfiguration(sliderClient);
+ describe("HBase site configuration from AM");
+ ConfigHelper.dumpConf(conf);
+ }
+
+ /**
+ * Wait for the hbase master to be live (or past it in the lifecycle)
+ * @param clustername cluster
+ * @param spintime time to wait
+ * @return true if the cluster came out of the sleep time live
+ * @throws IOException
+ * @throws org.apache.slider.core.exceptions.SliderException
+ */
+ public static boolean spinForClusterStartup(
+ SliderClient sliderClient,
+ long spintime,
+ String role = "master")
+ throws WaitTimeoutException, IOException, SliderException {
+ int state = sliderClient.waitForRoleInstanceLive(role, spintime);
+ return state == ClusterDescription.STATE_LIVE;
+ }
+
+ public static ClusterStatus basicHBaseClusterStartupSequence(
+ SliderClient sliderClient, int startupTime, int startupToLiveTime ) {
+ int state = sliderClient.waitForRoleInstanceLive(SliderKeys.COMPONENT_AM,
+ startupTime);
+ assert state == ClusterDescription.STATE_LIVE;
+ state = sliderClient.waitForRoleInstanceLive(HBaseKeys.ROLE_MASTER,
+ startupTime);
+ assert state == ClusterDescription.STATE_LIVE;
+ //sleep for a bit to give things a chance to go live
+ assert spinForClusterStartup(
+ sliderClient,
+ startupToLiveTime,
+ HBaseKeys.MASTER);
+
+ //grab the conf from the status and verify the ZK binding matches
+
+ ClusterStatus clustat = getHBaseClusterStatus(sliderClient);
+ describe("HBASE CLUSTER STATUS \n " + hbaseStatusToString(clustat));
+ return clustat;
+ }
+
+ /**
+ * Spin waiting for the RS count to match expected
+ * @param sliderClient client
+ * @param clustername cluster name
+ * @param regionServerCount RS count
+ * @param timeout timeout
+ */
+ public static ClusterStatus waitForHBaseRegionServerCount(
+ SliderClient sliderClient,
+ String clustername,
+ int regionServerCount,
+ int timeout) {
+ Duration duration = new Duration(timeout);
+ duration.start();
+ ClusterStatus clustat = null;
+ Configuration clientConf = createHBaseConfiguration(sliderClient)
+ while (true) {
+ clustat = getHBaseClusterStatus(clientConf);
+ int workerCount = clustat.servers.size();
+ if (workerCount >= regionServerCount) {
+ break;
+ }
+ if (duration.limitExceeded) {
+ describe("Cluster region server count of $regionServerCount not met:");
+ log.info(hbaseStatusToString(clustat));
+ ClusterDescription status = sliderClient.getClusterDescription(
+ clustername);
+ fail("Expected $regionServerCount YARN region servers," +
+ " but after $timeout millis saw $workerCount in ${hbaseStatusToString(clustat)}" +
+ " \n ${prettyPrint(status.toJsonString())}");
+ }
+ log.info(
+ "Waiting for $regionServerCount region servers -got $workerCount");
+ Thread.sleep(1000);
+ }
+ return clustat;
+ }
+
+ /**
+ * Probe to test for the HBasemaster being found
+ * @param clientConf client configuration
+ * @return true if the master was found in the hbase cluster status
+ */
+ public static boolean isHBaseMasterFound(Configuration clientConf) {
+ HConnection hbaseConnection
+ hbaseConnection = createHConnection(clientConf)
+ HBaseAdmin hBaseAdmin = new HBaseAdmin(hbaseConnection);
+ boolean masterFound
+ try {
+ ClusterStatus hBaseClusterStatus = hBaseAdmin.getClusterStatus();
+ masterFound = true;
+ } catch (RetriesExhaustedException e) {
+ masterFound = false;
+ }
+ return masterFound
+ }
+
+ /**
+ * attempt to talk to the hbase master; expect a failure
+ * @param clientConf client config
+ */
+ public static void assertNoHBaseMaster(
+ SliderClient sliderClient,
+ Configuration clientConf) {
+ boolean masterFound = isHBaseMasterFound(clientConf)
+ if (masterFound) {
+ def text = "HBase master is unexpectedly running"
+ dumpClusterStatus(sliderClient, text);
+ fail(text)
+ }
+ }
+
+ /**
+ * attempt to talk to the hbase master; expect success
+ * @param clientConf client config
+ */
+ public static void assertHBaseMasterFound(Configuration clientConf) {
+ boolean masterFound = isHBaseMasterFound(clientConf)
+ if (!masterFound) {
+ fail("HBase master not running")
+ }
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionExists.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionExists.groovy
new file mode 100644
index 0000000..b4a87f8
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionExists.groovy
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.actions
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * Test of RM creation. This is so the later test's prereq's can be met
+ */
+@CompileStatic
+@Slf4j
+
+class TestActionExists extends HBaseMiniClusterTestBase {
+
+ @Before
+ public void setup() {
+ super.setup()
+ createMiniCluster("TestActionExists", getConfiguration(), 1, false)
+ }
+
+ @Test
+ public void testExistsFailsWithUnknownCluster() throws Throwable {
+ log.info("RM address = ${RMAddr}")
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_EXISTS,
+ "unknown-cluster",
+ Arguments.ARG_MANAGER, RMAddr
+ ],
+ )
+ fail("expected an exception, got a status code "+ launcher.serviceExitCode)
+ } catch (UnknownApplicationInstanceException e) {
+
+ }
+ }
+
+ @Test
+ public void testExistsLiveCluster() throws Throwable {
+ //launch the cluster
+ String clustername = "testExistsLiveCluster"
+ ServiceLauncher launcher = createMasterlessAM(clustername, 0, true, false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(launcher)
+ ApplicationReport report = waitForClusterLive((SliderClient) launcher.service)
+
+ // exists holds when cluster is running
+ launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_EXISTS,
+ clustername,
+ Arguments.ARG_MANAGER, RMAddr
+ ],
+ )
+ assertSucceeded(launcher)
+
+ //and when cluster is running
+ launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_EXISTS,
+ clustername,
+ Arguments.ARG_LIVE,
+ Arguments.ARG_MANAGER, RMAddr
+ ],
+ )
+
+ assertSucceeded(launcher)
+
+ // assert that the cluster exists
+
+ assert 0 == sliderClient.actionExists(clustername, true)
+
+ // freeze the cluster
+ clusterActionFreeze(sliderClient, clustername)
+
+ //verify that exists(live) is now false
+ assert LauncherExitCodes.EXIT_FALSE == sliderClient.actionExists(clustername, true)
+
+ //but the cluster is still there for the default
+ assert 0 == sliderClient.actionExists(clustername, false)
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionList.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionList.groovy
new file mode 100644
index 0000000..fd3d7e6
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionList.groovy
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.actions
+
+import groovy.util.logging.Slf4j
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * Test of RM creation. This is so the later test's prereq's can be met
+ */
+@Slf4j
+
+class TestActionList extends HBaseMiniClusterTestBase {
+
+ @Before
+ public void setup() {
+ super.setup()
+ createMiniCluster("testActionList", getConfiguration(), 1, false)
+ }
+
+ @Test
+ public void testListThisUserNoClusters() throws Throwable {
+ log.info("RM address = ${RMAddr}")
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_LIST,
+ Arguments.ARG_MANAGER, RMAddr
+ ]
+ )
+ assert launcher.serviceExitCode == 0
+ SliderClient sliderClient = (SliderClient) launcher.service
+ }
+
+ @Test
+ public void testListAllUsersNoClusters() throws Throwable {
+ log.info("RM address = ${RMAddr}")
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_LIST,
+ Arguments.ARG_MANAGER, RMAddr,
+ ]
+ )
+ assert launcher.serviceExitCode == 0
+ }
+
+ @Test
+ public void testListLiveCluster() throws Throwable {
+ //launch the cluster
+ String clustername = "test_list_live_cluster"
+ ServiceLauncher launcher = createMasterlessAM(clustername, 0, true, false)
+ ApplicationReport report = waitForClusterLive((SliderClient) launcher.service)
+
+ //now list
+ launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_LIST,
+ ]
+ )
+ assert launcher.serviceExitCode == 0
+ //now look for the explicit sevice
+
+ //do the low level operations to get a better view of what is going on
+ SliderClient sliderClient = launcher.service
+ def serviceRegistryClient = sliderClient.YARNRegistryClient
+ ApplicationReport instance = serviceRegistryClient.findInstance(clustername)
+ assert instance != null
+ log.info(instance.toString())
+
+ //now list with the named cluster
+ launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_LIST, clustername
+ ]
+ )
+
+ }
+
+
+
+ @Test
+ public void testListMissingCluster() throws Throwable {
+ describe("create exec the status command against an unknown cluster")
+ //launch fake master
+ //launch the cluster
+ //exec the status command
+ ServiceLauncher launcher
+ try {
+ launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_LIST,
+ "testStatusMissingCluster"
+ ]
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (UnknownApplicationInstanceException e) {
+ //expected
+ }
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionStatus.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionStatus.groovy
new file mode 100644
index 0000000..cdd9fad
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionStatus.groovy
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.actions
+
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.core.exceptions.BadClusterStateException
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionStatusArgs
+import org.apache.slider.common.params.ClientArgs
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * Test of RM creation. This is so the later test's prereq's can be met
+ */
+//@CompileStatic
+@Slf4j
+class TestActionStatus extends HBaseMiniClusterTestBase {
+
+ @Before
+ public void setup() {
+ super.setup()
+ createMiniCluster("test_action_status", configuration, 1, false)
+ }
+
+ @Test
+ public void testStatusMissingCluster() throws Throwable {
+ describe("create exec the status command against an unknown cluster")
+ //launch fake master
+ //launch the cluster
+ //exec the status command
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ ClientArgs.ACTION_STATUS,
+ "test_status_missing_cluster",
+ Arguments.ARG_MANAGER, RMAddr
+ ]
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (UnknownApplicationInstanceException e) {
+ //expected
+ }
+
+ }
+
+ @Test
+ public void testStatusLiveCluster() throws Throwable {
+ describe("create a live cluster then exec the status command")
+ //launch fake master
+ String clustername = "test_status_live_cluster"
+
+ //launch the cluster
+ ServiceLauncher launcher = createMasterlessAM(clustername, 0, true, false)
+
+ ApplicationReport report = waitForClusterLive(launcher.service)
+
+ //do the low level operations to get a better view of what is going on
+ SliderClient sliderClient = (SliderClient) launcher.service
+
+ //now look for the explicit sevice
+
+ ActionStatusArgs statusArgs = new ActionStatusArgs()
+ int status = sliderClient.actionStatus(clustername, statusArgs)
+ assert status == SliderExitCodes.EXIT_SUCCESS
+
+ //now exec the status command
+ ServiceLauncher statusLauncher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ ClientArgs.ACTION_STATUS,
+ clustername,
+ Arguments.ARG_MANAGER, RMAddr,
+ ]
+
+ )
+ assert statusLauncher.serviceExitCode == 0
+
+ //status to a file
+ File tfile = new File("target/" + clustername + "/status.json")
+ statusArgs.output = tfile.absolutePath
+ sliderClient.actionStatus(clustername, statusArgs)
+ def text = tfile.text
+ ClusterDescription cd = new ClusterDescription();
+ cd.fromJson(text)
+
+ //status to a file via the command line : bin/slider status cl1 --out file.json
+ String path = "target/cluster.json"
+ statusLauncher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ ClientArgs.ACTION_STATUS,
+ clustername,
+ Arguments.ARG_MANAGER, RMAddr,
+ Arguments.ARG_OUTPUT, path
+ ]
+ )
+ assert statusLauncher.serviceExitCode == 0
+ tfile = new File(path)
+ ClusterDescription cd2 = new ClusterDescription();
+ cd2.fromJson(text)
+
+ clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
+ waitForAppToFinish(sliderClient)
+
+ //now expect the status to fail
+ try {
+ status = sliderClient.actionStatus(clustername, new ActionStatusArgs())
+ fail("expected an exception, but got the status $status")
+ } catch (BadClusterStateException e) {
+ assert e.toString().contains(ErrorStrings.E_APPLICATION_NOT_RUNNING)
+ }
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionVersion.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionVersion.groovy
new file mode 100644
index 0000000..6525e3b
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/actions/TestActionVersion.groovy
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.actions
+
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.test.YarnMiniClusterTestBase
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * Test of RM creation. This is so the later test's prereq's can be met
+ */
+@Slf4j
+
+class TestActionVersion extends YarnMiniClusterTestBase {
+
+ @Before
+ public void setup() {
+ super.setup()
+ }
+
+ @Test
+ public void testVersion() throws Throwable {
+
+ ServiceLauncher launcher = execSliderCommand(
+ new YarnConfiguration(),
+ [
+ SliderActions.ACTION_VERSION,
+ ]
+ )
+ assert launcher.serviceExitCode == 0
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
new file mode 100644
index 0000000..9cedbc2
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.hadoop.hbase.client.HConnection
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.hbase.HBaseTestUtils
+import org.apache.slider.test.YarnZKMiniClusterTestBase
+
+import static org.apache.slider.common.params.Arguments.*
+import static org.apache.slider.test.SliderTestUtils.*
+import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
+import static HBaseKeys.*
+/**
+ * test base for all hbase clusters
+ */
+@CompileStatic
+@Slf4j
+public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase {
+
+ public int hbaseClusterStartupTime = hbaseLaunchWaitTime
+
+ /**
+ * The time to sleep before trying to talk to the HBase Master and
+ * expect meaningful results.
+ */
+ public int hbaseClusterStartupToLiveTime = hbaseClusterStartupTime
+
+ public static final String HREGION = "HRegion"
+ public static final String HMASTER = "HMaster"
+ public static final String HB_HEAP = "256"
+
+
+ @Override
+ public String getTestConfigurationPath() {
+ return "src/main/resources/" + HBASE_CONF_RESOURCE;
+ }
+
+ @Override
+ void setup() {
+ super.setup()
+ def testConf = getTestConfiguration()
+ assumeBoolOption(testConf, KEY_TEST_HBASE_ENABLED, true)
+ assumeArchiveDefined();
+ assumeApplicationHome();
+ }
+
+ /**
+ * Teardown kills region servers
+ */
+ @Override
+ void teardown() {
+ super.teardown();
+ if (teardownKillall) {
+ killAllRegionServers();
+ killAllMasterServers();
+ }
+ }
+
+ /**
+ * Kill all the region servers
+ * <code>
+ * jps -l | grep HRegion | awk '{print $1}' | kill -9
+ * </code>
+ */
+ public void killAllRegionServers() {
+ killJavaProcesses(HREGION, SIGKILL);
+ }
+
+ /**
+ * Kill all master servers
+ */
+ public void killAllMasterServers() {
+ killJavaProcesses(HMASTER, SIGKILL);
+ }
+
+ /**
+ * Stop all the region servers
+ * <code>
+ * jps -l | grep HRegion | awk '{print $1}' | kill -19
+ * </code>
+ */
+ public void stopAllRegionServers() {
+ killJavaProcesses(HREGION, SIGTERM);
+ }
+
+
+ public static void assertHBaseMasterNotStopped(SliderClient sliderClient,
+ String clustername) {
+ String[] nodes = sliderClient.listNodeUUIDsByRole(ROLE_MASTER);
+ int masterNodeCount = nodes.length;
+ assert masterNodeCount > 0;
+ ClusterNode node = sliderClient.getNode(nodes[0]);
+ if (node.state >= ClusterDescription.STATE_STOPPED) {
+ //stopped, not what is wanted
+ log.error("HBase master has stopped");
+ log.error(node.toString());
+ fail("HBase master has stopped " + node.diagnostics);
+ }
+ }
+
+ /**
+ * Create an (unshared) HConnection talking to the hbase service that
+ * Slider should be running
+ * @param sliderClient slider client
+ * @param clustername the name of the Slider cluster
+ * @return the connection
+ */
+ public static HConnection createHConnection(Configuration clientConf) {
+ return HBaseTestUtils.createHConnection(clientConf)
+ }
+
+ /**
+ * get a string representation of an HBase cluster status
+ * @param status cluster status
+ * @return a summary for printing
+ */
+ public static String hbaseStatusToString(ClusterStatus status) {
+ return HBaseTestUtils.hbaseStatusToString(status)
+ }
+
+ public static ClusterStatus getHBaseClusterStatus(SliderClient sliderClient) {
+ return HBaseTestUtils.getHBaseClusterStatus(sliderClient)
+ }
+
+ public String getApplicationHomeKey() {
+ return KEY_TEST_HBASE_HOME
+ }
+
+ public String getArchiveKey() {
+ return KEY_TEST_HBASE_TAR
+ }
+
+ /**
+ * Create an HBase config to work with
+ * @param sliderClient slider client
+ * @param clustername cluster
+ * @return an hbase config extended with the custom properties from the
+ * cluster, including the binding to the HBase cluster
+ */
+ public static Configuration createHBaseConfiguration(SliderClient sliderClient) {
+ return HBaseTestUtils.createHBaseConfiguration(sliderClient)
+ }
+
+ /**
+ * Create a full cluster with a master & the requested no. of region servers
+ * @param clustername cluster name
+ * @param workers # of nodes
+ * @param extraArgs list of extra args to add to the creation command
+ * @param deleteExistingData should the data of any existing cluster
+ * of this name be deleted
+ * @param blockUntilRunning block until the AM is running
+ * @return launcher which will have executed the command.
+ */
+ public ServiceLauncher<SliderClient> createHBaseCluster(String clustername,
+ int workers,
+ List<String> extraArgs,
+ boolean deleteExistingData,
+ boolean blockUntilRunning) {
+ def masters = 1
+ return createHBaseCluster(
+ clustername,
+ masters,
+ workers,
+ extraArgs,
+ deleteExistingData,
+ blockUntilRunning)
+ }
+
+ /**
+ * Create a full cluster with a master & the requested no. of region servers
+ * @param clustername cluster name
+ * @param masters #of masters
+ * @param workers # of nodes
+ * @param extraArgs list of extra args to add to the creation command
+ * @param deleteExistingData should the data of any existing cluster
+ * of this name be deleted
+ * @param blockUntilRunning block until the AM is running
+ * @return launcher which will have executed the command.
+ */
+ public ServiceLauncher<SliderClient> createHBaseCluster(
+ String clustername,
+ int masters,
+ int workers,
+ List<String> extraArgs,
+ boolean deleteExistingData,
+ boolean blockUntilRunning) {
+ Map<String, Integer> roles = [
+ (ROLE_MASTER): masters,
+ (ROLE_WORKER): workers,
+ ];
+ extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY << YRAM
+ extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY << YRAM
+ extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+
+ return createCluster(clustername,
+ roles,
+ extraArgs,
+ deleteExistingData,
+ blockUntilRunning,
+ [:])
+ }
+
+ /**
+ * Create an AM without a master
+ * @param clustername AM name
+ * @param size # of nodes
+ * @param deleteExistingData should any existing cluster data be deleted
+ * @param blockUntilRunning block until the AM is running
+ * @return launcher which will have executed the command.
+ */
+ public ServiceLauncher<SliderClient> createMasterlessAM(String clustername, int size, boolean deleteExistingData, boolean blockUntilRunning) {
+ Map<String, Integer> roles = [
+ (ROLE_MASTER): 0,
+ (ROLE_WORKER): size,
+ ];
+ return createCluster(clustername,
+ roles,
+ [
+ ARG_PROVIDER, PROVIDER_HBASE
+ ],
+ deleteExistingData,
+ blockUntilRunning,
+ [:])
+ }
+
+ public ClusterStatus basicHBaseClusterStartupSequence(SliderClient sliderClient) {
+ return HBaseTestUtils.basicHBaseClusterStartupSequence(sliderClient,
+ hbaseClusterStartupTime,
+ hbaseClusterStartupToLiveTime)
+ }
+
+ /**
+ * Spin waiting for the RS count to match expected
+ * @param sliderClient client
+ * @param clustername cluster name
+ * @param regionServerCount RS count
+ * @param timeout timeout
+ */
+ public static ClusterStatus waitForHBaseRegionServerCount(SliderClient sliderClient,
+ String clustername,
+ int regionServerCount,
+ int timeout) {
+
+ return HBaseTestUtils.waitForHBaseRegionServerCount(sliderClient,
+ clustername,
+ regionServerCount,
+ timeout)
+ }
+
+ public boolean flexHBaseClusterTestRun(
+ String clustername,
+ int masters,
+ int masterFlexTarget,
+ int workers,
+ int flexTarget,
+ boolean testHBaseAfter) {
+ createMiniCluster(clustername, getConfiguration(),
+ 1,
+ true);
+ //now launch the cluster
+ SliderClient sliderClient;
+ ServiceLauncher<SliderClient> launcher = createCluster(clustername,
+ [
+ (ROLE_MASTER): masters,
+ (ROLE_WORKER): workers,
+ ],
+ [
+ ARG_RES_COMP_OPT , ROLE_MASTER, ResourceKeys.YARN_MEMORY, YRAM,
+ ARG_RES_COMP_OPT , ROLE_WORKER, ResourceKeys.YARN_MEMORY, YRAM,
+ ARG_PROVIDER , PROVIDER_HBASE
+ ],
+ true,
+ true,
+ [:]);
+ sliderClient = launcher.service;
+ try {
+ basicHBaseClusterStartupSequence(sliderClient);
+
+ describe("Waiting for initial worker count of $workers");
+
+ //verify the #of roles is as expected
+ //get the hbase status
+ waitForWorkerInstanceCount(sliderClient, workers, hbaseClusterStartupToLiveTime);
+ waitForSliderMasterCount(sliderClient, masters, hbaseClusterStartupToLiveTime);
+
+ log.info("Slider worker count at $workers, waiting for region servers to match");
+ waitForHBaseRegionServerCount(sliderClient, clustername, workers, hbaseClusterStartupToLiveTime);
+
+ //now flex
+ describe("Flexing masters:$masters -> $masterFlexTarget ; workers $workers -> $flexTarget");
+ boolean flexed;
+ flexed = 0 == sliderClient.flex(clustername,
+ [
+ (ROLE_WORKER): flexTarget,
+ (ROLE_MASTER): masterFlexTarget
+ ]
+ );
+ waitForWorkerInstanceCount(sliderClient, flexTarget, hbaseClusterStartupToLiveTime);
+ waitForSliderMasterCount(sliderClient, masterFlexTarget,
+ hbaseClusterStartupToLiveTime);
+
+ if (testHBaseAfter) {
+ waitForHBaseRegionServerCount(sliderClient, clustername, flexTarget,
+ hbaseClusterStartupToLiveTime);
+ }
+ return flexed;
+ } finally {
+ maybeStopCluster(sliderClient, null, "end of flex test run");
+ }
+
+ }
+
+ /**
+ * Spin waiting for the Slider worker count to match expected
+ * @param sliderClient client
+ * @param desiredCount RS count
+ * @param timeout timeout
+ */
+ public static ClusterDescription waitForWorkerInstanceCount(SliderClient sliderClient,
+ int desiredCount,
+ int timeout) {
+ return waitForRoleCount(sliderClient, ROLE_WORKER, desiredCount, timeout)
+ }
+
+ public static ClusterDescription waitForSliderMasterCount(SliderClient sliderClient,
+ int desiredCount,
+ int timeout) {
+ return waitForRoleCount(sliderClient, ROLE_MASTER, desiredCount, timeout)
+ }
+
+
+ /**
+ * attempt to talk to the hbase master; expect a failure
+ * @param clientConf client config
+ */
+ public void assertNoHBaseMaster(
+ SliderClient sliderClient, Configuration clientConf) {
+ HBaseTestUtils.assertNoHBaseMaster(sliderClient, clientConf)
+ }
+
+ /**
+ * attempt to talk to the hbase master; expect success
+ * @param clientConf client config
+ */
+ public void assertHBaseMasterFound(Configuration clientConf) {
+ HBaseTestUtils.assertHBaseMasterFound(clientConf)
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy
new file mode 100644
index 0000000..307a730
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestFreezeThawClusterFromArchive.groovy
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.archives
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * Test of RM creation. This is so the later test's prereq's can be met
+ */
+@CompileStatic
+@Slf4j
+class TestFreezeThawClusterFromArchive extends HBaseMiniClusterTestBase {
+
+
+ @Test
+ public void testFreezeThawClusterFromArchive() throws Throwable {
+ String clustername = "test_freeze_thaw_cluster_from_archive"
+ int regionServerCount = 1
+ createMiniCluster(clustername, configuration, 1, true)
+ switchToImageDeploy = true
+ ServiceLauncher<SliderClient> launcher = createHBaseCluster(clustername, regionServerCount, [], true, true)
+ SliderClient sliderClient = launcher.service
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+ log.info("${status.toJsonString()}")
+
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(sliderClient)
+
+ waitForHBaseRegionServerCount(sliderClient, clustername, regionServerCount,
+ hbaseClusterStartupToLiveTime)
+
+
+ clusterActionFreeze(sliderClient, clustername, "test freeze")
+ describe("Restarting cluster")
+ killAllRegionServers();
+
+ //now let's start the cluster up again
+ ServiceLauncher launcher2 = thawCluster(clustername, [], true);
+ SliderClient newCluster = launcher2.service
+ basicHBaseClusterStartupSequence(newCluster)
+
+ //get the hbase status
+ waitForHBaseRegionServerCount(newCluster, clustername, regionServerCount,
+ hbaseClusterStartupToLiveTime)
+
+ }
+
+
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestLiveClusterFromArchive.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestLiveClusterFromArchive.groovy
new file mode 100644
index 0000000..29c9033
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestLiveClusterFromArchive.groovy
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+
+package org.apache.slider.providers.hbase.minicluster.archives
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * create a live cluster from the image
+ */
+@CompileStatic
+@Slf4j
+class TestLiveClusterFromArchive extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testLiveClusterFromArchive() throws Throwable {
+ String clustername = getTestClusterName()
+ int regionServerCount = 1
+ createMiniCluster(clustername, getConfiguration(), regionServerCount + 1, 1, 1, true,
+ startHDFS())
+
+ //now launch the cluster
+ setupImageToDeploy()
+ ServiceLauncher launcher = createHBaseCluster(clustername, regionServerCount, [], true, true)
+
+ SliderClient sliderClient = (SliderClient) launcher.service
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(sliderClient)
+
+ //get the hbase status
+ waitForHBaseRegionServerCount(sliderClient, clustername, regionServerCount, hbaseClusterStartupToLiveTime)
+ waitForWorkerInstanceCount(sliderClient, regionServerCount, hbaseClusterStartupToLiveTime)
+
+ clusterActionFreeze(sliderClient, clustername,"end of run")
+ }
+
+ public void setupImageToDeploy() {
+ switchToImageDeploy = true
+ }
+
+ public String getTestClusterName() {
+ return "test_live_cluster_from_archive"
+ }
+
+ public boolean startHDFS() {
+ return false
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestLiveClusterFromArchiveOnHDFS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestLiveClusterFromArchiveOnHDFS.groovy
new file mode 100644
index 0000000..2245e2c
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/archives/TestLiveClusterFromArchiveOnHDFS.groovy
@@ -0,0 +1,45 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.archives
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+/**
+ * Test of RM creation. This is so the later test's prereq's can be met
+ */
+@CompileStatic
+@Slf4j
+class TestLiveClusterFromArchiveOnHDFS extends TestLiveClusterFromArchive {
+
+ @Override
+ String getTestClusterName() {
+ "test_live_cluster_from_archiveonhdfs"
+ }
+
+ @Override
+ boolean startHDFS() {
+ true
+ }
+
+ @Override
+ void setupImageToDeploy() {
+ enableTestRunAgainstUploadedArchive();
+ }
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildClusterM1W5.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildClusterM1W5.groovy
new file mode 100644
index 0000000..80178b1
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildClusterM1W5.groovy
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.build
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+import static HBaseKeys.PROVIDER_HBASE
+import static org.apache.slider.common.params.Arguments.ARG_PROVIDER
+
+@CompileStatic
+@Slf4j
+
+class TestBuildClusterM1W5 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testBuildCluster() throws Throwable {
+ String clustername = "test_build_cluster_m1_w5"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "verify that a build cluster is created but not started"
+
+ ServiceLauncher launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ clustername,
+ [
+ (HBaseKeys.ROLE_MASTER): 1,
+ (HBaseKeys.ROLE_WORKER): 5,
+ ],
+ [
+ ARG_PROVIDER, PROVIDER_HBASE
+ ],
+ true,
+ false,
+ [:])
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+
+ //verify that exists(live) is now false
+ assert LauncherExitCodes.EXIT_FALSE ==
+ sliderClient.actionExists(clustername, true)
+
+ //but the cluster is still there for the default
+ assert 0 == sliderClient.actionExists(clustername, false)
+
+ def serviceRegistryClient = sliderClient.YARNRegistryClient
+ ApplicationReport report = serviceRegistryClient.findInstance(clustername)
+ assert report == null;
+
+ //and a second attempt will fail as the cluster now exists
+ try {
+ createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ clustername,
+ [
+ (HBaseKeys.ROLE_MASTER): 1,
+ (HBaseKeys.ROLE_WORKER): 3,
+ ],
+ [
+ ARG_PROVIDER, PROVIDER_HBASE
+ ],
+ true,
+ false,
+ [:])
+ } catch (SliderException e) {
+ assert e.exitCode == SliderExitCodes.EXIT_INSTANCE_EXISTS
+ }
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
new file mode 100644
index 0000000..d1ef2d8
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.build
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+import static HBaseKeys.PROVIDER_HBASE
+import static org.apache.slider.common.params.Arguments.ARG_PROVIDER
+
+@CompileStatic
+@Slf4j
+
+class TestBuildThawClusterM1W1 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void test_build_thaw_cluster_m1_w1() throws Throwable {
+ String clustername = "test_build_thaw_cluster_m1_w1"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "verify that a built cluster can be thawed"
+
+ ServiceLauncher launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ clustername,
+ [
+ (HBaseKeys.ROLE_MASTER): 1,
+ (HBaseKeys.ROLE_WORKER): 1,
+ ],
+ [
+ ARG_PROVIDER, PROVIDER_HBASE
+ ],
+ true,
+ false,
+ [:])
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ def serviceRegistryClient = sliderClient.YARNRegistryClient
+ ApplicationReport report = serviceRegistryClient.findInstance(clustername)
+ assert report == null;
+
+ //thaw time
+ ServiceLauncher l2 = thawCluster(clustername, [], true)
+ SliderClient client2 = (SliderClient) l2.service
+ addToTeardown(client2);
+ waitForClusterLive(l2.service as SliderClient)
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestFailedRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestFailedRegionService.groovy
new file mode 100644
index 0000000..1c8d3d2
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestFailedRegionService.groovy
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.failures
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionKillContainerArgs
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * test create a live region service
+ */
+@CompileStatic
+@Slf4j
+class TestFailedRegionService extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testFailedRegionService() throws Throwable {
+ testRegionService("test_failed_region_service", true)
+ }
+
+ @Test
+ public void testStoppedRegionService() throws Throwable {
+ testRegionService("test_stopped_region_service", false)
+ }
+
+ private void testRegionService(String testName, boolean toKill) {
+ String clustername = testName
+ String action = toKill ? "kill" : "stop"
+ int regionServerCount = 2
+ createMiniCluster(clustername, getConfiguration(), 1, 1, 1, true, true)
+ describe("Create a single region service cluster then " + action + " the RS");
+
+ //now launch the cluster
+ ServiceLauncher launcher = createHBaseCluster(clustername, regionServerCount, [], true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(sliderClient)
+
+ status = waitForWorkerInstanceCount(sliderClient, regionServerCount, hbaseClusterStartupToLiveTime)
+ //get the hbase status
+ ClusterStatus hbaseStat = waitForHBaseRegionServerCount(sliderClient, clustername, regionServerCount, hbaseClusterStartupToLiveTime)
+
+ log.info("Initial cluster status : ${hbaseStatusToString(hbaseStat)}");
+ describe("running processes")
+ lsJavaProcesses()
+ describe("about to " + action + " servers")
+ if (toKill) {
+ killAllRegionServers()
+ } else {
+ stopAllRegionServers()
+ }
+
+ //sleep a bit
+ sleep(toKill ? 15000 : 30000);
+ lsJavaProcesses()
+
+ describe("waiting for recovery")
+
+ //and expect a recovery
+ status = waitForWorkerInstanceCount(sliderClient, regionServerCount, hbaseClusterStartupToLiveTime)
+
+ //now we expect the failure count to be two
+
+
+ hbaseStat = waitForHBaseRegionServerCount(sliderClient, clustername, regionServerCount, hbaseClusterStartupToLiveTime)
+
+ status = sliderClient.getClusterDescription()
+ assert status.roles[HBaseKeys.ROLE_WORKER][RoleKeys.ROLE_FAILED_INSTANCES] == "2"
+
+ log.info("Updated cluster status : ${hbaseStatusToString(hbaseStat)}");
+
+ //now attempt it by container kill command
+ def workers = status.instances[HBaseKeys.ROLE_WORKER]
+ assert workers.size() == 2
+ def worker1 = workers[0]
+ ActionKillContainerArgs args = new ActionKillContainerArgs();
+ args.id = worker1
+ assert 0 == sliderClient.actionKillContainer(clustername, args)
+ sleep(15000)
+ waitForWorkerInstanceCount(
+ sliderClient,
+ regionServerCount,
+ hbaseClusterStartupToLiveTime)
+ status = sliderClient.getClusterDescription()
+ assert status.roles[HBaseKeys.ROLE_WORKER][RoleKeys.ROLE_FAILED_INSTANCES] == "3"
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestFailureThreshold.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestFailureThreshold.groovy
new file mode 100644
index 0000000..ef073bc
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestFailureThreshold.groovy
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.failures
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.OptionKeys
+import org.apache.slider.core.exceptions.BadClusterStateException
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * test that if a container is killed too many times,
+ * the AM stays down
+ */
+@CompileStatic
+@Slf4j
+
+class TestFailureThreshold extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testFailedRegionService() throws Throwable {
+ failureThresholdTestRun("test_failure_threshold", true, 2, 5)
+ }
+
+
+
+ private void failureThresholdTestRun(
+ String testName,
+ boolean toKill,
+ int threshold,
+ int killAttempts) {
+ String clustername = testName
+ String action = toKill ? "kill" : "stop"
+ int regionServerCount = 2
+ createMiniCluster(clustername, configuration, 1, 1, 1, true, true)
+ describe(
+ "Create a single region service cluster then " + action + " the RS");
+
+ //now launch the cluster
+ ServiceLauncher<SliderClient> launcher = createHBaseCluster(
+ clustername,
+ regionServerCount,
+ [
+ Arguments.ARG_OPTION, OptionKeys.INTERNAL_CONTAINER_FAILURE_THRESHOLD,
+ Integer.toString(threshold)],
+ true,
+ true)
+ SliderClient client = launcher.service
+ addToTeardown(client);
+ ClusterDescription status = client.getClusterDescription(clustername)
+
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(client)
+ ClusterStatus hbaseStat
+ try {
+ for (restarts in 1..killAttempts) {
+ status = waitForWorkerInstanceCount(
+ client,
+ regionServerCount,
+ hbaseClusterStartupToLiveTime)
+ //get the hbase status
+/*
+ hbaseStat = waitForHBaseRegionServerCount(
+ client,
+ clustername,
+ regionServerCount,
+ HBASE_CLUSTER_STARTUP_TO_LIVE_TIME)
+
+ log.info("Initial cluster status : ${hbaseStatusToString(hbaseStat)}");
+*/
+ describe("running processes")
+ lsJavaProcesses()
+ describe("about to " + action + " servers")
+ if (toKill) {
+ killAllRegionServers()
+ } else {
+ stopAllRegionServers()
+ }
+
+ //sleep a bit
+ sleep(toKill ? 15000 : 25000);
+
+ describe("waiting for recovery")
+
+ //and expect a recovery
+ if (restarts < threshold) {
+
+ def restartTime = 1000
+ status = waitForWorkerInstanceCount(
+ client,
+ regionServerCount,
+ restartTime)
+ hbaseStat = waitForHBaseRegionServerCount(
+ client,
+ clustername,
+ regionServerCount,
+ restartTime)
+ } else {
+ //expect the cluster to have failed
+ try {
+ def finalCD = client.getClusterDescription(clustername)
+ dumpClusterDescription("expected the AM to have failed", finalCD)
+ fail("AM had not failed after $restarts worker kills")
+
+ } catch (BadClusterStateException e) {
+ assert e.toString().contains(ErrorStrings.E_APPLICATION_NOT_RUNNING)
+ assert e.exitCode == SliderExitCodes.EXIT_BAD_STATE
+ //success
+ break;
+ }
+ }
+ }
+ } catch (BadClusterStateException e) {
+ assert e.toString().contains(ErrorStrings.E_APPLICATION_NOT_RUNNING)
+ assert e.exitCode == SliderExitCodes.EXIT_BAD_STATE
+ }
+ ApplicationReport report = client.applicationReport
+ log.info(report.diagnostics)
+ assert report.finalApplicationStatus == FinalApplicationStatus.FAILED
+ assert report.diagnostics.contains(ErrorStrings.E_UNSTABLE_CLUSTER)
+
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledAM.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledAM.groovy
new file mode 100644
index 0000000..a378eae
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledAM.groovy
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.failures
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.hadoop.hbase.HConstants
+import org.apache.hadoop.hbase.client.HConnection
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionAMSuicideArgs
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * test create a live region service
+ */
+@CompileStatic
+@Slf4j
+class TestKilledAM extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testKilledAM() throws Throwable {
+ skip("failing")
+
+ String clustername = "test_killed_am"
+ int regionServerCount = 1
+
+
+ def conf = configuration
+ // patch the configuration for AM restart
+ conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, 3)
+
+ conf.setClass(YarnConfiguration.RM_SCHEDULER,
+ FifoScheduler, ResourceScheduler);
+ createMiniCluster(clustername, conf, 1, 1, 1, true, false)
+ describe(" Kill the AM, expect cluster to die");
+
+ //now launch the cluster
+ ServiceLauncher<SliderClient> launcher = createHBaseCluster(
+ clustername,
+ regionServerCount,
+ [],
+ true,
+ true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(sliderClient)
+
+
+ status = waitForWorkerInstanceCount(
+ sliderClient,
+ regionServerCount,
+ hbaseClusterStartupToLiveTime)
+ //get the hbase status
+ ClusterStatus hbaseStat = waitForHBaseRegionServerCount(
+ sliderClient,
+ clustername,
+ regionServerCount,
+ hbaseClusterStartupToLiveTime)
+
+ log.info("Initial cluster status : ${hbaseStatusToString(hbaseStat)}");
+
+ String hbaseMasterContainer = status.instances[HBaseKeys.ROLE_MASTER][0]
+
+ Configuration clientConf = createHBaseConfiguration(sliderClient)
+ clientConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
+ HConnection hbaseConnection
+ hbaseConnection = createHConnection(clientConf)
+
+
+
+ describe("running processes")
+ lsJavaProcesses()
+ describe("killing AM")
+
+ ActionAMSuicideArgs args = new ActionAMSuicideArgs()
+ args.message = "test AM"
+ args.waittime = 1000
+ args.exitcode = 1
+ sliderClient.actionAmSuicide(clustername, args)
+
+ killAllRegionServers();
+ waitWhileClusterLive(sliderClient);
+ // give yarn some time to notice
+ sleep(10000)
+
+ // await cluster startup
+ ApplicationReport report = sliderClient.applicationReport
+ assert report.yarnApplicationState != YarnApplicationState.FAILED;
+
+
+ def restartTime = 60000
+ status = waitForWorkerInstanceCount(
+ sliderClient,
+ regionServerCount,
+ restartTime)
+
+ dumpClusterDescription("post-restart status", status)
+ String restarted = status.getInfo(
+ StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ assert restarted != null
+ assert Integer.parseInt(restarted) == 1
+ assert null != status.instances[HBaseKeys.ROLE_MASTER]
+ assert 1 == status.instances[HBaseKeys.ROLE_MASTER].size()
+ assert hbaseMasterContainer == status.instances[HBaseKeys.ROLE_MASTER][0]
+
+ waitForHBaseRegionServerCount(
+ sliderClient,
+ clustername,
+ regionServerCount,
+ restartTime)
+
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseMaster.groovy
new file mode 100644
index 0000000..d05855f
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestKilledHBaseMaster.groovy
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.failures
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.hadoop.hbase.HConstants
+import org.apache.hadoop.hbase.ServerName
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * test create a live region service
+ */
+@CompileStatic
+@Slf4j
+class TestKilledHBaseMaster extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testKilledHBaseMaster() throws Throwable {
+ String clustername = "test_killed_hbase_master"
+ int regionServerCount = 1
+ createMiniCluster(clustername, getConfiguration(), 1, 1, 1, true, true)
+ describe("Kill the hbase master and expect a restart");
+
+ //now launch the cluster
+ ServiceLauncher launcher = createHBaseCluster(clustername, regionServerCount, [], true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(sliderClient)
+
+
+ status = waitForWorkerInstanceCount(sliderClient, regionServerCount, hbaseClusterStartupToLiveTime)
+ //get the hbase status
+ ClusterStatus hbaseStat = waitForHBaseRegionServerCount(sliderClient, clustername, regionServerCount, hbaseClusterStartupToLiveTime)
+ ServerName master = hbaseStat.master
+ log.info("HBase master providing status information at {}",
+ hbaseStat.master)
+
+ Configuration clientConf = createHBaseConfiguration(sliderClient)
+ clientConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10);
+ killAllMasterServers();
+ status = waitForRoleCount(
+ sliderClient, HBaseKeys.ROLE_MASTER, 1, hbaseClusterStartupToLiveTime)
+ hbaseStat = waitForHBaseRegionServerCount(
+ sliderClient,
+ clustername,
+ regionServerCount,
+ hbaseClusterStartupToLiveTime)
+
+ ServerName master2 = hbaseStat.master
+ log.info("HBase master providing status information again at {}",
+ master2)
+ assert master2 != master
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex0To1.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex0To1.groovy
new file mode 100644
index 0000000..587e41f
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex0To1.groovy
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@Slf4j
+class TestClusterFlex0To1 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterFlex0To1() throws Throwable {
+ assert flexHBaseClusterTestRun("test_cluster_flex_0To1", 1, 1, 0, 1, false)
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
new file mode 100644
index 0000000..3e0319f
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+
+class TestClusterFlex1To1 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterFlexPersistent() throws Throwable {
+ assert !flexHBaseClusterTestRun(
+ "test_cluster_flex_1to1",
+ 1,
+ 1,
+ 1,
+ 1
+ ,
+ true)
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To2.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To2.groovy
new file mode 100644
index 0000000..0df1c77
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To2.groovy
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlex1To2 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterFlex() throws Throwable {
+ assert flexHBaseClusterTestRun(
+ "test_cluster_flex_1to2",
+ 1,
+ 1,
+ 1,
+ 2
+ ,
+ true)
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2DownTo1.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2DownTo1.groovy
new file mode 100644
index 0000000..7596827
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2DownTo1.groovy
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlex2DownTo1 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterFlex2DownTo1() throws Throwable {
+ assert flexHBaseClusterTestRun(
+ "test_cluster_flex_2_down_to_1",
+ 1, 1,
+ 2,
+ 1
+ ,
+ false)
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
new file mode 100644
index 0000000..1fc41f6
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlex2To5 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterFlex() throws Throwable {
+ assert flexHBaseClusterTestRun(
+ "test_cluster_flex_2to5",
+ 1,
+ 1,
+ 2,
+ 5
+ ,
+ true)
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownToZero.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownToZero.groovy
new file mode 100644
index 0000000..12ad6a7
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownToZero.groovy
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlexDownToZero extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterFlexDownToZero() throws Throwable {
+ assert flexHBaseClusterTestRun(
+ "test_cluster_flex_down_to_zero",
+ 1, 1,
+ 1,
+ 0
+ ,
+ false)
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
new file mode 100644
index 0000000..1eae82e
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestHMasterFlex1To2 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterFlex() throws Throwable {
+ assert flexHBaseClusterTestRun(
+ "test_hmaster_flex_1to2",
+ 1,
+ 2,
+ 1,
+ 1
+ ,
+ true)
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeCommands.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeCommands.groovy
new file mode 100644
index 0000000..76da362
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeCommands.groovy
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.freezethaw
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * create masterless AMs and work with them. This is faster than
+ * bringing up full clusters
+ */
+@CompileStatic
+@Slf4j
+
+class TestFreezeCommands extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testFreezeCommands() throws Throwable {
+ String clustername = "test_freeze_commands"
+ YarnConfiguration conf = getConfiguration()
+ createMiniCluster(clustername, conf, 1, 1, 1, true, true)
+
+ describe "create a masterless AM, freeze it, try to freeze again"
+
+ ServiceLauncher launcher = createMasterlessAM(clustername, 0, true, true);
+ addToTeardown(launcher.service as SliderClient);
+
+
+ log.info("ListOp")
+ assertSucceeded(execSliderCommand(conf,
+ [SliderActions.ACTION_LIST,clustername]))
+
+ log.info("First Freeze command");
+ ServiceLauncher freezeCommand = execSliderCommand(conf,
+ [SliderActions.ACTION_FREEZE, clustername,
+ Arguments.ARG_WAIT, waitTimeArg]);
+ assertSucceeded(freezeCommand)
+
+ log.info("Second Freeze command");
+
+ ServiceLauncher freeze2 = execSliderCommand(conf,
+ [
+ SliderActions.ACTION_FREEZE, clustername,
+ Arguments.ARG_WAIT, waitTimeArg
+ ]);
+ assertSucceeded(freeze2)
+
+ log.info("First Exists");
+
+ //assert there is no running cluster
+ try {
+ ServiceLauncher exists1 = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ [
+ SliderActions.ACTION_EXISTS, clustername,
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ Arguments.ARG_LIVE
+ ],
+ )
+ assert 0 != exists1.serviceExitCode;
+ } catch (SliderException e) {
+ assert e.exitCode == LauncherExitCodes.EXIT_FALSE;
+ }
+
+ log.info("First Thaw");
+
+
+ def commands = [
+ SliderActions.ACTION_THAW, clustername,
+ Arguments.ARG_WAIT, waitTimeArg,
+ Arguments.ARG_FILESYSTEM, fsDefaultName
+ ]
+ commands.addAll(extraCLIArgs)
+
+ ServiceLauncher thawCommand = execSliderCommand(conf, commands);
+ assertSucceeded(thawCommand)
+ assertSucceeded(execSliderCommand(conf,
+ [SliderActions.ACTION_LIST, clustername]))
+ assertSucceeded(execSliderCommand(conf,
+ [SliderActions.ACTION_EXISTS, clustername]))
+
+ log.info("Freeze 3");
+
+ ServiceLauncher freeze3 = execSliderCommand(conf,
+ [
+ SliderActions.ACTION_FREEZE, clustername,
+ Arguments.ARG_WAIT, waitTimeArg
+ ]);
+ assertSucceeded(freeze3)
+
+ log.info("thaw2");
+ ServiceLauncher thaw2 = execSliderCommand(conf,
+ commands);
+ assert 0 == thaw2.serviceExitCode;
+ assertSucceeded(thaw2)
+
+ try {
+ log.info("thaw3 - should fail");
+ ServiceLauncher thaw3 = execSliderCommand(conf,
+ commands);
+ assert 0 != thaw3.serviceExitCode;
+ } catch (SliderException e) {
+ assertFailureClusterInUse(e);
+ }
+
+ //destroy should fail
+
+ log.info("destroy1");
+
+ try {
+ ServiceLauncher destroy1 = execSliderCommand(conf,
+ [
+ SliderActions.ACTION_DESTROY, clustername,
+ Arguments.ARG_FILESYSTEM, fsDefaultName
+ ]);
+ fail(
+ "expected a failure from the destroy, got error code ${destroy1.serviceExitCode}");
+ } catch (SliderException e) {
+ assertFailureClusterInUse(e);
+ }
+ log.info("freeze4");
+
+ //kill -19 the process to hang it, then force kill
+ killAM(SIGSTOP)
+
+ ServiceLauncher freeze4 = execSliderCommand(conf,
+ [
+ SliderActions.ACTION_FREEZE, clustername,
+ Arguments.ARG_FORCE,
+ Arguments.ARG_WAIT, waitTimeArg,
+ ]);
+ assertSucceeded(freeze4)
+
+ log.info("destroy2");
+ ServiceLauncher destroy2 = execSliderCommand(conf,
+ [
+ SliderActions.ACTION_DESTROY, clustername,
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ ]);
+ assertSucceeded(destroy2)
+
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
new file mode 100644
index 0000000..7264944
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.freezethaw
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.OptionKeys
+import org.apache.slider.core.build.InstanceIO
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.common.tools.ConfigHelper
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Test of RM creation. This is so the later test's prereq's can be met
+ */
+@CompileStatic
+@Slf4j
+class TestFreezeReconfigureThawLiveRegionService
+ extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testFreezeReconfigureThawLiveRegionService() throws Throwable {
+ String clustername = "test_freeze_reconfigure_thaw_live_regionservice"
+ int regionServerCount = 4
+ int nodemanagers = 3
+ YarnConfiguration conf = getConfiguration()
+ //one vcore per node
+ conf.setInt("yarn.nodemanager.resource.cpu-vcores", 1)
+ createMiniCluster(clustername, conf, nodemanagers, true)
+ describe(
+ "Create a $regionServerCount node cluster, freeze it, patch the configuration files," +
+ " thaw it and verify that it came back with the new settings")
+
+ ServiceLauncher launcher = createHBaseCluster(
+ clustername,
+ regionServerCount,
+ [],
+ true,
+ true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+ log.info("${status.toJsonString()}")
+
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(sliderClient)
+
+ clustat = waitForHBaseRegionServerCount(
+ sliderClient,
+ clustername,
+ regionServerCount,
+ hbaseClusterStartupToLiveTime)
+ describe("Cluster status")
+ log.info(hbaseStatusToString(clustat));
+
+
+ clusterActionFreeze(sliderClient, clustername)
+ killAllRegionServers();
+
+ //reconfigure time
+
+ //get the location of the cluster
+ HadoopFS dfs = HadoopFS.get(new URI(fsDefaultName), miniCluster.config)
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(dfs, miniCluster.config)
+ Path clusterDir = sliderFileSystem.buildClusterDirPath(clustername);
+ def instanceDefinition = InstanceIO.loadInstanceDefinitionUnresolved(
+ sliderFileSystem,
+ clusterDir)
+
+ def snapshotPath = instanceDefinition.getInternalOperations().get(
+ OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH)
+ assert snapshotPath != null
+
+ Path confdir = new Path(snapshotPath);
+ Path hbaseSiteXML = new Path(confdir, HBaseKeys.SITE_XML)
+ Configuration originalConf = ConfigHelper.loadTemplateConfiguration(
+ dfs,
+ hbaseSiteXML,
+ "");
+ //patch
+ String patchedText = "patched-after-freeze"
+ originalConf.setBoolean(patchedText, true);
+ //save
+ ConfigHelper.saveConfig(dfs, hbaseSiteXML, originalConf);
+
+ //now let's start the cluster up again
+ ServiceLauncher launcher2 = thawCluster(clustername, [], true);
+ SliderClient thawed = launcher2.service as SliderClient
+ clustat = basicHBaseClusterStartupSequence(thawed)
+
+ //get the options
+ ClusterDescription stat = thawed.clusterDescription
+ Map<String, String> properties = stat.clientProperties
+ log.info("Cluster properties: \n" + SliderUtils.stringifyMap(properties));
+ assert properties[patchedText] == "true";
+
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy
new file mode 100644
index 0000000..445e04b
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawLiveRegionService.groovy
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.freezethaw
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * Test of RM creation. This is so the later test's prereq's can be met
+ */
+@CompileStatic
+@Slf4j
+class TestFreezeThawLiveRegionService extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testFreezeThawLiveRegionService() throws Throwable {
+ String clustername = "test_freeze_thaw_live_regionservice"
+ int regionServerCount = 2
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+ describe("Create a cluster, freeze it, thaw it and verify that it came back ")
+ //use a smaller AM HEAP to include it in the test cycle
+ ServiceLauncher launcher = createHBaseCluster(clustername, regionServerCount,
+ [
+ Arguments.ARG_COMP_OPT, SliderKeys.COMPONENT_AM, RoleKeys.JVM_HEAP, "96M",
+ ],
+ true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+ log.info("${status.toJsonString()}")
+
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(sliderClient)
+
+ clustat = waitForHBaseRegionServerCount(sliderClient, clustername, regionServerCount,
+ hbaseClusterStartupToLiveTime)
+ describe("Cluster status")
+ log.info(hbaseStatusToString(clustat));
+
+
+ //verify you can't start a new cluster with that name
+ try {
+ ServiceLauncher launcher3 = createHBaseCluster(clustername, regionServerCount, [], false, false)
+ SliderClient cluster3 = launcher3.service as SliderClient
+ fail("expected a failure, got ${cluster3}")
+ } catch (SliderException e) {
+ assert e.exitCode == SliderExitCodes.EXIT_APPLICATION_IN_USE;
+ }
+
+
+ clusterActionFreeze(sliderClient, clustername)
+ killAllRegionServers();
+ //now let's start the cluster up again
+ ServiceLauncher launcher2 = thawCluster(clustername, [], true);
+ SliderClient newCluster = launcher2.service as SliderClient
+ basicHBaseClusterStartupSequence(newCluster)
+
+ //get the hbase status
+ waitForHBaseRegionServerCount(newCluster, clustername, regionServerCount,
+ hbaseClusterStartupToLiveTime)
+
+ // finally, attempt to thaw it while it is running
+ //now let's start the cluster up again
+ try {
+ ServiceLauncher launcher3 = thawCluster(clustername, [], true);
+ SliderClient cluster3 = launcher3.service as SliderClient
+ fail("expected a failure, got ${cluster3}")
+ } catch (SliderException e) {
+ assert e.exitCode == SliderExitCodes.EXIT_APPLICATION_IN_USE
+ }
+ }
+
+
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawMasterlessAM.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawMasterlessAM.groovy
new file mode 100644
index 0000000..827fd5e
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeThawMasterlessAM.groovy
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.freezethaw
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * create masterless AMs and work with them. This is faster than
+ * bringing up full clusters
+ */
+@CompileStatic
+@Slf4j
+
+class TestFreezeThawMasterlessAM extends HBaseMiniClusterTestBase {
+
+ File getConfDirFile() {
+ return new File("target/TestFreezeThawMasterlessAM/conf")
+ }
+
+ @Override
+ String getConfDir() {
+ return confDirFile.toURI().toString()
+ }
+
+ @Test
+ public void testFreezeThawMasterlessAM() throws Throwable {
+ String clustername = "test_freeze_thaw_masterless_am"
+ YarnConfiguration conf = getConfiguration()
+ createMiniCluster(clustername, conf, 1, 1, 1, true, true)
+
+ describe "create a masterless AM, freeze it, thaw it"
+ //copy the confdir somewhere
+ Path resConfPath = new Path(getResourceConfDirURI())
+ Path tempConfPath = new Path(confDir)
+ SliderUtils.copyDirectory(conf, resConfPath, tempConfPath, null)
+
+
+ ServiceLauncher launcher = createMasterlessAM(clustername, 0, true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+
+ assert 0 == clusterActionFreeze(sliderClient, clustername)
+
+
+ // here we do something devious: delete our copy of the configuration
+ // this makes sure the remote config gets picked up
+ HadoopFS localFS = HadoopFS.get(tempConfPath.toUri(), conf)
+ localFS.delete(tempConfPath,true)
+
+ //now start the cluster
+ ServiceLauncher launcher2 = thawCluster(clustername, [], true);
+ SliderClient newCluster = launcher.getService() as SliderClient
+ newCluster.getClusterDescription(clustername);
+
+ //freeze
+ assert 0 == clusterActionFreeze(sliderClient, clustername)
+
+ //freeze again
+ assert 0 == clusterActionFreeze(sliderClient, clustername)
+
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeUnknownCluster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeUnknownCluster.groovy
new file mode 100644
index 0000000..04c1a6e
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeUnknownCluster.groovy
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.freezethaw
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * create masterless AMs and work with them. This is faster than
+ * bringing up full clusters
+ */
+@CompileStatic
+@Slf4j
+
+class TestFreezeUnknownCluster extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testFreezeUnknownCluster() throws Throwable {
+ String clustername = "test_start_unknown_cluster"
+ YarnConfiguration conf = getConfiguration()
+ createMiniCluster(clustername, conf, 1, true)
+
+ describe "try to freeze a cluster that isn't defined"
+
+ //we are secretly picking up the RM details from the configuration file
+ try {
+ ServiceLauncher command = execSliderCommand(conf,
+ [
+ SliderActions.ACTION_FREEZE,
+ "no-such-cluster"
+ ]);
+ fail("Expected an error, got an exit code of ${command.serviceExitCode}")
+ } catch (UnknownApplicationInstanceException e) {
+ //expected
+ }
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestThawUnknownCluster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestThawUnknownCluster.groovy
new file mode 100644
index 0000000..9041e48
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestThawUnknownCluster.groovy
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.freezethaw
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * create masterless AMs and work with them. This is faster than
+ * bringing up full clusters
+ */
+@CompileStatic
+@Slf4j
+
+class TestThawUnknownCluster extends YarnZKMiniClusterTestBase {
+
+ @Test
+ public void testThawUnknownCluster() throws Throwable {
+ String clustername = "test_thaw_unknown_cluster"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "try to start a cluster that isn't defined"
+
+ try {
+ ServiceLauncher launcher = thawCluster(clustername, [], true);
+ fail("expected a failure, got ${launcher.serviceExitCode}")
+ } catch (UnknownApplicationInstanceException e) {
+ assert e.toString().contains(clustername)
+ }
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
new file mode 100644
index 0000000..69a0bbc
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/Test2Master2RS.groovy
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.providers.hbase.HBaseTestUtils
+import org.apache.slider.common.tools.Duration
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * test create a live region service
+ */
+@CompileStatic
+@Slf4j
+
+class Test2Master2RS extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void test2Master2RS() throws Throwable {
+
+ String clustername = "test2master2rs"
+ int regionServerCount = 2
+ createMiniCluster(clustername, getConfiguration(), 1, 1, 1, true, false)
+
+ describe(" Create a two master, two region service cluster");
+ //now launch the cluster
+ int masterCount = 2
+
+ ServiceLauncher launcher = createHBaseCluster(
+ clustername,
+ masterCount,
+ regionServerCount,
+ [],
+ true,
+ true)
+
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+ log.info("${status.toJsonString()}")
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(sliderClient)
+
+ status = waitForWorkerInstanceCount(
+ sliderClient,
+ regionServerCount,
+ hbaseClusterStartupToLiveTime)
+ //get the hbase status
+
+ Duration duration = new Duration(hbaseClusterStartupToLiveTime)
+ duration.start()
+
+ Configuration clientConf = HBaseTestUtils.createHBaseConfiguration(
+ sliderClient)
+
+
+ while (!duration.limitExceeded && clustat.backupMastersSize != 1) {
+ clustat = HBaseTestUtils.getHBaseClusterStatus(clientConf);
+ Thread.sleep(1000);
+ }
+ if (duration.limitExceeded) {
+ describe("Cluster region server count of ${regionServerCount} not met:");
+ log.info(HBaseTestUtils.hbaseStatusToString(clustat));
+ fail("Backup master count not reached");
+ }
+
+ }
+}
+
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
new file mode 100644
index 0000000..5abd95f
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.core.registry.zk.ZKIntegration
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLaunchException
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestHBaseMaster extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testHBaseMaster() throws Throwable {
+ String clustername = "test_hbase_master"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+ //make sure that ZK is up and running at the binding string
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
+ //now launch the cluster with 1 region server
+ int regionServerCount = 1
+ ServiceLauncher launcher = createHBaseCluster(clustername, regionServerCount,
+ [
+ Arguments.ARG_COMP_OPT, HBaseKeys.ROLE_MASTER, RoleKeys.JVM_HEAP, "256M",
+ Arguments.ARG_DEFINE, SliderXmlConfKeys.KEY_YARN_QUEUE + "=default"
+ ],
+ true,
+ true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+
+ //dumpFullHBaseConf(sliderClient, clustername)
+
+ basicHBaseClusterStartupSequence(sliderClient)
+
+ //verify the #of region servers is as expected
+ dumpClusterStatus(sliderClient, "post-hbase-boot status")
+
+ //get the hbase status
+ waitForHBaseRegionServerCount(sliderClient, clustername, 1, hbaseClusterStartupToLiveTime)
+ waitForWorkerInstanceCount(sliderClient, 1, hbaseClusterStartupToLiveTime)
+ waitForRoleCount(sliderClient, HBaseKeys.ROLE_MASTER, 1,
+ hbaseClusterStartupToLiveTime)
+ }
+
+ @Test
+ public void testHBaseMasterWithBadHeap() throws Throwable {
+ String clustername = "test_hbase_master_with_bad_heap"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "verify that bad Java heap options are picked up"
+ //now launch the cluster with 1 region server
+ int regionServerCount = 1
+ try {
+ ServiceLauncher launcher = createHBaseCluster(clustername, regionServerCount,
+ [Arguments.ARG_COMP_OPT, HBaseKeys.ROLE_WORKER, RoleKeys.JVM_HEAP, "invalid"], true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+
+ AggregateConf launchedInstance = sliderClient.launchedInstanceDefinition
+ AggregateConf liveInstance = sliderClient.launchedInstanceDefinition
+
+
+
+ basicHBaseClusterStartupSequence(sliderClient)
+ def report = waitForClusterLive(sliderClient)
+
+ ClusterStatus clustat = getHBaseClusterStatus(sliderClient);
+ // verify that region server cannot start
+ if (clustat.servers.size()) {
+ dumpClusterDescription("original",launchedInstance )
+ dumpClusterDescription("live", sliderClient.liveInstanceDefinition)
+ dumpClusterStatus(sliderClient,"JVM heap option not picked up")
+ }
+ assert 0 == clustat.servers.size()
+ } catch (ServiceLaunchException e) {
+ assertExceptionDetails(e, SliderExitCodes.EXIT_DEPLOYMENT_FAILED)
+ }
+ }
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
new file mode 100644
index 0000000..d4f52ff
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMasterOnHDFS.groovy
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * Test of RM creation. This is so the later test's prereq's can be met
+ */
+@CompileStatic
+@Slf4j
+class TestHBaseMasterOnHDFS extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testHBaseMasteOnHDFS() throws Throwable {
+ String clustername = "test_hbase_master_on_hdfs"
+ createMiniCluster(clustername, getConfiguration(), 1, 1, 1, true, true)
+ log.info("HDFS is at $fsDefaultName")
+ assert fsDefaultName.startsWith("hdfs://")
+ ServiceLauncher launcher = createHBaseCluster(clustername, 1, [], true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+ log.info("Status $status")
+
+ basicHBaseClusterStartupSequence(sliderClient)
+ dumpClusterStatus(sliderClient, "post-hbase-boot status")
+
+ //get the hbase status
+ status = waitForWorkerInstanceCount(sliderClient, 1, hbaseClusterStartupToLiveTime)
+ waitForHBaseRegionServerCount(sliderClient, clustername, 1, hbaseClusterStartupToLiveTime)
+
+ clusterActionFreeze(sliderClient, clustername)
+
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
new file mode 100644
index 0000000..2d935d7
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveRegionServiceOnHDFS.groovy
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.core.registry.zk.ZKIntegration
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * test create a live region service
+ */
+@CompileStatic
+@Slf4j
+class TestLiveRegionServiceOnHDFS extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testLiveRegionServiceOnHDFS() throws Throwable {
+ String clustername = "test_live_region_service_on_hdfs"
+ int regionServerCount = 1
+ createMiniCluster(clustername, getConfiguration(), 1, 1, 1, true, true)
+ describe(" Create a single region service cluster");
+
+ //make sure that ZK is up and running at the binding string
+ ZKIntegration zki = createZKIntegrationInstance(ZKBinding, clustername, false, false, 5000)
+ //now launch the cluster
+ ServiceLauncher launcher = createHBaseCluster(clustername, regionServerCount, [], true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+ log.info("${status.toJsonString()}")
+
+// dumpFullHBaseConf(sliderClient)
+
+ log.info("Running basic HBase cluster startup sequence")
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(sliderClient)
+
+
+ status = waitForWorkerInstanceCount(sliderClient, regionServerCount, hbaseClusterStartupToLiveTime)
+ describe("Cluster status")
+ log.info(prettyPrint(status.toJsonString()))
+ //get the hbase status
+ waitForHBaseRegionServerCount(sliderClient, clustername, regionServerCount, hbaseClusterStartupToLiveTime)
+
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
new file mode 100644
index 0000000..9b4b8a7
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestLiveTwoNodeRegionService.groovy
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.hbase.ClusterStatus
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * test create a live region service
+ */
+@CompileStatic
+@Slf4j
+
+class TestLiveTwoNodeRegionService extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testLiveTwoNodeRegionService() throws Throwable {
+
+ String clustername = "test_live_two_node_regionservice"
+ int regionServerCount = 2
+ createMiniCluster(clustername, configuration, 1, 1, 1, true, false)
+
+ describe(" Create a two node region service cluster");
+
+ //now launch the cluster
+ ServiceLauncher<SliderClient> launcher = createHBaseCluster(clustername, regionServerCount,
+ [
+ Arguments.ARG_COMP_OPT, HBaseKeys.ROLE_MASTER, RoleKeys.JVM_HEAP , HB_HEAP,
+ Arguments.ARG_COMP_OPT, HBaseKeys.ROLE_WORKER, RoleKeys.JVM_HEAP , HB_HEAP
+ ],
+ true,
+ true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.getClusterDescription(clustername)
+ dumpClusterDescription("initial status", status)
+
+ ClusterStatus clustat = basicHBaseClusterStartupSequence(sliderClient)
+
+ waitForWorkerInstanceCount(sliderClient, regionServerCount, hbaseClusterStartupToLiveTime)
+ //get the hbase status
+ waitForHBaseRegionServerCount(sliderClient, clustername, regionServerCount, hbaseClusterStartupToLiveTime)
+
+ //now log the final status
+ status = sliderClient.getClusterDescription(clustername)
+
+ dumpClusterDescription("final status", status)
+
+ String rootPage = sliderClient.applicationReport.originalTrackingUrl
+ assert rootPage
+ log.info("Slider root = ${rootPage}")
+ def page = fetchWebPage(rootPage)
+ log.info(page)
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
new file mode 100644
index 0000000..face330
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.live
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.server.services.curator.CuratorServiceInstance
+import org.apache.slider.server.services.curator.RegistryBinderService
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+
+class TestTwoLiveClusters extends HBaseMiniClusterTestBase {
+
+ /**
+ * Create two clusters simultaneously and verify that their lifecycle is
+ * independent.
+ * @throws Throwable
+ */
+ @Test
+ public void testTwoLiveClusters() throws Throwable {
+ createMiniCluster("TestTwoLiveClusters", configuration, 1, true)
+ String clustername1 = "testtwoliveclusters-a"
+ //now launch the cluster
+ int regionServerCount = 1
+ ServiceLauncher<SliderClient> launcher = createHBaseCluster(clustername1, regionServerCount, [], true, true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ basicHBaseClusterStartupSequence(sliderClient)
+
+ //verify the #of region servers is as expected
+ dumpClusterStatus(sliderClient, "post-hbase-boot status")
+
+ //get the hbase status
+ waitForWorkerInstanceCount(sliderClient, 1, hbaseClusterStartupToLiveTime)
+ waitForHBaseRegionServerCount(sliderClient, clustername1, 1, hbaseClusterStartupToLiveTime)
+
+ //now here comes cluster #2
+ String clustername2 = "testtwoliveclusters-b"
+
+
+ String zkpath = "/$clustername2"
+ launcher = createHBaseCluster(clustername2, regionServerCount,
+ [
+ Arguments.ARG_ZKPATH, zkpath
+ ],
+ true,
+ true)
+ SliderClient cluster2Client = launcher.service
+ addToTeardown(cluster2Client);
+
+ basicHBaseClusterStartupSequence(cluster2Client)
+ waitForWorkerInstanceCount(cluster2Client, 1, hbaseClusterStartupToLiveTime)
+ waitForHBaseRegionServerCount(cluster2Client, clustername2, 1, hbaseClusterStartupToLiveTime)
+
+ //and now verify that cluster 1 is still happy
+ waitForHBaseRegionServerCount(sliderClient, clustername1, 1, hbaseClusterStartupToLiveTime)
+
+ // registry instances def names = client.listRegistryNames(clustername)
+ describe "service registry names"
+ RegistryBinderService<ServiceInstanceData> registry = cluster2Client.registry
+ def names = registry.queryForNames();
+ dumpRegistryNames(names)
+
+ List<String> instanceIds = sliderClient.listRegistryInstanceIDs()
+
+
+ dumpRegistryInstanceIDs(instanceIds)
+ assert names.size() == 1
+ assert instanceIds.size() == 2
+
+
+ List<CuratorServiceInstance<ServiceInstanceData>> instances = sliderClient.listRegistryInstances(
+ )
+ dumpRegistryInstances(instances)
+ assert instances.size() == 2
+
+ clusterActionFreeze(cluster2Client, clustername2,"freeze cluster 2")
+ clusterActionFreeze(sliderClient, clustername1,"Freeze cluster 1")
+
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestBadArguments.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestBadArguments.groovy
new file mode 100644
index 0000000..2581e03
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestBadArguments.groovy
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.masterless
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.core.main.ServiceLaunchException
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+import static HBaseKeys.PROVIDER_HBASE
+import static Arguments.ARG_PROVIDER
+
+@CompileStatic
+@Slf4j
+
+class TestBadArguments extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testBadAMHeap() throws Throwable {
+ String clustername = "test_bad_am_heap"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "verify that bad Java heap options are picked up"
+
+ try {
+ ServiceLauncher launcher = createCluster(clustername,
+ [
+ (HBaseKeys.ROLE_MASTER): 0,
+ (HBaseKeys.ROLE_WORKER): 0,
+ ],
+ [
+ Arguments.ARG_COMP_OPT, SliderKeys.COMPONENT_AM, RoleKeys.JVM_HEAP, "invalid",
+ ARG_PROVIDER, PROVIDER_HBASE
+ ],
+ true,
+ false,
+ [:])
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+
+ ApplicationReport report = waitForClusterLive(sliderClient)
+ assert report.yarnApplicationState == YarnApplicationState.FAILED
+
+ } catch (ServiceLaunchException e) {
+ assertExceptionDetails(e, SliderExitCodes.EXIT_YARN_SERVICE_FAILED)
+ }
+
+ }
+
+ /**
+ * Test disabled because YARN queues don't get validated in the mini cluster
+ * @throws Throwable
+ */
+ public void DisabledtestBadYarnQueue() throws Throwable {
+ String clustername = "test_bad_yarn_queue"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "verify that bad Java heap options are picked up"
+
+ try {
+ ServiceLauncher launcher = createCluster(clustername,
+ [
+ (HBaseKeys.ROLE_MASTER): 0,
+ (HBaseKeys.ROLE_WORKER): 0,
+ ],
+ [
+ Arguments.ARG_DEFINE,
+ SliderXmlConfKeys.KEY_YARN_QUEUE + "=noqueue",
+ ARG_PROVIDER, PROVIDER_HBASE
+ ],
+ true,
+ false,
+ [:])
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+
+ ApplicationReport report = waitForClusterLive(sliderClient)
+ assert report.yarnApplicationState == YarnApplicationState.FAILED
+
+ } catch (ServiceLaunchException e) {
+ assertExceptionDetails(e, SliderExitCodes.EXIT_YARN_SERVICE_FAILED)
+ }
+
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestBadClusterName.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestBadClusterName.groovy
new file mode 100644
index 0000000..93eb3b9
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestBadClusterName.groovy
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.masterless
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.core.main.ServiceLaunchException
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+import static HBaseKeys.PROVIDER_HBASE
+import static org.apache.slider.common.params.Arguments.ARG_PROVIDER
+
+@CompileStatic
+@Slf4j
+
+class TestBadClusterName extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testBadClusterName() throws Throwable {
+ String clustername = "TestBadClusterName"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "verify that bad cluster are picked up"
+
+ try {
+ ServiceLauncher launcher = createCluster(clustername,
+ [
+ (HBaseKeys.ROLE_MASTER): 0,
+ (HBaseKeys.ROLE_WORKER): 0,
+ ],
+ [
+ ARG_PROVIDER, PROVIDER_HBASE
+ ],
+ true,
+ false,
+ [:])
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ fail("expected a failure")
+ } catch (ServiceLaunchException e) {
+ assertExceptionDetails(e, LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR)
+ }
+
+ }
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestCreateDuplicateLiveCluster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestCreateDuplicateLiveCluster.groovy
new file mode 100644
index 0000000..da501ee
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestCreateDuplicateLiveCluster.groovy
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.masterless
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * create masterless AMs and work with them. This is faster than
+ * bringing up full clusters
+ */
+@CompileStatic
+@Slf4j
+
+class TestCreateDuplicateLiveCluster extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testCreateClusterRunning() throws Throwable {
+ String clustername = "test_create_duplicate_live_cluster"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "create a masterless AM, while it is running, try to create" +
+ "a second cluster with the same name"
+
+ //launch fake master
+ ServiceLauncher launcher
+ launcher = createMasterlessAM(clustername, 0, true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+
+ //now try to create instance #2, and expect an in-use failure
+ try {
+ createMasterlessAM(clustername, 0, false, true)
+ fail("expected a failure")
+ } catch (SliderException e) {
+ assertFailureClusterInUse(e);
+ }
+
+
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestDestroyMasterlessAM.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestDestroyMasterlessAM.groovy
new file mode 100644
index 0000000..bf9addc
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestDestroyMasterlessAM.groovy
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.masterless
+
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.ActionEchoArgs
+import org.apache.slider.common.params.CommonArgs
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * create masterless AMs and work with them. This is faster than
+ * bringing up full clusters
+ */
+//@CompileStatic
+@Slf4j
+
+class TestDestroyMasterlessAM extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testDestroyMasterlessAM() throws Throwable {
+ String clustername = "test_destroy_masterless_am"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "create a masterless AM, stop it, try to create" +
+ "a second cluster with the same name, destroy it, try a third time"
+
+ ServiceLauncher launcher1 = launchClientAgainstMiniMR(
+ getConfiguration(),
+ [
+ CommonArgs.ACTION_DESTROY,
+ "no-cluster-of-this-name",
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ ])
+ assert launcher1.serviceExitCode == 0
+
+
+
+ ServiceLauncher launcher = createMasterlessAM(clustername, 0, true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+
+ SliderFileSystem sliderFileSystem = createSliderFileSystem()
+ def hdfs = sliderFileSystem.fileSystem
+ def instanceDir = sliderFileSystem.buildClusterDirPath(clustername)
+
+ assertPathExists(
+ hdfs,
+ "cluster path not found",
+ instanceDir)
+
+ sliderFileSystem.locateInstanceDefinition(clustername)
+ clusterActionFreeze(sliderClient, clustername,"stopping first cluster")
+ waitForAppToFinish(sliderClient)
+
+
+ describe "Warnings below are expected"
+
+ //now try to create instance #2, and expect an in-use failure
+ try {
+ createMasterlessAM(clustername, 0, false, false)
+ fail("expected a failure, got an AM")
+ } catch (SliderException e) {
+ assertExceptionDetails(e,
+ SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.E_ALREADY_EXISTS)
+ }
+
+ describe "END EXPECTED WARNINGS"
+
+
+ describe "destroying $clustername"
+ //now: destroy it
+
+ int exitCode = sliderClient.actionDestroy(clustername);
+ assert 0 == exitCode
+
+ describe "post destroy checks"
+ sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
+
+ describe "thaw expected to fail"
+ //expect thaw to now fail
+ try {
+ launcher = launch(SliderClient,
+ configuration,
+ [
+ CommonArgs.ACTION_THAW,
+ clustername,
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ Arguments.ARG_MANAGER, RMAddr,
+ ])
+ fail("expected an exception")
+ } catch (UnknownApplicationInstanceException e) {
+ //expected
+ }
+
+ describe "thaw completed, checking dir is still absent"
+ sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
+
+
+ describe "recreating $clustername"
+
+ //and create a new cluster
+ launcher = createMasterlessAM(clustername, 0, false, true)
+ SliderClient cluster2 = launcher.service
+
+ // do an echo here of a large string
+ // Hadoop RPC couldn't handle strings > 32K chars, this
+ // check here allows us to be confident that large JSON Reports are handled
+ StringBuilder sb = new StringBuilder()
+ for (int i = 0; i < 65536; i++) {
+ sb.append(Integer.toString(i, 16))
+ }
+ ActionEchoArgs args = new ActionEchoArgs();
+ args.message = sb.toString();
+ def echoed = cluster2.actionEcho(clustername, args)
+ assert echoed == args.message
+ log.info(
+ "Successful echo of a text document ${echoed.size()} characters long")
+ //try to destroy it while live
+ try {
+ int ec = cluster2.actionDestroy(clustername)
+ fail("expected a failure from the destroy, got error code $ec")
+ } catch (SliderException e) {
+ assertFailureClusterInUse(e);
+ }
+
+ //and try to destroy a completely different cluster just for the fun of it
+ assert 0 == sliderClient.actionDestroy("no-cluster-of-this-name")
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestKillMasterlessAM.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestKillMasterlessAM.groovy
new file mode 100644
index 0000000..fd06695
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestKillMasterlessAM.groovy
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.masterless
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+import static org.apache.slider.providers.hbase.HBaseKeys.PROVIDER_HBASE
+import static org.apache.slider.common.params.Arguments.*
+
+/**
+ * create masterless AMs and work with them. This is faster than
+ * bringing up full clusters
+ */
+@CompileStatic
+@Slf4j
+
+class TestKillMasterlessAM extends HBaseMiniClusterTestBase {
+
+
+ @Test
+ public void testKillMasterlessAM() throws Throwable {
+ String clustername = "test_kill_masterless_am"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "kill a masterless AM and verify that it shuts down"
+
+ Map<String, Integer> roles = [
+ (HBaseKeys.ROLE_MASTER): 0,
+ (HBaseKeys.ROLE_WORKER): 0,
+ ]
+ ServiceLauncher launcher = createCluster(clustername,
+ roles,
+ [
+/*
+ ARG_COMP_OPT, SliderKeys.COMPONENT_AM,
+ RoleKeys.JVM_OPTS, "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005",
+*/
+ ARG_PROVIDER, PROVIDER_HBASE
+ ],
+ true,
+ true,
+ [:])
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ describe("listing services")
+ lsJavaProcesses();
+ describe("killing services")
+ killServiceLaunchers(SIGTERM);
+ waitWhileClusterLive(sliderClient);
+ //give yarn some time to notice
+ sleep(2000)
+ describe("final listing")
+ lsJavaProcesses();
+ ApplicationReport report = sliderClient.applicationReport
+ assert report.yarnApplicationState == YarnApplicationState.FAILED;
+
+
+
+ clusterActionFreeze(sliderClient, clustername)
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRecreateMasterlessAM.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRecreateMasterlessAM.groovy
new file mode 100644
index 0000000..87f5c78
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRecreateMasterlessAM.groovy
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.masterless
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * create masterless AMs and work with them. This is faster than
+ * bringing up full clusters
+ */
+@CompileStatic
+@Slf4j
+
+class TestRecreateMasterlessAM extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testRecreateMasterlessAM() throws Throwable {
+ String clustername = "test_recreate_masterless_am"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "create a masterless AM, stop it, try to create" +
+ "a second cluster with the same name"
+
+ ServiceLauncher launcher = createMasterlessAM(clustername, 0, true, true)
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ clusterActionFreeze(sliderClient, clustername)
+
+ //now try to create instance #2, and expect an in-use failure
+ try {
+ createMasterlessAM(clustername, 0, false, false)
+ fail("expected a failure")
+ } catch (SliderException e) {
+ assertExceptionDetails(e,
+ SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.E_ALREADY_EXISTS)
+ }
+
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy
new file mode 100644
index 0000000..c44de16
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.masterless
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+import static HBaseKeys.PROVIDER_HBASE
+import static Arguments.ARG_PROVIDER
+
+@CompileStatic
+@Slf4j
+
+class TestRoleOptPropagation extends HBaseMiniClusterTestBase {
+
+ @Test
+
+ public void testRoleOptPropagation() throws Throwable {
+ skip("Disabled")
+ String clustername = "test_role_opt_propagation"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "verify that role options propagate down to deployed roles"
+
+ String ENV = "env.ENV_VAR"
+ ServiceLauncher launcher = createCluster(clustername,
+ [
+ (HBaseKeys.ROLE_MASTER): 0,
+ (HBaseKeys.ROLE_WORKER): 0,
+ ],
+ [
+ Arguments.ARG_COMP_OPT, HBaseKeys.ROLE_MASTER, ENV, "4",
+ ARG_PROVIDER, PROVIDER_HBASE
+ ],
+ true,
+ true,
+ [:])
+ SliderClient sliderClient = (SliderClient) launcher.service
+ addToTeardown(sliderClient);
+ ClusterDescription status = sliderClient.clusterDescription
+ Map<String, String> masterRole = status.getRole(HBaseKeys.ROLE_MASTER);
+ dumpClusterDescription("Remote CD", status)
+ assert masterRole[ENV] == "4"
+
+ }
+
+ @Test
+ public void testUnknownRole() throws Throwable {
+ String clustername = "test_unknown_role"
+ createMiniCluster(clustername, getConfiguration(), 1, true)
+
+ describe "verify that unknown role results in cluster creation failure"
+ try {
+ String MALLOC_ARENA = "env.MALLOC_ARENA_MAX"
+ ServiceLauncher launcher = createCluster(clustername,
+ [
+ (HBaseKeys.ROLE_MASTER): 0,
+ (HBaseKeys.ROLE_WORKER): 0,
+ ],
+ [
+ Arguments.ARG_COMP_OPT, SliderKeys.COMPONENT_AM, MALLOC_ARENA, "4",
+ Arguments.ARG_COMP_OPT, "unknown", MALLOC_ARENA, "3",
+ ARG_PROVIDER, PROVIDER_HBASE
+ ],
+ true,
+ true,
+ [:])
+ assert false
+ } catch (BadCommandArgumentsException bcae) {
+ /* expected */
+ }
+ }
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestSliderConfDirToMasterlessAM.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestSliderConfDirToMasterlessAM.groovy
new file mode 100644
index 0000000..45e5a0a
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestSliderConfDirToMasterlessAM.groovy
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.masterless
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+
+import org.junit.Test
+
+/**
+ * create masterless AMs and work with them. This is faster than
+ * bringing up full clusters
+ */
+@CompileStatic
+@Slf4j
+
+class TestSliderConfDirToMasterlessAM extends HBaseMiniClusterTestBase {
+
+
+ @Test
+ public void testSliderConfDirToMasterlessAM() throws Throwable {
+ String clustername = "test_slider_conf_dir_to_masterless_am"
+ YarnConfiguration conf = configuration
+ createMiniCluster(clustername, conf, 1, true)
+
+ describe "verify that a conf dir will propagate via the sytem proerpty"
+
+ File localConf = File.createTempDir("conf","dir")
+ String name = "slider.xml"
+ File sliderXML = new File(localConf, name)
+ def out = new FileWriter(sliderXML)
+ out.write(['a','b','c'] as char[])
+ out.write("as string")
+ out.flush()
+ out.close()
+ try {
+ System.setProperty(SliderKeys.PROPERTY_CONF_DIR,localConf.absolutePath);
+ ServiceLauncher<SliderClient> launcher = createMasterlessAM(clustername, 0, true, true)
+ SliderClient client = launcher.service
+ addToTeardown(client);
+ ApplicationReport report = waitForClusterLive(client)
+
+ ClusterDescription cd = waitForRoleCount(client,SliderKeys.COMPONENT_AM,
+ 1, hbaseClusterStartupTime)
+ HadoopFS fs = HadoopFS.getLocal(conf);
+
+ Path clusterDir = new SliderFileSystem(fs, conf).buildClusterDirPath(clustername)
+ assert fs.exists(clusterDir);
+ Path sliderConfDir = new Path(clusterDir, SliderKeys.SUBMITTED_CONF_DIR)
+ assert fs.exists(sliderConfDir);
+ Path remoteXml = new Path(sliderConfDir,name)
+ assert fs.exists(remoteXml)
+
+
+ clusterActionFreeze(client, clustername)
+ } finally {
+ SliderUtils.deleteDirectoryTree(localConf)
+ }
+
+
+
+ }
+
+
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/java/org/apache/slider/providers/hbase/StubCompile.java b/slider-providers/hbase/slider-hbase-provider/src/test/java/org/apache/slider/providers/hbase/StubCompile.java
new file mode 100644
index 0000000..7f062a4
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/java/org/apache/slider/providers/hbase/StubCompile.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase;
+
+class StubCompile {
+}
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties b/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a552a55
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+log4j.logger.org.apache.slider=DEBUG
+log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+
+
+
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
+log4j.logger.org.apache.hadoop.metrics2=ERROR
+log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
+log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN
+log4j.logger.org.apache.hadoop.security.token.delegation=WARN
+log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
diff --git a/src/site/html/blobstore-index.html b/src/site/html/blobstore-index.html
new file mode 100644
index 0000000..b2b092f
--- /dev/null
+++ b/src/site/html/blobstore-index.html
@@ -0,0 +1,54 @@
+<html>
+<head>
+ <meta charset="UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+ <meta name="Date-Revision-yyyymmdd" content="20131008"/>
+ <meta http-equiv="Content-Language" content="en"/>
+ <title>Slider: HBase on YARN</title>
+</head>
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+ <!--
+ Blobstore index.html
+
+ This file is for placement at the bottom of a blobstore serving
+ up the Slider artifacts.
+
+ -->
+<body>
+<h1>Slider Binary Releases </h1>
+
+<p></p>
+
+This is a repository of the released Slider artifacts
+
+<p>
+Slider is a Hadoop YARN application that can dynamically deploy
+Apache HBase and Apache Accumulo Column-table databases to a Hadoop 2.2+
+cluster.
+</p>
+
+For more details, please consult
+the <a href="https://github.com/hortonworks/slider">source repository</a>
+
+<p></p>
+
+<h2>Releases</h2>
+</body>
+
+</html>
diff --git a/src/site/markdown/agent_test_setup.md b/src/site/markdown/agent_test_setup.md
new file mode 100644
index 0000000..b0270a0
--- /dev/null
+++ b/src/site/markdown/agent_test_setup.md
@@ -0,0 +1,47 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Setting up for Agent test
+
+ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL
+ NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and
+ "OPTIONAL" in this document are to be interpreted as described in
+ RFC 2119.
+
+An agent is an application independent entity that can handle any application as long as its definition conforms to a specific format (TBD). While Slider will install the agent in its entirity, for now the host, where the agent is deployed, needs to be prepared for the agent. The following steps are necessary to setup the host for agent.
+
+## Install resource management library
+The resource management library is a common python library used by all application specification. The library is either pre-installed on the hosts or it is packaged along with the application package itself.
+
+Get resource_management.tar and install it at /usr/lib/python2.6/site-packages
+
+## Install the application spec
+The application spec is the declarative definition of a application that can be run on YARN. *At this point only HBase application is supported.*
+
+Get HDP-2.0.6.tar and install at /var/lib/ambari-agent/cache/stacks/HDP.
+
+## Install HBase using tarball
+Get the hbase tarball, hbase-0.96.1-hadoop2-bin.tar.tar.tar.gz, and expand it at /share/hbase.
+
+## Permissions
+Ensure that the user creating the hbase cluster has necessary permission for the resource management library and the application spec. Perform necessary **chown** and **chmod**.
+
+1. /share/hbase/hbase-0.96.1-hadoop2/conf
+2. /usr/lib/python2.6/site-packages/resource_management
+3. /var/lib/ambari-agent/cache/stacks/HDP
+4. /var/log/hbase, /var/run/hbase (or appropriate log and run directories)
+5. Ensure hbase root/staging HDFS directories have appropriate permission
diff --git a/src/site/markdown/app_needs.md b/src/site/markdown/app_needs.md
new file mode 100644
index 0000000..536a880
--- /dev/null
+++ b/src/site/markdown/app_needs.md
@@ -0,0 +1,140 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Slider's needs of an application
+
+Slider installs and runs applications in a YARN cluster -applications that
+do not need to be written for YARN.
+
+What they do need to be is deployable by Slider, which means installable by YARN,
+configurable by Slider, and, finally, executable by YARN. YARN will kill the
+executed process when destroying a container, so the deployed application
+must expect this to happen and be able to start up from a kill-initiated
+shutdown without any manual recovery process.
+
+They need to locate each other dynamically, both at startup and during execution,
+because the location of processes will be unknown at startup, and may change
+due to server and process failures.
+
+## Must
+
+* Install and run from a tarball -and be run from a user that is not root.
+
+* Be self contained or have all dependencies pre-installed.
+
+* Support dynamic discovery of nodes -such as via ZK.
+
+* Nodes to rebind themselves dynamically -so if nodes are moved, the application
+can continue
+
+* Handle kill as a normal shutdown mechanism.
+
+* Support multiple instances of the application running in the same cluster,
+ with processes from different application instances sharing
+the same servers.
+
+* Operate correctly when more than one role instance in the application is
+deployed on the same physical host. (If YARN adds anti-affinity options in
+container requests this will no longer be a requirement)
+
+* Dynamically allocate any RPC or web ports -such as supporting 0 as the number
+of the port to listen on in configuration options.
+
+* Be trusted. YARN does not run code in a sandbox.
+
+* If it talks to HDFS or other parts of Hadoop, be built against/ship with
+libaries compatible with the version of Hadoop running on the cluster.
+
+* Store persistent data in HDFS (directly or indirectly) with the exact storage location
+configurable. Specifically: not to the local filesystem, and not in a hard coded location
+such as `hdfs://app/data`. Slider creates per-instance directories for
+persistent data.
+
+* Be configurable as to where any configuration directory is (or simply relative
+to the tarball). The application must not require it to be in a hard-coded
+location such as `/etc`.
+
+* Not have a fixed location for log output -such as `/var/log/something`
+
+* Run until explicitly terminated. Slider treats an application termination
+(which triggers a container release) as a failure -and reacts to it by restarting
+the container.
+
+
+
+## MUST NOT
+
+* Require human intervention at startup or termination.
+
+## SHOULD
+
+These are the features that we'd like from a service:
+
+* Publish the actual RPC and HTTP ports in a way that can be picked up, such as via ZK
+or an admin API.
+
+* Be configurable via the standard Hadoop mechanisms: text files and XML configuration files.
+If not, custom parsers/configuration generators will be required.
+
+* Support an explicit parameter to define the configuration directory.
+
+* Take late bindings params via -D args or similar
+
+* Be possible to exec without running a complex script, so that process inheritance works everywhere, including (for testing) OS/X
+
+* Provide a way for Slider to get list of nodes in cluster and status. This will let Slider detect failed worker nodes and react to it.
+
+* FUTURE: If a graceful decommissioning is preferred, have an RPC method that a Slider provider can call to invoke this.
+
+* Be location aware from startup. Example: worker nodes to be allocated tables to serve based on which tables are
+stored locally/in-rack, rather than just randomly. This will accelerate startup time.
+
+* Support simple liveness probes (such as an HTTP GET operations).
+
+* Return a well documented set of exit codes, so that failures can be propagated
+ and understood.
+
+* Support cluster size flexing: the dynamic addition and removal of nodes.
+
+
+* Support a management platform such as Apache Ambari -so that the operational
+state of a Slider application can be monitored.
+
+## MAY
+
+* Include a single process that will run at a fixed location and whose termination
+can trigger application termination. Such a process will be executed
+in the same container as the Slider AM, and so known before all other containers
+are requested. If a live cluster is unable to handle restart/migration of
+such a process, then the Slider application will be unable to handle
+Slider AM restarts.
+
+* Ideally: report on load/cost of decommissioning.
+ E.g amount of data; app load.
+
+
+## MAY NOT
+
+* Be written for YARN.
+
+* Be (pure) Java. If the tarball contains native binaries for the cluster's hardware & OS,
+ they should be executable.
+
+* Be dynamically reconfigurable, except for the special requirement of handling
+movement of manager/peer containers in an application-specific manner.
+
+
diff --git a/src/site/markdown/architecture.md b/src/site/markdown/architecture.md
new file mode 100644
index 0000000..30a3816
--- /dev/null
+++ b/src/site/markdown/architecture.md
@@ -0,0 +1,139 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Architecture
+
+## Summary
+
+Slider is a YARN application to deploy non-YARN-enabled applications in a YARN cluster
+
+Slider consists of a YARN application master, the "Slider AM", and a client application which communicates with YARN and the Slider AM via remote procedure calls and/or REST requests. The client application offers command line access, as well as low-level API access for test purposes
+
+The deployed application must be a program that can be run across a pool of
+YARN-managed servers, dynamically locating its peers. It is not Slider's
+responsibility to configure up the peer servers, apart from some initial
+application-specific application instance configuration. (The full requirements
+of an application are [described in another document](app_needs.md).
+
+Every application instance is described as a set of one or more *component*; each
+component can have a different program/command, and a different set of configuration
+options and parameters.
+
+The AM takes the details on which roles to start, and requests a YARN container
+for each component; It then monitors the state of the application instance, receiving messages
+from YARN when a remotely executed process finishes. It then deploys another instance of
+that component.
+
+
+## Slider Packaging
+
+A key goal of Slider is to support the deployment of existing applications into
+a YARN application instance, without having to extend Slider itself.
+
+
+
+## AM Architecture
+
+The application master consists of
+
+ 1. The AM engine which handles all integration with external services, specifically YARN and any Slider clients
+ 1. A *provider* specific to deploying a class of applications.
+ 1. The Application State.
+
+The Application State is the model of the application instance, containing
+
+ 1. A specification of the desired state of the application instance -the number of instances of each role, their YARN and process memory requirements and some other options.
+ 1. A map of the current instances of each role across the YARN cluster, including reliability statistics of each node in the application instance used.
+ 1. [The Role History](rolehistory.html) -a record of which nodes roles were deployed on for re-requesting the same nodes in future. This is persisted to disk and re-read if present, for faster application startup times.
+ 1. Queues of track outstanding requests, released and starting nodes
+
+The Application Engine integrates with the outside world: the YARN Resource Manager ("the RM"), and the node-specific Node Managers, receiving events from the services, requesting or releasing containers via the RM, and starting applications on assigned containers.
+
+After any notification of a change in the state of the cluster (or an update to the client-supplied cluster specification), the Application Engine passes the information on to the Application State class, which updates its state and then returns a list of cluster operations to be submitted: requests for containers of different types -potentially on specified nodes, or requests to release containers.
+
+As those requests are met and allocation messages passed to the Application Engine, it works with the Application State to assign them to specific components, then invokes the provider to build up the launch context for that application.
+
+The provider has the task of populating container requests with the file references, environment variables and commands needed to start the provider's supported programs.
+
+The core provider deploys a minimal agent on the target containers, then, as the agent checks in to the agent provider's REST API, executes commands issued to it.
+
+The set of commands this agent executes focuses on downloading archives from HDFS, expanding them, then running Python scripts which perform the
+actual configuration and execution of the target problem -primarily through template expansion.
+
+
+To summarize: Slider is not an classic YARN analysis application, which allocates and schedules work across the cluster in short-to-medium life containers with the lifespan of a query or an analytics session, but instead for an application with a lifespan of days to months. Slider works to keep the actual state of its application cluster to match the desired state, while the application has the tasks of recovering from node failure, locating peer nodes and working with data in an HDFS filesystem.
+
+As such it is one of the first applications designed to use YARN as a platform for long-lived services -Samza being the other key example. These application's needs of YARN are different, and their application manager design is focused around maintaining the distributed application in its desired state rather than the ongoing progress of submitted work.
+
+The clean model-view-controller split was implemented to isolate the model and aid mock testing of large clusters with simulated scale, and hence increase confidence that Slider can scale to work in large YARN clusters and with larger application instances.
+
+
+
+### Failure Model
+
+The application master is designed to be a [crash-only application](https://www.usenix.org/legacy/events/hotos03/tech/full_papers/candea/candea.pdf), clients are free to terminate
+the application instance by asking YARN directly.
+
+There is an RPC call to stop the application instance - this is a nicety which includes a message in the termination log, and
+could, in future, perhaps warn the provider that the application instance is being torn down. That is a potentially dangerous feature
+to add -as provider implementors may start to expect the method to be called reliably. Slider is designed to fail without
+warning, to rebuild its state on a YARN-initiated restart, and to be manually terminated without any advance notice.
+
+### RPC Interface
+
+
+The RPC interface allows the client to query the current application state, and to update it by pushing out a new JSON specification.
+
+The core operations are
+
+* `getJSONClusterStatus()`: get the status of the application instance as a JSON document.
+* `flexCluster()` update the desired count of role instances in the running application instance.
+* `stopCluster` stop the application instance
+
+There are some other low-level operations for extra diagnostics and testing, but they are of limited importancs
+
+The `flexCluster()` call takes a JSON application instance specification and forwards it to the AM -which extracts the desired counts of each role to update the Application State. A change in the desired size of the application instance, is treated as any reported failure of node:
+it triggers a re-evaluation of the application state, building up the list of container add and release requests to make of
+the YARN resource manager.
+
+The final operation, `stopCluster()`, stops the application instance.
+
+### Security and Identity
+
+Slider's security model is described in detail in [an accompanying document](security.html)
+
+A Slider application instance is expected to access data belonging to the user creating the instance.
+
+In a secure YARN cluster, this is done by acquiring Kerberos tokens in the client when the application instance is updated, tokens which
+are propagated to the Slider AM and thence to the deployed application containers themselves. These
+tokens are valid for a finite time period.
+
+HBase has always required keytab files to be installed on every node in the Hadoop for it to have secure access -this requirement
+holds for Slider-deployed HBase clusters. Slider does not itself adopt the responsibility of preparing or distributing these files;
+this must be done via another channel.
+
+In Hadoop 2.2, the tokens for communication between the Slider AM and YARN expire after -by default- 72 hours. The
+HDFS tokens will also expire after some time period. This places an upper bound on the lifespan of a Slider application (or any
+other long-lived YARN application) in a secure Hadoop cluster.
+
+
+
+In an insecure Hadoopp cluster, the Slider AM and its containers are likely to run in a different OS account from the submitting user.
+To enable access to the database files as that submitting use, the identity of the user is provided when the AM is created; the
+AM will pass this same identity down to the created containers. This information *identifies* the user -but does not *authenticate* them: they are trusted to be who they claim to be.
+
+
diff --git a/src/site/markdown/building.md b/src/site/markdown/building.md
new file mode 100644
index 0000000..32ba758
--- /dev/null
+++ b/src/site/markdown/building.md
@@ -0,0 +1,368 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Building Slider
+
+
+Here's how to set this up.
+
+## Before you begin
+
+### Networking
+
+The network on the development system must be functional, with hostname lookup
+of the local host working. Tests will fail without this.
+
+### Maven
+
+You will need a version of Maven 3.0+, set up with enough memory
+
+ MAVEN_OPTS=-Xms256m -Xmx512m -Djava.awt.headless=true
+
+
+*Important*: As of October 6, 2013, Maven 3.1 is not supported due to
+[version issues](https://cwiki.apache.org/confluence/display/MAVEN/AetherClassNotFound).
+
+### Protoc
+
+You need a copy of the `protoc` compile
+
+1. OS/X: `brew install protobuf`
+1. Others: consult (Building Hadoop documentation)[http://wiki.apache.org/hadoop/HowToContribute].
+
+The version of protoc installed must be the same as that used by Hadoop itself.
+This is absolutely critical to prevent JAR version problems.
+
+## Building a compatible Hadoop version
+
+
+Slider is built against Hadoop 2 -you can download and install
+a copy from the [Apache Hadoop Web Site](http://hadoop.apache.org).
+
+
+During development, its convenient (but not mandatory)
+to have a local version of Hadoop -so that we can find and fix bugs/add features in
+Hadoop as well in Slider.
+
+
+To build and install locally, check out apache svn/github, branch `release-2.4.0`,
+and create a branch off that tag
+
+ git clone git://git.apache.org/hadoop-common.git
+ cd hadoop-common
+ git remote rename origin apache
+ git fetch --tags apache
+ git checkout release-2.4.0 --
+ git checkout -b release-2.4.0
+
+
+For the scripts below, set the `HADOOP_VERSION` variable to the version
+
+ export HADOOP_VERSION=2.4.0
+
+or, for building against a pre-release version of Hadoop 2.4
+
+ git checkout branch-2
+ export HADOOP_VERSION=2.4.0-SNAPSHOT
+
+To build and install it locally, skipping the tests:
+
+ mvn clean install -DskipTests
+
+To make a tarball for use in test runs:
+
+ #On osx
+ mvn clean install package -Pdist -Dtar -DskipTests -Dmaven.javadoc.skip=true
+
+ # on linux
+ mvn clean package -Pdist -Pnative -Dtar -DskipTests -Dmaven.javadoc.skip=true
+
+Then expand this
+
+ pushd hadoop-dist/target/
+ gunzip hadoop-$HADOOP_VERSION.tar.gz
+ tar -xvf hadoop-$HADOOP_VERSION.tar
+ popd
+
+This creates an expanded version of Hadoop. You can now actually run Hadoop
+from this directory. Do note that unless you have the native code built for
+your target platform, Hadoop will be slower.
+
+## building a compatible HBase version
+
+If you need to build a version of HBase -rather than use a released version,
+here are the instructions (for the hbase-0.98 release branch)
+
+Checkout the HBase `trunk` branch from apache svn/github.
+
+
+ git clone git://git.apache.org/hbase.git
+ cd hbase
+ git remote rename origin apache
+ git fetch --tags apache
+
+then
+
+ git checkout -b apache/0.98
+or
+
+ git checkout tags/0.98.1
+
+If you have already been building versions of HBase, remove the existing
+set of artifacts for safety:
+
+ rm -rf ~/.m2/repository/org/apache/hbase/
+
+The maven command for building hbase artifacts against this hadoop version is
+
+ mvn clean install assembly:single -DskipTests -Dmaven.javadoc.skip=true
+
+To use a different version of Hadoop from that defined in the `hadoop-two.version`
+property of`/pom.xml`:
+
+ mvn clean install assembly:single -DskipTests -Dmaven.javadoc.skip=true -Dhadoop-two.version=$HADOOP_VERSION
+
+This will create an hbase `tar.gz` file in the directory `hbase-assembly/target/`
+in the hbase source tree.
+
+ export HBASE_VERSION=0.98.1
+
+ pushd hbase-assembly/target
+ gunzip hbase-$HBASE_VERSION-bin.tar.gz
+ tar -xvf hbase-$HBASE_VERSION-bin.tar
+ gzip hbase-$HBASE_VERSION-bin.tar
+ popd
+
+This will create an untarred directory containing
+hbase. Both the `.tar.gz` and untarred file are needed for testing. Most
+tests just work directly with the untarred file as it saves time uploading
+and downloading then expanding the file.
+
+(and if you set `HBASE_VERSION` to something else, you can pick up that version
+-making sure that slider is in sync)
+
+For more information (including recommended Maven memory configuration options),
+see [HBase building](http://hbase.apache.org/book/build.html)
+
+For building just the JAR files:
+
+ mvn clean install -DskipTests -Dhadoop.profile=2.0 -Dhadoop-two.version=$HADOOP_VERSION
+
+*Tip:* you can force set a version in Maven by having it update all the POMs:
+
+ mvn versions:set -DnewVersion=0.98.1-SNAPSHOT
+
+## Building Accumulo
+
+Clone accumulo from apache;
+
+ git clone http://git-wip-us.apache.org/repos/asf/accumulo.git
+
+
+Check out branch 1.5.1-SNAPSHOT
+
+
+
+In the accumulo project directory, build it
+
+ mvn clean install -Passemble -DskipTests -Dmaven.javadoc.skip=true \
+ -Dhadoop.profile=2
+
+The default Hadoop version for accumulo-1.5.1 is hadoop 2.4.0; to build
+against a different version use the command
+
+ mvn clean install -Passemble -DskipTests -Dmaven.javadoc.skip=true \
+ -Dhadoop.profile=2 -Dhadoop.version=$HADOOP_VERSION
+
+This creates an accumulo tar.gz file in `assemble/target/`. Unzip then untar
+this, to create a .tar file and an expanded directory
+
+ accumulo/assemble/target/accumulo-1.5.1-SNAPSHOT-bin.tar
+
+ This can be done with the command sequence
+
+ export ACCUMULO_VERSION=1.5.1-SNAPSHOT
+
+ pushd assemble/target/
+ gunzip -f accumulo-$ACCUMULO_VERSION-bin.tar.gz
+ tar -xvf accumulo-$ACCUMULO_VERSION-bin.tar
+ popd
+
+Note that the final location of the accumulo files is needed for the configuration,
+it may be directly under target/ or it may be in a subdirectory, with
+a path such as `target/accumulo-$ACCUMULO_VERSION-dev/accumulo-$ACCUMULO_VERSION/`
+
+
+## Testing
+
+### Configuring Slider to locate the relevant artifacts
+
+You must have the file `src/test/resources/slider-test.xml` (this
+is ignored by git), declaring where HBase, accumulo, Hadoop and zookeeper are:
+
+ <configuration>
+
+ <property>
+ <name>slider.test.hbase.home</name>
+ <value>/home/slider/hbase/hbase-assembly/target/hbase-0.98.0-SNAPSHOT</value>
+ <description>HBASE Home</description>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <value>/home/hoya/hbase/hbase-assembly/target/hbase-0.98.0-SNAPSHOT-bin.tar.gz</value>
+ <description>HBASE archive URI</description>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.home</name>
+ <value>/home/hoya/accumulo/assemble/target/accumulo-1.5.1-SNAPSHOT/</value>
+ <description>Accumulo Home</description>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.tar</name>
+ <value>/home/hoya/accumulo/assemble/target/accumulo-1.5.1-SNAPSHOT-bin.tar.gz</value>
+ <description>Accumulo archive URI</description>
+ </property>
+
+ <property>
+ <name>zk.home</name>
+ <value>
+ /home/hoya/Apps/zookeeper</value>
+ <description>Zookeeper home dir on target systems</description>
+ </property>
+
+ <property>
+ <name>hadoop.home</name>
+ <value>
+ /home/hoya/hadoop-common/hadoop-dist/target/hadoop-2.3.0</value>
+ <description>Hadoop home dir on target systems</description>
+ </property>
+
+ </configuration>
+
+
+## Debugging a failing test
+
+1. Locate the directory `target/$TESTNAME` where TESTNAME is the name of the
+test case and or test method. This directory contains the Mini YARN Cluster
+logs. For example, `TestLiveRegionService` stores its data under
+`target/TestLiveRegionService`
+
+1. Look under that directory for `-logdir` directories, then an application
+and container containing logs. There may be more than node being simulated;
+every node manager creates its own logdir.
+
+1. Look for the `out.txt` and `err.txt` files for stdout and stderr log output.
+
+1. Slider uses SLF4J to log to `out.txt`; remotely executed processes may use
+either stream for logging
+
+Example:
+
+ target/TestLiveRegionService/TestLiveRegionService-logDir-nm-1_0/application_1376095770244_0001/container_1376095770244_0001_01_000001/out.txt
+
+1. The actual test log from JUnit itself goes to the console and into
+`target/surefire/`; this shows the events happening in the YARN services as well
+ as (if configured) HDFS and Zookeeper. It is noisy -everything after the *teardown*
+ message happens during cluster teardown, after the test itself has been completed.
+ Exceptions and messages here can generally be ignored.
+
+This is all a bit complicated -debugging is simpler if a single test is run at a
+time, which is straightforward
+
+ mvn clean test -Dtest=TestLiveRegionService
+
+
+### Building the JAR file
+
+You can create the JAR file and set up its directories with
+
+ mvn package -DskipTests
+
+# Development Notes
+
+
+## Git branch model
+
+The git branch model uses is
+[Git Flow](http://nvie.com/posts/a-successful-git-branching-model/).
+
+This is a common workflow model for Git, and built in to
+[Atlassian Source Tree](http://sourcetreeapp.com/).
+
+The command line `git-flow` tool is easy to install
+
+ brew install git-flow
+
+or
+
+ apt-get install git-flow
+
+You should then work on all significant features in their own branch and
+merge them back in when they are ready.
+
+
+ # until we get a public JIRA we're just using an in-house one. sorry
+ git flow feature start BUG-8192
+
+ # finishes merges back in to develop/
+ git flow feature finish BUG-8192
+
+ # release branch
+ git flow release start 0.4.0
+
+ git flow release finish 0.4.0
+
+## Attn OS/X developers
+
+YARN on OS/X doesn't terminate subprocesses the way it does on Linux, so
+HBase Region Servers created by the hbase shell script remain running
+even after the tests terminate.
+
+This causes some tests -especially those related to flexing down- to fail,
+and test reruns may be very confused. If ever a test fails because there
+are too many region servers running, this is the likely cause
+
+After every test run: do a `jps -v` to look for any leftover HBase services
+-and kill them.
+
+Here is a handy bash command to do this
+
+ jps -l | grep HRegion | awk '{print $1}' | xargs kill -9
+
+
+## Groovy
+
+Slider uses Groovy 2.x as its language for writing tests -for better assertions
+and easier handling of lists and closures. Although the first prototype
+used Groovy on the production source, this was dropped in favor of
+a Java-only production codebase.
+
+## Maven utils
+
+
+Here are some handy aliases to make maven easier
+
+ alias mci='mvn clean install -DskipTests'
+ alias mi='mvn install -DskipTests'
+ alias mvct='mvn clean test'
+ alias mvnsite='mvn site:site -Dmaven.javadoc.skip=true'
+ alias mvt='mvn test'
+
+
diff --git a/src/site/markdown/client-configuration.md b/src/site/markdown/client-configuration.md
new file mode 100644
index 0000000..9b13021
--- /dev/null
+++ b/src/site/markdown/client-configuration.md
@@ -0,0 +1,310 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Client Configuration
+
+This document covers how the client application is itself configured.
+
+## Summary
+
+The client application can be configured
+
+1. On the command line, which can set client options and JVM system properties.
+2. With Hadoop-style configuration options in the file `slider-client.xml`
+ in the configuration directory`conf/` dir
+2. Or, if the environment variable `SLIDER_CONF_DIR` is set, in the
+ file `$SLIDER_CONF_DIR/slider-client.xml`
+1. Logging is defined in the `log4j.properties` file in the same configuration
+directory.
+1. VM options can be defined in `SLIDER_JVM_OPTS`
+
+The options defined in a Slider cluster configuration are only used by the client
+when creating a cluster -not for the actual client itself.
+
+## Introduction
+
+The Slider client needs to be configured to talk to a Hadoop filesystem and a
+YARN resource manager ("the RM"). In a secure cluster it needs to be told the Kerberos
+identity, the *principal* of both the HDFS namenode and the YARN RM -and it may
+also need some JVM options set in order for Java's Kerberos module to
+correctly identify itself to these services.
+
+It cannot rely on local `HADOOP_PREFIX/conf/hadoop-site.xml` and
+`$YARN_PREFIX/conf/yarn-site.xml` files -because it is designed to
+work on client machines that may not have Hadoop and YARN installed.
+
+Instead all client-side (non-JVM) options can be predefined in the
+configuration file `slider-client.xml`.
+
+## Setting Slider JVM options
+
+Core JVM options can be set in the environment variable `SLIDER_JVM_OPTS`;
+if unset the `bin/slider` script will use the default values that were
+current when that version of Slider was released. These values may change
+across versions, and may in fact be.
+
+At the time of writing, the default values were:
+
+ "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Dslider.confdir=${confdir}"
+
+To allow some Java system properties to be set without editing this
+environment variable, such system properties may be set on the Slider command
+line through the `-S` parameter. For example, the following two operations are
+equivalent in terms of setting the system property `java.security.krb5.realm`
+to the value `LOCAL`.
+
+ export SLIDER_JVM_OPTS="-Djava.security.krb5.realm=LOCAL"
+
+and
+
+ slider -S java.security.krb5.realm=LOCAL
+
+Note that the first declaration invalidates all default JVM options; if any of
+those were desired, they should be included in the new definition.
+
+Multiple system property declarations are allowed on the command line -including
+duplicate declarations. In such a case the order of assignment is undefined.
+
+For any system property that the user expects to have to issue on every command
+-including any kerberos-related properties, adding them to the JVM options
+environment variable guarantees that they are always set.
+
+## Setting Slider client options on the command line with the `-D` parameter
+
+The slider client is configured via Hadoop-style configuration options.
+To be precise, all standard Hadoop-common, hadoop-hdfs client and hadoop-yar
+client-side options control how Slider communicates with the Hadoop YARN cluster.
+
+There are extra options specific to Slider itself, options which
+are again set as Hadoop configuration parameters.
+
+All Hadoop and Slider options can be set on the command line using the `-D`
+parameter followed by the appropriate `key=value` argument
+
+
+For example, here is a definition of the default Hadoop filesystem:
+
+ -D fs.defaultFS=hdfs://namenode:9000
+
+Multiple definitions are of course allowed on the command line
+
+ -D fs.defaultFS=hdfs://namenode:9000 -D dfs.namenode.kerberos.principal=hdfs/namenode@LOCAL
+
+Slider-specific options can be made the same way
+
+ -D slider.kerberos.principal=
+
+If duplicate declarations are made the order of assignment is undefined.
+
+# Setting common options through specific command-line arguments
+
+Some Hadoop and Slider options are so common that they have specific
+shortcut commands to aid their use
+
+`-m`, `--manager` : sets the YARN resource manager. Equivalent to setting the
+`yarn.resourcemanager.address` option
+
+`--fs`, `--filesystem`: defines the filesystem. Equivalent to setting the
+`fs.defaultFS` option
+
+If these shortcuts are used and the options are also defined via `-D`
+declarations, the order of assignment is undefined.
+
+# Defining Hadoop and Slider Options in the `slider-client.xml` file.
+
+In the Slider installation, alongside the `bin/slider` script is
+a configuration directory `conf`. This contains the files:
+
+1. `log4j.properties`
+1. `slider-client.xml`
+
+The `log4j.properties` file is not covered here -it is a standard Log4J file.
+At the time of writing, this log configuration file is used on both the
+client and the server.
+
+The `slider-client.xml` file is a hadoop-formatted XML options file, which
+is read by the Slider client -but not by they Slider Application Master.
+
+Here is an example file:
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>namenode:8033</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://namenode:9000</value>
+ </property>
+
+ <property>
+ <name>ipc.client.fallback-to-simple-auth-allowed</name>
+ <value>false</value>
+ </property>
+
+
+This defines both the filesystem and the YARN RM, and so obviates the need
+to declare either on the command line.
+
+If an option is defined in the `slider-client.xml` file and on the command line
+-be it by a `-D key=value` declaration or a `--manager` or `--filesystem`
+definition. (this holds even if the value is declared with `<final>true</final>`).
+
+## Selecting an alternate Slider configuration directory
+
+The environment variable `SLIDER_CONF_DIR` can be used to declare an alternate
+configuration directory. If set, the directory it identifies will be used
+as the source of the `log4j.properties` and `slider-client.xml` files.
+
+## Slider Client Configuration options
+
+As well as standard YARN and Hadoop configuration options, Slider supports
+a limited number of slider-specific configuration parameters.
+
+ <property>
+ <name>slider.zookeeper.quorum</name>
+ <value>localhost:2181,zookeeper2:4545</value>
+ </property>
+
+ <property>
+ <name>slider.yarn.queue</name>
+ <value>default</value>
+ </property>
+
+ <property>
+ <name>slider.security.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>slider.yarn.queue</name>
+ <value>default</value>
+ </property>
+
+ <property>
+ <name>slider.yarn.queue.priority</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>slider.yarn.restart.limit</name>
+ <value>5</value>
+ <description>How many times to start/restart the Slider AM</description>
+ </property>
+
+ <property>
+ <name>slider.cluster.directory.permissions</name>
+ <value>750</value>
+ </property>
+
+ <property>
+ <name>slider.data.directory.permissions</name>
+ <value>750</value>
+ </property>
+
+### `slider.zookeeper.quorum` - the zookeeper quorum.
+
+This defines the zookeeper quorum for this YARN cluster.
+
+It is used to locate the service registry, enable running instances to publish
+information about their application, and for clients to query this.
+
+It is also used as the default zookeeper binding for any application that
+uses zookeeper in its configuration -the value set when the application is
+defined will be copied into the instance definition file.
+
+### `"slider.registry.path"` - the zookeeper path for the service registry
+
+This declares the the zookeeper path for the service registry.
+
+### `slider.security.enabled` - enable security.
+
+This turns security on; consult [Security](security.html) for more information.
+
+
+### `slider.yarn.restart.limit` - set limit on Application Master Restarts
+
+This limits how many times YARN should start a failed application master.
+
+A short restart limit is useful when initially creating a cluster, as it
+ensures that YARN does not repeatedly try to restart a failing application.
+
+In production, however, a large number prevents YARN from halting a Slider
+application merely because failures in the underlying YARN cluster have
+triggered restarts.
+
+*Important:* The cluster-wide limit of `yarn.resourcemanager.am.max-attempts`
+places an upper limit on the number of retries that any application can request.
+If the application fails after less restarts than requested, check this cluster
+setting.
+
+### `slider.yarn.queue` - the name of the YARN queue for the cluster.
+
+This identifies the queue submit the application creation request to, which can
+define the priority, resource limits and other values of an application. All
+containers created in the Slider cluster will share this same queue.
+
+Default value: `default`.
+
+### `slider.yarn.queue.priority` - the name of the YARN queue for the cluster.
+
+This identifies the priority within the queue. The lower the value, the higher the
+priority
+
+Default value: `1`.
+
+ bin/slider thaw cl1 -D slider.yarn.queue.priority=5
+
+
+
+#### `slider.cluster.directory.permissions`
+
+An octal-format (`chmod`-style) permissions mask for the directory
+that contains the cluster specification `${user.home}/.slider/clusters/${clustername}`
+
+ <property>
+ <name>slider.cluster.directory.permissions</name>
+ <value>750</value>
+ </property>
+
+#### `slider.data.directory.permissions`
+
+An octal-format (`chmod`-style) permissions mask for the directory
+that contains the application data `${user.home}/.slider/clusters/${clustername}/database`
+
+ <property>
+ <name>slider.data.directory.permissions</name>
+ <value>750</value>
+ </property>
+
+
+## Debugging configuration issues
+
+If the slider packages are set to log at debug level in the log4j configuration
+file, details on properties will be part of the copious output.
+
+
+## How client options are passed down to created clusters.
+
+Apart from the filesystem bindings, Client configuration options are
+not passed down to the XML site specification of the created cluster.
+
+The sole options passed down are the HDFS bindings: `fs.defaultFS`,
+which is passed down both as that property and as `fs.default.name`,
+and, in a secure cluster, the security flag (`slider.security.enabled`)
+and the HDFS Kerberos principal.
+
diff --git a/src/site/markdown/configuration/core.md b/src/site/markdown/configuration/core.md
new file mode 100644
index 0000000..46e4b63
--- /dev/null
+++ b/src/site/markdown/configuration/core.md
@@ -0,0 +1,407 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Core Configuration Specification
+
+
+## Terminology
+
+
+*Application* A single application, such as an HBase cluster. An application
+is distribed across the YARN cluster.
+
+*Component* A single executable part of the larger application. An application
+may have multiple components, and multiple instances of each component.
+
+*YARN* Yet Another Resource Negotiator
+
+*YARN Resource Requirements* The requirements for a YARN resource request.
+Currently this consists of RAM and CPU requirements.
+
+*YARN Container*. An allocation portion of a servers resources granted
+to satisfy the requested YARN resource requirements. A process can be deployed
+to a container.
+
+
+*`resources.json`*: A file that describes the
+size of the application in terms of its component requirements: how many,
+and what their resource requirements are.
+
+*`application.json`*: A file that describes the
+size of the application in terms of its component requirements: how many,
+and what their resource requirements are.
+
+## Structure
+
+Configurations are stored in well-formed JSON files.
+1. Text MUST be saved in the UTF-8 format.
+1. Duplicate entries MUST NOT occur in any section.
+1. The ordering of elements is NOT significant.
+
+The JSON specification files all have a similar structure
+
+1. A `schema` string indicating version. Currently this is temporarily set to
+
+ "http://example.org/specification/v2.0.0"
+
+
+1. A global section, `/global` containing string properties
+1. A component section, `/components`.
+1. 0 or more sections under `/components` for each component, identified by component name,
+ containing string properties.
+1. 0 or 1 section `/metadata` containing arbitrary metadata (such as a description,
+author, or any other information that is not parsed or processed directly).
+
+
+The simplest valid specification file is
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ },
+ "components": {
+ }
+ }
+
+
+## Property inheritance model and *resolution*
+
+
+There is a simple global to component inheritance model.
+
+1. Properties defined in `/global` define parameters across the entire application.
+1. Properties defined a section under `/components` define parameters for
+a specific component in the application.
+1. All global properties are propagated to each component.
+1. A component section may override any global property.
+1. The final set of configuration properties for a component is the global
+properties extended and overridden by the global set.
+1. The process of expanding the properties is termed *resolution*; the *resolved*
+specification is the outcome.
+1. There is NO form of explicitly cross-referencing another attribute. This
+MAY be added in future.
+1. There is NO sharing of information from the different `.json` files in a
+an application configuration.
+
+### Example
+
+Here is an example configuration
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "components": {
+ "simple": {
+ },
+ "master": {
+ "name": "m",
+ "g1": "overridden"
+
+ },
+ "worker": {
+ "name": "w",
+ "g1": "overridden-by-worker",
+ "timeout": "1000"
+ }
+ }
+ }
+
+The `/global` section defines two properties
+
+ g1="a"
+ g2="b"
+
+These are the values visible to any part of the application which is
+not itself one of the components.
+
+
+There are three components defined, `simple`, `master` and `worker`.
+
+
+#### component `simple`:
+
+ g1="a"
+ g2="b"
+
+
+No settings have been defined specifically for the component; the global
+settings are applied.
+
+#### component `master`:
+
+ name="m",
+ g1="overridden"
+ g2="b"
+
+A new attribute, `name`, has been defined with the value `"m"`, and the
+global property `g1` has been overridden with the new value, `"overridden"`.
+The global property `g2` is passed down unchanged.
+
+
+#### component `worker`:
+
+ name="w",
+ g1="overridden-by-worker"
+ g2="b"
+ timeout: "1000"
+
+A new attribute, `name`, has been defined with the value `"w"`, and another,
+`timeout`, value "1000".
+
+The global property `g1` has been overridden with the new value, `"overridden-by-worker"`.
+
+The global property `g2` is passed down unchanged.
+
+This example shows some key points about the design
+
+* each component gets its own map of properties, which is independent from
+ that of other components.
+* all global properties are either present or overridden by a new value.
+ They can not be "undefined"
+* new properties defined in a component are not visible to any other component.
+
+The final *resolved* model is as follows
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "components": {
+ "simple": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "master": {
+ "name": "m",
+ "g1": "overridden",
+ "g2": "b"
+ },
+ "worker": {
+ "name": "m",
+ "g1": "overridden-by-worker",
+ "g2": "b",
+ "timeout": "1000"
+ }
+ }
+ }
+
+This the specification JSON that would have generate exactly the same result as
+in the example, without any propagation of data from the global section
+to individual components.
+
+Note that a resolved specification can still have the resolution operation applied
+to it -it just does not have any effect.
+
+## Metadata
+
+The metadata section can contain arbitrary string values for use in diagnostics
+and by other applications.
+
+To avoid conflict with other applications, please use a unique name in strings,
+such as java-style package names.
+
+# Resource Requirements: `resources.json`
+
+This file declares the resource requirements for YARN for the components
+of an application.
+
+`instances`: the number of instances of a role desired.
+`yarn.vcores`: number of "virtual" required by a component.
+`yarn.memory`: the number of megabytes required by a component.
+
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "example of a resources file"
+ },
+
+ "global": {
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ },
+
+ "components": {
+ "master": {
+ "instances": "1",
+ "yarn.memory": "1024"
+ },
+ "worker": {
+ "instances":"5"
+ }
+ }
+ }
+
+The resolved file would be
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "example of a resources file"
+ },
+
+ "global": {
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ },
+
+ "components": {
+ "master": {
+ "instances": "1",
+ "yarn.vcores": "1",
+ "yarn.memory": "1024"
+ },
+ "worker": {
+ "instances":"5",
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ }
+ }
+ }
+
+This declares this deployment of the application to consist of one instance of
+the master component, using 1 vcore and 1024MB of RAM, and five worker components
+each using one vcore and 512 MB of RAM.
+
+
+## Internal information, `internal.json`
+
+This contains internal data related to the deployment -it is not
+intended for manual editing.
+
+There MAY be a component, `diagnostics`. If defined, its content contains
+diagnostic information for support calls, and MUST NOT be interpreted
+during application deployment, (though it may be included in the generation
+of diagnostics reports)
+
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "Internal configuration DO NOT EDIT"
+ },
+ "global": {
+ "name": "small_cluster",
+ "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar"
+ },
+ "components": {
+
+ "diagnostics": {
+ "create.hadoop.deployed.info": "(release-2.3.0) @dfe463",
+ "create.hadoop.build.info": "2.3.0",
+ "create.time.millis": "1393512091276",
+ "create.time": "27 Feb 2014 14:41:31 GMT"
+ }
+ }
+ }
+
+
+## Deployment specification: `app_configuration.json`
+
+
+This defines parameters that are to be used when creating the instance of the
+application, and instances of the individual components.
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+ "components": {
+
+ "worker": {
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "jvm.heapsize": "512M"
+ }
+ }
+ }
+
+The resolved specification defines the values that are passed to the
+different components.
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+ "components": {
+
+ "worker": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "512M"
+ }
+ }
+ }
+
+The `site.` properties have been passed down to each component, components
+whose templates may generate local site configurations. The override model
+does not prevent any component from overriding global configuration so as
+to create local configurations incompatible with the global state. (i.e.,
+there is no way to declare an attribute as final). It is the responsibility
+of the author of the configuration file (and their tools) to detect such issues.
diff --git a/src/site/markdown/configuration/example-app_configuration-resolved.json b/src/site/markdown/configuration/example-app_configuration-resolved.json
new file mode 100644
index 0000000..5b90ba9
--- /dev/null
+++ b/src/site/markdown/configuration/example-app_configuration-resolved.json
@@ -0,0 +1,42 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+ "components": {
+
+ "worker": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "512M"
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/site/markdown/configuration/example-app_configuration.json b/src/site/markdown/configuration/example-app_configuration.json
new file mode 100644
index 0000000..489acda
--- /dev/null
+++ b/src/site/markdown/configuration/example-app_configuration.json
@@ -0,0 +1,25 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+ "components": {
+
+ "worker": {
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "jvm.heapsize": "512M"
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/site/markdown/configuration/example-empty.json b/src/site/markdown/configuration/example-empty.json
new file mode 100644
index 0000000..5c05163
--- /dev/null
+++ b/src/site/markdown/configuration/example-empty.json
@@ -0,0 +1,8 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ },
+ "components": {
+ }
+}
\ No newline at end of file
diff --git a/src/site/markdown/configuration/example-internal.json b/src/site/markdown/configuration/example-internal.json
new file mode 100644
index 0000000..8617d1f
--- /dev/null
+++ b/src/site/markdown/configuration/example-internal.json
@@ -0,0 +1,21 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "Internal configuration DO NOT EDIT"
+ },
+ "global": {
+ "application.name": "small_cluster",
+ "application.type": "hbase",
+ "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar"
+ },
+ "components": {
+
+ "diagnostics": {
+ "create.hadoop.deployed.info": "(release-2.3.0) @dfe463",
+ "create.hadoop.build.info": "2.3.0",
+ "create.time.millis": "1393512091276",
+ "create.time": "27 Feb 2014 14:41:31 GMT"
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/site/markdown/configuration/example-overridden-resolved.json b/src/site/markdown/configuration/example-overridden-resolved.json
new file mode 100644
index 0000000..2b810b5
--- /dev/null
+++ b/src/site/markdown/configuration/example-overridden-resolved.json
@@ -0,0 +1,25 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "components": {
+ "simple": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "master": {
+ "name": "m",
+ "g1": "overridden",
+ "g2": "b"
+ },
+ "worker": {
+ "name": "m",
+ "g1": "overridden-by-worker",
+ "g2": "b",
+ "timeout": "1000"
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/site/markdown/configuration/example-overridden.json b/src/site/markdown/configuration/example-overridden.json
new file mode 100644
index 0000000..9a74143
--- /dev/null
+++ b/src/site/markdown/configuration/example-overridden.json
@@ -0,0 +1,23 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "components": {
+ "simple": {
+ },
+ "master": {
+ "name": "m",
+ "g1": "overridden"
+
+ },
+ "worker": {
+ "name": "m",
+ "g1": "overridden-by-worker",
+ "timeout": "1000"
+
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/site/markdown/configuration/example-resources.json b/src/site/markdown/configuration/example-resources.json
new file mode 100644
index 0000000..06c3b54
--- /dev/null
+++ b/src/site/markdown/configuration/example-resources.json
@@ -0,0 +1,25 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "example of a resources file"
+ },
+
+ "global": {
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ },
+
+ "components": {
+ "master": {
+ "instances": "1",
+ "yarn.vcores": "1",
+ "yarn.memory": "1024"
+ },
+ "worker": {
+ "instances":"5",
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/site/markdown/configuration/index-markdown.md b/src/site/markdown/configuration/index-markdown.md
new file mode 100644
index 0000000..0616dfb
--- /dev/null
+++ b/src/site/markdown/configuration/index-markdown.md
@@ -0,0 +1,30 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+# Specification of an application instance, revision 2.0
+
+The specification of an applicaton, comprises
+1. The persistent description of an application's configuration
+1. The persistent description of the desired topology and YARN resource
+requirements.
+1. The dynamic description of the running application, including information
+on the location of components and aggregated statistics.
+
+
+1. [Redesign](redesign.md)
+1. [Specification](specification.md)
+1. [Example: current](original-hbase.json)
+1. [Example: proposed](proposed-hbase.json)
diff --git a/src/site/markdown/configuration/index.md b/src/site/markdown/configuration/index.md
new file mode 100644
index 0000000..ad81b4f
--- /dev/null
+++ b/src/site/markdown/configuration/index.md
@@ -0,0 +1,38 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Specification of an application instance, revision 2.0
+
+The specification of an applicaton, comprises
+1. The persistent description of an application's configuration
+1. The persistent description of the desired topology and YARN resource
+requirements.
+1. The dynamic description of the running application, including information
+on the location of components and aggregated statistics.
+
+The specifics of this are covered in the [Core Configuration Specification](core.md)
+
+
+## Historical References
+
+1. [Specification](specification.html)
+1. [Redesign](redesign.html)
+
+
+1. [Example: current](original-hbase.json)
+1. [Example: proposed](proposed-hbase.json)
+
diff --git a/src/site/markdown/configuration/original-hbase.json b/src/site/markdown/configuration/original-hbase.json
new file mode 100644
index 0000000..c01e376
--- /dev/null
+++ b/src/site/markdown/configuration/original-hbase.json
@@ -0,0 +1,139 @@
+{
+ "version": "1.0",
+ "name": "test_cluster_lifecycle",
+ "type": "hbase",
+ "state": 3,
+ "createTime": 1393512091276,
+ "updateTime": 1393512117286,
+ "originConfigurationPath": "hdfs://sandbox.hortonworks.com:8020/user/stevel/.hoya/cluster/test_cluster_lifecycle/snapshot",
+ "generatedConfigurationPath": "hdfs://sandbox.hortonworks.com:8020/user/stevel/.hoya/cluster/test_cluster_lifecycle/generated",
+ "dataPath": "hdfs://sandbox.hortonworks.com:8020/user/stevel/.hoya/cluster/test_cluster_lifecycle/database",
+ "options": {
+ "zookeeper.port": "2181",
+ "site.hbase.master.startup.retainassign": "true",
+ "hoya.cluster.application.image.path": "hdfs://sandbox.hortonworks.com:8020/hbase.tar.gz",
+ "site.fs.defaultFS": "hdfs://sandbox.hortonworks.com:8020",
+ "hoya.container.failure.threshold": "5",
+ "site.fs.default.name": "hdfs://sandbox.hortonworks.com:8020",
+ "hoya.cluster.directory.permissions": "0770",
+ "hoya.am.monitoring.enabled": "false",
+ "zookeeper.path": "/yarnapps_hoya_stevel_test_cluster_lifecycle",
+ "hoya.tmp.dir": "hdfs://sandbox.hortonworks.com:8020/user/stevel/.hoya/cluster/test_cluster_lifecycle/tmp/am",
+ "hoya.data.directory.permissions": "0770",
+ "zookeeper.hosts": "sandbox",
+ "hoya.container.failure.shortlife": "60"
+ },
+ "info": {
+ "create.hadoop.deployed.info": "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85",
+ "create.application.build.info": "Hoya Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
+ "create.hadoop.build.info": "2.3.0",
+ "create.time.millis": "1393512091276",
+ "create.time": "27 Feb 2014 14:41:31 GMT",
+ "hoya.am.restart.supported": "false",
+ "live.time": "27 Feb 2014 14:41:56 GMT",
+ "live.time.millis": "1393512116881",
+ "status.time": "27 Feb 2014 14:42:08 GMT",
+ "status.time.millis": "1393512128726",
+ "yarn.vcores": "32",
+ "yarn.memory": "2048",
+ "status.application.build.info": "Hoya Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
+ "status.hadoop.build.info": "2.3.0",
+ "status.hadoop.deployed.info": "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965"
+ },
+ "statistics": {
+ "worker": {
+ "containers.start.started": 0,
+ "containers.live": 0,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 0,
+ "containers.requested": 0
+ },
+ "hoya": {
+ "containers.unknown.completed": 0,
+ "containers.start.started": 0,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.surplus": 0
+ },
+ "master": {
+ "containers.start.started": 0,
+ "containers.live": 0,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 0,
+ "containers.requested": 0
+ }
+ },
+ "status": {
+ },
+ "instances": {
+ "hoya": [ "container_1393511571284_0002_01_000001" ]
+ },
+ "roles": {
+ "worker": {
+ "yarn.memory": "768",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "0",
+ "role.requested.instances": "0",
+ "role.name": "worker",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "0",
+ "jvm.heapsize": "512M",
+ "yarn.vcores": "1",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0",
+ "app.infoport": "0"
+ },
+ "hoya": {
+ "yarn.memory": "256",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "1",
+ "role.requested.instances": "0",
+ "role.name": "hoya",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "1",
+ "jvm.heapsize": "256M",
+ "yarn.vcores": "1",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0"
+ },
+ "master": {
+ "yarn.memory": "1024",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "0",
+ "role.requested.instances": "0",
+ "role.name": "master",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "0",
+ "jvm.heapsize": "512M",
+ "yarn.vcores": "1",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0",
+ "app.infoport": "0"
+ }
+ },
+ "clientProperties": {
+ "fs.defaultFS": "hdfs://sandbox.hortonworks.com:8020",
+ "hbase.cluster.distributed": "true",
+ "hbase.master.info.port": "0",
+ "hbase.master.port": "0",
+ "hbase.master.startup.retainassign": "true",
+ "hbase.regionserver.hlog.tolerable.lowreplication": "1",
+ "hbase.regionserver.info.port": "0",
+ "hbase.regionserver.port": "0",
+ "hbase.rootdir": "hdfs://sandbox.hortonworks.com:8020/user/stevel/.hoya/cluster/test_cluster_lifecycle/database",
+ "hbase.tmp.dir": "./hbase-tmp",
+ "hbase.zookeeper.property.clientPort": "2181",
+ "hbase.zookeeper.quorum": "sandbox",
+ "hoya.template.origin": "hdfs://sandbox.hortonworks.com:8020/user/stevel/.hoya/cluster/test_cluster_lifecycle/snapshot/hbase-site.xml",
+ "hoya.unused.option": "1",
+ "zookeeper.znode.parent": "/yarnapps_hoya_stevel_test_cluster_lifecycle"
+ }
+}
diff --git a/src/site/markdown/configuration/proposed-hbase.json b/src/site/markdown/configuration/proposed-hbase.json
new file mode 100644
index 0000000..934e574
--- /dev/null
+++ b/src/site/markdown/configuration/proposed-hbase.json
@@ -0,0 +1,273 @@
+{
+ "version": "2.0.0",
+ "name": "test_cluster_lifecycle",
+ "valid`": true,
+
+ "hoya-internal":{
+ "type": "hbase",
+ "createTime": 1393512091276,
+ "updateTime": 1393512117286,
+ "originConfigurationPath": "hdfs://sandbox.hortonworks.com:8020/user/hoya/.hoya/cluster/test_cluster_lifecycle/snapshot",
+ "generatedConfigurationPath": "hdfs://sandbox.hortonworks.com:8020/user/hoya/.hoya/cluster/test_cluster_lifecycle/generated",
+ "dataPath": "hdfs://sandbox.hortonworks.com:8020/user/hoya/.hoya/cluster/test_cluster_lifecycle/database",
+ "hoya.tmp.dir": "hdfs://sandbox.hortonworks.com:8020/user/hoya/.hoya/cluster/test_cluster_lifecycle/tmp/am",
+ "slider.cluster.directory.permissions": "0770",
+ "slider.data.directory.permissions": "0770"
+ },
+
+ "options": {
+ "hoya.am.monitoring.enabled": "false",
+ "hoya.cluster.application.image.path": "hdfs://sandbox.hortonworks.com:8020/hbase.tar.gz",
+ "hoya.container.failure.threshold": "5",
+ "hoya.container.failure.shortlife": "60",
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_hoya_hoya_test_cluster_lifecycle",
+ "zookeeper.hosts": "sandbox",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://sandbox.hortonworks.com:8020",
+ "site.fs.default.name": "hdfs://sandbox.hortonworks.com:8020",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+
+ "diagnostics": {
+ "create.hadoop.deployed.info": "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85",
+ "create.application.build.info": "Hoya Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by hoya",
+ "create.hadoop.build.info": "2.3.0",
+ "create.time.millis": "1393512091276",
+ "create.time": "27 Feb 2014 14:41:31 GMT"
+ },
+
+ "info": {
+ "hoya.am.restart.supported": "false",
+ "live.time": "27 Feb 2014 14:41:56 GMT",
+ "live.time.millis": "1393512116881",
+ "status.time": "27 Feb 2014 14:42:08 GMT",
+ "status.time.millis": "1393512128726",
+ "yarn.vcores": "32",
+ "yarn.memory": "2048",
+ "status.application.build.info": "Hoya Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by hoya",
+ "status.hadoop.build.info": "2.3.0",
+ "status.hadoop.deployed.info": "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965"
+ },
+
+ "statistics": {
+
+ "cluster": {
+ "containers.unknown.completed": 0,
+ "containers.start.completed": 3,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.surplus": 0
+
+ },
+ "roles": {
+ "worker": {
+ "containers.start.completed": 0,
+ "containers.live": 2,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 2,
+ "containers.requested": 0
+ },
+ "master": {
+ "containers.start.completed": 0,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 1,
+ "containers.requested": 0
+ }
+ }
+ },
+
+ "instances": {
+ "hoya": [ "container_1393511571284_0002_01_000001" ],
+ "master": [ "container_1393511571284_0002_01_000003" ],
+ "worker": [
+ "container_1393511571284_0002_01_000002",
+ "container_1393511571284_0002_01_000004"
+ ]
+ },
+
+ "roles": {
+ "worker": {
+ "yarn.memory": "768",
+ "role.instances": "0",
+ "role.name": "worker",
+ "jvm.heapsize": "512M",
+ "yarn.vcores": "1"
+ },
+ "hoya": {
+ "yarn.memory": "256",
+ "role.instances": "1",
+ "role.name": "hoya",
+ "jvm.heapsize": "256M",
+ "yarn.vcores": "1"
+ },
+ "master": {
+ "yarn.memory": "1024",
+ "role.instances": "0",
+ "role.name": "master",
+ "jvm.heapsize": "512M",
+ "yarn.vcores": "1"
+ }
+ },
+
+
+ "clientProperties": {
+ "fs.defaultFS": "hdfs://sandbox.hortonworks.com:8020",
+ "hbase.cluster.distributed": "true",
+ "hbase.master.info.port": "0",
+ "hbase.master.port": "0",
+ "hbase.master.startup.retainassign": "true",
+ "hbase.regionserver.hlog.tolerable.lowreplication": "1",
+ "hbase.regionserver.info.port": "0",
+ "hbase.regionserver.port": "0",
+ "hbase.rootdir": "hdfs://sandbox.hortonworks.com:8020/user/hoya/.hoya/cluster/test_cluster_lifecycle/database",
+ "hbase.tmp.dir": "./hbase-tmp",
+ "hbase.zookeeper.property.clientPort": "2181",
+ "hbase.zookeeper.quorum": "sandbox",
+ "zookeeper.znode.parent": "/yarnapps_hoya_hoya_test_cluster_lifecycle"
+ },
+
+
+ "clientfiles": {
+ "hbase-site.xml": "site information for HBase",
+ "log4.properties": "log4.property file"
+ },
+
+ "provider":{
+ "load":0.4,
+ "urls": {
+ "master": ["http://node4:28209"],
+ "worker": ["http://node4:28717", "http://node6:31268"]
+ }
+ },
+
+ "status": {
+ "live": {
+ "worker": {
+ "container_1394032374441_0001_01_000003": {
+ "name": "container_1394032374441_0001_01_000003",
+ "role": "worker",
+ "roleId": 1,
+ "createTime": 1394032384451,
+ "startTime": 1394032384503,
+ "released": false,
+ "host": "192.168.1.88",
+ "state": 3,
+ "exitCode": 0,
+ "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR regionserver start 1><LOG_DIR>/region-server.txt 2>&1 ; ",
+ "diagnostics": "",
+ "environment": [
+ "HADOOP_USER_NAME=\"hoya\"",
+ "HBASE_LOG_DIR=\"/tmp/hoya-hoya\"",
+ "HBASE_HEAPSIZE=\"256\"",
+ "MALLOC_ARENA_MAX=\"4\"",
+ "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\""
+ ]
+ },
+ "container_1394032374441_0001_01_000002": {
+ "name": "container_1394032374441_0001_01_000002",
+ "role": "worker",
+ "roleId": 1,
+ "createTime": 1394032384451,
+ "startTime": 1394032384552,
+ "released": false,
+ "host": "192.168.1.86",
+ "state": 3,
+ "exitCode": 0,
+ "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR regionserver start 1><LOG_DIR>/region-server.txt 2>&1 ; ",
+ "diagnostics": "",
+ "environment": [
+ "HADOOP_USER_NAME=\"hoya\"",
+ "HBASE_LOG_DIR=\"/tmp/hoya-hoya\"",
+ "HBASE_HEAPSIZE=\"256\"",
+ "MALLOC_ARENA_MAX=\"4\"",
+ "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\""
+ ]
+ }
+ },
+ "hoya": {
+ "container_1394032374441_0001_01_000001": {
+ "name": "container_1394032374441_0001_01_000001",
+ "role": "hoya",
+ "roleId": 0,
+ "createTime": 0,
+ "startTime": 0,
+ "released": false,
+ "host": "hoya-8.local",
+ "state": 3,
+ "exitCode": 0,
+ "command": "",
+ "diagnostics": ""
+ }
+ },
+ "master": {
+ "container_1394032374441_0001_01_000004": {
+ "name": "container_1394032374441_0001_01_000004",
+ "role": "master",
+ "roleId": 2,
+ "createTime": 1394032384451,
+ "startTime": 1394032384573,
+ "released": false,
+ "host": "192.168.1.86",
+ "state": 3,
+ "exitCode": 0,
+ "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR master start 1><LOG_DIR>/master.txt 2>&1 ; ",
+ "diagnostics": "",
+ "environment": [
+ "HADOOP_USER_NAME=\"hoya\"",
+ "HBASE_LOG_DIR=\"/tmp/hoya-hoya\"",
+ "HBASE_HEAPSIZE=\"256\"",
+ "MALLOC_ARENA_MAX=\"4\"",
+ "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\""
+ ]
+ }
+ }
+ },
+ "failed": {
+
+ },
+
+ "rolestatus": {
+ "worker": {
+ "role.instances": "2",
+ "role.requested.instances": "0",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "2",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "1"
+ },
+ "hoya": {
+ "role.instances": "1",
+ "role.requested.instances": "0",
+ "role.name": "hoya",
+ "role.actual.instances": "1",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0"
+ },
+ "master": {
+ "role.instances": "1",
+ "role.requested.instances": "1",
+ "role.name": "master",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "0",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0"
+ }
+ }
+ }
+
+
+
+
+}
diff --git a/src/site/markdown/configuration/redesign.md b/src/site/markdown/configuration/redesign.md
new file mode 100644
index 0000000..e3eba77
--- /dev/null
+++ b/src/site/markdown/configuration/redesign.md
@@ -0,0 +1,478 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Cluster Specification
+
+### Notation:
+
+In this document, a full path to a value is represented as a path
+`options/zookeeper.port` ; an assigment as `options/zookeeper.port=2181`.
+
+A wildcard indicates all entries matching a path: `options/zookeeper.*`
+or `/roles/*/yarn.memory`
+
+
+## History
+
+The Hoya cluster specification was implicitly defined in the file
+`org.apache.hoya.api.ClusterDescription`. It had a number of roles
+
+1. Persistent representaton of cluster state
+1. Internal model of desired cluster state within the Application Master.
+1. Dynamic representation of current cluster state when the AM
+was queried, marshalled over the network as JSON.
+1. Description of updated state when reconfiguring a running cluster.
+
+Initially the dynamic status included a complete history of all containers
+-this soon highlit some restrictions on the maximum size of a JSON-formatted
+string in Hadoop's "classic" RPC: 32K, after which the string was silently
+truncated. Accordingly, this history was dropped.
+
+Having moved to Protocol Buffers as the IPC wire format, with a web view
+alongside, this history could be reconsidered.
+
+The initial design place most values into the root entry, and relied
+on Jaxon introspection to set and retrieve the values -it was a
+Java-first specification, with no external specificatin or regression tests.
+
+As the number of entries in the root increased, the design switched to storing
+more attributes into specific sections *under* the root path:
+
+* `info`: read-only information about the cluster.
+* `statistics`: Numeric statistics about the cluster
+
+# Sections
+
+## Root
+
+Contains various string and integer values
+
+ "version": "1.0",
+ "name": "test_cluster_lifecycle",
+ "type": "hbase",
+ "state": 3,
+ "createTime": 1393512091276,
+ "updateTime": 1393512117286,
+ "originConfigurationPath": "hdfs://sandbox.hortonworks.com:8020/user/stevel/.hoya/cluster/test_cluster_lifecycle/snapshot",
+ "generatedConfigurationPath": "hdfs://sandbox.hortonworks.com:8020/user/stevel/.hoya/cluster/test_cluster_lifecycle/generated",
+ "dataPath": "hdfs://sandbox.hortonworks.com:8020/user/stevel/.hoya/cluster/test_cluster_lifecycle/database",
+
+
+* `version`: version of the JSON file. Not currently used
+to validate version compatibility; at this point in time
+releases may not be able to read existing .json files.
+
+* `name`: cluster name
+* `type`: reference to the provider type -this triggers a Hadoop configuration
+property lookup to find the implementation classes.
+* `state`: an enumeration value of the cluster state.
+
+ int STATE_INCOMPLETE = 0;
+ int STATE_SUBMITTED = 1;
+ int STATE_CREATED = 2;
+ int STATE_LIVE = 3;
+ int STATE_STOPPED = 4;
+ int STATE_DESTROYED = 5;
+
+ Only two states are persisted, "incomplete" and "created", though more
+ are used internally.
+ The `incomplete` state is used during cluster create/build,
+ allowing an incomplete JSON file to be written
+ -so minimising the window for race conditions on cluster construction.
+
+* `createTime` and `updateTime`: timestamps, informative only.
+ The `createTime` value is duplicated in `/info/createTimeMillis`
+* `originConfigurationPath`, `generatedConfigurationPath`, `dataPath` paths
+used internally -if changed the cluster may not start.
+
+*Proposed*:
+1. Move all state bar `name` and cluster state
+into a section `/hoya-internal`.
+1. The cluster state is moved from an enum to a simple
+ boolean, `valid`, set to true when the cluster JSON
+ has been fully constructed.
+
+## `/info`
+
+Read-only list of information about the application. Generally this is
+intended to be used for debugging and testing.
+
+### Persisted values: static information about the file history
+
+ "info" : {
+ "create.hadoop.deployed.info" : "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85",
+ "create.application.build.info" : "Hoya Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
+ "create.hadoop.build.info" : "2.3.0",
+ "create.time.millis" : "1393512091276",
+ },
+
+*Proposed*: move persisted info K-V pairs to a section `/diagnostics`.
+
+### Dynamic values:
+
+
+ whether the AM supports service restart without killing all the containers hosting
+ the role instances:
+
+ "hoya.am.restart.supported" : "false",
+
+ timestamps of the cluster going live, and when the status query was made
+
+ "live.time" : "27 Feb 2014 14:41:56 GMT",
+ "live.time.millis" : "1393512116881",
+ "status.time" : "27 Feb 2014 14:42:08 GMT",
+ "status.time.millis" : "1393512128726",
+
+ yarn data provided to the AM
+
+ "yarn.vcores" : "32",
+ "yarn.memory" : "2048",
+
+ information about the application and hadoop versions in use. Here
+ the application was built using Hadoop 2.3.0, but is running against the version
+ of Hadoop built for HDP-2.
+
+ "status.application.build.info" : "Hoya Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
+ "status.hadoop.build.info" : "2.3.0",
+ "status.hadoop.deployed.info" : "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965"
+
+
+ ## `instances`
+
+ Information about the live containers in a cluster
+
+ "instances": {
+ "hoya": [ "container_1393511571284_0002_01_000001" ],
+ "master": [ "container_1393511571284_0002_01_000003" ],
+ "worker": [
+ "container_1393511571284_0002_01_000002",
+ "container_1393511571284_0002_01_000004"
+ ]
+ },
+
+There's no information about location, nor is there any history about containers
+that are no longer part of the cluster (i.e. failed & released containers).
+
+It could be possible to include a list of previous containers,
+though Hoya would need to be selective about how many to store
+(or how much detail to retain) on those previous containers.
+
+Perhaps the list could be allowed to grow without limit, but detail
+only preserved on the last 100. If more containers fail than that,
+there is likely to be a problem which the most recent containers
+will also display.
+
+*Proposed*
+
+1. Return to the full serialization of container state -but only for running containers.
+1. Have a list of failed containers, but only include last 8; make it a rolling
+buffer. This avoids a significantly failing role to overload the status document.
+
+
+ ## `statistics`
+
+ Statistics on each role.
+
+ They can be divided into counters that only increase
+
+ "containers.start.completed": 0,
+ "containers.start.failed": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.requested": 0
+
+and those that vary depending upon the current state
+
+ "containers.live": 0,
+ "containers.active.requests": 0,
+ "containers.desired": 0,
+
+
+* Propose: move these values out of statistics into some other section, as they
+are state, not statistics*
+
+
+ "statistics": {
+ "worker": {
+ "containers.start.completed": 0,
+ "containers.live": 2,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 2,
+ "containers.requested": 0
+ },
+ "hoya": {
+ "containers.unknown.completed": 0,
+ "containers.start.completed": 3,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.surplus": 0
+ },
+ "master": {
+ "containers.start.completed": 0,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 1,
+ "containers.requested": 0
+ }
+ },
+
+The `/statistics/hoya` section is unusual in that it provides the aggregate statistics
+of the cluster -this is not obvious. A different name could be used -but
+again, there's a risk of clash with or confusion with a role.
+
+Better to have a specific `/statistics/cluster` element,
+and to move the roles' statistics under `/statistics/roles`:
+
+ "statistics": {
+ "cluster": {
+ "containers.unknown.completed": 0,
+ "containers.start.completed": 3,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.surplus": 0
+
+ },
+ "roles": {
+ "worker": {
+ "containers.start.completed": 0,
+ "containers.live": 2,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 2,
+ "containers.requested": 0
+ },
+ "master": {
+ "containers.start.completed": 0,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 1,
+ "containers.requested": 0
+ }
+ }
+ },
+
+This approach allows extra statistics sections to be added (perhaps
+by providers), without any changes to the toplevel section.
+
+## Options
+
+A list of options used by Hoya and its providers to build up the AM
+and the configurations of the deployed service components
+
+
+ "options": {
+ "zookeeper.port": "2181",
+ "site.hbase.master.startup.retainassign": "true",
+ "hoya.cluster.application.image.path": "hdfs://sandbox.hortonworks.com:8020/hbase.tar.gz",
+ "site.fs.defaultFS": "hdfs://sandbox.hortonworks.com:8020",
+ "hoya.container.failure.threshold": "5",
+ "site.fs.default.name": "hdfs://sandbox.hortonworks.com:8020",
+ "slider.cluster.directory.permissions": "0770",
+ "hoya.am.monitoring.enabled": "false",
+ "zookeeper.path": "/yarnapps_hoya_stevel_test_cluster_lifecycle",
+ "hoya.tmp.dir": "hdfs://sandbox.hortonworks.com:8020/user/stevel/.hoya/cluster/test_cluster_lifecycle/tmp/am",
+ "slider.data.directory.permissions": "0770",
+ "zookeeper.hosts": "sandbox",
+ "hoya.container.failure.shortlife": "60"
+ },
+
+Some for these options have been created by hoya itself ("hoya.tmp.dir")
+for internal use -and are cluster specific. If/when the ability to use
+an existing json file as a template for a new cluster is added, having these
+options in the configuration will create problems
+
+
+# Proposed Changes
+
+
+## Move Hoya internal state to `/hoya-internal`
+
+Move all hoya "private" data to an internal section,`/hoya-internal`
+including those in the toplevel directory and in `/options`
+
+## Allow `/options` and `roles/*/` options entries to take the value "null".
+
+This would be a definition that the value must be defined before the cluster
+can start. Provider templates could declare this.
+
+## Make client configuration retrieval hierarchical -and maybe move out of the
+status
+
+The current design assumes that it is a -site.xml file being served up. This
+does not work for alternate file formats generated by the Provider.
+
+## Role Options
+
+The `/roles/$ROLENAME/` clauses each provide options for a
+specific role.
+
+This includes
+1. `role.instances`: defines the number of instances of a role to create
+1. `env.` environment variables for launching the container
+1. `yarn.` properties to configure YARN requests.
+1. `jvm.heapsize`: an option supported by some providers to
+fix the heap size of a component.
+1. `app.infoport`: an option supported by some providers (e.g. HBase)
+to fix the port to which a role (master or worker) binds its web UI.
+
+
+
+ "worker": {
+ "yarn.memory": "768",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "0",
+ "role.name": "worker",
+ "jvm.heapsize": "512M",
+ "yarn.vcores": "1",
+ "app.infoport": "0"
+ },
+
+In a live cluster, the role information also includes status information
+about the cluster.
+
+ "master": {
+ "yarn.memory": "1024",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "0",
+ "role.requested.instances": "0",
+ "role.name": "master",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "0",
+ "jvm.heapsize": "512M",
+ "yarn.vcores": "1",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0",
+ "app.infoport": "0"
+ }
+
+The role `hoya` represents the Hoya Application Master itself.
+
+
+ "hoya": {
+ "yarn.memory": "256",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "1",
+ "role.name": "hoya",
+ "jvm.heapsize": "256M",
+ "yarn.vcores": "1",
+ },
+
+### Proposed:
+1. move all dynamic role status to its own clauses.
+1. use a simple inheritance model from `/options`
+1. don't allow role entries to alter the cluster state.
+
+### Proposed: `/clientProperties` continues return Key-val pairs
+
+The `/clientProperties` section will remain, with key-val pairs of type
+string, the expectation being this is where providers can insert specific
+single attributes for client applications.
+
+These values can be converted to application-specific files on the client,
+in code -as done today in the Hoya CLI-, or via template expansion (beyond
+the scope of this document.
+
+
+
+### Proposed: alongside `/clientProperties` comes `/clientfiles`
+
+This section will list all files that an application instance can generate
+for clients, along with with a description.
+
+ "/clientfiles/hbase-site.xml": "site information for HBase"
+ "/clientfiles/log4.properties": "log4.property file"
+
+A new CLI command would be added to retrieve a client file.
+1. The specific file must be named.
+1. If it is not present, an error must be raised.
+1. If it is present, it is downloaded and output to the console/to a named
+destination file/directory `--outfile <file>` and `--outdir <dir>`
+1. If the `--list` argument is provided, the list of available files is
+returned (e.g.)
+
+ hbase-site.xml: site information for HBase
+ log4.properties: log4.property file
+
+*No attempt to parse/process the body of the messages will be returned.*
+
+In a REST implementation of the client API, /clientconf would be a path
+to the list of options; each file a path underneath.
+
+Client configuration file retrieval outside the status completely;
+the status just lists the possible values; a separate call returns them.
+
+This will permit binary content to be retrieved, and avoid any marshalling
+problems and inefficiencies.
+
+With this change, there will now be two ways to generate client configuration
+files
+
+* Client-side: as today
+* Server-side: via the provider
+
+Client side is more extensible as it allows for arbitrary clients; server-side
+is restricted to those files which the application provider is capable of
+generating. The advantage of the server-side option is that for those files
+about which the provider is aware of, they will be visible through the
+REST and Web UIs, so trivially retrieved.
+
+### Stop intermixing role specification with role current state
+
+Create a new section, `rolestatus`, which lists the current status
+of the roles: how many are running vs requested, how many are being
+released.
+
+There's some overlap here with the `/statistics` field, so we should
+either merge them or clearly separate the two. Only the `role.failed`
+properties match entries in the statistics -perhaps they should be cut.
+
+#### provider-specific status
+
+Allow providers to publish information to the status, in their
+own section.
+
+There already is support for providers updating the cluster status
+in Hoya 12.1 and earlier, but it has flaws
+
+A key one is that it is done sychronously on a `getStatus()` call;
+as providers may perform a live query of their status (example, the HBase
+provider looks up the Web UI ports published by HBase to zookeeper),
+there's overhead, and if the operation blocks (example: when HBase hasn't
+ever been deployed and the zookeeper path is empty), then the status
+call blocks.
+
+*Proposed:*
+
+1. There is a specific `/provider` section
+1. There's no restriction on what JSON is permitted in this section.
+1. Providers may make their own updates to the application state to read and
+write this block -operations that are asynchronous to any status queries.
diff --git a/src/site/markdown/configuration/resolved-resources.json b/src/site/markdown/configuration/resolved-resources.json
new file mode 100644
index 0000000..5299897
--- /dev/null
+++ b/src/site/markdown/configuration/resolved-resources.json
@@ -0,0 +1,22 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "example of a resources file"
+ },
+
+ "global": {
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ },
+
+ "components": {
+ "master": {
+ "instances": "1",
+ "yarn.memory": "1024"
+ },
+ "worker": {
+ "instances":"5"
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/site/markdown/configuration/specification.md b/src/site/markdown/configuration/specification.md
new file mode 100644
index 0000000..8c97000
--- /dev/null
+++ b/src/site/markdown/configuration/specification.md
@@ -0,0 +1,512 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Specification of the "Cluster Description"
+
+* This is partially obsolete. Slider still returns the Hoya Cluster Description
+as changing it will break most of the unit tests -once these are updated
+this document will be completely obsolete and replaced with a new one.
+
+
+### Notation:
+
+In this document, a full path to a value is represented as a path
+`options/zookeeper.port` ; an assigment as `options/zookeeper.port=2181`.
+
+A wildcard indicates all entries matching a path: `options/zookeeper.*`
+or `/roles/*/yarn.memory`
+
+
+## Core Concepts
+
+The specificaton of an application instance is defined in an application instance
+directory, `${user.home}/.slidera/clusters/${clustername}/cluster.json`)
+
+
+## Sections for specifying and describing cluster state
+
+The cluster desciption is hierarchal, with standardized sections.
+
+Different sections have one of three roles.
+
+1. Storage and specification of internal properties used to define a cluster -properties
+that should not be modified by users -doing so is likely to render the
+cluster undeployable.
+
+1. Storage and specification of the components deployed by Hoya.
+These sections define options for the deployed application, the size of
+the deployed application, attributes of the deployed roles, and customizable
+aspects of the Hoya application master.
+
+ This information defines the *desired state* of a cluster.
+
+ Users may edit these sections, either via the CLI, or by directly editing the `cluster.json` file of
+ a frozen cluster.
+
+1. Status information provided by a running cluster. These include:
+ information about the cluster, statistics, information about reach role in
+ the cluster -as well as other aspects of the deployment.
+
+ This information describes the *actual state* of a cluster.
+
+Using a common format for both the specification and description of a cluster
+may be confusing, but it is designed to unify the logic needed to parse
+and process cluster descriptions. There is only one JSON file to parse
+-merely different sections of relevance at different times.
+
+## Role-by-role subsections
+
+A hoya-deployed application consists of the single Hoya application master,
+and one or more roles -specific components in the actual application.
+
+The `/roles` section contains a listing for each role,
+declaring the number of instances of each role desired,
+possibly along with some details defining the actual execution of the application.
+
+The `/statistics/roles/` section returns statistics on each role,
+while `/instances` has a per-role entry listing the YARN
+containers hosting instances.
+
+
+## Cluster information for applications
+
+The AM/application provider may generate information for use by client applications.
+
+There are three ways to provide this
+
+1. A section in which simple key-value pairs are provided for interpretation
+by client applications -usually to generate configuration documents
+2. A listing of files that may be provided directly to a client. The API to provide these files is not covered by this document.
+3. A provider-specific section in which arbitrary values and structures may be defined. This allows greater flexibility in the information that a provider can publish -though it does imply custom code to process this data on the client.
+
+
+# Persistent Specification Sections
+
+## "/" : root
+
+The root contains a limited number of key-value pairs,
+
+* `version`: string; required.
+The version of the JSON file, as an `x.y.z` version string.
+ 1. Applications MUST verify that they can process a specific version.
+ 1. The version number SHOULD be incremented in the minor "z" value
+ after enhancements that are considered backwards compatible.
+ Incompatible updates MUST be updated with a new "y" value.
+ The final, "x" number, is to be reserved for major reworkings
+ of the cluster specification itself (this document or its
+ successors).
+
+* `name`: string; required. Cluster name;
+* `type`: string; required.
+Reference to the provider type -this triggers a Hadoop configuration
+property lookup to find the implementation classes.
+* `valid`: boolean; required.
+Flag to indicate whether or not a specification is considered valid.
+If false, the rest of the document is in an unknown state.
+
+## `/hoya-internal`: internal confiugration
+
+Stores internal configuration options. These parameters
+are not defined in this document.
+
+## `/diagnostics`: diagnostics sections
+
+Persisted list of information about Hoya.
+
+Static information about the file history
+
+ "diagnostics" : {
+ "create.hadoop.deployed.info" :
+ "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85",
+ "create.application.build.info" :
+ "Hoya Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
+ "create.hadoop.build.info" : "2.3.0",
+ "create.time.millis" : "1393512091276",
+ },
+
+This information is not intended to provide anything other
+than diagnostics to an application; the values and their meaning
+are not defined. All applications MUST be able to process
+an empty or absent `/diagnostics` section.
+
+## Options: cluster options
+
+A persisted list of options used by Hoya and its providers to build up the AM
+and the configurations of the deployed service components
+
+
+ "options": {
+ "hoya.am.monitoring.enabled": "false",
+ "hoya.cluster.application.image.path": "hdfs://sandbox.hortonworks.com:8020/hbase.tar.gz",
+ "hoya.container.failure.threshold": "5",
+ "hoya.container.failure.shortlife": "60",
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_hoya_stevel_test_cluster_lifecycle",
+ "zookeeper.hosts": "sandbox",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://sandbox.hortonworks.com:8020",
+ "site.fs.default.name": "hdfs://sandbox.hortonworks.com:8020",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+
+Many of the properties are automatically set by Hoya when a cluster is constructed.
+They may be edited afterwards.
+
+
+### Standard Option types
+
+All option values MUST be strings.
+
+#### `hoya.`
+All options that begin with `hoya.` are intended for use by hoya and
+providers to configure the Hoya application master itself, and the
+application. For example, `hoya.container.failure.threshold` defines
+the number of times a container must fail before the role (and hence the cluster)
+is considered to have failed. As another example, the zookeeper bindings
+such as `zookeeper.hosts` are read by the HBase and Ambari providers, and
+used to modify the applications' site configurations with application-specific
+properties.
+
+#### `site.`
+
+These are properties that are expected to be propagated to an application's
+ `site` configuration -if such a configuration is created. For HBase, the
+ site file is `hbase-site.xml`; for Accumulo it is `accumulo-site.xml`
+
+1. The destination property is taken by removing the prefix `site.`, and
+setting the shortened key with the defined value.
+1. Not all applications have the notion of a site file; These applications MAY
+ignore the settings.
+1. Providers MAY validate site settings to recognise invalid values. This
+aids identifying and diagnosing startup problems.
+
+#### `env.`
+
+These are options to configure environment variables in the roles. When
+a container is started, all `env.` options have the prefix removed, and
+are then set as environment variables in the target context.
+
+1. The Hoya AM uses these values to configure itself, after following the
+option/role merge process.
+1. Application providers SHOULD follow the same process.
+
+
+## '/roles': role declarations
+
+The `/roles/$ROLENAME/` clauses each provide options for a
+specific role.
+
+This includes
+1. `role.instances`: defines the number of instances of a role to create
+1. `env.` environment variables for launching the container
+1. `yarn.` properties to configure YARN requests.
+1. `jvm.heapsize`: an option supported by some providers to
+fix the heap size of a component.
+
+
+ "worker": {
+ "yarn.memory": "768",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "0",
+ "role.name": "worker",
+ "role.failed.starting.instances": "0",
+ "jvm.heapsize": "512M",
+ "yarn.vcores": "1",
+ },
+
+
+The role `hoya` represents the Hoya Application Master itself.
+
+
+ "hoya": {
+ "yarn.memory": "256",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "1",
+ "role.name": "hoya",
+ "jvm.heapsize": "256M",
+ "yarn.vcores": "1",
+ },
+
+Providers may support a fixed number of roles -or they may support a dynamic
+number of roles defined at run-time, potentially from other data sources.
+
+## How `/options` and role options are merged.
+
+The options declared for a specific role are merged with the cluster-wide options
+to define the final options for a role. This is implemented in a simple
+override model: role-specific options can override any site-wide options.
+
+1. The options defined in `/options` are used to create the initial option
+map for each role.
+1. The role's options are then applied to the map -this may overwrite definitions
+from the `/options` section.
+1. There is no way to "undefine" a cluster option, merely overwrite it.
+1. The merged map is then used by the provider to create the component.
+1. The special `hoya` role is used in the CLI to define the attributes of the AM.
+
+Options set on a role do not affect any site-wide options: they
+are specific to the invidual role being created.
+
+As such, overwriting a `site.` option may have no effect -or it it may
+change the value of a site configuration document *in that specific role instance*.
+
+### Standard role options
+
+* `role.instances` : number; required.
+ The number of instances of that role desired in the application.
+* `yarn.vcores` : number.
+ The number of YARN "virtual cores" to request for each role instance.
+ The larger the number, the more CPU allocation -and potentially the longer
+ time to satisfy the request and so instantiate the node.
+ If the value '"-1"` is used -for any role but `hoya`-the maximum value
+ available to the application is requested.
+* `yarn.memory` : number.
+ The number in Megabytes of RAM to request for each role instance.
+ The larger the number, the more memory allocation -and potentially the longer
+ time to satisfy the request and so instantiate the node.
+ If the value '"-1"` is used -for any role but `hoya`-the maximum value
+ available to the application is requested.
+
+* `env.` environment variables.
+String environment variables to use when setting up the container
+
+### Provider-specific role options
+
+* `jvm.heapsize` -the amount of memory for a provider to allocate for
+ a processes JVM. Example "512M". This option MAY be implemented by a provider.
+
+
+
+
+
+# Dynamic Information Sections
+
+These are the parts of the document that provide dynamic run-time
+information about an application. They are provided by the
+Hoya Application Master when a request for the cluster status is issued.
+
+## `/info`
+
+Dynamic set of string key-value pairs containing
+information about the running application -as provided by th
+
+The values in this section are not normatively defined.
+
+Here are some standard values
+
+* `hoya.am.restart.supported"` whether the AM supports service restart without killing all the containers hosting
+ the role instances:
+
+ "hoya.am.restart.supported" : "false",
+
+* timestamps of the cluster going live, and when the status query was made
+
+ "live.time" : "27 Feb 2014 14:41:56 GMT",
+ "live.time.millis" : "1393512116881",
+ "status.time" : "27 Feb 2014 14:42:08 GMT",
+ "status.time.millis" : "1393512128726",
+
+* yarn data provided to the AM
+
+ "yarn.vcores" : "32",
+ "yarn.memory" : "2048",
+
+* information about the application and hadoop versions in use. Here
+ the application was built using Hadoop 2.3.0, but is running against the version
+ of Hadoop built for HDP-2.
+
+ "status.application.build.info" : "Hoya Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
+ "status.hadoop.build.info" : "2.3.0",
+ "status.hadoop.deployed.info" : "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965"
+
+
+As with the `/diagnostics` section, this area is primarily intended
+for debugging.
+
+ ## `/instances`: instance list
+
+ Information about the live containers in a cluster
+
+ "instances": {
+ "hoya": [ "container_1393511571284_0002_01_000001" ],
+ "master": [ "container_1393511571284_0002_01_000003" ],
+ "worker": [
+ "container_1393511571284_0002_01_000002",
+ "container_1393511571284_0002_01_000004"
+ ]
+ },
+
+
+## `/status`: detailed dynamic state
+
+This provides more detail on the application including live and failed instances
+
+### `/status/live`: live role instances by container
+
+ "cluster": {
+ "live": {
+ "worker": {
+ "container_1394032374441_0001_01_000003": {
+ "name": "container_1394032374441_0001_01_000003",
+ "role": "worker",
+ "roleId": 1,
+ "createTime": 1394032384451,
+ "startTime": 1394032384503,
+ "released": false,
+ "host": "192.168.1.88",
+ "state": 3,
+ "exitCode": 0,
+ "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR regionserver start 1><LOG_DIR>/region-server.txt 2>&1 ; ",
+ "diagnostics": "",
+ "environment": [
+ "HADOOP_USER_NAME=\"hoya\"",
+ "HBASE_LOG_DIR=\"/tmp/hoya-hoya\"",
+ "HBASE_HEAPSIZE=\"256\"",
+ "MALLOC_ARENA_MAX=\"4\"",
+ "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\""
+ ]
+ }
+ }
+ failed : {}
+ }
+
+All live instances MUST be described in `/status/live`
+
+Failed clusters MAY be listed in the `/status/failed` section, specifically,
+a limited set of recently failed clusters SHOULD be provided.
+
+Future versions of this document may introduce more sections under `/status`.
+
+### `/status/rolestatus`: role status information
+
+This lists the current status of the roles:
+How many are running vs requested, how many are being
+released.
+
+
+ "rolestatus": {
+ "worker": {
+ "role.instances": "2",
+ "role.requested.instances": "0",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "2",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "1"
+ },
+ "hoya": {
+ "role.instances": "1",
+ "role.requested.instances": "0",
+ "role.name": "hoya",
+ "role.actual.instances": "1",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0"
+ },
+ "master": {
+ "role.instances": "1",
+ "role.requested.instances": "1",
+ "role.name": "master",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "0",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0"
+ }
+ }
+
+
+### `/status/provider`: provider-specific information
+
+Providers MAY publish information to the `/status/provider` section.
+
+1. There's no restriction on what JSON is permitted in this section.
+1. Providers may make their own updates to the application state to read and
+write this block -operations that are asynchronous to any status queries.
+
+
+
+## `/statistics`: aggregate statistics
+
+Statistics on the cluster and each role in the cluster
+
+Better to have a specific `/statistics/cluster` element,
+and to move the roles' statistics under `/statistics/roles`:
+
+ "statistics": {
+ "cluster": {
+ "containers.unknown.completed": 0,
+ "containers.start.completed": 3,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.surplus": 0
+ },
+ "roles": {
+ "worker": {
+ "containers.start.completed": 0,
+ "containers.live": 2,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 2,
+ "containers.requested": 0
+ },
+ "master": {
+ "containers.start.completed": 0,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 1,
+ "containers.requested": 0
+ }
+ }
+ },
+
+`/statistics/cluster` provides aggregate statistics for the entire cluster.
+
+Under `/statistics/roles` MUST come an entry for each role in the cluster.
+
+All simple values in statistics section are integers.
+
+
+### `/clientProperties`
+
+The `/clientProperties` section contains key-val pairs of type
+string, the expectation being this is where providers can insert specific
+single attributes for client applications.
+
+These values can be converted to application-specific files on the client,
+in code -as done today in the Hoya CLI-, or via template expansion (beyond
+the scope of this document.
+
+
+### `/clientfiles`
+
+This section list all files that an application instance MAY generate
+for clients, along with with a description.
+
+ "/clientfiles/hbase-site.xml": "site information for HBase"
+ "/clientfiles/log4.properties": "log4.property file"
+
+Client configuration file retrieval is by other means; this
+status operation merely lists files that are available;
+
+
diff --git a/src/site/markdown/debugging.md b/src/site/markdown/debugging.md
new file mode 100644
index 0000000..df6de58
--- /dev/null
+++ b/src/site/markdown/debugging.md
@@ -0,0 +1,69 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Debugging Slider
+There are a number of options available to you for debugging Slider applications. They include:
+
+* Using Slider logging
+* IDE-based remote debugging of the Application Master
+
+## Using Slider logging
+There are a number of options for viewing the generated log files:
+
+1. Using a web browser
+2. Accessing the host machine
+
+### Using a web browser
+
+The log files are accessible via the Yarn Resource Manager UI. From the main page (e.g. http://YARN_RESOURCE_MGR_HOST:8088), click on the link for the application instance of interest, and then click on the "logs" link. This will present you with a page with links to the slider-err.txt file and the slider-out.txt file. The former is the file you should select. Once the log page is presented, click on the link at the top of the page ("Click here for full log") to view the entire file.
+
+### Accessing the host machine
+
+If access to other log files is required, there is the option of logging in to the host machine on which the application component is running. The root directory for all Yarn associated files is the value of "yarn.nodemanager.log-dirs" in yarn-site.xml - e.g. /hadoop/yarn/log. Below the root directory you will find an application and container sub-directory (e.g. /application_1398372047522_0009/container_1398372047522_0009_01_000001/). Below the container directory you will find any log files associated with the processes running in the given Yarn container.
+
+Within a container log the following files are useful while debugging the application.
+
+**agent.log**
+
+E.g. application_1398098639743_0024/container_1398098639743_0024_01_000003/infra/log/agent.log
+This file contains the logs from the Slider-Agent.
+
+**application component log**
+
+E.g. ./log/application_1398098639743_0024/container_1398098639743_0024_01_000003/app/log/hbase-yarn-regionserver-c6403.ambari.apache.org.log
+
+The location of the application log is defined by the application. "${AGENT_LOG_ROOT}" is a symbol available to the app developers to use as a root folder for logging.
+
+**agent operations log**
+
+E.g. ./log/application_1398098639743_0024/container_1398098639743_0024_01_000003/app/command-log/
+
+The command logs produced by the slider-agent are available in the "command-log" folder relative to "${AGENT_LOG_ROOT}"/app
+
+
+## IDE-based remote debugging of the Application Master
+
+For situtations in which the logging does not yield enough information to debug an issue, the user has the option of specifying JVM command line options for the Application Master that enable attaching to the running process with a debugger (e.g. the remote debugging facilities in Eclipse or Intellij IDEA). In order to specify the JVM options, edit the application configuration file (the file specified as the --template argument value on the command line for cluster creation) and specify the "jvm.opts" property for the "slider-appmaster" component:
+
+ `"components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M",
+ "jvm.opts": "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"
+ },
+ ...`
+
+You may specify "suspend=y" in the line above if you wish to have the application master process wait for the debugger to attach before beginning its processing.
diff --git a/src/site/markdown/examples.md b/src/site/markdown/examples.md
new file mode 100644
index 0000000..9813165
--- /dev/null
+++ b/src/site/markdown/examples.md
@@ -0,0 +1,159 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Examples
+
+
+## Setup
+
+### Setting up a YARN cluster
+
+For simple local demos, a Hadoop pseudo-distributed cluster will suffice -if on a VM then
+its configuration should be changed to use a public (machine public) IP.
+
+# The examples below all assume there is a cluster node called 'master', which
+hosts the HDFS NameNode and the YARN Resource Manager
+
+
+# preamble
+
+ export HADOOP_CONF_DIR=~/conf
+ export PATH=~/hadoop/bin:/~/hadoop/sbin:~/zookeeper-3.4.5/bin:$PATH
+
+ hdfs namenode -format master
+
+
+
+
+# start all the services
+
+ nohup hdfs --config $HADOOP_CONF_DIR namenode &
+ nohup hdfs --config $HADOOP_CONF_DIR datanode &
+
+
+ nohup yarn --config $HADOOP_CONF_DIR resourcemanager &
+ nohup yarn --config $HADOOP_CONF_DIR nodemanager &
+
+# using hadoop/sbin service launchers
+
+ hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start namenode
+ hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start datanode
+ yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
+ yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
+
+ ~/zookeeper/bin/zkServer.sh start
+
+
+# stop them
+
+ hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop namenode
+ hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop datanode
+
+ yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager
+ yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager
+
+
+
+NN up on [http://master:50070/dfshealth.jsp](http://master:50070/dfshealth.jsp)
+RM yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
+
+ ~/zookeeper/bin/zkServer.sh start
+
+
+ # shutdown
+ ~/zookeeper/bin/zkServer.sh stop
+
+
+Tip: after a successful run on a local cluster, do a quick `rm -rf $HADOOP_HOME/logs`
+to keep the log bloat under control.
+
+## get hbase in
+
+copy to local
+
+ get hbase-0.98.0-bin.tar on
+
+
+ hdfs dfs -rm hdfs://master:9090/hbase.tar
+ hdfs dfs -copyFromLocal hbase-0.98.0-bin.tar hdfs://master:9090/hbase.tar
+
+or
+
+ hdfs dfs -copyFromLocal hbase-0.96.0-bin.tar hdfs://master:9090/hbase.tar
+ hdfs dfs -ls hdfs://master:9090/
+
+
+### Optional: point bin/slider at your chosen cluster configuration
+
+export SLIDER_CONF_DIR=~/Projects/slider/slider-core/src/test/configs/ubuntu-secure/slider
+
+## Optional: Clean up any existing slider cluster details
+
+This is for demos only, otherwise you lose the clusters and their databases.
+
+ hdfs dfs -rm -r hdfs://master:9090/user/home/stevel/.hoya
+
+## Create a Slider Cluster
+
+
+ slider create cl1 \
+ --component worker 1 --component master 1 \
+ --manager master:8032 --filesystem hdfs://master:9090 \
+ --zkhosts localhost:2181 --image hdfs://master:9090/hbase.tar
+
+ # create the cluster
+
+ slider create cl1 \
+ --component worker 4 --component master 1 \
+ --manager master:8032 --filesystem hdfs://master:9090 --zkhosts localhost \
+ --image hdfs://master:9090/hbase.tar \
+ --appconf file:////Users/slider/Hadoop/configs/master/hbase \
+ --compopt master jvm.heap 128 \
+ --compopt master env.MALLOC_ARENA_MAX 4 \
+ --compopt worker jvm.heap 128
+
+ # freeze the cluster
+ slider freeze cl1 \
+ --manager master:8032 --filesystem hdfs://master:9090
+
+ # thaw a cluster
+ slider thaw cl1 \
+ --manager master:8032 --filesystem hdfs://master:9090
+
+ # destroy the cluster
+ slider destroy cl1 \
+ --manager master:8032 --filesystem hdfs://master:9090
+
+ # list clusters
+ slider list cl1 \
+ --manager master:8032 --filesystem hdfs://master:9090
+
+ slider flex cl1 --component worker 2
+ --manager master:8032 --filesystem hdfs://master:9090 \
+ --component worker 5
+
+## Create an Accumulo Cluster
+
+ slider create accl1 --provider accumulo \
+ --component master 1 --component tserver 1 --component gc 1 --component monitor 1 --component tracer 1 \
+ --manager localhost:8032 --filesystem hdfs://localhost:9000 \
+ --zkhosts localhost:2181 --zkpath /local/zookeeper \
+ --image hdfs://localhost:9000/user/username/accumulo-1.6.0-SNAPSHOT-bin.tar \
+ --appconf hdfs://localhost:9000/user/username/accumulo-conf \
+ -O zk.home /local/zookeeper -O hadoop.home /local/hadoop \
+ -O site.monitor.port.client 50095 -O accumulo.password secret
+
diff --git a/src/site/markdown/exitcodes.md b/src/site/markdown/exitcodes.md
new file mode 100644
index 0000000..05a369a
--- /dev/null
+++ b/src/site/markdown/exitcodes.md
@@ -0,0 +1,156 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Client Exit Codes
+
+Here are the exit codes returned
+
+Exit code values 1 and 2 are interpreted by YARN -in particular converting the
+"1" value from an error into a successful shut down. Slider
+converts the -1 error code from a forked process into `EXIT_MASTER_PROCESS_FAILED`;
+no. 72.
+
+
+ /**
+ * 0: success
+ */
+ int EXIT_SUCCESS = 0;
+
+ /**
+ * -1: generic "false" response. The operation worked but
+ * the result was not true
+ */
+ int EXIT_FALSE = -1;
+
+ /**
+ * Exit code when a client requested service termination:
+ */
+ int EXIT_CLIENT_INITIATED_SHUTDOWN = 1;
+
+ /**
+ * Exit code when targets could not be launched:
+ */
+ int EXIT_TASK_LAUNCH_FAILURE = 2;
+
+ /**
+ * Exit code when an exception was thrown from the service:
+ */
+ int EXIT_EXCEPTION_THROWN = 32;
+
+ /**
+ * Exit code when a usage message was printed:
+ */
+ int EXIT_USAGE = 33;
+
+ /**
+ * Exit code when something happened but we can't be specific:
+ */
+ int EXIT_OTHER_FAILURE = 34;
+
+ /**
+ * Exit code when a control-C, kill -3, signal was picked up:
+ */
+
+ int EXIT_INTERRUPTED = 35;
+
+ /**
+ * Exit code when the command line doesn't parse:, or
+ * when it is otherwise invalid.
+ */
+ int EXIT_COMMAND_ARGUMENT_ERROR = 36;
+
+ /**
+ * Exit code when the configurations in valid/incomplete:
+ */
+ int EXIT_BAD_CONFIGURATION = 37;
+
+ /**
+ * Exit code when the configurations in valid/incomplete:
+ */
+ int EXIT_CONNECTIVTY_PROBLEM = 38;
+
+ /**
+ * internal error: {@value}
+ */
+ int EXIT_INTERNAL_ERROR = 64;
+
+ /**
+ * Unimplemented feature: {@value}
+ */
+ int EXIT_UNIMPLEMENTED = 65;
+
+ /**
+ * service entered the failed state: {@value}
+ */
+ int EXIT_YARN_SERVICE_FAILED = 66;
+
+ /**
+ * service was killed: {@value}
+ */
+ int EXIT_YARN_SERVICE_KILLED = 67;
+
+ /**
+ * timeout on monitoring client: {@value}
+ */
+ int EXIT_TIMED_OUT = 68;
+
+ /**
+ * service finished with an error: {@value}
+ */
+ int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 69;
+
+ /**
+ * the application instance is unknown: {@value}
+ */
+ int EXIT_UNKNOWN_INSTANCE = 70;
+
+ /**
+ * the application instance is in the wrong state for that operation: {@value}
+ */
+ int EXIT_BAD_STATE = 71;
+
+ /**
+ * A spawned master process failed
+ */
+ int EXIT_PROCESS_FAILED = 72;
+
+ /**
+ * The cluster failed -too many containers were
+ * failing or some other threshold was reached
+ */
+ int EXIT_DEPLOYMENT_FAILED = 73;
+
+ /**
+ * The application is live -and the requested operation
+ * does not work if the cluster is running
+ */
+ int EXIT_APPLICATION_IN_USE = 74;
+
+ /**
+ * There already is an application instance of that name
+ * when an attempt is made to create a new instance
+ */
+ int EXIT_INSTANCE_EXISTS = 75;
+
+## Other exit codes
+
+YARN itself can fail containers, here are some of the causes we've seen
+
+
+ 143: Appears to be triggered by the container exceeding its cgroup memory
+ limits
+
diff --git a/src/site/markdown/hoya_cluster_descriptions-old.md b/src/site/markdown/hoya_cluster_descriptions-old.md
new file mode 100644
index 0000000..388a213
--- /dev/null
+++ b/src/site/markdown/hoya_cluster_descriptions-old.md
@@ -0,0 +1,210 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Hoya Cluster Specification
+
+A Hoya Cluster Specification is a JSON file which describes a cluster to
+Hoya: what application is to be deployed, which archive file contains the
+application, specific cluster-wide options, and options for the individual
+roles in a cluster.
+
+## Hoya Options
+
+These are options read by the Hoya Application Master; the program deployed
+in the YARN cluster to start all the other roles.
+
+When the AM is started, all entries in the cluster-wide are loaded as Hadoop
+configuration values.
+
+Only those related to Hadoop, the filesystem in use, YARN and hoya will be
+used; others are likely to be ignored.
+
+## Cluster Options
+
+Cluster wide options are used to configure the application itself.
+
+These are specified at the command line with the `-O key=value` syntax
+
+All options beginning with the prefix `site.` are converted into
+site XML options for the specific application (assuming the application uses
+a site XML configuration file)
+
+Standard keys are defined in the class `org.apache.hoya.api.OptionKeys`.
+
+#### `hoya.test`
+
+A boolean value to indicate this is a test run, not a production run. In this
+mode Hoya opts to fail fast, rather than retry container deployments when
+they fail. It is primarily used for internal tests.
+
+
+#### `hoya.container.failure.shortlife`
+
+An integer stating the time in milliseconds before which a failed container is
+considered 'short lived'.
+
+A failure of a short-lived container is treated as a sign of a problem with
+the role configuration and/or another aspect of the Hoya cluster -or
+a problem with the specific node on which the attempt to run
+the container was made.
+
+
+
+#### `hoya.container.failure.threshold`
+
+An integer stating the number of failures tolerated in a single role before
+the cluster is considered to have failed.
+
+
+
+#### `hoya.am.monitoring.enabled`
+
+A boolean flag to indicate whether application-specific health monitoring
+should take place. Until this monitoring is completed the option
+is ignored -it is added as `false` by default on clusters created.
+
+
+## Roles
+
+A Hoya application consists of the Hoya Application Master, "the AM", which
+manages the cluster, and a number of instances of the "roles" of the actual
+application.
+
+For HBase the roles are `master` and `worker`; Accumulo has more.
+
+For every role, the cluster specification can define
+1. How many instances of that role are desired.
+1. Some options with well known names for configuring the runtimes
+of the roles.
+1. Environment variables needed to help configure and run the process.
+1. Options for YARN
+
+### Standard Role Options
+
+#### Desired instance count `role.instances`
+
+#### Additional command-line arguments `role.additional.args`
+
+This argument is meant to provide a configuration-based static option
+that is provided to every instance of the given role. For example, this is
+useful in providing a binding address to Accumulo's Monitor process.
+
+Users can override the option on the hoya executable using the roleopt argument:
+
+ --roleopt monitor role.additional.args "--address 127.0.0.1"
+
+#### YARN container memory `yarn.memory`
+
+The amount of memory in MB for the YARN container hosting
+that role. Default "256".
+
+The special value `max` indicates that Hoya should request the
+maximum value that YARN allows containers to have -a value
+determined dynamically after the cluster starts.
+
+Examples:
+
+ --roleopt master yarn.memory 2048
+ --roleopt worker yarn.memory max
+
+If a YARN cluster is configured to set process memory limits via the OS,
+and the application tries to use more memory than allocated, it will fail
+with the exit code "143".
+
+#### YARN vCores `yarn.vcores`
+
+Number of "Cores" for the container hosting
+a role. Default value: "1"
+
+The special value `max` indicates that Hoya should request the
+maximum value that YARN allows containers to have -a value
+determined dynamically after the cluster starts.
+
+As well as being able to specify the numeric values of memory and cores
+in role, via the `--roleopt` argument, you can now ask for the maximum
+allowed value by using the parameter `max`
+
+Examples:
+
+ --roleopt master yarn.vcores 2
+ --roleopt master yarn.vcores max
+
+#### Master node web port `app.infoport`
+
+The TCP socket port number to use for the master node web UI. This is translated
+into an application-specific site.xml property for both Accumulo and HBase.
+
+If set to a number other than the default, "0", then if the given port is in
+use, the role instance will not start. This will occur if YARN is already
+running a master node on that server, or if another application is using
+the same TCP port.
+
+#### JVM Heapsize `jvm.heapsize`
+
+Heapsize as a JVM option string, such as `"256M"` or `"2G"`
+
+ --roleopt worker jvm.heapsize 8G
+
+This is not correlated with the YARN memory -changes in the YARN memory allocation
+are not reflected in the JVM heapsize -and vice versa.
+
+### Environment variables
+
+
+All role options beginning with `env.` are automatically converted to
+environment variables which will be set for all instances of that role.
+
+ --roleopt worker env.MALLOC_ARENA 4
+
+## Accumulo-Specific Options
+
+### Accumulo cluster options
+
+Here are options specific to Accumulo clusters.
+
+#### Mandatory: Zookeeper Home: `zk.home`
+
+Location of Zookeeper on the target machine. This is needed by the
+Accumulo startup scripts.
+
+#### Mandatory: Hadoop Home `hadoop.home`
+
+Location of Hadoop on the target machine. This is needed by the
+Accumulo startup scripts.
+
+#### Mandatory: Accumulo database password `accumulo.password`
+
+This is the password used to control access to the accumulo data.
+A random password (from a UUID, hence very low-entropy) is chosen when
+the cluster is created. A more rigorous password can be set on the command
+line _at the time of cluster creation_.
+
+
+## Hoya AM Role Options
+
+The Hoya Application Master has its own role, `hoya`, which can also
+be configured with role options. Currently only JVM and YARN options
+are supported:
+
+ --roleopt hoya jvm.heapsize 256M
+ --roleopt hoya jvm.opts "-Djvm.property=true"
+ --roleopt hoya yarn.memory 512
+
+Normal memory requirements of the AM are low, except in the special case of
+starting an accumulo cluster for the first time. In this case, `bin\accumulo init`
+needs to be run: the extra memory requirements of the accumulo process
+need to be included in the hoya role's `yarn.memory` values.
diff --git a/src/site/markdown/images/app_config_folders_01.png b/src/site/markdown/images/app_config_folders_01.png
new file mode 100644
index 0000000..4e78b63
--- /dev/null
+++ b/src/site/markdown/images/app_config_folders_01.png
Binary files differ
diff --git a/src/site/markdown/images/app_package_sample_04.png b/src/site/markdown/images/app_package_sample_04.png
new file mode 100644
index 0000000..170256b
--- /dev/null
+++ b/src/site/markdown/images/app_package_sample_04.png
Binary files differ
diff --git a/src/site/markdown/images/managed_client.png b/src/site/markdown/images/managed_client.png
new file mode 100644
index 0000000..9c094b1
--- /dev/null
+++ b/src/site/markdown/images/managed_client.png
Binary files differ
diff --git a/src/site/markdown/images/slider-container.png b/src/site/markdown/images/slider-container.png
new file mode 100644
index 0000000..2e02833
--- /dev/null
+++ b/src/site/markdown/images/slider-container.png
Binary files differ
diff --git a/src/site/markdown/images/unmanaged_client.png b/src/site/markdown/images/unmanaged_client.png
new file mode 100644
index 0000000..739d56d
--- /dev/null
+++ b/src/site/markdown/images/unmanaged_client.png
Binary files differ
diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md
new file mode 100644
index 0000000..7ababc7
--- /dev/null
+++ b/src/site/markdown/index.md
@@ -0,0 +1,92 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+# Slider: Dynamic YARN Applicatins
+
+
+
+Slider is a YARN application to deploy existing distributed applications on YARN,
+monitor them and make them larger or smaller as desired -even while
+the cluster is running.
+
+
+Slider has a plug-in *provider* architecture to support different applications,
+and currently supports Apache HBase and Apache Accumulo.
+
+Clusters can be stopped, "frozen" and restarted, "thawed" later; the distribution
+of the deployed application across the YARN cluster is persisted -enabling
+a best-effort placement close to the previous locations on a cluster thaw.
+Applications which remember the previous placement of data (such as HBase)
+can exhibit fast start-up times from this feature.
+
+YARN itself monitors the health of 'YARN containers" hosting parts of
+the deployed application -it notifies the Slider manager application of container
+failure. Slider then asks YARN for a new container, into which Slider deploys
+a replacement for the failed component. As a result, Slider can keep the
+size of managed applications consistent with the specified configuration, even
+in the face of failures of servers in the cluster -as well as parts of the
+application itself
+
+Some of the features are:
+
+* Allows users to create on-demand Apache HBase and Apache Accumulo clusters
+
+* Allow different users/applicatins to run different versions of the application.
+
+* Allow users to configure different Hbase instances differently
+
+* Stop / Suspend / Resume clusters as needed
+
+* Expand / shrink clusters as needed
+
+The Slider tool is a Java command line application.
+
+The tool persists the information as a JSON document into the HDFS.
+It also generates the configuration files assuming the passed configuration
+directory as a base - in particular, the HDFS and ZooKeeper root directories
+for the new HBase instance has to be generated (note that the underlying
+HDFS and ZooKeeper are shared by multiple cluster instances). Once the
+cluster has been started, the cluster can be made to grow or shrink
+using the Slider commands. The cluster can also be stopped, *frozen*
+and later resumed, *thawed*.
+
+Slider implements all its functionality through YARN APIs and the existing
+application shell scripts. The goal of the application was to have minimal
+code changes and as of this writing, it has required few changes.
+
+## Using
+
+* [Announcement](announcement.html)
+* [Installing](installing.html)
+* [Man Page](manpage.html)
+* [Examples](examples.html)
+* [Client Configuration](hoya-client-configuration.html)
+* [Client Exit Codes](exitcodes.html)
+* [Cluster Descriptions](hoya_cluster_descriptions.html)
+* [Security](security.html)
+* [Logging](logging.html)
+
+## Developing
+
+* [Architecture](architecture.html)
+* [Application Needs](app_needs.html)
+* [Building](building.html)
+* [Releasing](releasing.html)
+* [Role history](rolehistory.html)
+* [Specification](specification/index.html)
+* [Application configuration model](configuration/index.html)
diff --git a/src/site/markdown/installing.md b/src/site/markdown/installing.md
new file mode 100644
index 0000000..5efca36
--- /dev/null
+++ b/src/site/markdown/installing.md
@@ -0,0 +1,26 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Installing and Running Slider
+
+
+1. Unzip/Untar the archive
+1. Add `slider/bin` to the path
+1. The logging settings are set in `conf/log4j.properties`
+1. Standard configuration options may be set as defined in
+[Slider Client Configuration] (client-configuration.html)
+
diff --git a/src/site/markdown/manpage.md b/src/site/markdown/manpage.md
new file mode 100644
index 0000000..09f97a5
--- /dev/null
+++ b/src/site/markdown/manpage.md
@@ -0,0 +1,428 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# slider: YARN-hosted applications
+
+## NAME
+
+slider -YARN-hosted applications
+
+## SYNOPSIS
+
+Slider enables applications to be dynamically created on a YARN-managed datacenter.
+The program can be used to create, pause, and shutdown the application. It can also be used to list current active
+and existing but not running "frozen" application instances.
+
+## CONCEPTS
+
+1. A *Slider application* is an application packaged to be deployed by Slider. It consists of one or more distributed *components*
+
+1. A *Slider application instance* is a slider application configured to be deployable on a specific YARN cluster, with a specific configuration. An instance can be *live* -actually running- or *frozen*. When frozen all its configuration details and instance-specific data are preserved on HDFS.
+
+1. An *image* is a *tar.gz* file containing binaries used to create the application. 1. Images are kept in the HDFS filesystem and identified by their path names; filesystem permissions can be used to share images amongst users.
+
+1. An *image configuration* is a directory that is overlaid file-by-file onto the conf/ directory inside the HBase image.
+
+1. Users can have multiple image configurations -they too are kept in HDFS, identified by their path names, and can be shared by setting the appropriate permissions, along with a configuration template file.
+
+1. Only those files provided in the image configuration directory overwrite the default values contained in the image; all other configuration files are retained.
+
+1. Late-binding properties can also be provided at create time.
+
+1. Slider can overwrite some of the configuration properties to enable the dynamically created components to bind correctly to each other.
+
+1. An *instance directory* is a directory created in HDFS describing the application instance; it records the configuration -both user specified, application-default and any dynamically created by slider.
+
+1. A user can create an application instance.
+
+1. A live instances can be *frozen*, saving its final state to its application instance state directory. All running components are shut down.
+
+1. A frozen instance can be *thawed* -a its components started on or near the servers where they were previously running.
+
+1. A frozen instance can be *destroyed*.
+
+1. Running instances can be listed.
+
+1. An instance consists of a set of components
+
+1. The supported component types depends upon the slider application.
+
+1. the count of each component must initially be specified when a application instance is created.
+
+1. Users can flex an application instance: adding or removing components dynamically.
+If the application instance is live, the changes will have immediate effect. If not, the changes will be picked up when the instance is next thawed.
+
+
+<!--- ======================================================================= -->
+
+## Invoking Slider
+
+
+ slider <ACTION> [<name>] [<OPTIONS>]
+
+
+<!--- ======================================================================= -->
+
+## COMMON COMMAND-LINE OPTIONS
+
+### `--conf configuration.xml`
+
+Configure the Slider client. This allows the filesystem, zookeeper instance and other properties to be picked up from the configuration file, rather than on the command line.
+
+Important: *this configuration file is not propagated to the application. It is purely for configuring the client itself.
+
+### `-D name=value`
+
+Define a Hadoop configuration option which overrides any options in the client configuration XML files.
+
+
+### `-m, --manager url`
+
+URL of the YARN resource manager
+
+
+### `--fs filesystem-uri`
+
+Use the specific filesystem URI as an argument to the operation.
+
+
+
+
+
+<!--- ======================================================================= -->
+
+
+<!--- ======================================================================= -->
+
+## Actions
+
+COMMANDS
+
+
+
+### `build <name>`
+
+Build an instance of the given name, with the specific options.
+
+It is not started; this can be done later with a `thaw` command.
+
+### `create <name>`
+
+Build and run an applicaton instance of the given name
+
+The `--wait` parameter, if provided, specifies the time to wait until the YARN application is actually running. Even after the YARN application has started, there may be some delay for the instance to start up.
+
+### Arguments for `build` and `create`
+
+
+
+##### `--package <uri-to-package>`
+
+This define the slider application package to be deployed.
+
+
+##### `--option <name> <value>`
+
+Set a application instance option.
+
+Example:
+
+Set an option to be passed into the `-site.xml` file of the target system, reducing
+the HDFS replication factor to 2. (
+
+ --option site.dfs.blocksize 128m
+
+Increase the number of YARN containers which must fail before the Slider application instance
+itself fails.
+
+ -O slider.container.failure.threshold 16
+
+##### `--appconf dfspath`
+
+A URI path to the configuration directory containing the template application specification. The path must be on a filesystem visible to all nodes in the YARN cluster.
+
+1. Only one configuration directory can be specified.
+1. The contents of the directory will only be read when the application instance is created/built.
+
+Example:
+
+ --appconf hdfs://namenode/users/slider/conf/hbase-template
+ --appconf file://users/accumulo/conf/template
+
+
+
+##### `--apphome localpath`
+
+A path to the home dir of a pre-installed application. If set when a Slider
+application instance is created, the instance will run with the binaries pre-installed
+on the nodes at this location
+
+*Important*: this is a path in the local filesystem which must be present
+on all hosts in the YARN cluster
+
+Example
+
+ --apphome /usr/hadoop/hbase
+
+##### `--template <filename>`
+
+Filename for the template application instance configuration. This
+will be merged with -and can overwrite- the built-in configuration options, and can
+then be overwritten by any command line `--option` and `--compopt` arguments to
+generate the final application configuration.
+
+##### `--resources <filename>`
+
+Filename for the resources configuration. This
+will be merged with -and can overwrite- the built-in resource options, and can
+then be overwritten by any command line `--resopt`, `--rescompopt` and `--component`
+arguments to generate the final resource configuration.
+
+
+##### `--image path`
+
+The full path in Hadoop HDFS to a `.tar` or `.tar.gz` file containing
+the binaries needed to run the target application.
+
+Example
+
+ --image hdfs://namenode/shared/binaries/hbase-0.96.tar.gz
+
+##### `--component <name> <count>`
+
+The desired number of instances of a component
+
+
+Example
+
+ --component worker 16
+
+This just sets the `component.instances` value of the named component's resource configuration.
+it is exactly equivalent to
+
+ --rco worker component.instances 16
+
+
+
+#### `--compopt <component> <option> <value>`
+
+Provide a specific application configuration option for a component
+
+Example
+
+ --compopt master env.TIMEOUT 10000
+
+These options are saved into the `app_conf.json` file; they are not used to configure the YARN Resource
+allocations, which must use the `--rco` parameter
+
+#### Resource Component option `--rescompopt` `--rco`
+
+`--rescompopt <component> <option> <value>`
+
+Set any role-specific option, such as its YARN memory requirements.
+
+Example
+
+ --rco worker master yarn.memory 2048
+ --rco worker worker yarn.memory max
+
+
+##### `--zkhosts host1:port1,[host2:port2,host3:port3, ...] `
+
+The zookeeper quorum.
+
+Example
+
+ --zkhosts zk1:2181,zk2:2181,zk3:4096
+
+If unset, the zookeeper quorum defined in the property `slider.zookeeper.quorum`
+is used
+
+### `destroy <name>`
+
+Destroy a (stopped) applicaton instance .
+
+Important: This deletes all persistent data
+
+Example
+
+ slider destroy instance1
+
+### `exists <name> [--live]`
+
+Probe the existence of the named Slider application instance. If the `--live` flag is set, the instance
+must actually be running
+
+If not, an error code is returned.
+
+When the --live` flag is unset, the command looks for the application instance to be
+defined in the filesystem -its operation state is not checked.
+
+Return codes
+
+ 0 : application instance is defined in the filesystem
+ 70 : application instance is unknown
+
+Example:
+
+ slider exists instance4
+
+#### Live Tests
+
+When the `--live` flag is set, the application instance must be running for the command
+to succeed
+
+1. The probe does not check the status of any Slider-deployed services, merely that a application instance has been deployed
+1. A application instance that is finished or failed is not considered to be live.
+
+Return codes
+
+ 0 : application instance is running
+ -1 : application instance exists but is not running
+ 70 : application instance is unknown
+
+
+Example:
+
+ slider exists instance4 --live
+
+### `flex <name> [--component component count]* `
+
+Flex the number of workers in an application instance to the new value. If greater than before, new copies of the component will be requested. If less, component instances will be destroyed.
+
+This operation has a return value of 0 if the size of a running instance was changed.
+
+It returns -1 if there is no running application instance, or the size of the flexed instance matches that of the original -in which case its state does not change.
+
+Example
+
+ slider flex instance1 --component worker 8 --filesystem hdfs://host:port
+ slider flex instance1 --component master 2 --filesystem hdfs://host:port
+
+
+### `freeze <name> [--force] [--wait time] [--message text]`
+
+freeze the application instance. The running application is stopped. Its settings are retained in HDFS.
+
+The `--wait` argument can specify a time in seconds to wait for the application instance to be frozen.
+
+The `--force` flag causes YARN asked directly to terminate the application instance.
+The `--message` argument supplies an optional text message to be used in
+the request: this will appear in the application's diagnostics in the YARN RM UI.
+
+If an unknown (or already frozen) application instance is named, no error is returned.
+
+Examples
+
+ slider freeze instance1 --wait 30
+ slider freeze instance2 --force --message "maintenance session"
+
+
+### `list <name>`
+
+List running Slider application instances visible to the user.
+
+If an instance name is given and there is no running instance with that name, an error is returned.
+
+Example
+
+ slider list
+ slider list instance1
+
+### `status <name> [--out <filename>]`
+
+Get the status of the named application instance in JSON format. A filename can be used to
+specify the destination.
+
+Examples:
+
+ slider status instance1 --manager host:port
+
+ slider status instance2 --manager host:port --out status.json
+
+
+
+### `thaw <name> [--wait time`]
+
+Resume a frozen application instance, recreating it from its previously saved state. This will include a best-effort attempt to create the same number of nodes as before, though their locations may be different.
+
+Examples:
+
+ slider thaw instance2
+ slider thaw instance1 --wait 60
+
+
+If the application instance is already running, this command will not affect it.
+
+
+### `version`
+
+The command `slider version` prints out information about the compiled
+Slider application, the version of Hadoop against which it was built -and
+the version of Hadoop that is currently on its classpath.
+
+Note that this is the client-side Hadoop version, not that running on the server, though
+that can be obtained in the status operation
+
+
+
+## Commands for testing
+
+
+These are clearly abnormal operations; they are here primarily for testing
+-and documented for completeness.
+
+### `kill-container <name> --id container-id`
+
+Kill a YARN container belong to the application. This is useful primarily for testing the
+resilience to failures.
+
+Container IDs can be determined from the application instance status JSON document.
+
+
+### `am-suicide <name> [--exitcode code] [--message message] [--wait time]`
+
+This operation is purely for testing Slider Application Master restart;
+it triggers an asynchronous self-destruct operation in the AM -an
+operation that does not make any attempt to cleanly shut down the process.
+
+If the application has not exceeded its restart limit (as set by
+`slider.yarn.restart.limit`), YARN will attempt to restart the failed application.
+
+Example
+
+ slider am-suicide --exitcode 1 --wait 5000 -message "test"
+
+<!--- ======================================================================= -->
+
+
+## Instance Naming
+
+Application instance names must:
+
+1. be at least one character long
+1. begin with a lower case letter
+1. All other characters must be in the range \[a-z,0-9,_]
+1. All upper case characters are converted to lower case
+
+Example valid names:
+
+ slider1
+ storm4
+ hbase_instance
+ accumulo_m1_tserve4
+
diff --git a/src/site/markdown/registry/registry-model.md b/src/site/markdown/registry/registry-model.md
new file mode 100644
index 0000000..31bd592
--- /dev/null
+++ b/src/site/markdown/registry/registry-model.md
@@ -0,0 +1,75 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Registry
+
+The service registry model is designed to support dynamically
+deployed Slider applications, *and* statically deployed versions
+of the same application -provided the latter also registers itself,
+its public network services, and any configurations and files
+that it wishes clients to be able to retrieve.
+
+The architecture and implementation of this registry is not defined
+here -instead the view of it seen by clients.
+
+1. A 'service registry' exists in the YARN cluster into which
+services can be registered.
+
+1. There is no restriction on the number of services that can be registered in
+the registry, the type of service that may register, or even on how many
+registered services an application running in the YARN cluster may register.
+
+1. Services are registered by their type, owner and name. As an example,
+Alice's slider-managed HBase cluster `ingress` would have a type `org.apache.hbase`,
+owner `alice` and name `ingress`.
+
+1. In the case of Slider-managed services, there is a separate slider instance
+registration which publishes information about slider itself. In the example
+above, this would be (`org.apache.slider`,`alice`,`ingress`).
+
+1. Services can publish information about themselves, with common entries being:
+
+ * service name and description.
+ * URLs of published web UIs and web service APIs
+ * network address of other protocols
+
+1. Services may also publish.
+
+ * URLs to configuration details
+ * URLs documents published for use as client-side configuration data -either
+ directly or through some form of processing.
+ * public service-specific data, for use by applications that are aware of
+ the specific service type.
+ * internal service-specific data -for use by the components that comprise
+ an application. This allows the registry to be used to glue together
+ the application itself.
+
+1. Services can be listed and examined.
+
+1. Service-published configuration key-value pairs can be retrieved by clients
+
+1. Service-published documents (and packages of such documents) can be
+retrieved by clients.
+
+1. There's no requirement for service instances to support any standard protocols;
+
+1. Some protocols are defined which they MAY implement. For example, the protocol
+to enumerate and retrieve configuration documents is designed to be implemented
+by any service that wishes to publish such content.
+
+1. In a secure cluster, the registry data may be restricted, along with any
+service protocols offered by the registered services.
diff --git a/src/site/markdown/release_notes/release-0.22.0.md b/src/site/markdown/release_notes/release-0.22.0.md
new file mode 100644
index 0000000..b68aafd
--- /dev/null
+++ b/src/site/markdown/release_notes/release-0.22.0.md
@@ -0,0 +1,48 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Slider Release 0.22.0
+
+April 2014
+
+This release is built against the Apache Hadoop 2.4.0, HBase-0.98.0RC1
+and Accumulo 1.5.0 artifacts.
+
+Download: [slider-0.22.0-all.tar.gz](http://public-repo-1.hortonworks.com/slider/slider-0.22.0-all.tar.gz)
+
+
+## Key changes
+
+### Added support for Slider App Package
+
+Slider AppPackages are a declarative definition of an application for application management. Slider can deploy and manage any application described in the App Package format.
+
+
+### Added support for Slider Agent
+
+Slider agent is a generic provider that can process Slider App Packages.
+
+### Added documentation on developing and using Slider App Packages
+
+### Added Service Registry and associated REST resources
+
+### Enhanced Agent Provider AM Web UI with additional application info
+
+Added links to WADL listing expose AM REST management resources as well as the exposed Registry REST resources
+
+
+
diff --git a/src/site/markdown/release_notes/release-0.24.0.md b/src/site/markdown/release_notes/release-0.24.0.md
new file mode 100644
index 0000000..21578ea
--- /dev/null
+++ b/src/site/markdown/release_notes/release-0.24.0.md
@@ -0,0 +1,50 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Slider Release 0.24.0
+
+May 2014
+
+This release is built against the Apache Hadoop 2.4.0, HBase-0.98.1
+and Accumulo 1.5.1 artifacts.
+
+Download: []()
+
+
+## Key changes
+
+### Added support for Slider App Package
+
+Slider AppPackages are a declarative definition of an application for application management. Slider can deploy and manage any application described in the App Package format.
+
+
+### Added support for Slider Agent
+
+Slider agent is a generic provider that can process Slider App Packages.
+
+### Added documentation on developing and using Slider App Packages
+
+### Added Service Registry and associated REST resources
+
+### Enhanced Agent Provider AM Web UI with additional application info
+
+Added links to WADL listing expose AM REST management resources as well as the exposed Registry REST resources
+
+
+## Other changes
+
+1. Security is now enabled by setting the configuration option `hadoop.security.authorization`
diff --git a/src/site/markdown/releasing.md b/src/site/markdown/releasing.md
new file mode 100644
index 0000000..a396a28
--- /dev/null
+++ b/src/site/markdown/releasing.md
@@ -0,0 +1,192 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+# Release Process
+
+Here is our release process.
+
+### Before you begin
+
+Check out the latest version of the develop branch,
+run the tests. This should be done on a checked out
+version of the code that is not the one you are developing on
+(ideally, a clean VM), to ensure that you aren't releasing a slightly
+modified version of your own, and that you haven't accidentally
+included passwords or other test run details into the build resource
+tree.
+
+The `slider-funtest` functional test package is used to run functional
+tests against a running Hadoop YARN cluster. It needs to be configured
+according to the instructions in [testing](testing.html) to
+create HBase and Accumulo clusters in the YARN cluster.
+
+*Make sure that the functional tests are passing (and not being skipped) before
+starting to make a release*
+
+
+
+**Step #1:** Create a JIRA for the release, estimate 3h
+(so you don't try to skip the tests)
+
+ export SLIDER_RELEASE_JIRA=BUG-13927
+
+**Step #2:** Check everything in. Git flow won't let you progress without this.
+
+**Step #3:** Git flow: create a release branch
+
+ export SLIDER_RELEASE=0.5.2
+
+ git flow release start slider-$SLIDER_RELEASE
+
+**Step #4:** in the new branch, increment those version numbers using (the maven
+versions plugin)[http://mojo.codehaus.org/versions-maven-plugin/]
+
+ mvn versions:set -DnewVersion=$SLIDER_RELEASE
+
+
+**Step #5:** commit the changed POM files
+
+ git add <changed files>
+ git commit -m "$SLIDER_RELEASE_JIRA updating release POMs for $SLIDER_RELEASE"
+
+
+**Step #6:** Do a final test run to make sure nothing is broken
+
+In the `slider` directory, run:
+
+ mvn clean install -DskipTests
+
+Once everything is built- including .tar files, run the tests
+
+ mvn test
+
+This will run the functional tests as well as the `slider-core` tests.
+
+It is wise to reset any VMs here, and on live clusters kill all running jobs.
+This stops functional tests failing because the job doesn't get started before
+the tests time out.
+
+As the test run takes 30-60+ minutes, now is a good time to consider
+finalizing the release notes.
+
+
+**Step #7:** Build the release package
+
+Run
+
+ mvn clean site:site site:stage package -DskipTests
+
+
+
+**Step #8:** validate the tar file
+
+Look in `slider-assembly/target` to find the `.tar.gz` file, and the
+expanded version of it. Inspect that expanded version to make sure that
+everything looks good -and that the versions of all the dependent artifacts
+look good too: there must be no `-SNAPSHOT` dependencies.
+
+
+**Step #9:** Build the release notes
+
+Create a a one-line plain text release note for commits and tags
+And a multi-line markdown release note, which will be used for artifacts.
+
+
+ Release against hadoop 2.4.0, HBase-0.98.1 and Accumulo 1.5.1 artifacts.
+
+The multi-line release notes should go into `slider/src/site/markdown/release_notes`.
+
+
+These should be committed
+
+ git add --all
+ git commit -m "$SLIDER_RELEASE_JIRA updating release notes"
+
+**Step #10:** End the git flow
+
+Finish the git flow release, either in the SourceTree GUI or
+the command line:
+
+
+ git flow release finish slider-$SLIDER_RELEASE
+
+
+On the command line you have to enter the one-line release description
+prepared earlier.
+
+You will now be back on the `develop` branch.
+
+**Step #11:** update mvn versions
+
+Switch back to `develop` and update its version number past
+the release number
+
+
+ export SLIDER_RELEASE=0.6.0-SNAPSHOT
+ mvn versions:set -DnewVersion=$SLIDER_RELEASE
+ git commit -a -m "$SLIDER_RELEASE_JIRA updating development POMs to $SLIDER_RELEASE"
+
+**Step #12:** Push the release and develop branches to github
+
+ git push origin master develop
+
+(assuming that `origin` maps to `git@github.com:hortonworks/hoya.git`;
+ you can check this with `git remote -v`
+
+
+The `git-flow` program automatically pushes up the `release/slider-X.Y` branch,
+before deleting it locally.
+
+If you are planning on any release work of more than a single test run,
+consider having your local release branch track the master.
+
+
+**Step #13:** ### Release on github small artifacts
+
+Browse to https://github.com/hortonworks/slider/releases/new
+
+Create a new release on the site by following the instructions
+
+Files under 5GB can be published directly. Otherwise, follow step 14
+
+**Step #14:** For releasing via an external CDN (e.g. Rackspace Cloud)
+
+Using the web GUI for your particular distribution network, upload the
+`.tar.gz` artifact
+
+After doing this, edit the release notes on github to point to the
+tar file's URL.
+
+Example:
+ [Download slider-0.10.1-all.tar.gz](http://dffeaef8882d088c28ff-185c1feb8a981dddd593a05bb55b67aa.r18.cf1.rackcdn.com/slider-0.10.1-all.tar.gz)
+
+**Step #15:** Announce the release
+
+**Step #16:** Finish the JIRA
+
+Log the time, close the issue. This should normally be the end of a
+sprint -so wrap that up too.
+
+**Step #17:** Get back to developing!
+
+Check out the develop branch and purge all release artifacts
+
+ git checkout develop
+ git pull origin
+ mvn clean
+
diff --git a/src/site/markdown/rolehistory.md b/src/site/markdown/rolehistory.md
new file mode 100644
index 0000000..83f2010
--- /dev/null
+++ b/src/site/markdown/rolehistory.md
@@ -0,0 +1,1010 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Role History: how Slider brings back nodes in the same location
+
+### Last updated 2013-12-06
+
+* This document uses the pre-slider terminology of role/cluster and not
+component and application instance *
+
+
+## Outstanding issues
+
+1. Can we use the history to implement anti-affinity: for any role with this flag,
+use our knowledge of the cluster to ask for all nodes that aren't in use already
+
+1. How to add blacklisting here? We are tracking failures and startup failures
+per node (not persisted), but not using this in role placement requests yet.
+
+## Introduction
+
+Slider needs to bring up instances of a given role on the machine(s) on which
+they last ran -it should remember after shrinking or freezing a cluster which
+servers were last used for a role -and use this (persisted) data to select
+clusters next time
+
+It does this in the basis that the role instances prefer node-local
+access to data previously persisted to HDFS. This is precisely the case
+for Apache HBase, which can use Unix Domain Sockets to talk to the DataNode
+without using the TCP stack. The HBase master persists to HDFS the tables
+assigned to specific Region Servers, and when HBase is restarted its master
+tries to reassign the same tables back to Region Servers on the same machine.
+
+For this to work in a dynamic cluster, Slider needs to bring up Region Servers
+on the previously used hosts, so that the HBase Master can re-assign the same
+tables.
+
+Note that it does not need to care about the placement of other roles, such
+as the HBase masters -there anti-affinity between other instances is
+the key requirement.
+
+### Terminology
+
+* **Role Instance** : a single instance of a role.
+* **Node** : A server in the YARN Physical (or potentially virtual) Cluster of servers.
+* **Slider Cluster**: The set of role instances deployed by Slider so as to
+ create a single aggregate application.
+* **Slider AM**: The Application Master of Slider: the program deployed by YARN to
+manage its Slider Cluster.
+* **RM** YARN Resource Manager
+
+### Assumptions
+
+Here are some assumptions in Slider's design
+
+1. Instances of a specific role should preferably be deployed onto different
+servers. This enables Slider to only remember the set of server nodes onto
+which instances were created, rather than more complex facts such as "two Region
+Servers were previously running on Node #17. On restart Slider can simply request
+one instance of a Region Server on a specific node, leaving the other instance
+to be arbitrarily deployed by YARN. This strategy should help reduce the *affinity*
+in the role deployment, so increase their resilience to failure.
+
+1. There is no need to make sophisticated choices on which nodes to request
+re-assignment -such as recording the amount of data persisted by a previous
+instance and prioritizing nodes based on such data. More succinctly 'the
+only priority needed when asking for nodes is *ask for the most recently used*.
+
+1. Different roles are independent: it is not an issue if a role of one type
+ (example, an Accumulo Monitor and an Accumulo Tablet Server) are on the same
+ host. This assumption allows Slider to only worry about affinity issues within
+ a specific role, rather than across all roles.
+
+1. After a cluster has been started, the rate of change of the cluster is
+low: both node failures and cluster flexing happen at the rate of every few
+hours, rather than every few seconds. This allows Slider to avoid needing
+data structures and layout persistence code designed for regular and repeated changes.
+
+1. Instance placement is best-effort: if the previous placement cannot be satisfied,
+the application will still perform adequately with role instances deployed
+onto new servers. More specifically, if a previous server is unavailable
+for hosting a role instance due to lack of capacity or availability, Slider
+will not decrement the number of instances to deploy: instead it will rely
+on YARN to locate a new node -ideally on the same rack.
+
+1. If two instances of the same role do get assigned to the same server, it
+is not a failure condition. (This may be problematic for some roles
+-we may need a role-by-role policy here, so that master nodes can be anti-affine)
+[specifically, >1 HBase master mode will not come up on the same host]
+
+1. If a role instance fails on a specific node, asking for a container on
+that same node for the replacement instance is a valid recovery strategy.
+This contains assumptions about failure modes -some randomness here may
+be a valid tactic, especially for roles that do not care about locality.
+
+1. Tracking failure statistics of nodes may be a feature to add in future;
+designing the Role History datastructures to enable future collection
+of rolling statistics on recent failures would be a first step to this
+
+### The Role History
+
+The `RoleHistory` is a datastructure which models the role assignment, and
+can persist it to and restore it from the (shared) filesystem.
+
+* For each role, there is a list of cluster nodes which have supported this role
+used in the past.
+
+* This history is used when selecting a node for a role.
+
+* This history remembers when nodes were allocated. These are re-requested
+when thawing a cluster.
+
+* It must also remember when nodes were released -these are re-requested
+when returning the cluster size to a previous size during flex operations.
+
+* It has to track nodes for which Slider has an outstanding container request
+with YARN. This ensures that the same node is not requested more than once
+due to outstanding requests.
+
+* It does not retain a complete history of the role -and does not need to.
+All it needs to retain is the recent history for every node onto which a role
+instance has been deployed. Specifically, the last allocation or release
+operation on a node is all that needs to be persisted.
+
+* On AM startup, all nodes in the history are considered candidates, even those nodes currently marked
+as active -as they were from the previous instance.
+
+* On AM restart, nodes in the role history marked as active have to be considered
+still active -the YARN RM will have to provide the full list of which are not.
+
+* During cluster flexing, nodes marked as released -and for which there is no
+outstanding request - are considered candidates for requesting new instances.
+
+* When choosing a candidate node for hosting a role instance, it from the head
+of the time-ordered list of nodes that last ran an instance of that role
+
+### Persistence
+
+The state of the role is persisted to HDFS on changes -but not on cluster
+termination.
+
+1. When nodes are allocated, the Role History is marked as dirty
+1. When container release callbacks are received, the Role History is marked as dirty
+1. When nodes are requested or a release request made, the Role History is *not*
+ marked as dirty. This information is not relevant on AM restart.
+
+As at startup, a large number of allocations may arrive in a short period of time,
+the Role History may be updated very rapidly -yet as the containers are
+only recently activated, it is not likely that an immediately restarted Slider
+cluster would gain by re-requesting containers on them -their historical
+value is more important than their immediate past.
+
+Accordingly, the role history may be persisted to HDFS asynchronously, with
+the dirty bit triggering an flushing of the state to HDFS. The datastructure
+will still need to be synchronized for cross thread access, but the
+sync operation will not be a major deadlock, compared to saving the file on every
+container allocation response (which will actually be the initial implementation).
+
+There's no need to persist the format in a human-readable form; while protobuf
+might seem the approach most consistent with the rest of YARN, it's not
+an easy structure to work with.
+
+The initial implementation will use Apache Avro as the persistence format,
+with the data saved in JSON or compressed format.
+
+
+## Weaknesses in this design
+
+**Blacklisting**: even if a node fails repeatedly, this design will still try to re-request
+instances on this node; there is no blacklisting. As a central blacklist
+for YARN has been proposed, it is hoped that this issue will be addressed centrally,
+without Slider having to remember which nodes are unreliable *for that particular
+Slider cluster*.
+
+**Anti-affinity**: If multiple role instances are assigned to the same node,
+Slider has to choose on restart or flexing whether to ask for multiple
+nodes on that node again, or to pick other nodes. The assumed policy is
+"only ask for one node"
+
+**Bias towards recent nodes over most-used**: re-requesting the most
+recent nodes, rather than those with the most history of use, may
+push Slider to requesting nodes that were only briefly in use -and so have
+on a small amount of local state, over nodes that have had long-lived instances.
+This is a problem that could perhaps be addressed by preserving more
+history of a node -maintaining some kind of moving average of
+node use and picking the heaviest used, or some other more-complex algorithm.
+This may be possible, but we'd need evidence that the problem existed before
+trying to address it.
+
+# The NodeMap: the core of the Role History
+
+The core data structure, the `NodeMap` is a map of every known node in the cluster, tracking
+how many containers are allocated to specific roles in it, and, when there
+are no active instances, when it was last used. This history is used to
+choose where to request new containers. Because of the asynchronous
+allocation and release of containers, the Role History also needs to track
+outstanding release requests --and, more critically, outstanding allocation
+requests. If Slider has already requested a container for a specific role
+on a host, then asking for another container of that role would break
+anti-affinity requirements. Note that not tracking outstanding requests would
+radically simplify some aspects of the design, especially the complexity
+of correlating allocation responses with the original requests -and so the
+actual hosts originally requested.
+
+1. Slider builds up a map of which nodes have recently been used.
+1. Every node counts the number. of active containers in each role.
+1. Nodes are only chosen for allocation requests when there are no
+active or requested containers on that node.
+1. When choosing which instances to release, Slider could pick the node with the
+most containers on it. This would spread the load.
+1. When there are no empty nodes to request containers on, a request would
+let YARN choose.
+
+#### Strengths
+
+* Handles the multi-container on one node problem
+* By storing details about every role, cross-role decisions could be possible
+* Simple counters can track the state of pending add/release requests
+* Scales well to a rapidly flexing cluster
+* Simple to work with and persist
+* Easy to view and debug
+* Would support cross-role collection of node failures in future
+
+#### Weaknesses
+
+* Size of the data structure is `O(nodes * role-instances`). This
+could be mitigated by regular cleansing of the structure. For example, at
+thaw time (or intermittently) all unused nodes > 2 weeks old could be dropped.
+* Locating a free node could take `O(nodes)` lookups -and if the criteria of "newest"
+is included, will take exactly `O(nodes)` lookups. As an optimization, a list
+of recently explicitly released nodes can be maintained.
+* Need to track outstanding requests against nodes, so that if a request
+was satisfied on a different node, the original node's request count is
+ decremented, *not that of the node actually allocated*.
+* In a virtual cluster, may fill with node entries that are no longer in the cluster.
+Slider should query the RM (or topology scripts?) to determine if nodes are still
+parts of the YARN cluster.
+
+## Data Structures
+
+### RoleHistory
+
+ startTime: long
+ saveTime: long
+ dirty: boolean
+ nodemap: NodeMap
+ roles: RoleStatus[]
+ outstandingRequests: transient OutstandingRequestTracker
+ availableNodes: transient List<NodeInstance>[]
+
+This is the aggregate data structure that is persisted to/from file
+
+### NodeMap
+
+ clusterNodes: Map: NodeId -> NodeInstance
+ clusterNodes(): Iterable<NodeInstance>
+ getOrCreate(NodeId): NodeInstance
+
+ Maps a YARN NodeID record to a Slider `NodeInstance` structure
+
+### NodeInstance
+
+Every node in the cluster is modeled as an ragged array of `NodeEntry` instances, indexed
+by role index -
+
+ NodeEntry[roles]
+ get(roleId): NodeEntry or null
+ create(roleId): NodeEntry
+ getNodeEntries(): NodeEntry[roles]
+ getOrCreate(roleId): NodeEntry
+ remove(roleId): NodeEntry
+
+This could be implemented in a map or an indexed array; the array is more
+efficient but it does mandate that the number of roles are bounded and fixed.
+
+### NodeEntry
+
+Records the details about all of a roles containers on a node. The
+`active` field records the number of containers currently active.
+
+ active: int
+ requested: transient int
+ releasing: transient int
+ last_used: long
+
+ NodeEntry.available(): boolean = active - releasing == 0 && requested == 0
+
+The two fields `releasing` and `requested` are used to track the ongoing
+state of YARN requests; they do not need to be persisted across freeze/thaw
+cycles. They may be relevant across AM restart, but without other data
+structures in the AM, not enough to track what the AM was up to before
+it was restarted. The strategy will be to ignore unexpected allocation
+responses (which may come from pre-restart) requests, while treating
+unexpected container release responses as failures.
+
+The `active` counter is only decremented after a container release response
+has been received.
+
+### RoleStatus
+
+This is the existing `org.apache.hoya.yarn.appmaster.state.RoleStatus` class
+
+### RoleList
+
+A list mapping role to int enum is needed to index NodeEntry elements in
+the NodeInstance arrays. Although such an enum is already implemented in the Slider
+Providers, explicitly serializing and deserializing it would make
+the persistent structure easier to parse in other tools, and resilient
+to changes in the number or position of roles.
+
+This list could also retain information about recently used/released nodes,
+so that the selection of containers to request could shortcut a search
+
+
+### ContainerPriority
+
+The container priority field (a 32 bit integer) is used by Slider (0.5.x)
+to index the specific role in a container so as to determine which role
+has been offered in a container allocation message, and which role has
+been released on a release event.
+
+The Role History needs to track outstanding requests, so that
+when an allocation comes in, it can be mapped back to the original request.
+Simply looking up the nodes on the provided container and decrementing
+its request counter is not going to work -the container may be allocated
+on a different node from that requested.
+
+**Proposal**: The priority field of a request is divided by Slider into 8 bits for
+`roleID` and 24 bits for `requestID`. The request ID will be a simple
+rolling integer -Slider will assume that after 2^24 requests per role, it can be rolled,
+-though as we will be retaining a list of outstanding requests, a clash should not occur.
+The main requirement is: not have > 2^24 outstanding requests for instances of a specific role,
+which places an upper bound on the size of a Slider cluster.
+
+The splitting and merging will be implemented in a ContainerPriority class,
+for uniform access.
+
+### OutstandingRequest ###
+
+Tracks an outstanding request. This is used to correlate an allocation response
+(whose Container Priority file is used to locate this request), with the
+node and role used in the request.
+
+ roleId: int
+ requestID : int
+ node: string (may be null)
+ requestedTime: long
+ priority: int = requestID << 24 | roleId
+
+The node identifier may be null -which indicates that a request was made without
+a specific target node
+
+### OutstandingRequestTracker ###
+
+Contains a map from requestID to the specific `OutstandingRequest` made,
+and generates the request ID
+
+ nextRequestId: int
+ requestMap(RequestID) -> OutstandingRequest
+
+Operations
+
+ addRequest(NodeInstance, RoleId): OutstandingRequest
+ (and an updated request Map with a new entry)
+ lookup(RequestID): OutstandingRequest
+ remove(RequestID): OutstandingRequest
+ listRequestsForNode(ClusterID): [OutstandingRequest]
+
+The list operation can be implemented inefficiently unless it is found
+to be important -if so a more complex structure will be needed.
+
+### AvailableNodes
+
+This is a field in `RoleHistory`
+
+ availableNodes: List<NodeInstance>[]
+
+
+For each role, lists nodes that are available for data-local allocation,
+ordered by more recently released - To accelerate node selection
+
+The performance benefit is most significant when requesting multiple nodes,
+as the scan for M locations from N nodes is reduced from `M*N` comparisons
+to 1 Sort + M list lookups.
+
+Each list can be created off the Node Map by building for each role a sorted
+list of all Nodes which are available for an instance of that role,
+using a comparator that places the most recently released node ahead of older
+nodes.
+
+This list is not persisted -when a Slider Cluster is frozen it is moot, and when
+an AM is restarted this structure will be rebuilt.
+
+1. When a node is needed for a new request, this list is consulted first.
+1. After the request is issued it can be removed from the list
+1. Whenever a container is released, if the node is now available for
+requests for that node, should be added to to the front
+of the list for that role.
+
+If the list is empty during a container request operation, it means
+that the Role History does not know of any nodes
+in the cluster that have hosted instances of that role and which are not
+in use. There are then two possible strategies to select a role
+
+1. Ask for an instance anywhere in the cluster (policy in Slider 0.5)
+1. Search the node map to identify other nodes which are (now) known about,
+but which are not hosting instances of a specific role -this can be used
+as the target for the next resource request.
+
+Strategy #1 is simpler; Strategy #2 *may* decrease the affinity in the cluster,
+as the AM will be explicitly requesting an instance on a node which it knows
+is not running an instance of that role.
+
+
+#### ISSUE What to do about failing nodes?
+
+Should a node whose container just failed be placed at the
+top of the stack, ready for the next request?
+
+If the container failed due to an unexpected crash in the application, asking
+for that container back *is the absolute right strategy* -it will bring
+back a new role instance on that machine.
+
+If the container failed because the node is now offline, the container request
+will not be satisfied by that node.
+
+If there is a problem with the node, such that containers repeatedly fail on it,
+then re-requesting containers on it will amplify the damage.
+
+## Actions
+
+### Bootstrap
+
+1. Persistent Role History file not found; empty data structures created.
+
+### Thaw
+
+When thawing, the Role History should be loaded -if it is missing Slider
+must revert to the bootstrap actions.
+
+If found, the Role History will contain Slider's view of the Slider Cluster's
+state at the time the history was saved, explicitly recording the last-used
+time of all nodes no longer hosting a role's container. By noting which roles
+were actually being served, it implicitly notes which nodes have a `last_used`
+value greater than any of the `last_used` fields persisted in the file. That is:
+all node entries listed as having active nodes at the time the history was
+saved must have more recent data than those nodes listed as inactive.
+
+When rebuilding the data structures, the fact that nodes were active at
+save time must be converted into the data that indicates that the nodes
+were at least in use *at the time the data was saved*. The state of the cluster
+after the last save is unknown.
+
+1: Role History loaded; Failure => Bootstrap.
+2: Future: if role list enum != current enum, remapping could take place. Until then: fail.
+3: Mark all nodes as active at save time to that of the
+
+ //define a threshold
+ threshold = rolehistory.saveTime - 7*24*60*60* 1000
+
+
+ for (clusterId, clusternode) in rolehistory.clusterNodes().entries() :
+ for (role, nodeEntry) in clusterNode.getNodeEntries():
+ nodeEntry.requested = 0
+ nodeEntry.releasing = 0
+ if nodeEntry.active > 0 :
+ nodeEntry.last_used = rolehistory.saveTime;
+ nodeEntry.n.active = 0
+ if nodeEntry.last_used < threshold :
+ clusterNode.remove(role)
+ else:
+ availableNodes[role].add(clusterId)
+ if clusterNode.getNodeEntries() isEmpty :
+ rolehistory.clusterNodes.remove(clusterId)
+
+
+ for availableNode in availableNodes:
+ sort(availableNode,new last_used_comparator())
+
+After this operation, the structures are purged with all out of date entries,
+and the available node list contains a sorted list of the remainder.
+
+### AM Restart
+
+
+1: Create the initial data structures as the thaw operation
+2: update the structure with the list of live nodes, removing those nodes
+from the list of available nodes
+
+ now = time()
+ activeContainers = RM.getActiveContainers()
+
+ for container in activeContainers:
+ nodeId = container.nodeId
+ clusterNode = roleHistory.nodemap.getOrCreate(nodeId)
+ role = extractRoleId(container.getPriority)
+ nodeEntry = clusterNode.getOrCreate(role)
+ nodeEntry.active++
+ nodeEntry.last_used = now
+ availableNodes[role].remove(nodeId)
+
+There's no need to resort the available node list -all that has happened
+is that some entries have been removed
+
+
+**Issue**: what if requests come in for a `(role, requestID)` for
+the previous instance of the AM? Could we just always set the initial
+requestId counter to a random number and hope the collision rate is very, very
+low (2^24 * #(outstanding_requests)). If YARN-1041 ensures that
+a restarted AM does not receive outstanding requests, this issue goes away.
+
+
+### Teardown
+
+1. If dirty, save role history to its file.
+1. Issue release requests
+1. Maybe update data structures on responses, but do not mark Role History
+as dirty or flush it to disk.
+
+This strategy is designed to eliminate the expectation that there will ever
+be a clean shutdown -and so that the startup-time code should expect
+the Role History to have been written during shutdown. Instead the code
+should assume that the history was saved to disk at some point during the life
+of the Slider Cluster -ideally after the most recent change, and that the information
+in it is only an approximate about what the previous state of the cluster was.
+
+### Flex: Requesting a container in role `role`
+
+
+ node = availableNodes[roleId].pop()
+ if node != null :
+ node.nodeEntry[roleId].requested++;
+ outstanding = outstandingRequestTracker.addRequest(node, roleId)
+ request.node = node
+ request.priority = outstanding.priority
+
+ //update existing Slider role status
+ roleStatus[roleId].incRequested();
+
+
+There is a bias here towards previous nodes, even if the number of nodes
+in the cluster has changed. This is why a node is picked where the number
+of `active-releasing == 0 and requested == 0`, rather than where it is simply the lowest
+value of `active + requested - releasing`: if there is no node in the nodemap that
+is not running an instance of that role, it is left to the RM to decide where
+the role instance should be instantiated.
+
+This bias towards previously used nodes also means that (lax) requests
+will be made of nodes that are currently unavailable either because they
+are offline or simply overloaded with other work. In such circumstances,
+the node will have an active count of zero -so the search will find these
+nodes and request them -even though the requests cannot be satisfied.
+As a result, the request will be downgraded to a rack-local or cluster-wide,
+request -an acceptable degradation on a cluster where all the other entries
+in the nodemap have instances of that specific node -but not when there are
+empty nodes.
+
+
+#### Solutions
+
+1. Add some randomness in the search of the datastructure, rather than simply
+iterate through the values. This would prevent the same unsatisfiable
+node from being requested first.
+
+1. Keep track of requests, perhaps through a last-requested counter -and use
+this in the selection process. This would radically complicate the selection
+algorithm, and would not even distinguish "node recently released that was
+also the last requested" from "node that has not recently satisfied requests
+even though it was recently requested".
+
+1. Keep track of requests that weren't satisfied, so identify a node that
+isn't currently satisfying requests.
+
+
+#### History Issues
+
+Without using that history, there is a risk that a very old assignment
+is used in place of a recent one and the value of locality decreased.
+
+But there are consequences:
+
+**Performance**:
+
+Using the history to pick a recent node may increase selection times on a
+large cluster, as for every instance needed, a scan of all nodes in the
+nodemap is required (unless there is some clever bulk assignment list being built
+up), or a sorted version of the nodemap is maintained, with a node placed
+at the front of this list whenever its is updated.
+
+**Thaw-time problems**
+
+There is also the risk that while thawing, the `rolehistory.saved`
+flag may be updated while the cluster flex is in progress, so making the saved
+nodes appear out of date. Perhaps the list of recently released nodes could
+be rebuilt at thaw time.
+
+The proposed `recentlyReleasedList` addresses this, though it creates
+another data structure to maintain and rebuild at cluster thaw time
+from the last-used fields in the node entries.
+
+### AM Callback : onContainersAllocated
+
+ void onContainersAllocated(List<Container> allocatedContainers)
+
+This is the callback received when containers have been allocated.
+Due to (apparently) race conditions, the AM may receive duplicate
+container allocations -Slider already has to recognize this and
+currently simply discards any surplus.
+
+If the AM tracks outstanding requests made for specific hosts, it
+will need to correlate allocations with the original requests, so as to decrement
+the node-specific request count. Decrementing the request count
+on the allocated node will not work, as the allocation may not be
+to the node originally requested.
+
+ assignments = []
+ operations = []
+ for container in allocatedContainers:
+ cid = container.getId();
+ roleId = container.priority & 0xff
+ nodeId = container.nodeId
+ outstanding = outstandingRequestTracker.remove(C.priority)
+ roleStatus = lookupRoleStatus(container);
+ roleStatus.decRequested();
+ allocated = roleStatus.incActual();
+ if outstanding == null || allocated > desired :
+ operations.add(new ContainerReleaseOperation(cid))
+ surplusNodes.add(cid);
+ surplusContainers++
+ roleStatus.decActual();
+ else:
+ assignments.add(new ContainerAssignment(container, role))
+ node = nodemap.getOrCreate(nodeId)
+ nodeentry = node.get(roleId)
+ if nodeentry == null :
+ nodeentry = new NodeEntry()
+ node[roleId] = nodeentry
+ nodeentry.active = 1
+ else:
+ if nodeentry.requested > 0 :
+ nodeentry.requested--
+ nodeentry.active++
+ nodemap.dirty = true
+
+ // work back from request ID to node where the
+ // request was outstanding
+ requestID = outstanding != null? outstanding.nodeId : null
+ if requestID != null:
+ reqNode = nodeMap.get(requestID)
+ reqNodeEntry = reqNode.get(roleId)
+ reqNodeEntry.requested--
+ if reqNodeEntry.available() :
+ availableNodeList.insert(reqNodeEntry)
+
+
+
+1. At end of this, there is a node in the nodemap, which has recorded that
+there is now an active node entry for that role. The outstanding request has
+been removed.
+
+1. If a callback comes in for which there is no outstanding request, it is rejected
+(logged, ignored, etc). This handles duplicate responses as well as any
+other sync problem.
+
+1. The node selected for the original request has its request for a role instance
+decremented, so that it may be viewed as available again. The node is also
+re-inserted into the AvailableNodes list -not at its head, but at its position
+in the total ordering of the list.
+
+### NMClientAsync Callback: onContainerStarted()
+
+
+ onContainerStarted(ContainerId containerId)
+
+The AM uses this as a signal to remove the container from the list
+of starting containers, moving it into the map of live nodes; the counters
+in the associated `RoleInstance` are updated accordingly; the node entry
+adjusted to indicate it has one more live node and one less starting node.
+
+
+### NMClientAsync Callback: onContainerStartFailed()
+
+
+The AM uses this as a signal to remove the container from the list
+of starting containers -the count of starting containers for the relevant
+NodeEntry is decremented. If the node is now available for instances of this
+container, it is returned to the queue of available nodes.
+
+
+### Flex: Releasing a role instance from the cluster
+
+Simple strategy: find a node with at least one active container
+
+ select a node N in nodemap where for NodeEntry[roleId]: active > releasing;
+ nodeentry = node.get(roleId)
+ nodeentry.active--;
+
+Advanced Strategy:
+
+ Scan through the map looking for a node where active >1 && active > releasing.
+ If none are found, fall back to the previous strategy
+
+This is guaranteed to release a container on any node with >1 container in use,
+if such a node exists. If not, the scan time has increased to #(nodes).
+
+Once a node has been identified
+
+1. a container on it is located (via the existing container map). This container
+must: be of the target role, and not already be queued for release.
+1. A release operation is queued trigger a request for the RM.
+1. The (existing) `containersBeingReleased` Map has the container inserted into it
+
+After the AM processes the request, it triggers a callback
+
+### AM callback onContainersCompleted:
+
+ void onContainersCompleted(List<ContainerStatus> completedContainers)
+
+This callback returns a list of containers that have completed.
+
+These need to be split into successful completion of a release request
+and containers which have failed.
+
+This is currently done by tracking which containers have been queued
+for release, as well as which were rejected as surplus before even having
+any role allocated onto them.
+
+A container is considered to have failed if it was an active container which
+has completed although it wasn't on the list of containers to release
+
+ shouldReview = false
+ for container in completedContainers:
+ containerId = container.containerId
+ nodeId = container.nodeId
+ node = nodemap.get(nodeId)
+ if node == null :
+ // unknown node
+ continue
+ roleId = node.roleId
+ nodeentry = node.get(roleId)
+ nodeentry.active--
+ nodemap.dirty = true
+ if getContainersBeingReleased().containsKey(containerId) :
+ // handle container completion
+ nodeentry.releasing --
+
+ // update existing Slider role status
+ roleStatus[roleId].decReleasing();
+ containersBeingReleased.remove(containerId)
+ else:
+ //failure of a live node
+ roleStatus[roleId].decActual();
+ shouldReview = true
+
+ if nodeentry.available():
+ nodentry.last_used = now()
+ availableNodes[roleId].insert(node)
+ //trigger a comparison of actual vs desired
+ if shouldReview :
+ reviewRequestAndReleaseNodes()
+
+By calling `reviewRequestAndReleaseNodes()` the AM triggers
+a re-evaluation of how many instances of each node a cluster has, and how many
+it needs. If a container has failed and that freed up all role instances
+on that node, it will have been inserted at the front of the `availableNodes` list.
+As a result, it is highly likely that a new container will be requested on
+the same node. (The only way a node the list would be newer is
+be if other containers were completed in the same callback)
+
+
+
+### Implementation Notes ###
+
+Notes made while implementing the design.
+
+`OutstandingRequestTracker` should also track requests made with
+no target node; this makes seeing what is going on easier. `ARMClientImpl`
+is doing something similar, on a priority-by-priority basis -if many
+requests are made, each with their own priority, that base class's hash tables
+may get overloaded. (it assumes a limited set of priorities)
+
+Access to the role history datastructures was restricted to avoid
+synchronization problems. Protected access is permitted so that a
+test subclass can examine (and change?) the internals.
+
+`NodeEntries need to add a launching value separate from active so that
+when looking for nodes to release, no attempt is made to release
+a node that has been allocated but is not yet live.
+
+We can't reliably map from a request to a response. Does that matter?
+If we issue a request for a host and it comes in on a different port, do we
+care? Yes -but only because we are trying to track nodes which have requests
+outstanding so as not to issue new ones. But if we just pop the entry
+off the available list, that becomes moot.
+
+Proposal: don't track the requesting numbers in the node entries, just
+in the role status fields.
+
+but: this means that we never re-insert nodes onto the available list if a
+node on them was requested but not satisfied.
+
+Other issues: should we place nodes on the available list as soon as all the entries
+have been released? I.e. Before YARN has replied
+
+RoleStats were removed -left in app state. Although the rolestats would
+belong here, leaving them where they were reduced the amount of change
+in the `AppState` class, so risk of something breaking.
+
+## MiniYARNCluster node IDs
+
+Mini YARN cluster NodeIDs all share the same hostname , at least when running
+against file://; so mini tests with >1 NM don't have a 1:1 mapping of
+`NodeId:NodeInstance`. What will happen is that
+`NodeInstance getOrCreateNodeInstance(Container container) '
+will always return the same (now shared) `NodeInstance`.
+
+## Releasing Containers when shrinking a cluster
+
+When identifying instances to release in a bulk downscale operation, the full
+list of targets must be identified together. This is not just to eliminate
+multiple scans of the data structures, but because the containers are not
+released until the queued list of actions are executed -the nodes' release-in-progress
+counters will not be incremented until after all the targets have been identified.
+
+It also needs to handle the scenario where there are many role instances on a
+single server -it should prioritize those.
+
+
+The NodeMap/NodeInstance/NodeEntry structure is adequate for identifying nodes,
+at least provided there is a 1:1 mapping of hostname to NodeInstance. But it
+is not enough to track containers in need of release: the AppState needs
+to be able to work backwards from a NodeEntry to container(s) stored there.
+
+The `AppState` class currently stores this data in a `ConcurrentMap<ContainerId, RoleInstance>`
+
+To map from NodeEntry/NodeInstance to containers to delete, means that either
+a new datastructure is created to identify containers in a role on a specific host
+(e.g a list of ContainerIds under each NodeEntry), or we add an index reference
+in a RoleInstance that identifies the node. We already effectively have that
+in the container
+
+### dropping any available nodes that are busy
+
+When scanning the available list, any nodes that are no longer idle for that
+role should be dropped from the list.
+
+This can happen when an instance was allocated on a different node from
+that requested.
+
+### Finding a node when a role has instances in the cluster but nothing
+known to be available
+
+One condition found during testing is the following:
+
+1. A role has one or more instances running in the cluster
+1. A role has no entries in its available list: there is no history of the
+role ever being on nodes other than which is currently in use.
+1. A new instance is requested.
+
+In this situation, the `findNodeForNewInstance` method returns null: there
+is no recommended location for placement. However, this is untrue: all
+nodes in the cluster `other` than those in use are the recommended nodes.
+
+It would be possible to build up a list of all known nodes in the cluster that
+are not running this role and use that in the request, effectively telling the
+AM to pick one of the idle nodes. By not doing so, we increase the probability
+that another instance of the same role will be allocated on a node in use,
+a probability which (were there capacity on these nodes and placement random), be
+`1/(clustersize-roleinstances)`. The smaller the cluster and the bigger the
+application, the higher the risk.
+
+This could be revisited, if YARN does not support anti-affinity between new
+requests at a given priority and existing ones: the solution would be to
+issue a relaxed placement request listing all nodes that are in the NodeMap and
+which are not running an instance of the specific role. [To be even more rigorous,
+the request would have to omit those nodes for which an allocation has already been
+made off the available list and yet for which no container has yet been
+granted].
+
+
+## Reworked Outstanding Request Tracker
+
+The reworked request tracker behaves as follows
+
+1. outstanding requests with specific placements are tracked by `(role, hostname)`
+1. container assigments are attempted to be resolved against the same parameters.
+1. If found: that request is considered satisfied *irrespective of whether or not
+the request that satisfied the allocation was the one that requested that location.
+1. When all instances of a specific role have been allocated, the hostnames of
+all outstanding requests are returned to the available node list on the basis
+that they have been satisifed elswhere in the YARN cluster. This list is
+then sorted.
+
+This strategy returns unused hosts to the list of possible hosts, while retaining
+the ordering of that list in most-recent-first.
+
+### Weaknesses
+
+if one or more container requests cannot be satisifed, then all the hosts in
+the set of outstanding requests will be retained, so all these hosts in the
+will be considered unavailable for new location-specific requests.
+This may imply that new requests that could be explicity placed will now only
+be randomly placed -however, it is moot on the basis that if there are outstanding
+container requests it means the RM cannot grant resources: new requests at the
+same priority (i.e. same Slider Role ID) will not be granted either.
+
+The only scenario where this would be different is if the resource requirements
+of instances of the target role were decreated during a cluster flex such that
+the placement could now be satisfied on the target host. This is not considered
+a significant problem.
+
+# Persistence
+
+The initial implementation uses the JSON-formatted Avro format; while significantly
+less efficient than a binary format, it is human-readable
+
+Here are sequence of entries from a test run on a single node cluster; running 1 HBase Master
+and two region servers.
+
+Initial save; the instance of Role 1 (HBase master) is live, Role 2 (RS) is not.
+
+ {"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183475949,"savedx":"14247c3aeed","roles":3}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":0}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
+
+At least one RS is live:
+
+ {"entry":{"org.apache.hoya.avro.RoleHistoryFooter":{"count":2}}}{"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183476010,"savedx":"14247c3af2a","roles":3}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":0}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":true,"last_used":0}}}
+
+Another entry is saved -presumably the second RS is now live, which triggered another write
+
+ {"entry":{"org.apache.hoya.avro.RoleHistoryFooter":{"count":2}}}{"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183476028,"savedx":"14247c3af3c","roles":3}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":0}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":true,"last_used":0}}}
+
+At this point the cluster was frozen and thawed. Slider does not save the cluster state
+at freeze time, but does as it is rebuilt.
+
+When the cluster is restarted, every node that was active for a role at the time the file was saved `1384183476028`
+is given a last_used timestamp of that time.
+
+When the history is next saved, the master has come back onto the (single) node,
+it is active while its `last_used` timestamp is the previous file's timestamp.
+No region servers are yet live.
+
+ {"entry":{"org.apache.hoya.avro.RoleHistoryFooter":{"count":2}}}{"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183512173,"savedx":"14247c43c6d","roles":3}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":1384183476028}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":1384183476028}}}
+
+Here a region server is live
+
+ {"entry":{"org.apache.hoya.avro.RoleHistoryFooter":{"count":2}}}{"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183512199,"savedx":"14247c43c87","roles":3}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":1384183476028}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":true,"last_used":1384183476028}}}
+
+And here, another region server has started. This does not actually change the contents of the file
+
+ {"entry":{"org.apache.hoya.avro.RoleHistoryFooter":{"count":2}}}{"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183512217,"savedx":"14247c43c99","roles":3}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":1384183476028}}}
+ {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":true,"last_used":1384183476028}}}
+
+The `last_used` timestamps will not be changed until the cluster is shrunk or thawed, as the `active` flag being set
+implies that the server is running both roles at the save time of `1384183512217`.
+
+## Resolved issues
+
+> How best to distinguish at thaw time from nodes used just before thawing
+from nodes used some period before? Should the RoleHistory simply forget
+about nodes which are older than some threshold when reading in the history?
+
+we just track last used times
+
+
+> Is there a way to avoid tracking the outstanding requests?
+
+No
+
+> What will the strategy of picking the most-recently-used node do if
+that node creates the container and then fails to start it up. Do we need
+to add blacklisting too? Or actually monitor the container start time, and
+if a container hasn't been there for very long, don't pick it.
+
+Startup failures drop the node from the ready-to-use list; the node is no longer
+trusted. We don't blacklist it (yet)
+
+
+> Should we prioritise a node that was used for a long session ahead of
+a node that was used more recently for a shorter session? Maybe, but
+it complicates selection as generating a strict order of nodes gets
+significantly harder.
+
+No: you need to start tracking aggregate execution time, for the last session.
+In a stable state, all servers recorded in the history will have spread the
+data amongst them, so its irrelevant.
diff --git a/src/site/markdown/security.md b/src/site/markdown/security.md
new file mode 100644
index 0000000..ea9ccb7
--- /dev/null
+++ b/src/site/markdown/security.md
@@ -0,0 +1,197 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Security
+
+This document discusses the design, implementation and use of Slider
+to deploy secure applications on a secure Hadoop cluster.
+
+### Important:
+
+This document does not cover Kerberos, how to secure a Hadoop cluster, Kerberos
+command line tools or how Hadoop uses delegation tokens to delegate permissions
+round a cluster. These are assumed, though some links to useful pages are
+listed at the bottom.
+
+
+## Concepts
+
+Slider runs in secure clusters, but with restrictions
+
+1. The keytabs to allow a worker to authenticate with the master must
+ be distributed in advance: Slider does not attempt to pass these around.
+1. Until the location of Slider node instances can be strictly limited to
+ a set of nodes (a future YARN feature), the keytabs must be passed to
+ all the nodes in the cluster in advance, *and made available to the
+ user creating the cluster*
+1. due to the way that HBase and accumulo authenticate worker nodes to
+ the masters, any HBase node running on a server must authenticate as
+ the same principal, and so have equal access rights to the HBase cluster.
+1. As the data directories for a slider cluster are created under the home
+ directories of that user, the principals representing all role instances
+ in the clusters *MUST* have read/write access to these files. This can be
+ done with a shortname that matches that of the user, or by requesting
+ that Slider create a directory with group write permissions -and using LDAP
+ to indentify the application principals as members of the same group
+ as the user.
+
+
+## Requirements
+
+
+### Needs
+* Slider and HBase to work against secure HDFS
+* Slider to work with secure YARN.
+* Slider to start a secure HBase cluster
+* Kerberos and ActiveDirectory to perform the authentication.
+* Slider to only allow cluster operations by authenticated users -command line and direct RPC.
+* Any Slider Web UI and REST API for Ambari to only allow access to authenticated users.
+* The Slider database in ~/.hoya/clusters/$name/data to be writable by HBase
+
+
+### Short-lived Clusters
+* Cluster to remain secure for the duration of the Kerberos tokens issued to Slider.
+
+
+### Long-lived Clusters
+
+* Slider application instance and HBase instance to remain functional and secure over an indefinite period of time.
+
+### Initial Non-requirements
+* secure audit trail of cluster operations.
+* multiple authorized users being granted rights to a Slider Cluster (YARN admins can always kill the Slider application instance.
+* More than one HBase cluster in the YARN cluster belonging to a single user (irrespective of how they are started).
+* Any way to revoke certificates/rights of running containers.
+
+### Assumptions
+* Kerberos is running and that HDFS and YARN are running Kerberized.
+* LDAP cannot be assumed.
+* Credentials needed for HBase can be pushed out into the local filesystems of
+ the of the worker nodes via some external mechanism (e.g. scp), and protected by
+ the access permissions of the native filesystem. Any user with access to these
+ credentials is considered to have been granted such rights.
+* These credentials can outlive the duration of the HBase containers
+* The user running HBase has the same identity as that of the HBase cluster.
+
+## Design
+
+
+1. The user is expected to have their own Kerberos principal, and have used `kinit`
+ or equivalent to authenticate with Kerberos and gain a (time-bounded) TGT
+1. The user is expected to have their own principals for every host in the cluster of the form
+ username/hostname@REALM
+1. A keytab must be generated which contains all these principals -and distributed
+ to all the nodes in the cluster with read access permissions to the user.
+1. When the user creates a secure cluster, they provide the standard HBase kerberos options
+ to identify the principals to use and the keytab location.
+
+The Slider Client will talk to HDFS and YARN authenticating itself with the TGT,
+talking to the YARN and HDFS principals which it has been configured to expect.
+
+This can be done as described in [Client Configuration] (client-configuration.html) on the command line as
+
+ -D yarn.resourcemanager.principal=yarn/master@LOCAL
+ -D dfs.namenode.kerberos.principal=hdfs/master@LOCAL
+
+The Slider Client will create the cluster data directory in HDFS with `rwx` permissions for
+user `r-x` for the group and `---` for others. (these can be configurable as part of the cluster options),
+
+It will then deploy the AM, which will (somehow? for how long?) retain the access
+rights of the user that created the cluster.
+
+The Application Master will read in the JSON cluster specification file, and instantiate the
+relevant number of componentss.
+
+
+## Securing communications between the Slider Client and the Slider AM.
+
+When the AM is deployed in a secure cluster,
+it automatically uses Kerberos-authorized RPC channels. The client must acquire a
+token to talk the AM.
+
+This is provided by the YARN Resource Manager when the client application
+wishes to talk with the HoyaAM -a token which is only provided after
+the caller authenticates itself as the user that has access rights
+to the cluster
+
+To allow the client to freeze a Slider application instance while they are unable to acquire
+a token to authenticate with the AM, use the `--force` option.
+
+### How to enable a secure Slider client
+
+Slider can be placed into secure mode by setting the Hadoop security options:
+
+This can be done in `slider-client.xml`:
+
+
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
+
+
+Or it can be done on the command line
+
+ -D hadoop.security.authorization=true -D hadoop.security.authentication=kerberos
+
+### Adding Kerberos binding properties to the Slider Client JVM
+
+The Java Kerberos library needs to know the Kerberos controller and
+realm to use. This should happen automatically if this is set up as the
+default Kerberos binding (on a Unix system this is done in `/etc/krb5.conf`.
+
+If is not set up, a stack trace with kerberos classes at the top and
+the message `java.lang.IllegalArgumentException: Can't get Kerberos realm`
+will be printed -and the client will then fail.
+
+The realm and controller can be defined in the Java system properties
+`java.security.krb5.realm` and `java.security.krb5.kdc`. These can be fixed
+in the JVM options, as described in the [Client Configuration] (hoya-client-configuration.html)
+documentation.
+
+They can also be set on the Slider command line itself, using the `-S` parameter.
+
+ -S java.security.krb5.realm=MINICLUSTER -S java.security.krb5.kdc=hadoop-kdc
+
+### Java Cryptography Exceptions
+
+
+When trying to talk to a secure, cluster you may see the message:
+
+ No valid credentials provided (Mechanism level: Illegal key size)]
+
+This means that the JRE does not have the extended cryptography package
+needed to work with the keys that Kerberos needs. This must be downloaded
+from Oracle (or other supplier of the JVM) and installed according to
+its accompanying instructions.
+
+## Useful Links
+
+1. [Adding Security to Apache Hadoop](http://hortonworks.com/wp-content/uploads/2011/10/security-design_withCover-1.pdf)
+1. [The Role of Delegation Tokens in Apache Hadoop Security](http://hortonworks.com/blog/the-role-of-delegation-tokens-in-apache-hadoop-security/)
+1. [Chapter 8. Secure Apache HBase](http://hbase.apache.org/book/security.html)
+1. Hadoop Operations p135+
+1. [Java Kerberos Requirements](http://docs.oracle.com/javase/7/docs/technotes/guides/security/jgss/tutorials/KerberosReq.htmla)
+1. [Troubleshooting Kerberos on Java](http://docs.oracle.com/javase/7/docs/technotes/guides/security/jgss/tutorials/Troubleshooting.html)
+1. For OS/X users, the GUI ticket viewer is `/System/Library/CoreServices/Ticket\ Viewer.app`
+
+
diff --git a/src/site/markdown/slider_specs/app_developer_guideline.md b/src/site/markdown/slider_specs/app_developer_guideline.md
new file mode 100644
index 0000000..3232b58
--- /dev/null
+++ b/src/site/markdown/slider_specs/app_developer_guideline.md
@@ -0,0 +1,140 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Slider's needs of an application
+
+Slider installs and runs applications in a YARN cluster -applications that
+do not need to be written for YARN.
+
+What they do need to be is deployable by Slider, which means installable by YARN,
+configurable by Slider, and, finally, executable by YARN. YARN will kill the
+executed process when destroying a container, so the deployed application
+must expect this to happen and be able to start up from a kill-initiated
+shutdown without any manual recovery process.
+
+They need to locate each other dynamically, both at startup and during execution,
+because the location of processes will be unknown at startup, and may change
+due to server and process failures.
+
+## Must
+
+* Install and run from a tarball -and be run from a user that is not root.
+
+* Be self contained or have all dependencies pre-installed.
+
+* Support dynamic discovery of nodes -such as via ZK.
+
+* Nodes to rebind themselves dynamically -so if nodes are moved, the application
+can continue
+
+* Handle kill as a normal shutdown mechanism.
+
+* Support multiple instances of the application running in the same cluster,
+ with processes from different application instances sharing
+the same servers.
+
+* Operate correctly when more than one role instance in the application is
+deployed on the same physical host. (If YARN adds anti-affinity options in
+container requests this will no longer be a requirement)
+
+* Dynamically allocate any RPC or web ports -such as supporting 0 as the number
+of the port to listen on in configuration options.
+
+* Be trusted. YARN does not run code in a sandbox.
+
+* If it talks to HDFS or other parts of Hadoop, be built against/ship with
+libaries compatible with the version of Hadoop running on the cluster.
+
+* Store persistent data in HDFS (directly or indirectly) with the exact storage location
+configurable. Specifically: not to the local filesystem, and not in a hard coded location
+such as `hdfs://app/data`. Slider creates per-Slider application directories for
+persistent data.
+
+* Be configurable as to where any configuration directory is (or simply relative
+to the tarball). The application must not require it to be in a hard-coded
+location such as `/etc`.
+
+* Not have a fixed location for log output -such as `/var/log/something`
+
+* Run until explicitly terminated. Slider treats an application termination
+(which triggers a container release) as a failure -and reacts to it by restarting
+the container.
+
+
+
+## MUST NOT
+
+* Require human intervention at startup or termination.
+
+## SHOULD
+
+These are the features that we'd like from a service:
+
+* Publish the actual RPC and HTTP ports in a way that can be picked up, such as via ZK
+or an admin API.
+
+* Be configurable via the standard Hadoop mechanisms: text files and XML configuration files.
+If not, custom parsers/configuration generators will be required.
+
+* Support an explicit parameter to define the configuration directory.
+
+* Take late bindings params via -D args or similar
+
+* Be possible to exec without running a complex script, so that process inheritance works everywhere, including (for testing) OS/X
+
+* Provide a way for Slider to get list of nodes in cluster and status. This will let Slider detect failed worker nodes and react to it.
+
+* FUTURE: If a graceful decommissioning is preferred, have an RPC method that a Slider provider can call to invoke this.
+
+* Be location aware from startup. Example: worker nodes to be allocated tables to serve based on which tables are
+stored locally/in-rack, rather than just randomly. This will accelerate startup time.
+
+* Support simple liveness probes (such as an HTTP GET operations).
+
+* Return a well documented set of exit codes, so that failures can be propagated
+ and understood.
+
+* Support cluster size flexing: the dynamic addition and removal of nodes.
+
+
+* Support a management platform such as Apache Ambari -so that the operational
+state of a Slider application can be monitored.
+
+## MAY
+
+* Include a single process that will run at a fixed location and whose termination
+can trigger application termination. Such a process will be executed
+in the same container as the Slider AM, and so known before all other containers
+are requested. If a live cluster is unable to handle restart/migration of
+such a process, then the Slider application will be unable to handle
+Slider AM restarts.
+
+* Ideally: report on load/cost of decommissioning.
+ E.g amount of data; app load.
+
+
+## MAY NOT
+
+* Be written for YARN.
+
+* Be (pure) Java. If the tarball contains native binaries for the cluster's hardware & OS,
+ they should be executable.
+
+* Be dynamically reconfigurable, except for the special requirement of handling
+movement of manager/peer containers in an application-specific manner.
+
+
diff --git a/src/site/markdown/slider_specs/application_configuration.md b/src/site/markdown/slider_specs/application_configuration.md
new file mode 100644
index 0000000..55b78ae
--- /dev/null
+++ b/src/site/markdown/slider_specs/application_configuration.md
@@ -0,0 +1,82 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+#Application Configuration
+
+App Configuration captures the default configuration associated with the application. *Details of configuration management is discussed in a separate spec*. The default configuration is modified based on user provided InstanceConfiguration, cluster specific details (e.g. HDFS root, local dir root), container allocated resources (port and hostname), and dependencies (e.g. ZK quorom hosts) and handed to the component instances.
+
+App Configuration is a folder containing all configuration needed by the application. Config files include any site.xml, log4j properties file, etc.
+
+In addition, application may have configuration parameters that do not necessarily go into a config files. Such configurations may be used during template expansion (parameters in env.sh files), as environment variables (e.g. JAVA_HOME), customize user names (for runas). These configurations can be provided as user inputs or are automatically inferred from the environment. Such configurations are stored in a file named "app_config.xml".
+
+
+
+A config file is of the form:
+
+```
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ ...
+ </property>
+</configuration>
+```
+
+
+Each configuration property is specified as follows:
+
+```
+<property>
+ <name>storm.zookeeper.session.timeout</name>
+ <value>20000</value>
+ <description>The session timeout for clients to ZooKeeper.</description>
+ <required>false</required>
+ <valueRestriction>0-30000</valueRestriction>
+ </property>
+ <property>
+ <name>storm.zookeeper.root</name>
+ <value>/storm</value>
+ <description>The root location at which Storm stores data in ZK.</description>
+ <required>true</required>
+ </property>
+ <property>
+ <name>jvm.heapsize</name>
+ <value>256</value>
+ <description>The default JVM heap size for any component instance.</description>
+ <required>true</required>
+ </property>
+ <property>
+ <name>nimbus.host</name>
+ <value>localhost</value>
+ <description>The host that the master server is running on.</description>
+ <required>true</required>
+ <clientVisible>true</clientVisible>
+ </property>
+ ```
+
+
+* name: name of the parameter
+
+* value: default value of the parameter
+
+* description: a short description of the parameter
+
+* required: if the parameter is mandatory in which case it must have a value - default is "false"
+
+* clientVisible: if the property must be exported for a client
+
diff --git a/src/site/markdown/slider_specs/application_definition.md b/src/site/markdown/slider_specs/application_definition.md
new file mode 100644
index 0000000..e6bd510
--- /dev/null
+++ b/src/site/markdown/slider_specs/application_definition.md
@@ -0,0 +1,171 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Application Definition
+
+App definition is a declarative definition of a YARN application describing its content. The AppDefinition is used in conjunction with the [AppPackage](application_package.md).
+
+## Structure
+
+*Non-mandatory fields are described in **italics**.*
+
+The fields to describe an application is as follows:
+
+* **name**: the name of the application
+
+* **version**: the version of the application. name and version together uniquely identify an application.
+
+* **type**: the type of the application. "YARN-APP" identifies an application definition suitable for YARN.
+
+* **minHadoopVersion**: the minimum version of hadoop on which the app can run
+
+* **components**: the list of component that the application is comprised of
+
+* **osSpecifics**: OS specific package information for the application
+
+* *commandScript*: application wide commands may also be defined. The command is executed on a component instance that is a client
+
+* *dependencies*: application can define a list of dependencies. Dependencies can be on the base services such as HDFS, ZOOKEEPER, YARN which are infrastructure services or GANGLIA, NAGIOS, etc. which are monitoring/alert services. The dependencies are parsed by the management infrastructure to provide the necessary configurations to allow the app to access the services. For example, a HDFS folder could be requested by the app to store its data, a ZOOKEEPER node to co-ordinate among components.
+
+An application contains several component. The fields associated with a component are:
+
+* **name**: name of the component
+
+* **category**: type of the component - MASTER, SLAVE, and CLIENT
+
+* **minInstanceCount**: the minimum number of instances required for this component
+
+* *maxInstanceCount*: maximum number of instances allowed for a component
+
+* **commandScript**: the script that implements the commands.
+
+ * **script**: the script location - relative to the AppPackage root
+
+ * **scriptType**: type of the script
+
+ * **timeout**: default timeout of the script
+
+* *customCommands*: any additional commands available for the component and their implementation
+
+An application definition also includes the package used to install the application. Its typically a tarball or some other form of package that does not require root access to install. The details of what happens during install is captured in the command script.
+
+* **osSpecific**: details on a per OS basis
+
+* **osType**: "any" refers to any OS ~ typical for tarballs
+
+* **packages**: list of packages that needs to be deployed
+
+* **type**: type of package
+
+* **name**: name of the package
+
+* **location**: location of the package (can be a relative folder within the parent AppPackage)
+
+Application can define a set of dependencies. The dependencies are parsed by Slider to provide additional configuration parameters to the command scripts. For example, an application may need to ZooKeeper quorum hosts to communicate with ZooKeeper. In this case, ZooKeeper is a "base" service available in the cluster.
+
+* **dependency**: an application can specify more than one dependency
+
+* **name**: a well-known name of the base service (or another application) on which the dependency is defined
+
+* **scope**: is the dependent service/application expected on the same cluster or it needs to be on the same hosts where components are instantiated
+
+* **requirement**: a set of requirements that lets Slider know what properties are required by the app command scripts
+
+```
+ <metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <application>
+ <name>HBASE</name>
+ <version>0.96.0.2.1.1</version>
+ <type>YARN-APP</type>
+ <minHadoopVersion>2.1.0</minHadoopVersion>
+ <components>
+ <component>
+ <name>HBASE_MASTER</name>
+ <category>MASTER</category>
+ <minInstanceCount>1</minInstanceCount>
+ <maxInstanceCount>2</maxInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>GRACEFUL_STOP</name>
+ <commandScript>
+ <script>scripts/hbase_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1800</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ </component>
+
+ <component>
+ <name>HBASE_REGIONSERVER</name>
+ <category>SLAVE</category>
+ <minInstanceCount>1</minInstanceCount>
+ ...
+ </component>
+
+ <component>
+ <name>HBASE_CLIENT</name>
+ <category>CLIENT</category>
+ ...
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>hbase-0.96.1-tar.gz</name>
+ <location>package/files</location>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/app_health_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <dependencies>
+ <dependency>
+ <name>ZOOKEEPER</name>
+ <scope>cluster</scope>
+ <requirement>client,zk_quorom_hosts</requirement>
+ </dependency>
+ </dependencies>
+
+ </application>
+ </metainfo>
+```
+
+
+## Open Questions
+
+1. Applications may need some information from other applications or base services such as ZK, YARN, HDFS. Additionally, they may need a dedicated ZK node, a HDFS working folder, etc. How do we capture this requirement? There needs to be a well-known way to ask for these information e.g. fs.default.name, zk_hosts.
+
+2. Similar to the above there are common parameters such as JAVA_HOME and other environment variables. Application should be able to refer to these parameters and Slider should be able to provide them.
+
+3. Composite application definition: Composite application definition would require a spec that refers to this spec and binds multiple applications together.
+
diff --git a/src/site/markdown/slider_specs/application_instance_configuration.md b/src/site/markdown/slider_specs/application_instance_configuration.md
new file mode 100644
index 0000000..6ccdece
--- /dev/null
+++ b/src/site/markdown/slider_specs/application_instance_configuration.md
@@ -0,0 +1,60 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+#App Instance Configuration
+
+App Instance Configuration is the configuration override provided by the application owner when creating an application instance using Slider. This configuration values override the default configuration available in the App Package.
+
+Instance configuration is a JSON formatted doc in the following form:
+
+```
+{
+ "configurations": {
+ "app-global-config": {
+ },
+ "config-type-1": {
+ },
+ "config-type-2": {
+ },
+ }
+}
+```
+
+
+The configuration overrides are organized in a two level structure where name-value pairs are grouped on the basis of config types they belong to. App instantiator can provide arbitrary custom name-value pairs within a config type defined in the AppPackage or can create a completely new config type that does not exist in the AppAPackage. The interpretation of the configuration is entirely up to the command implementations present in the AppPackage. Slider will simply merge the configs with the InstanceConfiguration being higher priority than that default configuration and hand it off to the app commands.
+
+A sample config for hbase may be as follows:
+
+
+```
+{
+ "configurations": {
+ "hbase-log4j": {
+ "log4j.logger.org.apache.zookeeper": "INFO",
+ "log4j.logger.org.apache.hadoop.hbase": "DEBUG"
+ },
+ "hbase-site": {
+ "hbase.hstore.flush.retries.number": "120",
+ "hbase.regionserver.info.port": "",
+ "hbase.master.info.port": "60010"
+ }
+}
+```
+
+
+The above config overwrites few parameters in hbase-site and hbase-log4j files. Several config properties such as "hbase.zookeeper.quorum" for hbase may not be known to the user at the time of app instantiation. These configurations will be provided by the Slider infrastructure in a well-known form so that the app implementation can read and set them while instantiating component instances..
+
diff --git a/src/site/markdown/slider_specs/application_package.md b/src/site/markdown/slider_specs/application_package.md
new file mode 100644
index 0000000..691c4d1
--- /dev/null
+++ b/src/site/markdown/slider_specs/application_package.md
@@ -0,0 +1,149 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+#App Package
+
+All application artifacts, app definition, app configuration, scripts are packaged into a structured single package that can be handed off to any YARN application deployment tool including Slider
+
+## Overall Structure
+
+App package is a tarball containing all application artifacts. App package contains the following items:
+
+* **app definition file**
+application structure, content, definition, supported platforms, version, etc.
+
+* **default configurations folder**
+various configurations and configuration files associated with the application
+
+* **cmd_impl folder**
+management operations for the application/component
+
+ * **scripts folder**
+various scripts that implement management operations
+
+ * **templates folder**
+various templates used by the application
+
+ * **files folder**
+other scripts, txt files, tarballs, etc.
+
+
+
+
+The example above shows a semi-expanded view of an application "HBASE-YARN-APP" and the package structure for OOZIE command scripts.
+
+## app definition
+
+App definition is a file named "metainfo.xml". The file contains application definition as described in [Application Definition](application_definition.md).
+
+## default configurations
+
+This folder consists of various config files containing default configuration as described in [App Configuration](application_configuration.md).
+
+## package folder
+
+package includes the "implementation" of all management operations. The folders within are divided into scripts, templates, and files.
+
+### scripts folder
+
+Scripts are the implementation of management operations. There are five default operations and a composite operation. "restart" can be redefined to have a custom implementation.
+
+1. install
+
+2. configure
+
+3. start
+
+4. stop
+
+5. status
+
+6. restart (by default calls stop + start)
+
+The script specified in the metainfo is expected to understand the command. It can choose to call other scripts based on how the application author organizes the code base. For example:
+
+```
+class OozieServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ oozie(is_server=True)
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ oozie_service(action='start')
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+ oozie_service(action='stop')
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.pid_file)
+```
+
+
+The scripts are invoked in the following manner:
+
+`python SCRIPT COMMAND JSON_FILE PACKAGE_ROOT STRUCTURED_OUT_FILE`
+
+* SCRIPT is the top level script that implements the commands for the component.
+
+* COMMAND is one of the six commands listed above or can be a custom command as defined in Application Definition
+
+* JSON_FILE includes all configuration parameters and the values
+
+* PACKAGE_ROOT is the root folder of the package. From this folder, its possible to access files, scripts, templates, packages (e.g. tarballs), etc. The App author has complete control over the structure of the package
+
+* STRUCTURED_OUT_FILE is the file where the script can output structured data. The management infrastructure is expected to automatically reports back STD_OUT and STD_ERR.
+
+A separate document (link TBD) discusses how the scripts are developed and the structure of the JSON_FILE containing the parameters.
+
+### templates folder
+
+templates are configurable text files that are NOT regular config files. *A library has been developed that can materialize a complete site configuration file from a property bag and therefore are not generated from templates.* Other files such as env sh files, log4j properties file, etc. may be derived from a template. Again, the implementor can choose to create these files from scratch and not use templates. The following operations are allowed during template expansion:
+
+* variable expansion
+
+* if condition
+
+* for loop
+
+* ...
+
+Sample template file for dfs.exclude file to list excluded/decommissioned hosts. hdfs_exclude_files in the property defined in params.py which is populated from config parameters defined in JSON_FILE.
+
+```
+{% if hdfs_exclude_file %}
+{% for host in hdfs_exclude_file %}
+{{host}}
+{% endfor %}
+{% endif %}
+```
+
+
+### files folder
+
+files is a directory to store any other files that are needed for management operations. Sample files stored here are tarballs used to install the application, shell scripts used by various operations.
+
diff --git a/src/site/markdown/slider_specs/apps_on_yarn_cli.md b/src/site/markdown/slider_specs/apps_on_yarn_cli.md
new file mode 100644
index 0000000..d834115
--- /dev/null
+++ b/src/site/markdown/slider_specs/apps_on_yarn_cli.md
@@ -0,0 +1,92 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+#Slider CLI
+
+This document describes the CLI to deploy and manage YARN applications using Slider.
+
+## Operations
+
+### `create <application> <--app.package packagelocation> <--resource resourcespec> <--app.instance.configuration appconfiguration> <--options sliderconfiguration> [--provider providername]`
+
+Build an application specification by laying out the application artifacts in HDFS and prepares it for *start*. This involves specifying the application name, application package, YARN resource requirements, application specific configuration overrides, options for Slider, and optionally declaring the provider, etc. The provider is invoked during the build process, and can set default values for components.
+
+The default application configuration and components would be built from the application metadata contained in the package. *create* performs a structural validation of the application package and validates the supplied resources specification and instance configuration against the application package.
+
+**parameters**
+
+* application: name of the application instance, must be unique within the Hadoop cluster
+* packagelocation: the application package on local disk or HDFS
+* resourcespec: YARN resource requirements for application components
+* appconfiguration: configuration override for the application
+* sliderconfiguration: configuration for Slider itself such as location of YARN resource manager, HDFS file system, ZooKeeper quorom nodes, etc.
+* providername: name of the any non-default provider. Agent provider is default.
+
+### `destroy <application>`
+
+destroy a (stopped) application. The stop check is there to prevent accidentally destroying an application in use
+
+### `start <application>`
+
+Start an application instance that is already created through *create*
+
+### `stop <application> [--force]`
+
+Stop the application instance.
+
+The --force operation tells YARN to kill the application instance without involving the AppMaster.
+
+### `flex <application> [--component componentname count]* <--resource resourcespec>`
+
+Update component instance count
+
+### `configure <application> <--app.instance.configuration appconfiguration>`
+
+Modify application instance configuration. Updated configuration is only applied after the application is restarted.
+
+### `status <application>`
+
+Report the status of an application instance. If there is a record of an application instance in a failed/finished state AND there is no live application instance, the finished application is reported. Otherwise, the running application's status is reported.
+
+If there a no instances of an application in the YARN history, the application is looked up in the applications directory, and the status is listed if present.
+
+
+### `listapplications [--accepted] [--started] [--live] [--finished] [--failed] [--stopped]`
+
+List all applications, optionally the ones in the named specific states.
+
+
+### `getconfig <application> [--config filename [--dir destdir|--outfile destfile]]`
+
+list/retrieve any configs published by the application.
+
+if no --config option is provided all available configs are listed
+
+If a --file is specified, it is downloaded to the current directory with the specified filename, unless a destination directory/filename is provided
+
+
+### `history <application>`
+
+Lists all life-cycle events of the application instance since the last *start*
+
+### `kill --containers [containers] --components [components] --nodes [nodes]`
+
+Kill listed containers, everything in specific components, or on specific nodes. This can be used to trigger restart of services and decommission of nodes
+
+### `wait <application> [started|live|stopped] --timeout <time>`
+
+Block waiting for a application to enter the specififed state. Can fail if the application stops while waiting for it to be started/live
diff --git a/src/site/markdown/slider_specs/canonical_scenarios.md b/src/site/markdown/slider_specs/canonical_scenarios.md
new file mode 100644
index 0000000..3576b91
--- /dev/null
+++ b/src/site/markdown/slider_specs/canonical_scenarios.md
@@ -0,0 +1,165 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Guidelines for Clients and Client Applications
+
+This document will define the canonical scenarios for the deployment and management of Slider hosted applications. It will define the types of applications supported, the sequence of events for deploying the application types, and the management facilities exposed for the deployed applications.
+
+## Deployable Application Types
+
+The server-side components of an application (alternatively referred to as the application components) will be deployed and managed using a fairly uniform mechanism. However, applications can be distinguished by their associated client implementation. Specifically, each different client application type can yield different development, deployment, and management approaches. There are two primary application client types:
+
+1. **New application client** - the application deployed to Slider will interact with a newly coded application client leveraging the Slider supplied client API and management facilities. The client API will make use of distributed management facilities such as Zookeeper to provide the application client with the configuration information required for invoking remote application components. For example, the application client (or an application hosted component, e.g. a web application) will leverage the API to lookup the appropriate host(s) and port(s) for RPC based communication. Alternatively, if annotation libraries or an "app container" environment is provided, the appropriate values will be injected into the client process.
+
+2. **Existing, configurable client** - the application client predates deployment in the Slider environment, but is implemented in a manner that can be integrated with the Slider application client support facilities (APIs etc). This case is probably very similar in nature to the new application client in terms of the mechanisms it will use for component discovery, but is distinguished by the fact that it’s development pre-dates Slider. There are two possible variants of application client in this category:
+
+ 1. A client that is static - the client is dependent on existing configuration properties to communicate with master components and the existing code can not been altered (at least in the short term). This type of client would require a support infrastructure that may be relatively complex (e.g. templating of configuration files, proxying of server components).
+
+ 2. A client that can be enhanced - a client that can have its code altered can leverage a number of mechanisms (REST APIs, Zookeeper, a provided client discover API) to obtain the information required to invoke the master components of the application.
+
+## Deployment Scenarios
+
+There are two primary deployment mechanisms to examine: application component and client-side (application client) component deployment.
+
+## Application Component Deployment
+
+Applications generally are composed of one or more components. In the deployment steps below, be advised that there may be a need to repeat some of the configuration/definition steps for each component.
+
+The process of deploying applications (specifically, the non-client components of an application) is:
+
+1. Compose an application package that contains:
+
+ 1. An application definition that provides the following items:
+
+ 1. name and version of application
+
+ 2. component type(s) and role(s)
+
+ 3. system requirements (RAM, CPUs, disk space etc)
+
+ 4. ports required for RPC, UI
+
+ 5. software dependencies (HDP deployed services required in the cluster, JDK versions, python versions, etc)
+
+ 6. configurable properties including:
+
+ 1. application-specific properties. If the properties designate port numbers, the general recommendation would be to set them to zero to allow for the system to assign a dynamically selected port number that subsequently can be published to zookeeper. Other properties designating remote components running on other hosts may need some additional support (floating IPs, discovery APIs, etc). The processing of these properties by the infrastructure requires agreed upon mechanisms for identification (e.g. all properties beginning with the application name are passed through as application properties with the application name prefix stripped)
+
+ 2. Slider-specific properties. The following Slider properties are currently available:
+
+ 1. yarn.memory
+
+ 2. Environment variables specified with a "env." prefix (e.g. env.MALLOC_ARENA_MAX)
+
+ 3. role.instances
+
+ 4. role.name
+
+ 5. yarn.vcores
+
+ 2. Other application artifacts (binaries etc)
+
+2. Install the application
+
+ 1. The application package will be made available in a designated HDFS location
+
+ 2. If there is a managed application client component it will be deployed to selected nodes as defined in the cluster specification.
+
+ 3. Slider interacts with yarn (resource manager and node manager(s)) to populate the local resources on the designated nodes.
+
+ 4. Some properties important for facilitating remote interaction with the deployed components are advertised via zookeeper (though this may actually take place during component start up as the assignment of ports is a late-binding operation. Alternatively, developers may be encouraged to store these values in a registry rather than as direct-application configuration properties).
+
+## Client Application Deployment
+
+Although application clients are components that are deployed using mechanisms similar to other application components (especially in the managed case), there are a number of features that distinguish them:
+
+1. **configuration** - client applications generally require some information (host names, ports, etc) to ascertain the correct networking values required to communicate with the application's server components. In order to work in a yarn deployment, this configuration may need to be manipulated to allow proper operation (e.g. the configuration files updated with correct values, the configuration properties ascertained dynamically from services such as Zookeeper)
+
+2. **execution context **- it may be necessary to provide an execution environment for application clients that allows for discovery mechanisms (dependency injection, annotation libraries, etc)
+
+For each of these application client types there are two possible deployment modes:
+
+* **Managed** - the application client is deployed via Slider mechanisms. Clients, in this context, differ from the other application components in that they are not running, daemon processes. However, in a managed environment there is the expectation that the appropriate binaries and application elements will be distributed to the designated client hosts, and the configuration on those hosts will be updated to allow for execution of requests to the application’s master/server components. Therefore, client components should be defined in the application specification as elements that the management infrastructure supports (Figure 1).
+
+
+Figure 1 - Managed Application Client and associated Slider Application
+
+* **Unmanaged** - the application client is run as a process outside of Slider/yarn, although it may leverage Slider provided libraries that allow for server component discovery etc (Figure 2). These libraries would primarily be client bindings providing access to the registry leveraged by Slider (e.g. Java and python bindings to Zookeeper)
+
+
+Figure 2 - Unmanaged Application Client and associated Slider Application
+
+### Managed Application Client
+
+A managed application client is a component defined as part of the Slider/yarn application (i.e. it is part of the application definition presented to Slider). As such, it is deployed and managed via standard Slider/yarn mechanisms. This type of application client is more than likely already configured and written to work in a yarn environment.
+
+There are two primary needs to be met for a properly functioning client:
+
+1. **Discovery** - as a client, it is important that the client application retrieve the information it requires in order to communicate with the remote application components. As application components are spawned they (or the associated container agent) will advertise the relevant information using zookeeper. It will be up to the client (or the associated Slider client library) to contact zookeeper and retrieve the requested information.
+
+2. **Configuration** - there may be use cases in which a large number of configuration items are required by the client for its processing. In such cases it is more appropriate for a client to perform a bulk download of the application component(s) configuration as a JSON or XML document (via zookeeper or Slider-app comm?)
+
+Whether ascertained via discovery or bulk configuration retrieval, the attributes that the client obtains will more than likely need to be populated into the client’s configuration files. Therefore, a templating facility or the like should be provided to allow for such configuration file manipulation.
+
+### Unmanaged Application Client
+
+An unmanaged application client is a standalone application that leverages application components deployed into the Slider/yarn environment. It is not possible to predict the deployment mechanisms or network topology employed for an unmanaged application. However, it is feasible to provide some guidance and/or software (APIs etc) to allow for application component discovery and communication.
+
+## Application Management
+
+Post deployment, the Slider infrastructure will provide the requisite set of administrative facilities required for application management, including the ability to start/stop the application, monitor the application, and reconfigure the application.
+
+### General Management
+
+There is one general management command:
+
+* List Yarn Apps - returns a listing of deployed yarn apps and associated information:
+
+ * name and version
+
+ * dependencies (required HDP services and versions, etc)
+
+ * configuration properties
+
+ * components/roles and associated configuration
+
+### Application/Component Management
+
+The following administrative functions are supported for applications:
+
+* Install the application - the installation command will take care of the population of the application resources into the pre-determined application resource directory in HDFS. The required configuration and binary directories will also be created.
+
+* start/thaw the application - Slider/Yarn runtime negotiates and instantiates the number of component containers designated by the cluster description and the components are started.
+
+* stop/freeze the application - similar to stopping, applicaiton (or a subset of their components) can be stopped.
+
+* get application status - the retrieval of application status may take a number of forms, including:
+
+ * liveness of service components
+
+ * operational metrics (application-level or component-level)
+
+ * viewing of logs
+
+* get application configuration - the configuration of application components is retrieved (JSON or XML form)
+
+* get cluster configuration - the cluster configuration is retrieved (number of various application components, associated hosts etc)
+
+* get cluster history
+
+* re-configure cluster
+
diff --git a/src/site/markdown/slider_specs/creating_app_definitions.md b/src/site/markdown/slider_specs/creating_app_definitions.md
new file mode 100644
index 0000000..40ae707
--- /dev/null
+++ b/src/site/markdown/slider_specs/creating_app_definitions.md
@@ -0,0 +1,148 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Define and use Slider AppPackage
+
+Slider AppPackages are a declarative definition of an application for application management. AppPackage is not a packaging scheme for application binaries and artifacts. Tarball, zip files, rpms etc. are available for that purpose. Instead AppPackage includes the application binaries along with other artifacts necessary for application management.
+
+An application instance consists of several active component such as one or more masters and several slaves. There may be a number of accompanying processes in addition to the basic master/slave processes - lets refer to all processes as app component instances. When run in the context of Yarn, the application specific processes are activated within individual Yarn Container. If you pry into an Yarn container (created through Slider) it will be apparent as to what is the role of Slider-Agent and the actual application components. The following image provides an high-level view. Within a container there are at least two processes - and instance of a slider agent and an instance of an application component. The application can itself spawn more procsses if needed.
+
+
+
+Figure 1 - High-level view of a container
+
+For example:
+
+* yarn 8849 -- python ./infra/agent/slider-agent/agent/main.py --label container_1397675825552_0011_01_000003___HBASE_REGIONSERVER --host AM_HOST --port 47830
+* yarn 9085 -- bash /hadoop/yarn/local/usercache/yarn/appcache/application_1397675825552_0011/ ... internal_start regionserver
+* yarn 9114 -- /usr/jdk64/jdk1.7.0_45/bin/java -Dproc_regionserver -XX:OnOutOfMemoryError=...
+
+Shows three processes, the Slider-Agent process, the bash script to start HBase Region Server and the HBase Region server itself. Three of these together constitute the container.
+
+## Using an AppPackage
+The following command creates an HBase application using the AppPackage for HBase.
+
+ "./slider create cl1 --zkhosts zk1,zk2 --image hdfs://NN:8020/slider/agent/slider-agent-0.21.tar --option agent.conf hdfs://NN:8020/slider/agent/conf/agent.ini --template /work/appConf.json --resources /work/resources.json --option application.def hdfs://NN:8020/slider/hbase_v096.tar"
+
+Lets analyze various parameters from the perspective of app creation:
+
+* **--image**: its the slider agent tarball
+* **--option agent.conf**: the configuration file for the agent instance
+* **--option app.def**: app def (AppPackage)
+* **--template**: app configuration
+* **--resources**: yarn resource requests
+* … other parameters are described in accompanying docs.
+
+### AppPackage
+The structure of an AppPackage is described at [AppPackage](application_package.md).
+
+In the enlistment there are three example AppPackages
+
+* app-packages/hbase-v0_96
+* app-packages/accumulo-v1_5
+* app-packages/storm-v0_91
+
+The application tarball, containing the binaries/artifacts of the application itself is a component within the AppPackage. They are:
+
+* For hbase-v0_96 - app-packages/hbase-v0_96/package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE
+* For accumulo-v1_5 - app-packages/accumulo-v1_5/package/files/accumulo-1.5.1-bin.tar.gz.REPLACE
+* For storm-v0_91 - app-packages/storm-v0_91/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.placeholder
+
+They are placehoder files, mostly because the files themselves are too large as well as users are free to use their own version of the package. To create a Slider AppPackage - replace the file with an actual application tarball and then ensure that the metainfo.xml has the correct file name. After that create a tarball using standard tar commands and ensure that the package has the metainfo.xml file at the root folder.
+
+### appConf.json
+An appConf.json contains the application configuration. The sample below shows configuration for HBase.
+
+```
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ "config_types": "core-site,hdfs-site,hbase-site",
+
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "package_list": "files/hbase-0.96.1-hadoop2-bin.tar",
+
+ "site.global.app_user": "yarn",
+ "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
+ "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+ "site.global.security_enabled": "false",
+
+ "site.hbase-site.hbase.hstore.flush.retries.number": "120",
+ "site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
+ "site.hbase-site.hbase.hstore.compactionThreshold": "3",
+ "site.hbase-site.hbase.rootdir": "${NN_URI}/apps/hbase/data",
+ "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+ "site.hbase-site.hbase.regionserver.port": "0",
+
+ "site.core-site.fs.defaultFS": "${NN_URI}",
+ "site.hdfs-site.dfs.namenode.https-address": "${NN_HOST}:50470",
+ "site.hdfs-site.dfs.namenode.http-address": "${NN_HOST}:50070"
+ }
+}
+```
+appConf.jso allows you to pass in arbitrary set of configuration that Slider will forward to the application component instances.
+
+* Variables of the form "site.xx.yy" translates to variables by the name "yy" within the group "xx" and are typically converted to site config files by the name "xx" containing variable "yy". For example, "site.hbase-site.hbase.regionserver.port":"" will be sent to the Slider-Agent as "hbase-site" : { "hbase.regionserver.port": ""} and app def scripts can access all variables under "hbase-site" as a single property bag.
+* Similarly, "site.core-site.fs.defaultFS" allows you to pass in the default fs. *This specific variable is automatically made available by Slider but its shown here as an example.*
+* Variables of the form "site.global.zz" are sent in the same manner as other site variables except these variables are not expected to get translated to a site xml file. Usually, variables needed for template or other filter conditions (such as security_enabled = true/false) can be sent in as "global variable".
+
+### --resources resources.json
+The resources.json file encodes the Yarn resource count requirement for the application instance.
+
+The components section lists the two application component for an HBase application.
+
+* wait.heartbeat: a crude mechanism to control the order of component activation. A heartbeat is ~10 seconds.
+* role.priority: each component must be assigned unique priority
+* component.instances: number of instances for this component type
+* role.script: the script path for the role *a temporary work-around as this will eventually be gleaned from metadata.xml*
+
+Sample:
+
+```
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components" : {
+ "HBASE_MASTER" : {
+ "wait.heartbeat" : "5",
+ "role.priority" : "1",
+ "component.instances" : "1",
+ "role.script" : "scripts/hbase_master.py"
+ },
+ "slider-appmaster" : {
+ "jvm.heapsize" : "256M"
+ },
+ "HBASE_REGIONSERVER" : {
+ "wait.heartbeat" : "3",
+ "role.priority" : "2",
+ "component.instances" : "1",
+ "role.script" : "scripts/hbase_regionserver.py"
+ }
+ }
+}
+```
+
+## Creating AppPackage
+Refer to [App Command Scripts](writing_app_command_scripts) for details on how to write scripts for a AppPackage. These scripts are in the package/script folder within the AppPackage. *Use the checked in samples for HBase/Storm/Accumulo as reference for script development.*
+
+
+
diff --git a/src/site/markdown/slider_specs/index.md b/src/site/markdown/slider_specs/index.md
new file mode 100644
index 0000000..eb63ec3
--- /dev/null
+++ b/src/site/markdown/slider_specs/index.md
@@ -0,0 +1,54 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+PROJECT SLIDER
+===
+
+Introduction
+---
+
+**SLIDER: A collection of tools and technologies to simplify the packaging, deployment and management of long-running applications on YARN.**
+
+- Availability (always-on) - YARN works with the application to ensure recovery or restart of running application components.
+- Flexibility (dynamic scaling) - YARN provides the application with the facilities to allow for scale-up or scale-down
+- Resource Mgmt (optimization) - YARN handles allocation of cluster resources.
+
+Terminology
+---
+
+- **Apps on YARN**
+ - Application written to run directly on YARN
+ - Packaging, deployment and lifecycle management are custom built for each application
+
+- **Slider Apps**
+ - Applications deployed and managed on YARN using Slider
+ - Use of slider minimizes custom code for deployment + lifecycle management
+ - Requires apps to follow Slider guidelines and packaging ("Sliderize")
+
+Specifications
+---
+
+The entry points to leverage Slider are:
+
+- [Specifications for AppPackage](application_package.md)
+- [Documentation for the SliderCLI](apps_on_yarn_cli.md)
+- [Specifications for Application Definition](application_definition.md)
+- [Specifications for Configuration](application_configuration.md)
+- [Specification of Resources](resource_specification.md)
+- [Specifications InstanceConfiguration](application_instance_configuration.md)
+- [Guidelines for Clients and Client Applications](canonical_scenarios.md)
+- [Documentation for "General Developer Guidelines"](app_developer_guideline.md)
diff --git a/src/site/markdown/slider_specs/resource_specification.md b/src/site/markdown/slider_specs/resource_specification.md
new file mode 100644
index 0000000..a3f62ec
--- /dev/null
+++ b/src/site/markdown/slider_specs/resource_specification.md
@@ -0,0 +1,55 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+#Resource Specification
+Resource specification is an input to Slider to specify the Yarn resource needs for each component type that belong to the application.
+
+Some parameters that can be specified for a component instance include:
+
+* yarn.memory: amount of memory requried for the component instance
+* env.MALLOC_ARENA_MAX: maximum number of memory pools used, arbitrary environment settings can be provided through format env.NAME_OF_THE_VARIABLE
+* component.instances: number of instances requested
+* component.name: name of the component
+* yarn.vcores: number of vcores requested
+
+An example resource requirement for an application that has two components "master" and "worker" is as follows. Slider will automatically add the requirements for the AppMaster for the application. This compoent is named "slider".
+
+```
+"components" : {
+ "worker" : {
+ "yarn.memory" : "768",
+ "env.MALLOC_ARENA_MAX" : "4",
+ "component.instances" : "1",
+ "component.name" : "worker",
+ "yarn.vcores" : "1"
+ },
+ "slider" : {
+ "yarn.memory" : "256",
+ "env.MALLOC_ARENA_MAX" : "4",
+ "component.instances" : "1",
+ "component.name" : "slider",
+ "yarn.vcores" : "1"
+ },
+ "master" : {
+ "yarn.memory" : "1024",
+ "env.MALLOC_ARENA_MAX" : "4",
+ "component.instances" : "1",
+ "component.name" : "master",
+ "yarn.vcores" : "1"
+ }
+ }
+```
diff --git a/src/site/markdown/slider_specs/slider-specifications-draft-20140310.zip b/src/site/markdown/slider_specs/slider-specifications-draft-20140310.zip
new file mode 100644
index 0000000..3dd4feb
--- /dev/null
+++ b/src/site/markdown/slider_specs/slider-specifications-draft-20140310.zip
Binary files differ
diff --git a/src/site/markdown/slider_specs/writing_app_command_scripts.md b/src/site/markdown/slider_specs/writing_app_command_scripts.md
new file mode 100644
index 0000000..4b97c0f
--- /dev/null
+++ b/src/site/markdown/slider_specs/writing_app_command_scripts.md
@@ -0,0 +1,218 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+#Developing App Command Scripts
+
+App command implementations follow a standard structure so that they can be invoked in an uniform manner. For any command, the python scripts are invoked as:
+
+`python SCRIPT COMMAND JSON_FILE PACKAGE_ROOT STRUCTURED_OUT_FILE`
+
+* SCRIPT is the top level script that implements the commands for the component.
+
+* COMMAND is one of the following default commands - START, STOP, INSTALL, CONFIG, RESTART, STATUS or any custom commands.
+
+* JSON_FILE includes all configuration parameters and the values.
+
+* PACKAGE_ROOT is the root folder of the package. From this folder, its possible to access files, scripts, templates, packages (e.g. tarballs), etc. The Yarn-App author has complete control over the structure of the package as long as the PACKAGE_ROOT and SCRIPT path is known to the management tool.
+
+* STRUCTURED_OUT_FILE is the file where the script can output structured data.
+
+The management infrastructure is expected to automatically reports back STD_OUT and STD_ERR.
+
+Sample:
+
+```
+python /apps/HBASE_ON_YARN/package/scripts/hbase_regionserver.py START /apps/commands/cmd_332/command.json /apps/HBASE_ON_YARN/package /apps/commands/cmd_332/strout.txt
+```
+
+**Note**: The above is how Slider-Agent invokes the scripts. Its provided as a reference for developing the scripts themselves as well as a way to test/debug the scripts.
+
+## Structure of JSON formatted parameter
+
+The parameters are organized as multi-layer name-value pairs.
+
+```
+{
+ "commandId": "Command Id as assigned by Slider",
+ "command": "Command being executed",
+ "commandType": "Type of command",
+ "clusterName": "Name of the cluster",
+ "appName": "Name of the app",
+ "component": "Name of the component",
+ "hostname": "Name of the host",
+ "public_hostname": "FQDN of the host",
+ "hostParams": {
+ "host specific parameters common to all commands"
+ },
+ "componentParams": {
+ "component specific parameters, if any"
+ },
+ "commandParams": {
+ "command specific parameters, usually used in case of custom commands"
+ },
+ "configurations": {
+ "app-global-config": {
+ },
+ "config-type-2": {
+ },
+ "config-type-2": {
+ }
+ }
+}
+```
+
+
+## Sample configuration parameters
+
+```
+{
+ "commandId": "2-2",
+ "command": "START",
+ "commandType": "EXECUTION_COMMAND",
+ "clusterName": "c1",
+ "appName": "HBASE",
+ "componentName": "HBASE_MASTER",
+ "hostParams": {
+ "java_home": "/usr/jdk64/jdk1.7.0_45"
+ },
+ "componentParams": {},
+ "commandParams": {},
+ "hostname": "c6403.ambari.apache.org",
+ "public_hostname": "c6403.ambari.apache.org",
+ "configurations": {
+ "hbase-log4j": {
+ "log4j.threshold": "ALL",
+ "log4j.rootLogger": "${hbase.root.logger}",
+ "log4j.logger.org.apache.zookeeper": "INFO",
+ "log4j.logger.org.apache.hadoop.hbase": "DEBUG",
+ "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher": "INFO",
+ "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil": "INFO",
+ "log4j.category.SecurityLogger": "${hbase.security.logger}",
+ "log4j.appender.console": "org.apache.log4j.ConsoleAppender",
+ "log4j.appender.console.target": "System.err",
+ "log4j.appender.console.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.console.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "log4j.appender.RFAS": "org.apache.log4j.RollingFileAppender",
+ "log4j.appender.RFAS.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.RFAS.layout.ConversionPattern": "%d{ISO8601} %p %c: %m%n",
+ "log4j.appender.RFAS.MaxFileSize": "${hbase.security.log.maxfilesize}",
+ "log4j.appender.RFAS.MaxBackupIndex": "${hbase.security.log.maxbackupindex}",
+ "log4j.appender.RFAS.File": "${hbase.log.dir}/${hbase.security.log.file}",
+ "log4j.appender.RFA": "org.apache.log4j.RollingFileAppender",
+ "log4j.appender.RFA.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.RFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "log4j.appender.RFA.MaxFileSize": "${hbase.log.maxfilesize}",
+ "log4j.appender.RFA.MaxBackupIndex": "${hbase.log.maxbackupindex}",
+ "log4j.appender.RFA.File": "${hbase.log.dir}/${hbase.log.file}",
+ "log4j.appender.NullAppender": "org.apache.log4j.varia.NullAppender",
+ "log4j.appender.DRFA": "org.apache.log4j.DailyRollingFileAppender",
+ "log4j.appender.DRFA.layout": "org.apache.log4j.PatternLayout",
+ "log4j.appender.DRFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "log4j.appender.DRFA.File": "${hbase.log.dir}/${hbase.log.file}",
+ "log4j.appender.DRFA.DatePattern": ".yyyy-MM-dd",
+ "log4j.additivity.SecurityLogger": "false",
+ "hbase.security.logger": "INFO,console",
+ "hbase.security.log.maxfilesize": "256MB",
+ "hbase.security.log.maxbackupindex": "20",
+ "hbase.security.log.file": "SecurityAuth.audit",
+ "hbase.root.logger": "INFO,console",
+ "hbase.log.maxfilesize": "256MB",
+ "hbase.log.maxbackupindex": "20",
+ "hbase.log.file": "hbase.log",
+ "hbase.log.dir": "."
+ },
+ "app-global-config": {
+ "security_enabled": "false",
+ "pid_dir": "/hadoop/yarn/log/application_1394053491953_0003/run",
+ "log_dir": "/hadoop/yarn/log/application_1394053491953_0003/log",
+ "tmp_dir": "/hadoop/yarn/log/application_1394053491953_0003/tmp",
+ "user_group": "hadoop",
+ "user": "hbase",
+ "hbase_regionserver_heapsize": "1024m",
+ "hbase_master_heapsize": "1024m",
+ "fs_default_name": "hdfs://c6403.ambari.apache.org:8020",
+ "hdfs_root": "/apps/hbase/instances/01",
+ "zookeeper_node": "/apps/hbase/instances/01",
+ "zookeeper_quorom_hosts": "c6403.ambari.apache.org",
+ "zookeeper_port": "2181",
+ },
+ "hbase-site": {
+ "hbase.hstore.flush.retries.number": "120",
+ "hbase.client.keyvalue.maxsize": "10485760",
+ "hbase.hstore.compactionThreshold": "3",
+ "hbase.rootdir": "hdfs://c6403.ambari.apache.org:8020/apps/hbase/instances/01/data",
+ "hbase.stagingdir": "hdfs://c6403.ambari.apache.org:8020/apps/hbase/instances/01/staging",
+ "hbase.regionserver.handler.count": "60",
+ "hbase.regionserver.global.memstore.lowerLimit": "0.38",
+ "hbase.hregion.memstore.block.multiplier": "2",
+ "hbase.hregion.memstore.flush.size": "134217728",
+ "hbase.superuser": "yarn",
+ "hbase.zookeeper.property.clientPort": "2181",
+ "hbase.regionserver.global.memstore.upperLimit": "0.4",
+ "zookeeper.session.timeout": "30000",
+ "hbase.tmp.dir": "/hadoop/yarn/log/application_1394053491953_0003/tmp",
+ "hbase.hregion.max.filesize": "10737418240",
+ "hfile.block.cache.size": "0.40",
+ "hbase.security.authentication": "simple",
+ "hbase.defaults.for.version.skip": "true",
+ "hbase.zookeeper.quorum": "c6403.ambari.apache.org",
+ "zookeeper.znode.parent": "/apps/hbase/instances/01",
+ "hbase.hstore.blockingStoreFiles": "10",
+ "hbase.hregion.majorcompaction": "86400000",
+ "hbase.security.authorization": "false",
+ "hbase.cluster.distributed": "true",
+ "hbase.hregion.memstore.mslab.enabled": "true",
+ "hbase.client.scanner.caching": "100",
+ "hbase.zookeeper.useMulti": "true",
+ "hbase.regionserver.info.port": "",
+ "hbase.master.info.port": "60010"
+ }
+ }
+}
+```
+
+
+## Sample command script
+
+```
+class OozieServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ oozie(is_server=True)
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ oozie_service(action='start')
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+ oozie_service(action='stop')
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.pid_file)
+```
+
+
diff --git a/src/site/markdown/specification/cli-actions.md b/src/site/markdown/specification/cli-actions.md
new file mode 100644
index 0000000..96115b5
--- /dev/null
+++ b/src/site/markdown/specification/cli-actions.md
@@ -0,0 +1,675 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# CLI Actions
+
+
+## Important
+
+1. This document is still being updated from the original hoya design
+2. The new cluster model of separated specification files for internal, resource and application configuration
+has not been incorporated.
+1. What is up to date is the CLI command list and arguments
+
+## client configuration
+
+As well as the CLI options, the `conf/slider-client.xml` XML file can define arguments used to communicate with the Application instance
+
+
+#### `fs.defaultFS`
+
+Equivalent to setting the filesystem with `--filesystem`
+
+
+
+## Common
+
+### System Properties
+
+Arguments of the form `-S key=value` define JVM system properties.
+
+These are supported primarily to define options needed for some Kerberos configurations.
+
+### Definitions
+
+Arguments of the form `-D key=value` define JVM system properties.
+
+These can define client options that are not set in `conf/hoya-client.xml` - or to override them.
+
+### Cluster names
+
+All actions that must take an instance name will fail with `EXIT_UNKNOWN_INSTANCE`
+if one is not provided.
+
+## Action: Build
+
+Builds a cluster -creates all the on-filesystem datastructures, and generates a cluster description
+that is both well-defined and deployable -*but does not actually start the cluster*
+
+ build (instancename,
+ options:List[(String,String)],
+ components:List[(String, int)],
+ componentOptions:List[(String,String, String)],
+ resourceOptions:List[(String,String)],
+ resourceComponentOptions:List[(String,String, String)],
+ confdir: URI,
+ provider: String
+ zkhosts,
+ zkport,
+ image
+ apphome
+ appconfdir
+
+
+#### Preconditions
+
+(Note that the ordering of these preconditions is not guaranteed to remain constant)
+
+The instance name is valid
+
+ if not valid-instance-name(instancename) : raise SliderException(EXIT_COMMAND_ARGUMENT_ERROR)
+
+The instance must not be live. This is purely a safety check as the next test should have the same effect.
+
+ if slider-instance-live(YARN, instancename) : raise SliderException(EXIT_CLUSTER_IN_USE)
+
+The instance must not exist
+
+ if is-dir(HDFS, instance-path(FS, instancename)) : raise SliderException(EXIT_CLUSTER_EXISTS)
+
+The configuration directory must exist it does not have to be the instance's HDFS instance,
+as it will be copied there -and must contain only files
+
+ let FS = FileSystem.get(appconfdir)
+ if not isDir(FS, appconfdir) raise SliderException(EXIT_COMMAND_ARGUMENT_ERROR)
+ forall f in children(FS, appconfdir) :
+ if not isFile(f): raise IOException
+
+There's a race condition at build time where between the preconditions being met and the instance specification being saved, the instance
+is created by another process. This addressed by creating a lock file, `writelock.json` in the destination directory. If the file
+exists, no other process may acquire the lock.
+
+There is a less exclusive readlock file, `readlock.json` which may be created by any process that wishes to read the configuration.
+If it exists when another process wishes to access the files, the subsequent process may read the data, but MUST NOT delete it
+afterwards. A process attempting to acquire the writelock must check for the existence of this file before AND after creating the
+writelock file, failing if its present. This retains a small race condition: a second or later reader may still be reading the data
+when a process successfully acquires the write lock. If this proves to be an issue, a stricter model could be implemented, with each reading process creating a unique named readlock- file.
+
+
+
+
+#### Postconditions
+
+All the instance directories exist
+
+ is-dir(HDFS', instance-path(HDFS', instancename))
+ is-dir(HDFS', original-conf-path(HDFS', instancename))
+ is-dir(HDFS', generated-conf-path(HDFS', instancename))
+
+The application cluster specification saved is well-defined and deployable
+
+ let instance-description = parse(data(HDFS', instance-json-path(HDFS', instancename)))
+ well-defined-instance(instance-description)
+ deployable-application-instance(HDFS', instance-description)
+
+More precisely: the specification generated before it is saved as JSON is well-defined and deployable; no JSON file will be created
+if the validation fails.
+
+Fields in the cluster description have been filled in
+
+ internal.global["internal.provider.name"] == provider
+ app_conf.global["zookeeper.port"] == zkport
+ app_conf.global["zookeeper.hosts"] == zkhosts
+
+
+ package => app_conf.global["agent.package"] = package
+
+
+
+Any `apphome` and `image` properties have propagated
+
+ apphome == null or clusterspec.options["cluster.application.home"] == apphome
+ image == null or clusterspec.options["cluster.application.image.path"] == image
+
+(The `well-defined-application-instance()` requirement above defines the valid states
+of this pair of options)
+
+
+All role sizes have been mapped to `component.instances` fields
+
+ forall (name, size) in components :
+ resources.components[name]["components.instances"] == size
+
+
+
+
+All option parameters have been added to the `options` map in the specification
+
+ forall (opt, val) in options :
+ app_conf.global[opt] == val
+
+ forall (opt, val) in resourceOptions :
+ resource.global[opt] == val
+
+All component option parameters have been added to the specific components's option map
+in the relevant configuration file
+
+ forall (name, opt, val) in componentOptions :
+ app_conf.components[name][opt] == val
+
+ forall (name, opt, val) in resourceComponentOptions :
+ resourceComponentOptions.components[name][opt] == val
+
+To avoid some confusion as to where keys go, all options beginning with the
+prefix `component.` are automatically copied into the resources file:
+
+ forall (opt, val) in options where startswith(opt, "component.")
+ or startswith(opt, "role.")
+ or startswith(opt, "yarn."):
+ resource.global[opt] == val
+
+ forall (name, opt, val) in componentOptions where startswith(opt, "component.")
+ or startswith(opt, "role.")
+ or startswith(opt, "yarn."):
+ resourceComponentOptions.components[name][opt] == val
+
+
+There's no explicit rejection of duplicate options, the outcome of that
+state is 'undefined'.
+
+What is defined is that if Slider or its provider provided a default option value,
+the command-line supplied option will override it.
+
+All files that were in the configuration directory are now copied into the "original" configuration directory
+
+ let FS = FileSystem.get(appconfdir)
+ let dest = original-conf-path(HDFS', instancename)
+ forall [c in children(FS, confdir) :
+ data(HDFS', dest + [filename(c)]) == data(FS, c)
+
+All files that were in the configuration directory now have equivalents in the generated configuration directory
+
+ let FS = FileSystem.get(appconfdir)
+ let dest = generated-conf-path(HDFS', instancename)
+ forall [c in children(FS, confdir) :
+ isfile(HDFS', dest + [filename(c)])
+
+
+## Action: Thaw
+
+ thaw <instancename> [--wait <timeout>]
+
+Thaw takes an application instance with configuration and (possibly) data on disk, and
+attempts to create a live application with the specified number of nodes
+
+#### Preconditions
+
+ if not valid-instance-name(instancename) : raise SliderException(EXIT_COMMAND_ARGUMENT_ERROR)
+
+The cluster must not be live. This is purely a safety check as the next test should have the same effect.
+
+ if slider-instance-live(YARN, instancename) : raise SliderException(EXIT_CLUSTER_IN_USE)
+
+The cluster must not exist
+
+ if is-dir(HDFS, application-instance-path(FS, instancename)) : raise SliderException(EXIT_CLUSTER_EXISTS)
+
+The cluster specification must exist, be valid and deployable
+
+ if not is-file(HDFS, cluster-json-path(HDFS, instancename)) : SliderException(EXIT_UNKNOWN_INSTANCE)
+ if not well-defined-application-instance(HDFS, application-instance-path(HDFS, instancename)) : raise SliderException(EXIT_BAD_CLUSTER_STATE)
+ if not deployable-application-instance(HDFS, application-instance-path(HDFS, instancename)) : raise SliderException(EXIT_BAD_CLUSTER_STATE)
+
+### Postconditions
+
+
+After the thaw has been performed, there is now a queued request in YARN
+for the chosen (how?) queue
+
+ YARN'.Queues'[amqueue] = YARN.Queues[amqueue] + [launch("slider", instancename, requirements, context)]
+
+If a wait timeout was specified, the cli waits until the application is considered
+running by YARN (the AM is running), the wait timeout has been reached, or
+the application has failed
+
+ waittime < 0 or (exists a in slider-running-application-instances(yarn-application-instances(YARN', instancename, user))
+ where a.YarnApplicationState == RUNNING)
+
+
+## Outcome: AM-launched state
+
+Some time after the AM was queued, if the relevant
+prerequisites of the launch request are met, the AM will be deployed
+
+#### Preconditions
+
+* The resources referenced in HDFS (still) are accessible by the user
+* The requested YARN memory and core requirements could be met on the YARN cluster and
+specific YARN application queue.
+* There is sufficient capacity in the YARN cluster to create a container for the AM.
+
+#### Postconditions
+
+Define a YARN state at a specific time `t` as `YARN(t)`; the fact that
+an AM is launched afterwards
+
+The AM is deployed if there is some time `t` after the submission time `t0`
+where the application is listed
+
+ exists t1 where t1 > t0 and slider-instance-live(YARN(t1), user, instancename)
+
+At which time there is a container in the cluster hosting the AM -it's
+context is the launch context
+
+ exists c in containers(YARN(t1)) where container.context = launch.context
+
+There's no way to determine when this time `t1` will be reached -or if it ever
+will -its launch may be postponed due to a lack of resources and/or higher priority
+requests using resources as they become available.
+
+For tests on a dedicated YARN cluster, a few tens of seconds appear to be enough
+for the AM-launched state to be reached, a failure to occur, or to conclude
+that the resource requirements are unsatisfiable.
+
+## Outcome: AM-started state
+
+A (usually short) time after the AM is launched, it should start
+
+* The node hosting the container is working reliably
+* The supplied command line could start the process
+* the localized resources in the context could be copied to the container (which implies
+that they are readable by the user account the AM is running under)
+* The combined classpath of YARN, extra JAR files included in the launch context,
+and the resources in the slider client 'conf' dir contain all necessary dependencies
+to run Slider.
+* There's no issue with the cluster specification that causes the AM to exit
+with an error code.
+
+Node failures/command line failures are treated by YARN as an AM failure which
+will trigger a restart attempt -this may be on the same or a different node.
+
+#### preconditions
+
+The AM was launched at an earlier time, `t1`
+
+ exists t1 where t1 > t0 and am-launched(YARN(t1)
+
+
+#### Postconditions
+
+The application is actually started if it is listed in the YARN application list
+as being in the state `RUNNING`, an RPC port has been registered with YARN (visible as the `rpcPort`
+attribute in the YARN Application Report,and that port is servicing RPC requests
+from authenticated callers.
+
+ exists t2 where:
+ t2 > t1
+ and slider-instance-live(YARN(t2), YARN, instancename, user)
+ and slider-live-instances(YARN(t2))[0].rpcPort != 0
+ and rpc-connection(slider-live-instances(YARN(t2))[0], HoyaClusterProtocol)
+
+A test for accepting cluster requests is querying the cluster status
+with `HoyaClusterProtocol.getJSONClusterStatus()`. If this returns
+a parseable cluster description, the AM considers itself live.
+
+## Outcome: Applicaton Instance operational state
+
+Once started, Slider enters the operational state of trying to keep the numbers
+of live role instances matching the numbers specified in the cluster specification.
+
+The AM must request the a container for each desired instance of a specific roles of the
+application, wait for those requests to be granted, and then instantiate
+the specific application roles on the allocated containers.
+
+Such a request is made on startup, whenever a failure occurs, or when the
+cluster size is dynamically updated.
+
+The AM releases containers when the cluster size is shrunk during a flex operation,
+or during teardown.
+
+### steady state condition
+
+The steady state of a Slider cluster is that the number of live instances of a role,
+plus the number of requested instances , minus the number of instances for
+which release requests have been made must match that of the desired number.
+
+If the internal state of the Slider AM is defined as `AppState`
+
+ forall r in clusterspec.roles :
+ r["component.instances"] ==
+ AppState.Roles[r].live + AppState.Roles[r].requested - AppState.Roles[r].released
+
+The `AppState` represents Slider's view of the external YARN system state, based on its
+history of notifications received from YARN.
+
+It is indirectly observable from the cluster state which an AM can be queried for
+
+
+ forall r in AM.getJSONClusterStatus().roles :
+ r["component.instances"] ==
+ r["role.actual.instances"] + r["role.requested.instances"] - r["role.releasing.instances"]
+
+Slider does not consider it an error if the number of actual instances remains below
+the desired value (i.e. outstanding requests are not being satisfied) -this is
+an operational state of the cluster that Slider cannot address.
+
+### Cluster startup
+
+On a healthy dedicated test cluster, the time for the requests to be satisfied is
+a few tens of seconds at most: a failure to achieve this state is a sign of a problem.
+
+### Node or process failure
+
+After a container or node failure, a new container for a new instance of that role
+is requested.
+
+The failure count is incremented -it can be accessed via the `"role.failed.instances"`
+attribute of a role in the status report.
+
+The number of failures of a role is tracked, and used by Slider as to when to
+conclude that the role is somehow failing consistently -and it should fail the
+entire application.
+
+This has initially been implemented as a simple counter, with the cluster
+option: `"hoya.container.failure.threshold"` defining that threshold.
+
+ let status = AM.getJSONClusterStatus()
+ forall r in in status.roles :
+ r["role.failed.instances"] < status.options["hoya.container.failure.threshold"]
+
+
+### Instance startup failure
+
+
+Startup failures are measured alongside general node failures.
+
+A container is deemed to have failed to start if either of the following conditions
+were met:
+
+1. The AM received an `onNodeManagerContainerStartFailed` event.
+
+1. The AM received an `onCompletedNode` event on a node that started less than
+a specified number of seconds earlier -a number given in the cluster option
+`"hoya.container.failure.shortlife"`.
+
+More sophisticated failure handling logic than is currently implemented may treat
+startup failures differently from ongoing failures -as they can usually be
+treated as a sign that the container is failing to launch the program reliably -
+either the generated command line is invalid, or the application is failing
+to run/exiting on or nearly immediately.
+
+## Action: Create
+
+Create is simply `build` + `thaw` in sequence - the postconditions from the first
+action are intended to match the preconditions of the second.
+
+## Action: Freeze
+
+ freeze instancename [--wait time] [--message message]
+
+The *freeze* action "freezes" the cluster: all its nodes running in the YARN
+cluster are stopped, leaving all the persistent state.
+
+The operation is intended to be idempotent: it is not an error if
+freeze is invoked on an already frozen cluster
+
+#### Preconditions
+
+The cluster name is valid and it matches a known cluster
+
+ if not valid-instance-name(instancename) : raise SliderException(EXIT_COMMAND_ARGUMENT_ERROR)
+
+ if not is-file(HDFS, application-instance-path(HDFS, instancename)) :
+ raise SliderException(EXIT_UNKNOWN_INSTANCE)
+
+#### Postconditions
+
+If the cluster was running, an RPC call has been sent to it `stopCluster(message)`
+
+If the `--wait` argument specified a wait time, then the command will block
+until the cluster has finished or the wait time was exceeded.
+
+If the `--message` argument specified a message -it must appear in the
+YARN logs as the reason the cluster was frozen.
+
+
+The outcome should be the same:
+
+ not slider-instance-live(YARN', instancename)
+
+## Action: Flex
+
+Flex the cluster size: add or remove roles.
+
+ flex instancename
+ components:List[(String, int)]
+
+1. The JSON cluster specification in the filesystem is updated
+1. if the cluster is running, it is given the new cluster specification,
+which will change the desired steady-state of the application
+
+#### Preconditions
+
+ if not is-file(HDFS, cluster-json-path(HDFS, instancename)) :
+ raise SliderException(EXIT_UNKNOWN_INSTANCE)
+
+#### Postconditions
+
+ let originalSpec = data(HDFS, cluster-json-path(HDFS, instancename))
+
+ let updatedSpec = originalspec where:
+ forall (name, size) in components :
+ updatedSpec.roles[name]["component.instances"] == size
+ data(HDFS', cluster-json-path(HDFS', instancename)) == updatedSpec
+ rpc-connection(slider-live-instances(YARN(t2))[0], HoyaClusterProtocol)
+ let flexed = rpc-connection(slider-live-instances(YARN(t2))[0], HoyaClusterProtocol).flexClusterupdatedSpec)
+
+
+#### AM actions on flex
+
+ boolean HoyaAppMaster.flexCluster(ClusterDescription updatedSpec)
+
+If the cluster is in a state where flexing is possible (i.e. it is not in teardown),
+then `AppState` is updated with the new desired role counts. The operation will
+return once all requests to add or remove role instances have been queued,
+and be `True` iff the desired steady state of the cluster has been changed.
+
+#### Preconditions
+
+ well-defined-application-instance(HDFS, updatedSpec)
+
+
+#### Postconditions
+
+ forall role in AppState.Roles.keys:
+ AppState'.Roles'[role].desiredCount = updatedSpec[roles]["component.instances"]
+ result = AppState' != AppState
+
+
+The flexing may change the desired steady state of the cluster, in which
+case the relevant requests will have been queued by the completion of the
+action. It is not possible to state whether or when the requests will be
+satisfied.
+
+## Action: Destroy
+
+Idempotent operation to destroy a frozen cluster -it succeeds if the
+cluster has already been destroyed/is unknown, but not if it is
+actually running.
+
+#### Preconditions
+
+ if not valid-instance-name(instancename) : raise SliderException(EXIT_COMMAND_ARGUMENT_ERROR)
+
+ if slider-instance-live(YARN, instancename) : raise SliderException(EXIT_CLUSTER_IN_USE)
+
+
+#### Postconditions
+
+The cluster directory and all its children do not exist
+
+ not is-dir(HDFS', application-instance-path(HDFS', instancename))
+
+
+## Action: Status
+
+ status instancename [--out outfile]
+ 2
+#### Preconditions
+
+ if not slider-instance-live(YARN, instancename) : raise SliderException(EXIT_UNKNOWN_INSTANCE)
+
+#### Postconditions
+
+The status of the application has been successfully queried and printed out:
+
+ let status = slider-live-instances(YARN).rpcPort.getJSONClusterStatus()
+
+if the `outfile` value is not defined then the status appears part of stdout
+
+ status in STDOUT'
+
+otherwise, the outfile exists in the local filesystem
+
+ (outfile != "") ==> data(LocalFS', outfile) == body
+ (outfile != "") ==> body in STDOUT'
+
+## Action: Exists
+
+This probes for a named cluster being defined or actually being in the running
+state.
+
+In the running state; it is essentially the status
+operation with only the exit code returned
+
+#### Preconditions
+
+
+ if not is-file(HDFS, application-instance-path(HDFS, instancename)) :
+ raise SliderException(EXIT_UNKNOWN_INSTANCE)
+
+#### Postconditions
+
+The operation succeeds if the cluster is running and the RPC call returns the cluster
+status.
+
+ if live and not slider-instance-live(YARN, instancename):
+ retcode = -1
+ else:
+ retcode = 0
+
+## Action: getConf
+
+This returns the live client configuration of the cluster -the
+site-xml file.
+
+ getconf --format (xml|properties) --out [outfile]
+
+*We may want to think hard about whether this is needed*
+
+#### Preconditions
+
+ if not slider-instance-live(YARN, instancename) : raise SliderException(EXIT_UNKNOWN_INSTANCE)
+
+
+#### Postconditions
+
+The operation succeeds if the cluster status can be retrieved and saved to
+the named file/printed to stdout in the format chosen
+
+ let status = slider-live-instances(YARN).rpcPort.getJSONClusterStatus()
+ let conf = status.clientProperties
+ if format == "xml" :
+ let body = status.clientProperties.asXmlDocument()
+ else:
+ let body = status.clientProperties.asProperties()
+
+ if outfile != "" :
+ data(LocalFS', outfile) == body
+ else
+ body in STDOUT'
+
+## Action: list
+
+ list [instancename]
+
+Lists all clusters of a user, or only the one given
+
+#### Preconditions
+
+If a instancename is specified it must be in YARNs list of active or completed applications
+of that user:
+
+ if instancename != "" and [] == yarn-application-instances(YARN, instancename, user)
+ raise SliderException(EXIT_UNKNOWN_INSTANCE)
+
+
+#### Postconditions
+
+If no instancename was given, all hoya applications of that user are listed,
+else only the one running (or one of the finished ones)
+
+ if instancename == "" :
+ forall a in yarn-application-instances(YARN, user) :
+ a.toString() in STDOUT'
+ else
+ let e = yarn-application-instances(YARN, instancename, user)
+ e.toString() in STDOUT'
+
+## Action: killcontainer
+
+This is an operation added for testing. It will kill a container in the cluster
+*without flexing the cluster size*. As a result, the cluster will detect the
+failure and attempt to recover from the failure by instantiating a new instance
+of the cluster
+
+ killcontainer cluster --id container-id
+
+#### Preconditions
+
+ if not slider-instance-live(YARN, instancename) : raise SliderException(EXIT_UNKNOWN_INSTANCE)
+
+ exists c in hoya-app-containers(YARN, instancename, user) where c.id == container-id
+
+ let status := AM.getJSONClusterStatus()
+ exists role = status.instances where container-id in status.instances[role].values
+
+
+#### Postconditions
+
+The container is not in the list of containers in the cluster
+
+ not exists c in containers(YARN) where c.id == container-id
+
+And implicitly, not in the running containers of that application
+
+ not exists c in hoya-app-containers(YARN', instancename, user) where c.id == container-id
+
+At some time `t1 > t`, the status of the application (`AM'`) will be updated to reflect
+that YARN has notified the AM of the loss of the container
+
+
+ let status' = AM'.getJSONClusterStatus()
+ len(status'.instances[role]) < len(status.instances[role])
+ status'.roles[role]["role.failed.instances"] == status'.roles[role]["role.failed.instances"]+1
+
+
+At some time `t2 > t1` in the future, the size of the containers of the application
+in the YARN cluster `YARN''` will be as before
+
+ let status'' = AM''.getJSONClusterStatus()
+ len(status''.instances[r] == len(status.instances[r])
diff --git a/src/site/markdown/specification/index.md b/src/site/markdown/specification/index.md
new file mode 100644
index 0000000..d732959
--- /dev/null
+++ b/src/site/markdown/specification/index.md
@@ -0,0 +1,41 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Specification of Slider behaviour
+
+This is a a "more rigorous" definition of the behavior of Slider in terms
+of its state and its command-line operations -by defining a 'formal' model
+of HDFS, YARN and Slider's internal state, then describing the operations
+that can take place in terms of their preconditions and postconditions.
+
+This is to show what tests we can create to verify that an action
+with a valid set of preconditions results in an outcome whose postconditions
+can be verified. It also makes more apparent what conditions should be
+expected to result in failures, as well as what the failure codes should be.
+
+Specifying the behavior has also helped identify areas where there was ambiguity,
+where clarification and more tests were needed.
+
+The specification depends on ongoing work in [HADOOP-9361](https://issues.apache.org/jira/browse/HADOOP-9361):
+to define the Hadoop Filesytem APIs --This specification uses [the same notation](https://github.com/steveloughran/hadoop-trunk/blob/stevel/HADOOP-9361-filesystem-contract/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/notation.md)
+
+
+1. [Model: YARN And Slider](slider-model.html)
+1. [CLI actions](cli-actions.html)
+
+Exceptions and operations may specify exit codes -these are listed in
+[Client Exit Codes](../exitcodes.html)
diff --git a/src/site/markdown/specification/slider-model.md b/src/site/markdown/specification/slider-model.md
new file mode 100644
index 0000000..269ce68
--- /dev/null
+++ b/src/site/markdown/specification/slider-model.md
@@ -0,0 +1,286 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Formal Slider Model
+
+This is the model of Slider and YARN for the rest of the specification.
+
+## File System
+
+A File System `HDFS` represents a Hadoop FileSystem -either HDFS or another File
+System which spans the cluster. There are also other filesystems that
+can act as sources of data that is then copied into HDFS. These will be marked
+as `FS` or with the generic `FileSystem` type.
+
+
+There's ongoing work in [HADOOP-9361](https://issues.apache.org/jira/browse/HADOOP-9361)
+to define the Hadoop Filesytem APIs using the same notation as here,
+the latest version being available on [github](https://github.com/steveloughran/hadoop-trunk/tree/stevel/HADOOP-9361-filesystem-contract/hadoop-common-project/hadoop-common/src/site/markdown/filesystem)
+Two key references are
+
+ 1. [The notation reused in the Slider specifications](https://github.com/steveloughran/hadoop-trunk/blob/stevel/HADOOP-9361-filesystem-contract/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/notation.md)
+ 1. [The model of the filesystem](https://github.com/steveloughran/hadoop-trunk/blob/stevel/HADOOP-9361-filesystem-contract/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/model.md)
+
+ The model and its predicates and invariants will be used in these specifications.
+
+## YARN
+
+From the perspective of YARN application, The YARN runtime is a state, `YARN`,
+comprised of: ` (Apps, Queues, Nodes)`
+
+ Apps: Map[AppId, ApplicationReport]
+
+An application has a name, an application report and a list of outstanding requests
+
+ App: (Name, report: ApplicationReport, Requests:List[AmRequest])
+
+An application report contains a mixture of static and dynamic state of the application
+and the AM.
+
+ ApplicationReport: AppId, Type, User, YarnApplicationState, AmContainer, RpcPort, TrackingURL,
+
+YARN applications have a number of states. These are ordered such that if the
+`state.ordinal() > RUNNING.ordinal() ` then the application has entered an exit state.
+
+ YarnApplicationState : [NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED ]
+
+AMs can request containers to be added or released
+
+ AmRequest = { add-container(priority, requirements), release(containerId)}
+
+Job queues are named queues of job requests; there is always a queue called `"default"`
+
+ Queues: Map[String:Queue]
+ Queue: List[Requests]
+ Request = {
+ launch(app-name, app-type, requirements, context)
+ }
+ Context: (localized-resources: Map[String,URL], command)
+
+
+This is doesn't completely model the cluster from the AM perspective -there's no
+notion of node operations (launching code in a container) or events coming from YARN.
+
+The `Nodes` structure models the nodes in a cluster
+
+ Nodes: Map[nodeID,(name, containers:List[Container])]
+
+A container contains some state
+
+ Container: (containerId, appId, context)
+
+The containers in a cluster are the aggregate set of all containers across
+all nodes
+
+ def containers(YARN) =
+ [c for n in keys(YARN.Nodes) for c in YARN.Nodes[n].Containers ]
+
+
+The containers of an application are all containers that are considered owned by it,
+
+ def app-containers(YARN, appId: AppId) =
+ [c in containers(YARN) where c.appId == appId ]
+
+### Operations & predicates used the specifications
+
+
+ def applications(YARN, type) =
+ [ app.report for app in YARN.Apps.values where app.report.Type == type]
+
+ def user-applications(YARN, type, user)
+ [a in applications(YARN, type) where: a.User == user]
+
+
+## UserGroupInformation
+
+Applications are launched and executed on hosts computers: either client machines
+or nodes in the cluster, these have their own state which may need modeling
+
+ HostState: Map[String, String]
+
+A key part of the host state is actually the identity of the current user,
+which is used to define the location of the persistent state of the cluster -including
+its data, and the identity under which a deployed container executes.
+
+In a secure cluster, this identity is accompanied by kerberos tokens that grant the caller
+access to the filesystem and to parts of YARN itself.
+
+This specification does not currently explicitly model the username and credentials.
+If it did they would be used throughout the specification to bind to a YARN or HDFS instance.
+
+`UserGroupInformation.getCurrentUser(): UserGroupInformation`
+
+Returns the current user information. This information is immutable and fixed for the duration of the process.
+
+
+
+## Slider Model
+
+### Cluster name
+
+A valid cluster name is a name of length > 1 which follows the internet hostname scheme of letter followed by letter or digit
+
+ def valid-cluster-name(c) =
+ len(c)> 0
+ and c[0] in ['a'..'z']
+ and c[1] in (['a'..'z'] + ['-'] + ['0..9'])
+
+### Persistent Cluster State
+
+A Slider cluster's persistent state is stored in a path
+
+ def cluster-path(FS, clustername) = user-home(FS) + ["clusters", clustername]
+ def cluster-json-path(FS, clustername) = cluster-path(FS, clustername) + ["cluster.json"]
+ def original-conf-path(FS, clustername) = cluster-path(FS, clustername) + ["original"]
+ def generated-conf-path(FS, clustername) = cluster-path(FS, clustername) + ["generated"]
+ def data-path(FS, clustername) = cluster-path(FS, clustername) + ["data"]
+
+When a cluster is built/created the specified original configuration directory
+is copied to `original-conf-path(FS, clustername)`; this is patched for the
+specific instance bindings and saved into `generated-conf-path(FS, clustername)`.
+
+A cluster *exists* if all of these paths are found:
+
+ def cluster-exists(FS, clustername) =
+ is-dir(FS, cluster-path(FS, clustername))
+ and is-file(FS, cluster-json-path(FS, clustername))
+ and is-dir(FS, original-conf-path(FS, clustername))
+ and generated-conf-path(FS, original-conf-path(FS, clustername))
+
+A cluster is considered `running` if there is a Slider application type belonging to the current user in one of the states
+`{NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING}`.
+
+ def final-yarn-states = {FINISHED, FAILED, KILLED }
+
+ def slider-app-instances(YARN, clustername, user) =
+ [a in user-applications(YARN, "slider", user) where:
+ and a.Name == clustername]
+
+ def slider-app-running-instances(YARN, clustername, user) =
+ [a in slider-app-instances(YARN, user, clustername) where:
+ not a.YarnApplicationState in final-yarn-state]
+
+ def slider-app-running(YARN, clustername, user) =
+ [] != slider-app-running-instances(YARN, clustername, user)
+
+ def slider-app-live-instances(YARN, clustername, user) =
+ [a in slider-app-instances(YARN, user, clustername) where:
+ a.YarnApplicationState == RUNNING]
+
+ def slider-app-live(YARN, clustername, user) =
+ [] != slider-app-live-instances(YARN, clustername, user)
+
+### Invariant: there must never be more than one running instance of a named Slider cluster
+
+
+There must never be more than one instance of the same Slider cluster running:
+
+ forall a in user-applications(YARN, "slider", user):
+ len(slider-app-running-instances(YARN, a.Name, user)) <= 1
+
+There may be multiple instances in a finished state, and one running instance alongside multiple finished instances -the applications
+that work with Slider MUST select a running cluster ahead of any terminated clusters.
+
+### Containers of an application
+
+
+The containers of a slider application are the set of containers of that application
+
+ def slider-app-containers(YARN, clustername, user) =
+ app-containers(YARN, appid where
+ appid = slider-app-running-instances(YARN, clustername, user)[0])
+
+
+
+
+### RPC Access to a slider cluster
+
+
+ An application is accepting RPC requests for a given protocol if there is a port binding
+ defined and it is possible to authenticate a connection using the specified protocol
+
+ def rpc-connection(appReport, protocol) =
+ appReport.host != null
+ appReport.rpcPort != 0
+ and RPC.getProtocolProxy(appReport.host, appReport.rpcPort, protocol)
+
+ Being able to open an RPC port is the strongest definition of liveness possible
+ to make: if the AM responds to RPC operations, it is doing useful work.
+
+### Valid Cluster Description
+
+The `cluster.json` file of a cluster configures Slider to deploy the application.
+
+#### well-defined-cluster(cluster-description)
+
+A Cluster Description is well-defined if it is valid JSON and required properties are present
+
+**OBSOLETE**
+
+
+Irrespective of specific details for deploying the Slider AM or any provider-specific role instances,
+a Cluster Description defined in a `cluster.json` file at the path `cluster-json-path(FS, clustername)`
+is well-defined if
+
+1. It is parseable by the jackson JSON parser.
+1. Root elements required of a Slider cluster specification must be defined, and, where appropriate, non-empty
+1. It contains the extensible elements required of a Slider cluster specification. For example, `options` and `roles`
+1. The types of the extensible elements match those expected by Slider.
+1. The `version` element matches a supported version
+1. Exactly one of `options/cluster.application.home` and `options/cluster.application.image.path` must exist.
+1. Any cluster options that are required to be integers must be integers
+
+This specification is very vague here to avoid duplication: the cluster description structure is currently implicitly defined in
+`org.apache.slider.api.ClusterDescription`
+
+Currently Slider ignores unknown elements during parsing. This may be changed.
+
+The test for this state does not refer to the cluster filesystem
+
+#### deployable-cluster(FS, cluster-description)
+
+A Cluster Description defines a deployable cluster if it is well-defined cluster and the contents contain valid information to deploy a cluster
+
+This defines how a cluster description is valid in the extends the valid configuration with
+
+* The entry `name` must match a supported provider
+* Any elements that name the cluster match the cluster name as defined by the path to the cluster:
+
+ originConfigurationPath == original-conf-path(FS, clustername)
+ generatedConfigurationPath == generated-conf-path(FS, clustername)
+ dataPath == data-path(FS, clustername)
+
+* The paths defined in `originConfigurationPath` , `generatedConfigurationPath` and `dataPath` must all exist.
+* `options/zookeeper.path` must be defined and refer to a path in the ZK cluster
+defined by (`options/zookeeper.hosts`, `zookeeper.port)` to which the user has write access (required by HBase and Accumulo)
+* If `options/cluster.application.image.path` is defined, it must exist and be readable by the user.
+* It must declare a type that maps to a provider entry in the Slider client's XML configuration:
+
+ len(clusterspec["type"]) > 0
+ clientconfig["slider.provider."+ clusterspec["type"]] != null
+
+* That entry must map to a class on the classpath which can be instantiated
+and cast to `HoyaProviderFactory`.
+
+ let classname = clientconfig["slider.provider."+ clusterspec["type"]]
+ (Class.forName(classname).newInstance()) instanceof HoyaProviderFactory
+
+#### valid-for-provider(cluster-description, provider)
+
+A provider considers a specification valid if its own validation logic is satisfied. This normally
+consists of rules about the number of instances of different roles; it may include other logic.
+
diff --git a/src/site/markdown/testing.md b/src/site/markdown/testing.md
new file mode 100644
index 0000000..7bee6e1
--- /dev/null
+++ b/src/site/markdown/testing.md
@@ -0,0 +1,442 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Testing
+
+ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL
+ NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and
+ "OPTIONAL" in this document are to be interpreted as described in
+ RFC 2119.
+
+## Standalone Tests
+
+Slider core contains a suite of tests that are designed to run on the local machine,
+using Hadoop's `MiniDFSCluster` and `MiniYARNCluster` classes to create small,
+one-node test clusters. All the YARN/HDFS code runs in the JUnit process; the
+AM and spawned HBase and Accumulo processes run independently.
+
+Requirements
+* A copy of hbase.tar.gz in the local filesystem
+* A an expanded hbase.tar.gz in the local filesystem
+
+* A copy of accumulo.tar.gz in the local filesystem,
+* An expanded accumulo.tar.gz in the local filesystem,
+* an expanded Zookeeper installation
+
+All of these need to be defined in the file `slider-core/src/test/resources/slider-test.xml`
+
+Here's
+
+ <configuration>
+
+ <property>
+ <name>slider.test.hbase.enabled</name>
+ <description>Flag to enable/disable HBase tests</description>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.home</name>
+ <value>/home/slider/hbase-0.98.0</value>
+ <description>HBASE Home</description>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <value>/home/slider/Projects/hbase-0.98.0-bin.tar.gz</value>
+ <description>HBASE archive URI</description>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.enabled</name>
+ <description>Flag to enable/disable Accumulo tests</description>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.home</name>
+ <value>
+ /home/slider/accumulo-1.6.0-SNAPSHOT/</value>
+ <description>Accumulo Home</description>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.tar</name>
+ <value>/home/slider/accumulo-1.6.0-SNAPSHOT-bin.tar</value>
+ <description>Accumulo archive URI</description>
+ </property>
+
+ <property>
+ <name>slider.test.am.restart.time</name>
+ <description>Time in millis to await an AM restart</description>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>zk.home</name>
+ <value>/home/slider/zookeeper</value>
+ <description>Zookeeper home dir on target systems</description>
+ </property>
+
+ <property>
+ <name>hadoop.home</name>
+ <value>/home/slider/hadoop-2.2.0</value>
+ <description>Hadoop home dir on target systems</description>
+ </property>
+
+ </configuration>
+
+*Important:* For the local tests, a simple local filesystem path is used for
+all the values.
+
+For the functional tests, the accumulo and hbase tar properties will
+need to be set to a URL of a tar file that is accessible to all the
+nodes in the cluster -which usually means HDFS, and so an `hdfs://` URL
+
+
+
+## Functional Tests
+
+The functional test suite is designed to run the executables against
+a live cluster.
+
+For these to work you need
+1. A YARN Cluster -secure or insecure
+1. A `slider-client.xml` file configured to interact with the cluster
+1. HBase `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
+directory
+1. Accumulo `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
+directory
+
+## Configuration of functional tests
+
+Maven needs to be given
+1. A path to the expanded test archive
+1. A path to a slider configuration directory for the cluster
+
+The path for the expanded test is automatically calculated as being the directory under
+`..\slider-assembly\target` where an untarred slider distribution can be found.
+If it is not present, the tests will fail
+
+The path to the configuration directory must be supplied in the property
+`slider.conf.dir` which can be set on the command line
+
+ mvn test -Dhoya.conf.dir=src/test/configs/sandbox/slider
+
+It can also be set in the (optional) file `slider-funtest/build.properties`:
+
+ slider.conf.dir=src/test/configs/sandbox/slider
+
+This file is loaded whenever a slider build or test run takes place
+
+## Configuration of `slider-client.xml`
+
+The `slider-client.xml` must have extra configuration options for both the HBase and
+Accumulo tests, as well as a common set for actually talking to a YARN cluster.
+
+## Disabling the functional tests entirely
+
+All functional tests which require a live YARN cluster
+can be disabled through the property `slider.funtest.enabled`
+
+ <property>
+ <name>slider.funtest.enabled</name>
+ <value>false</value>
+ </property>
+
+There is a configuration do do exactly this in
+`src/test/configs/offline/slider`:
+
+ slider.conf.dir=src/test/configs/offline/slider
+
+Tests which do not require a live YARN cluster will still run;
+these verify that the `bin/slider` script works.
+
+### Non-mandatory options
+
+The following test options may be added to `slider-client.xml` if the defaults
+need to be changed
+
+ <property>
+ <name>slider.test.zkhosts</name>
+ <description>comma separated list of ZK hosts</description>
+ <value>localhost</value>
+ </property>
+
+ <property>
+ <name>slider.test.thaw.wait.seconds</name>
+ <description>Time to wait in seconds for a thaw to result in a running AM</description>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <name>slider.test.freeze.wait.seconds</name>
+ <description>Time to wait in seconds for a freeze to halt the cluster</description>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <name>slider.test.timeout.millisec</name>
+ <description>Time out in milliseconds before a test is considered to have failed.
+ There are some maven properties which also define limits and may need adjusting</description>
+ <value>180000</value>
+ </property>
+
+
+
+Note that while the same properties need to be set in
+`slider-core/src/test/resources/slider-client.xml`, those tests take a file in the local
+filesystem -here a URI to a path visible across all nodes in the cluster are required
+the tests do not copy the .tar/.tar.gz files over. The application configuration
+directories may be local or remote -they are copied into the `.slider` directory
+during cluster creation.
+
+##
+
+Provider-specific parameters
+
+An individual provider can pick up settings from their own
+`src/test/resources/slider-client.xml` file, or the one in `slider-core`.
+We strongly advice placing all the values in the `slider-core` file.
+
+1. All uncertainty about which file is picked up on the class path first goes
+away
+2. There's one place to keep all the configuration values in sync.
+
+### HBase Parameters
+
+The HBase tests can be enabled or disabled
+
+ <property>
+ <name>slider.test.hbase.enabled</name>
+ <description>Flag to enable/disable HBase tests</description>
+ <value>true</value>
+ </property>
+
+Mandatory test parameters must be added to `slider-client.xml`
+
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <description>Path to the HBase Tar file in HDFS</description>
+ <value>hdfs://sandbox.hortonworks.com:8020/user/slider/hbase.tar.gz</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.appconf</name>
+ <description>Path to the directory containing the HBase application config</description>
+ <value>file://${user.dir}/src/test/configs/sandbox/hbase</value>
+ </property>
+
+Optional parameters:
+
+ <property>
+ <name>slider.test.hbase.launch.wait.seconds</name>
+ <description>Time to wait in seconds for HBase to start</description>
+ <value>180000</value>
+ </property>
+
+
+#### Accumulo configuration options
+
+Enable/disable the tests
+
+ <property>
+ <name>slider.test.accumulo.enabled</name>
+ <description>Flag to enable/disable Accumulo tests</description>
+ <value>true</value>
+ </property>
+
+
+Optional parameters
+
+ <property>
+ <name>slider.test.accumulo.launch.wait.seconds</name>
+ <description>Time to wait in seconds for Accumulo to start</description>
+ <value>180000</value>
+ </property>
+
+
+
+### Configuring the YARN cluster for tests
+
+
+Here are the configuration options we use in `yarn-site.xml` for testing:
+
+These tell YARN to ignore memory requirements in allocating VMs, and
+to keep the log files around after an application run.
+
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>1</value>
+ </property>
+ <property>
+ <description>Whether physical memory limits will be enforced for
+ containers.
+ </description>
+ <name>yarn.nodemanager.pmem-check-enabled</name>
+ <value>false</value>
+ </property>
+ <!-- we really don't want checking here-->
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ </property>
+
+ <!-- how long after a failure to see what is left in the directory-->
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>60000</value>
+ </property>
+
+ <!--ten seconds before the process gets a -9 -->
+ <property>
+ <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
+ <value>30000</value>
+ </property>
+
+
+### Testing against a secure cluster
+
+To test against a secure cluster
+
+1. `slider-client.xml` must be configured as per [Security](security.html).
+1. the client must have the kerberos tokens issued so that the user running
+the tests has access to HDFS and YARN.
+
+If there are problems authenticating (including the cluster being offline)
+the tests appear to hang
+
+### Validating the configuration
+
+ mvn test -Dtest=TestBuildSetup
+
+### Using relative paths in test configurations
+
+When you are sharing configurations across machines via SCM or similar,
+its impossible to have absolute paths in the configuration options to
+the location of items in the local filesystem (e.g. configuration directories).
+
+There's two techniques
+
+1. Keep the data in HDFS and refer to it there. This works if there is a shared,
+persistent HDFS cluster.
+
+1. Use the special property `slider.test.conf.dir` that is set to the path
+of the directory, and which can then be used to create an absolute path
+from paths relative to the configuration dir:
+
+ <property>
+ <name>slider.test.hbase.appconf</name>
+ <description>Path to the directory containing the HBase application config
+ </description>
+ <value>file://${slider.test.conf.dir}/../hbase</value>
+ </property>
+
+
+If the actual XML file path is required, a similar property
+`slider.test.conf.xml` is set.
+
+
+## Parallel execution
+
+Attempts to run test cases in parallel failed -even with a configuration
+to run methods in a class sequentially, but separate classes independently.
+
+Even after identifying and eliminating some unintended sharing of static
+mutable variables, trying to run test cases in parallel seemed to hang
+tests and produce timeouts.
+
+For this reason parallel tests have been disabled. To accelerate test runs
+through parallelization, run different tests on different hosts instead.
+
+## Other constraints
+
+* Port assignments SHOULD NOT be fixed, as this will cause clusters to fail if
+there are too many instances of a role on a same host, or if other tests are
+using the same port.
+* If a test does need to fix a port, it MUST be for a single instance of a role,
+and it must be different from all others. The assignment should be set in
+`org.apache.slider.funtest.itest.PortAssignments` so as to ensure uniqueness
+over time. Otherwise: use the value of `0` to allow the OS to assign free ports
+on demand.
+
+## Test Requirements
+
+
+1. Test cases should be written so that each class works with exactly one
+Slider-deployed cluster
+1. Every test MUST have its own cluster name -preferably derived from the
+classname.
+1. This cluster should be deployed in an `@BeforeClass` method.
+1. The `@AfterClass` method MUST tear this cluster down.
+1. Tests must skip their execution if functional tests -or the
+specific hbase or accumulo categories- are disabled.
+1. Tests within the suite (i.e. class) must be designed to be independent
+-to work irrespectively of the ordering of other tests.
+
+## Running and debugging the functional tests.
+
+The functional tests all
+
+1. In the root `slider` directory, build a complete Slider release
+
+ mvn install -DskipTests
+1. Start the YARN cluster/set up proxies to connect to it, etc.
+
+1. In the `slider-funtest` dir, run the test
+
+ mvn test -Dtest=TestHBaseCreateCluster
+
+A common mistake during development is to rebuild the `slider-core` JARs
+then the `slider-funtest` tests without rebuilding the `slider-assembly`.
+In this situation, the tests are in sync with the latest build of the code
+-including any bug fixes- but the scripts executed by those tests are
+of a previous build of `slider-core.jar`. As a result, the fixes are not picked
+up.
+
+#### To propagate changes in slider-core through to the funtest classes for
+testing, you must build/install all the slider packages from the root assembly.
+
+ mvn clean install -DskipTests
+
+## Limitations of slider-funtest
+
+1. All tests run from a single client -workload can't scale
+1. Output from failed AM and containers aren't collected
+
+## Troubleshooting the functional tests
+
+1. If you are testing in a local VM and stops responding, it'll have been
+swapped out to RAM. Rebooting can help, but for a long term fix go through
+all the Hadoop configurations (HDFS, YARN, Zookeeper) and set their heaps to
+smaller numbers, like 256M each. Also: turn off unused services (hcat, oozie,
+webHDFS)
+
+1. The YARN UI will list the cluster launches -look for the one
+with a name close to the test and view its logs
+
+1. Container logs will appear "elsewhere". The log lists
+the containers used -you may be able to track the logs
+down from the specific nodes.
+
+1. If you browse the filesystem, look for the specific test clusters
+in `~/.slider/cluster/$testname`
+
+1. If you are using a secure cluster, make sure that the clocks
+are synchronized, and that you have a current token -`klist` will
+tell you this. In a VM: install and enable `ntp`.
diff --git a/src/site/markdown/troubleshooting.md b/src/site/markdown/troubleshooting.md
new file mode 100644
index 0000000..4e5a1ed
--- /dev/null
+++ b/src/site/markdown/troubleshooting.md
@@ -0,0 +1,154 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Troubleshooting
+
+Slider can be tricky to start using, because it combines the need to set
+up a YARN application, with the need to have an HBase configuration
+that works
+
+
+### Common problems
+
+## Classpath for Slider AM wrong
+
+The Slider Application Master, the "Slider AM" builds up its classpath from
+those JARs it has locally, and the JARS pre-installed on the classpath
+
+This often surfaces in an exception that can be summarized as
+"hadoop-common.jar is not on the classpath":
+
+ Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/util/ExitUtil$ExitException
+ Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.util.ExitUtil$ExitException
+ at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
+ at java.security.AccessController.doPrivileged(Native Method)
+ at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
+ at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
+ at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
+ at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
+ Could not find the main class: org.apache.hadoop.yarn.service.launcher.ServiceLauncher. Program will exit.
+
+
+For ambari-managed deployments, we recommend the following
+
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>
+ /etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
+ </value>
+ </property>
+
+The `yarn-site.xml` file for the site will contain the relevant value.
+
+### Application Instantiation fails, "TriggerClusterTeardownException: Unstable Cluster"
+
+Slider gives up if it cannot keep enough instances of a role running -or more
+precisely, if they keep failing.
+
+If this happens on cluster startup, it means that the application is not working
+
+ org.apache.hoya.exceptions.TriggerClusterTeardownException: Unstable Cluster:
+ - failed with role worker failing 4 times (4 in startup); threshold is 2
+ - last failure: Failure container_1386872971874_0001_01_000006 on host 192.168.1.86,
+ see http://hor12n22.gq1.ygridcore.net:19888/jobhistory/logs/192.168.1.86:45454/container_1386872971874_0001_01_000006/ctx/yarn
+
+This message warns that a role -here worker- is failing to start and it has failed
+more than the configured failure threshold is. What it doesn't do is say why it failed,
+because that is not something the AM knows -that is a fact hidden in the logs on
+the container that failed.
+
+The final bit of the exception message can help you track down the problem,
+as it points you to the logs.
+
+In the example above the failure was in `container_1386872971874_0001_01_000006`
+on the host `192.168.1.86`. If you go to then node manager on that machine (the YARN
+RM web page will let you do this), and look for that container,
+you may be able to grab the logs from it.
+
+A quicker way is to browse to the URL on the next line.
+Note: the URL depends on yarn.log.server.url being properly configured.
+
+It is from those logs that the cause of the problem -because they are the actual
+output of the actual application which Slider is trying to deploy.
+
+
+
+### Not all the containers start -but whenever you kill one, another one comes up.
+
+This is often caused by YARN not having enough capacity in the cluster to start
+up the requested set of containers. The AM has submitted a list of container
+requests to YARN, but only when an existing container is released or killed
+is one of the outstanding requests granted.
+
+Fix #1: Ask for smaller containers
+
+edit the `yarn.memory` option for roles to be smaller: set it 64 for a smaller
+YARN allocation. *This does not affect the actual heap size of the
+application component deployed*
+
+Fix #2: Tell YARN to be less strict about memory consumption
+
+Here are the properties in `yarn-site.xml` which we set to allow YARN
+to schedule more role instances than it nominally has room for.
+
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>1</value>
+ </property>
+ <property>
+ <description>Whether physical memory limits will be enforced for
+ containers.
+ </description>
+ <name>yarn.nodemanager.pmem-check-enabled</name>
+ <value>false</value>
+ </property>
+ <!-- we really don't want checking here-->
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ </property>
+
+If you create too many instances, your hosts will start swapping and
+performance will collapse -we do not recommend using this in production.
+
+
+### Configuring YARN for better debugging
+
+
+One configuration to aid debugging is tell the nodemanagers to
+keep data for a short period after containers finish
+
+ <!-- 10 minutes after a failure to see what is left in the directory-->
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>600</value>
+ </property>
+
+You can then retrieve logs by either the web UI, or by connecting to the
+server (usually by `ssh`) and retrieve the logs from the log directory
+
+
+We also recommend making sure that YARN kills processes
+
+ <!--time before the process gets a -9 -->
+ <property>
+ <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
+ <value>30000</value>
+ </property>
+
+
diff --git a/src/site/resources/hoya_am_architecture.png b/src/site/resources/hoya_am_architecture.png
new file mode 100644
index 0000000..191a8db
--- /dev/null
+++ b/src/site/resources/hoya_am_architecture.png
Binary files differ
diff --git a/src/site/site.xml b/src/site/site.xml
new file mode 100644
index 0000000..892d2b2
--- /dev/null
+++ b/src/site/site.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project name="Slider ${project.version}">
+<!--
+
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-stylus-skin</artifactId>
+ <version>1.2</version>
+ </skin>
+
+-->
+
+
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-fluido-skin</artifactId>
+ <version>1.3.0</version>
+ </skin>
+
+ <custom>
+ <fluidoSkin>
+ <topBarEnabled>true</topBarEnabled>
+ <sideBarEnabled>false</sideBarEnabled>
+ </fluidoSkin>
+ </custom>
+
+ <version position="right"/>
+
+ <body>
+ <menu ref="reports"/>
+
+ <menu name="Documents">
+ <item name="announcement" href="/announcement.html"/>
+ <item name="application Needs" href="/app_needs.html"/>
+ <item name="architecture" href="/architecture.html"/>
+ <item name="building" href="/building.html"/>
+ <item name="examples" href="/examples.html"/>
+ <item name="exitcodes" href="/exitcodes.html"/>
+ <item name="installing" href="/installing.html"/>
+ <item name="manpage" href="/manpage.html"/>
+ <item name="rolehistory" href="/rolehistory.html"/>
+ </menu>
+ </body>
+</project>
diff --git a/src/test/clusters/configs.md b/src/test/clusters/configs.md
new file mode 100644
index 0000000..3fb7adb
--- /dev/null
+++ b/src/test/clusters/configs.md
@@ -0,0 +1,20 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# test/configs
+
+This package just contains some configurations for Hadoop clusters/miniclusters
+used in some demo, test & jenkins situations, such as machine-local VM clusters.
+
+These can be generally ignored by others
\ No newline at end of file
diff --git a/src/test/clusters/local/README.md b/src/test/clusters/local/README.md
new file mode 100644
index 0000000..7107992
--- /dev/null
+++ b/src/test/clusters/local/README.md
@@ -0,0 +1,41 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+ # README
+
+ These are just some templates for a pseudo-local cluster
+
+
+ * Namenode [http://http://localhost:50070/dfshealth.jsp](http://localhost:50070/dfshealth.jsp)
+ * YARN RM [http://localhost:9081/cluster](http://localhost:9081/cluster)
+
+ # Core settings
+
+ <configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://localhost:9000</value>
+ </property>
+ </configuration>
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>localhost:8032</value>
+ </property>
+
+
+ For the hoya command line
+
+ --manager localhost:9080 --filesystem hdfs://localhost:9000 --zkhosts localhost
+
+
\ No newline at end of file
diff --git a/src/test/clusters/local/capacity-scheduler.xml b/src/test/clusters/local/capacity-scheduler.xml
new file mode 100644
index 0000000..80a9fec
--- /dev/null
+++ b/src/test/clusters/local/capacity-scheduler.xml
@@ -0,0 +1,111 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-applications</name>
+ <value>10000</value>
+ <description>
+ Maximum number of applications that can be pending and running.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+ <value>0.1</value>
+ <description>
+ Maximum percent of resources in the cluster which can be used to run
+ application masters i.e. controls number of concurrent running
+ applications.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.resource-calculator</name>
+ <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
+ <description>
+ The ResourceCalculator implementation to be used to compare
+ Resources in the scheduler.
+ The default i.e. DefaultResourceCalculator only uses Memory while
+ DominantResourceCalculator uses dominant-resource to compare
+ multi-dimensional resources such as Memory, CPU etc.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ <value>default</value>
+ <description>
+ The queues at the this level (root is the root queue).
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.capacity</name>
+ <value>100</value>
+ <description>Default queue target capacity.</description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
+ <value>1</value>
+ <description>
+ Default queue user limit a percentage from 0.0 to 1.0.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
+ <value>100</value>
+ <description>
+ The maximum capacity of the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.state</name>
+ <value>RUNNING</value>
+ <description>
+ The state of the default queue. State can be one of RUNNING or STOPPED.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
+ <value>*</value>
+ <description>
+ The ACL of who can submit jobs to the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
+ <value>*</value>
+ <description>
+ The ACL of who can administer jobs on the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.node-locality-delay</name>
+ <value>-1</value>
+ <description>
+ Number of missed scheduling opportunities after which the CapacityScheduler
+ attempts to schedule rack-local containers.
+ Typically this should be set to number of racks in the cluster, this
+ feature is disabled by default, set to -1.
+ </description>
+ </property>
+
+</configuration>
diff --git a/src/test/clusters/local/configuration.xsl b/src/test/clusters/local/configuration.xsl
new file mode 100644
index 0000000..d50d80b
--- /dev/null
+++ b/src/test/clusters/local/configuration.xsl
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="html"/>
+<xsl:template match="configuration">
+<html>
+<body>
+<table border="1">
+<tr>
+ <td>name</td>
+ <td>value</td>
+ <td>description</td>
+</tr>
+<xsl:for-each select="property">
+<tr>
+ <td><a name="{name}"><xsl:value-of select="name"/></a></td>
+ <td><xsl:value-of select="value"/></td>
+ <td><xsl:value-of select="description"/></td>
+</tr>
+</xsl:for-each>
+</table>
+</body>
+</html>
+</xsl:template>
+</xsl:stylesheet>
diff --git a/src/test/clusters/local/container-executor.cfg b/src/test/clusters/local/container-executor.cfg
new file mode 100644
index 0000000..65ba15b
--- /dev/null
+++ b/src/test/clusters/local/container-executor.cfg
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodemanager.linux-container-executor.group
+banned.users=#comma separated list of users who can not run applications
+min.user.id=1000#Prevent other super-users
diff --git a/src/test/clusters/local/core-site.xml b/src/test/clusters/local/core-site.xml
new file mode 100644
index 0000000..2ca4a77
--- /dev/null
+++ b/src/test/clusters/local/core-site.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://localhost:9000</value>
+ </property>
+ </configuration>
+</configuration>
diff --git a/src/test/clusters/local/hadoop-env.cmd b/src/test/clusters/local/hadoop-env.cmd
new file mode 100644
index 0000000..05badc2
--- /dev/null
+++ b/src/test/clusters/local/hadoop-env.cmd
@@ -0,0 +1,81 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Set Hadoop-specific environment variables here.
+
+@rem The only required environment variable is JAVA_HOME. All others are
+@rem optional. When running a distributed configuration it is best to
+@rem set JAVA_HOME in this file, so that it is correctly defined on
+@rem remote nodes.
+
+@rem The java implementation to use. Required.
+set JAVA_HOME=%JAVA_HOME%
+
+@rem The jsvc implementation to use. Jsvc is required to run secure datanodes.
+@rem set JSVC_HOME=%JSVC_HOME%
+
+@rem set HADOOP_CONF_DIR=
+
+@rem Extra Java CLASSPATH elements. Automatically insert capacity-scheduler.
+if exist %HADOOP_HOME%\contrib\capacity-scheduler (
+ if not defined HADOOP_CLASSPATH (
+ set HADOOP_CLASSPATH=%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
+ ) else (
+ set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
+ )
+)
+
+@rem The maximum amount of heap to use, in MB. Default is 1000.
+@rem set HADOOP_HEAPSIZE=
+@rem set HADOOP_NAMENODE_INIT_HEAPSIZE=""
+
+@rem Extra Java runtime options. Empty by default.
+@rem set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true
+
+@rem Command specific options appended to HADOOP_OPTS when specified
+if not defined HADOOP_SECURITY_LOGGER (
+ set HADOOP_SECURITY_LOGGER=INFO,RFAS
+)
+if not defined HDFS_AUDIT_LOGGER (
+ set HDFS_AUDIT_LOGGER=INFO,NullAppender
+)
+
+set HADOOP_NAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_NAMENODE_OPTS%
+set HADOOP_DATANODE_OPTS=-Dhadoop.security.logger=ERROR,RFAS %HADOOP_DATANODE_OPTS%
+set HADOOP_SECONDARYNAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_SECONDARYNAMENODE_OPTS%
+
+@rem The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+set HADOOP_CLIENT_OPTS=-Xmx128m %HADOOP_CLIENT_OPTS%
+@rem set HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData %HADOOP_JAVA_PLATFORM_OPTS%"
+
+@rem On secure datanodes, user to run the datanode as after dropping privileges
+set HADOOP_SECURE_DN_USER=%HADOOP_SECURE_DN_USER%
+
+@rem Where log files are stored. %HADOOP_HOME%/logs by default.
+@rem set HADOOP_LOG_DIR=%HADOOP_LOG_DIR%\%USERNAME%
+
+@rem Where log files are stored in the secure data environment.
+set HADOOP_SECURE_DN_LOG_DIR=%HADOOP_LOG_DIR%\%HADOOP_HDFS_USER%
+
+@rem The directory where pid files are stored. /tmp by default.
+@rem NOTE: this should be set to a directory that can only be written to by
+@rem the user that will run the hadoop daemons. Otherwise there is the
+@rem potential for a symlink attack.
+set HADOOP_PID_DIR=%HADOOP_PID_DIR%
+set HADOOP_SECURE_DN_PID_DIR=%HADOOP_PID_DIR%
+
+@rem A string representing this instance of hadoop. %USERNAME% by default.
+set HADOOP_IDENT_STRING=%USERNAME%
diff --git a/src/test/clusters/local/hadoop-env.sh b/src/test/clusters/local/hadoop-env.sh
new file mode 100644
index 0000000..5836a8a
--- /dev/null
+++ b/src/test/clusters/local/hadoop-env.sh
@@ -0,0 +1,77 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME. All others are
+# optional. When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use.
+export JAVA_HOME=${JAVA_HOME}
+
+# The jsvc implementation to use. Jsvc is required to run secure datanodes.
+#export JSVC_HOME=${JSVC_HOME}
+
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
+
+# Extra Java CLASSPATH elements. Automatically insert capacity-scheduler.
+for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
+ if [ "$HADOOP_CLASSPATH" ]; then
+ export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
+ else
+ export HADOOP_CLASSPATH=$f
+ fi
+done
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+#export HADOOP_HEAPSIZE=
+#export HADOOP_NAMENODE_INIT_HEAPSIZE=""
+
+# Extra Java runtime options. Empty by default.
+export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
+
+# Command specific options appended to HADOOP_OPTS when specified
+export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
+export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
+
+export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
+
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"
+#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
+
+# On secure datanodes, user to run the datanode as after dropping privileges
+export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
+
+# Where log files are stored. $HADOOP_HOME/logs by default.
+#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
+
+# Where log files are stored in the secure data environment.
+export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
+
+# The directory where pid files are stored. /tmp by default.
+# NOTE: this should be set to a directory that can only be written to by
+# the user that will run the hadoop daemons. Otherwise there is the
+# potential for a symlink attack.
+export HADOOP_PID_DIR=${HADOOP_PID_DIR}
+export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
+
+# A string representing this instance of hadoop. $USER by default.
+export HADOOP_IDENT_STRING=$USER
diff --git a/src/test/clusters/local/hadoop-metrics.properties b/src/test/clusters/local/hadoop-metrics.properties
new file mode 100644
index 0000000..bb4393b
--- /dev/null
+++ b/src/test/clusters/local/hadoop-metrics.properties
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configuration of the "dfs" context for null
+dfs.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "dfs" context for file
+#dfs.class=org.apache.hadoop.metrics.file.FileContext
+#dfs.period=10
+#dfs.fileName=/tmp/dfsmetrics.log
+
+# Configuration of the "dfs" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# dfs.period=10
+# dfs.servers=localhost:8649
+
+
+# Configuration of the "mapred" context for null
+mapred.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "mapred" context for file
+#mapred.class=org.apache.hadoop.metrics.file.FileContext
+#mapred.period=10
+#mapred.fileName=/tmp/mrmetrics.log
+
+# Configuration of the "mapred" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# mapred.period=10
+# mapred.servers=localhost:8649
+
+
+# Configuration of the "jvm" context for null
+#jvm.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "jvm" context for file
+#jvm.class=org.apache.hadoop.metrics.file.FileContext
+#jvm.period=10
+#jvm.fileName=/tmp/jvmmetrics.log
+
+# Configuration of the "jvm" context for ganglia
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# jvm.period=10
+# jvm.servers=localhost:8649
+
+# Configuration of the "rpc" context for null
+rpc.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "rpc" context for file
+#rpc.class=org.apache.hadoop.metrics.file.FileContext
+#rpc.period=10
+#rpc.fileName=/tmp/rpcmetrics.log
+
+# Configuration of the "rpc" context for ganglia
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rpc.period=10
+# rpc.servers=localhost:8649
+
+
+# Configuration of the "ugi" context for null
+ugi.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "ugi" context for file
+#ugi.class=org.apache.hadoop.metrics.file.FileContext
+#ugi.period=10
+#ugi.fileName=/tmp/ugimetrics.log
+
+# Configuration of the "ugi" context for ganglia
+# ugi.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# ugi.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# ugi.period=10
+# ugi.servers=localhost:8649
+
diff --git a/src/test/clusters/local/hadoop-metrics2.properties b/src/test/clusters/local/hadoop-metrics2.properties
new file mode 100644
index 0000000..c3ffe31
--- /dev/null
+++ b/src/test/clusters/local/hadoop-metrics2.properties
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# syntax: [prefix].[source|sink].[instance].[options]
+# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
+
+*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink
+# default sampling period, in seconds
+*.period=10
+
+# The namenode-metrics.out will contain metrics from all context
+#namenode.sink.file.filename=namenode-metrics.out
+# Specifying a special sampling period for namenode:
+#namenode.sink.*.period=8
+
+#datanode.sink.file.filename=datanode-metrics.out
+
+# the following example split metrics of different
+# context to different sinks (in this case files)
+#jobtracker.sink.file_jvm.context=jvm
+#jobtracker.sink.file_jvm.filename=jobtracker-jvm-metrics.out
+#jobtracker.sink.file_mapred.context=mapred
+#jobtracker.sink.file_mapred.filename=jobtracker-mapred-metrics.out
+
+#tasktracker.sink.file.filename=tasktracker-metrics.out
+
+#maptask.sink.file.filename=maptask-metrics.out
+
+#reducetask.sink.file.filename=reducetask-metrics.out
+
diff --git a/src/test/clusters/local/hadoop-policy.xml b/src/test/clusters/local/hadoop-policy.xml
new file mode 100644
index 0000000..491dbe7
--- /dev/null
+++ b/src/test/clusters/local/hadoop-policy.xml
@@ -0,0 +1,219 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+
+ Copyright 2011 The Apache Software Foundation
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ClientProtocol, which is used by user code
+ via the DistributedFileSystem.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.client.datanode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
+ for block recovery.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.datanode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for DatanodeProtocol, which is used by datanodes to
+ communicate with the namenode.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.inter.datanode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for InterDatanodeProtocol, the inter-datanode protocol
+ for updating generation timestamp.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.namenode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for NamenodeProtocol, the protocol used by the secondary
+ namenode to communicate with the namenode.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.operations.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for AdminOperationsProtocol. Used for admin commands.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.refresh.usertogroups.mappings.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for RefreshUserMappingsProtocol. Used to refresh
+ users mappings. The ACL is a comma-separated list of user and
+ group names. The user and group list is separated by a blank. For
+ e.g. "alice,bob users,wheel". A special value of "*" means all
+ users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.refresh.policy.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
+ dfsadmin and mradmin commands to refresh the security policy in-effect.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.ha.service.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HAService protocol used by HAAdmin to manage the
+ active and stand-by states of namenode.</description>
+ </property>
+
+ <property>
+ <name>security.zkfc.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for access to the ZK Failover Controller
+ </description>
+ </property>
+
+ <property>
+ <name>security.qjournal.service.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for QJournalProtocol, used by the NN to communicate with
+ JNs when using the QuorumJournalManager for edit logs.</description>
+ </property>
+
+ <property>
+ <name>security.mrhs.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HSClientProtocol, used by job clients to
+ communciate with the MR History Server job status etc.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <!-- YARN Protocols -->
+
+ <property>
+ <name>security.resourcetracker.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ResourceTrackerProtocol, used by the
+ ResourceManager and NodeManager to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.resourcemanager-administration.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ResourceManagerAdministrationProtocol, for admin commands.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.applicationclient.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ApplicationClientProtocol, used by the ResourceManager
+ and applications submission clients to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.applicationmaster.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ApplicationMasterProtocol, used by the ResourceManager
+ and ApplicationMasters to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.containermanagement.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ContainerManagementProtocol protocol, used by the NodeManager
+ and ApplicationMasters to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.resourcelocalizer.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ResourceLocalizer protocol, used by the NodeManager
+ and ResourceLocalizer to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.job.task.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
+ tasks to communicate with the parent tasktracker.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.job.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for MRClientProtocol, used by job clients to
+ communciate with the MR ApplicationMaster to query job status etc.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+</configuration>
diff --git a/src/test/clusters/local/hbase/hadoop-metrics.properties b/src/test/clusters/local/hbase/hadoop-metrics.properties
new file mode 100644
index 0000000..9c2e229
--- /dev/null
+++ b/src/test/clusters/local/hbase/hadoop-metrics.properties
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See http://wiki.apache.org/hadoop/GangliaMetrics
+# Make sure you know whether you are using ganglia 3.0 or 3.1.
+# If 3.1, you will have to patch your hadoop instance with HADOOP-4675
+# And, yes, this file is named hadoop-metrics.properties rather than
+# hbase-metrics.properties because we're leveraging the hadoop metrics
+# package and hadoop-metrics.properties is an hardcoded-name, at least
+# for the moment.
+#
+# See also http://hadoop.apache.org/hbase/docs/current/metrics.html
+# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia
+# meta daemon (gmetad) service is running
+
+# Configuration of the "hbase" context for NullContextWithUpdateThread
+# NullContextWithUpdateThread is a null context which has a thread calling
+# periodically when monitoring is started. This keeps the data sampled
+# correctly.
+hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+hbase.period=10
+
+# Configuration of the "hbase" context for file
+# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# hbase.fileName=/tmp/metrics_hbase.log
+
+# HBase-specific configuration to reset long-running stats (e.g. compactions)
+# If this variable is left out, then the default is no expiration.
+hbase.extendedperiod = 3600
+
+# Configuration of the "hbase" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# hbase.period=10
+# hbase.servers=GMETADHOST_IP:8649
+
+# Configuration of the "jvm" context for null
+jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+jvm.period=10
+
+# Configuration of the "jvm" context for file
+# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# jvm.fileName=/tmp/metrics_jvm.log
+
+# Configuration of the "jvm" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# jvm.period=10
+# jvm.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rpc" context for null
+rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+rpc.period=10
+
+# Configuration of the "rpc" context for file
+# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# rpc.fileName=/tmp/metrics_rpc.log
+
+# Configuration of the "rpc" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rpc.period=10
+# rpc.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rest" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rest.period=10
+# rest.servers=GMETADHOST_IP:8649
diff --git a/src/test/clusters/local/hbase/hbase-env.sh b/src/test/clusters/local/hbase/hbase-env.sh
new file mode 100644
index 0000000..da53a27
--- /dev/null
+++ b/src/test/clusters/local/hbase/hbase-env.sh
@@ -0,0 +1,106 @@
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Set environment variables here.
+
+# This script sets variables multiple times over the course of starting an hbase process,
+# so try to keep things idempotent unless you want to take an even deeper look
+# into the startup scripts (bin/hbase, etc.)
+
+# The java implementation to use. Java 1.6 required.
+# export JAVA_HOME=/usr/java/jdk1.6.0/
+
+# Extra Java CLASSPATH elements. Optional.
+# export HBASE_CLASSPATH=
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+
+# Uncomment below to enable java garbage collection logging for the server-side processes
+# this enables basic gc logging for the server processes to the .out file
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M $HBASE_GC_OPTS"
+
+# Uncomment below to enable java garbage collection logging for the client processes in the .out file.
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS).
+# This applies to both the server and client GC options above
+# export HBASE_USE_GC_LOGFILE=true
+
+
+# Uncomment below if you intend to use the EXPERIMENTAL off heap cache.
+# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize="
+# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value.
+
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers
+
+# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default.
+# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored. $HBASE_HOME/logs by default.
+# export HBASE_LOG_DIR=${HBASE_HOME}/logs
+
+# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073"
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+# export HBASE_PID_DIR=/var/hadoop/pids
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+# export HBASE_MANAGES_ZK=true
diff --git a/src/test/clusters/local/hbase/hbase-policy.xml b/src/test/clusters/local/hbase/hbase-policy.xml
new file mode 100644
index 0000000..e45f23c
--- /dev/null
+++ b/src/test/clusters/local/hbase/hbase-policy.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HRegionInterface protocol implementations (ie.
+ clients talking to HRegionServers)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterInterface protocol implementation (ie.
+ clients talking to HMaster for admin operations).
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterRegionInterface protocol implementations
+ (for HRegionServers communicating with HMaster)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+</configuration>
diff --git a/src/test/clusters/local/hbase/hbase-site.xml b/src/test/clusters/local/hbase/hbase-site.xml
new file mode 100644
index 0000000..4084d30
--- /dev/null
+++ b/src/test/clusters/local/hbase/hbase-site.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+ <!--
+ This is a template HBase site file that
+ does not include all the properties
+ required for a valid configuration -
+ the remainder are injected during
+ conversion from a template to
+ actual file
+ -->
+<configuration>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>/tmp/hbase-tmp</value>
+ </property>
+ <property>
+ <name>hbase.regionserver.hlog.tolerable.lowreplication</name>
+ <value>1</value>
+ </property>
+</configuration>
diff --git a/src/test/clusters/local/hbase/log4j.properties b/src/test/clusters/local/hbase/log4j.properties
new file mode 100644
index 0000000..8765555
--- /dev/null
+++ b/src/test/clusters/local/hbase/log4j.properties
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,DRFA
+hbase.security.logger=INFO,DRFA
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+#
+# Security audit appender
+#
+hbase.security.log.file=SecurityAuth.audit
+log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file}
+log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of .META. messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
diff --git a/src/test/clusters/local/hbase/regionservers b/src/test/clusters/local/hbase/regionservers
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/test/clusters/local/hbase/regionservers
diff --git a/src/test/clusters/local/hdfs-site.xml b/src/test/clusters/local/hdfs-site.xml
new file mode 100644
index 0000000..f6d72b9
--- /dev/null
+++ b/src/test/clusters/local/hdfs-site.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>/tmp/hdfs/namenode</value>
+ </property>
+
+
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/tmp/hdfs/datanode</value>
+ </property>
+
+
+</configuration>
\ No newline at end of file
diff --git a/src/test/clusters/local/httpfs-env.sh b/src/test/clusters/local/httpfs-env.sh
new file mode 100644
index 0000000..84c67b7
--- /dev/null
+++ b/src/test/clusters/local/httpfs-env.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# Set httpfs specific environment variables here.
+
+# Settings for the Embedded Tomcat that runs HttpFS
+# Java System properties for HttpFS should be specified in this variable
+#
+# export CATALINA_OPTS=
+
+# HttpFS logs directory
+#
+# export HTTPFS_LOG=${HTTPFS_HOME}/logs
+
+# HttpFS temporary directory
+#
+# export HTTPFS_TEMP=${HTTPFS_HOME}/temp
+
+# The HTTP port used by HttpFS
+#
+# export HTTPFS_HTTP_PORT=14000
+
+# The Admin port used by HttpFS
+#
+# export HTTPFS_ADMIN_PORT=`expr ${HTTPFS_HTTP_PORT} + 1`
+
+# The hostname HttpFS server runs on
+#
+# export HTTPFS_HTTP_HOSTNAME=`hostname -f`
diff --git a/src/test/clusters/local/httpfs-log4j.properties b/src/test/clusters/local/httpfs-log4j.properties
new file mode 100644
index 0000000..284a819
--- /dev/null
+++ b/src/test/clusters/local/httpfs-log4j.properties
@@ -0,0 +1,35 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# If the Java System property 'httpfs.log.dir' is not defined at HttpFSServer start up time
+# Setup sets its value to '${httpfs.home}/logs'
+
+log4j.appender.httpfs=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.httpfs.DatePattern='.'yyyy-MM-dd
+log4j.appender.httpfs.File=${httpfs.log.dir}/httpfs.log
+log4j.appender.httpfs.Append=true
+log4j.appender.httpfs.layout=org.apache.log4j.PatternLayout
+log4j.appender.httpfs.layout.ConversionPattern=%d{ISO8601} %5p %c{1} [%X{hostname}][%X{user}:%X{doAs}] %X{op} %m%n
+
+log4j.appender.httpfsaudit=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.httpfsaudit.DatePattern='.'yyyy-MM-dd
+log4j.appender.httpfsaudit.File=${httpfs.log.dir}/httpfs-audit.log
+log4j.appender.httpfsaudit.Append=true
+log4j.appender.httpfsaudit.layout=org.apache.log4j.PatternLayout
+log4j.appender.httpfsaudit.layout.ConversionPattern=%d{ISO8601} %5p [%X{hostname}][%X{user}:%X{doAs}] %X{op} %m%n
+
+log4j.logger.httpfsaudit=INFO, httpfsaudit
+
+log4j.logger.org.apache.hadoop.fs.http.server=INFO, httpfs
+log4j.logger.org.apache.hadoop.lib=INFO, httpfs
diff --git a/src/test/clusters/local/httpfs-signature.secret b/src/test/clusters/local/httpfs-signature.secret
new file mode 100644
index 0000000..56466e9
--- /dev/null
+++ b/src/test/clusters/local/httpfs-signature.secret
@@ -0,0 +1 @@
+hadoop httpfs secret
diff --git a/src/test/clusters/local/httpfs-site.xml b/src/test/clusters/local/httpfs-site.xml
new file mode 100644
index 0000000..4a718e1
--- /dev/null
+++ b/src/test/clusters/local/httpfs-site.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+</configuration>
diff --git a/src/test/clusters/local/log4j.properties b/src/test/clusters/local/log4j.properties
new file mode 100644
index 0000000..2ca667f
--- /dev/null
+++ b/src/test/clusters/local/log4j.properties
@@ -0,0 +1,224 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+hadoop.root.logger=INFO,console
+hadoop.log.dir=.
+hadoop.log.file=hadoop.log
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshold=ALL
+
+# Null Appender
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# Rolling File Appender - cap space usage at 5gb.
+#
+hadoop.log.maxfilesize=256MB
+hadoop.log.maxbackupindex=20
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize}
+log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex}
+
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# Daily Rolling File Appender
+#
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+#
+# TaskLog Appender
+#
+
+#Default values
+hadoop.tasklog.taskid=null
+hadoop.tasklog.iscleanup=false
+hadoop.tasklog.noKeepSplits=4
+hadoop.tasklog.totalLogFileSize=100
+hadoop.tasklog.purgeLogSplits=true
+hadoop.tasklog.logsRetainHours=12
+
+log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
+log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
+
+log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+# HDFS block state change log from block manager
+#
+# Uncomment the following to suppress normal block state change
+# messages from BlockManager in NameNode.
+#log4j.logger.BlockStateChange=WARN
+
+#
+#Security appender
+#
+hadoop.security.logger=INFO,NullAppender
+hadoop.security.log.maxfilesize=256MB
+hadoop.security.log.maxbackupindex=20
+log4j.category.SecurityLogger=${hadoop.security.logger}
+hadoop.security.log.file=SecurityAuth-${user.name}.audit
+log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
+log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
+log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
+log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
+
+#
+# Daily Rolling Security appender
+#
+log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
+log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
+
+#
+# hdfs audit logging
+#
+hdfs.audit.logger=INFO,NullAppender
+hdfs.audit.log.maxfilesize=256MB
+hdfs.audit.log.maxbackupindex=20
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
+log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
+log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender
+log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
+log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize}
+log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex}
+
+#
+# mapred audit logging
+#
+mapred.audit.logger=INFO,NullAppender
+mapred.audit.log.maxfilesize=256MB
+mapred.audit.log.maxbackupindex=20
+log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
+log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
+log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender
+log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
+log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize}
+log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
+
+# Custom Logging levels
+
+#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
+#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
+#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG
+
+# Jets3t library
+log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+
+#
+# Event Counter Appender
+# Sends counts of logging messages at different severity levels to Hadoop Metrics.
+#
+log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
+
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file :
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+hadoop.mapreduce.jobsummary.log.maxfilesize=256MB
+hadoop.mapreduce.jobsummary.log.maxbackupindex=20
+log4j.appender.JSA=org.apache.log4j.RollingFileAppender
+log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
+log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize}
+log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex}
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
+log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
+
+#
+# Yarn ResourceManager Application Summary Log
+#
+# Set the ResourceManager summary log filename
+yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log
+# Set the ResourceManager summary log level and appender
+yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
+#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
+
+# To enable AppSummaryLogging for the RM,
+# set yarn.server.resourcemanager.appsummary.logger to
+# <LEVEL>,RMSUMMARY in hadoop-env.sh
+
+# Appender for ResourceManager Application Summary Log
+# Requires the following properties to be set
+# - hadoop.log.dir (Hadoop Log directory)
+# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
+# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
+
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
+log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
+log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
+log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
+log4j.appender.RMSUMMARY.MaxFileSize=256MB
+log4j.appender.RMSUMMARY.MaxBackupIndex=20
+log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
+log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
diff --git a/src/test/clusters/local/mapred-env.cmd b/src/test/clusters/local/mapred-env.cmd
new file mode 100644
index 0000000..610d593
--- /dev/null
+++ b/src/test/clusters/local/mapred-env.cmd
@@ -0,0 +1,20 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+set HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
+
+set HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+
diff --git a/src/test/clusters/local/mapred-env.sh b/src/test/clusters/local/mapred-env.sh
new file mode 100644
index 0000000..6be1e27
--- /dev/null
+++ b/src/test/clusters/local/mapred-env.sh
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+
+export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
+
+export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+
+#export HADOOP_JOB_HISTORYSERVER_OPTS=
+#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.
+#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.
+#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.
+#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default
+#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.
diff --git a/src/test/clusters/local/mapred-queues.xml.template b/src/test/clusters/local/mapred-queues.xml.template
new file mode 100644
index 0000000..ce6cd20
--- /dev/null
+++ b/src/test/clusters/local/mapred-queues.xml.template
@@ -0,0 +1,92 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- This is the template for queue configuration. The format supports nesting of
+ queues within queues - a feature called hierarchical queues. All queues are
+ defined within the 'queues' tag which is the top level element for this
+ XML document. The queue acls configured here for different queues are
+ checked for authorization only if the configuration property
+ mapreduce.cluster.acls.enabled is set to true. -->
+<queues>
+
+ <!-- Configuration for a queue is specified by defining a 'queue' element. -->
+ <queue>
+
+ <!-- Name of a queue. Queue name cannot contain a ':' -->
+ <name>default</name>
+
+ <!-- properties for a queue, typically used by schedulers,
+ can be defined here -->
+ <properties>
+ </properties>
+
+ <!-- State of the queue. If running, the queue will accept new jobs.
+ If stopped, the queue will not accept new jobs. -->
+ <state>running</state>
+
+ <!-- Specifies the ACLs to check for submitting jobs to this queue.
+ If set to '*', it allows all users to submit jobs to the queue.
+ If set to ' '(i.e. space), no user will be allowed to do this
+ operation. The default value for any queue acl is ' '.
+ For specifying a list of users and groups the format to use is
+ user1,user2 group1,group2
+
+ It is only used if authorization is enabled in Map/Reduce by setting
+ the configuration property mapreduce.cluster.acls.enabled to true.
+
+ Irrespective of this ACL configuration, the user who started the
+ cluster and cluster administrators configured via
+ mapreduce.cluster.administrators can do this operation. -->
+ <acl-submit-job> </acl-submit-job>
+
+ <!-- Specifies the ACLs to check for viewing and modifying jobs in this
+ queue. Modifications include killing jobs, tasks of jobs or changing
+ priorities.
+ If set to '*', it allows all users to view, modify jobs of the queue.
+ If set to ' '(i.e. space), no user will be allowed to do this
+ operation.
+ For specifying a list of users and groups the format to use is
+ user1,user2 group1,group2
+
+ It is only used if authorization is enabled in Map/Reduce by setting
+ the configuration property mapreduce.cluster.acls.enabled to true.
+
+ Irrespective of this ACL configuration, the user who started the
+ cluster and cluster administrators configured via
+ mapreduce.cluster.administrators can do the above operations on all
+ the jobs in all the queues. The job owner can do all the above
+ operations on his/her job irrespective of this ACL configuration. -->
+ <acl-administer-jobs> </acl-administer-jobs>
+ </queue>
+
+ <!-- Here is a sample of a hierarchical queue configuration
+ where q2 is a child of q1. In this example, q2 is a leaf level
+ queue as it has no queues configured within it. Currently, ACLs
+ and state are only supported for the leaf level queues.
+ Note also the usage of properties for the queue q2.
+ <queue>
+ <name>q1</name>
+ <queue>
+ <name>q2</name>
+ <properties>
+ <property key="capacity" value="20"/>
+ <property key="user-limit" value="30"/>
+ </properties>
+ </queue>
+ </queue>
+ -->
+</queues>
diff --git a/src/test/clusters/local/mapred-site.xml b/src/test/clusters/local/mapred-site.xml
new file mode 100644
index 0000000..1be6975
--- /dev/null
+++ b/src/test/clusters/local/mapred-site.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>mapred.job.tracker</name>
+ <value>localhost:9001</value>
+ </property>
+</configuration>
diff --git a/src/test/clusters/local/slaves b/src/test/clusters/local/slaves
new file mode 100644
index 0000000..2fbb50c
--- /dev/null
+++ b/src/test/clusters/local/slaves
@@ -0,0 +1 @@
+localhost
diff --git a/src/test/clusters/local/ssl-client.xml.example b/src/test/clusters/local/ssl-client.xml.example
new file mode 100644
index 0000000..a50dce4
--- /dev/null
+++ b/src/test/clusters/local/ssl-client.xml.example
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+<property>
+ <name>ssl.client.truststore.location</name>
+ <value></value>
+ <description>Truststore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.password</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.location</name>
+ <value></value>
+ <description>Keystore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.password</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.keypassword</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+</configuration>
diff --git a/src/test/clusters/local/ssl-server.xml.example b/src/test/clusters/local/ssl-server.xml.example
new file mode 100644
index 0000000..4b363ff
--- /dev/null
+++ b/src/test/clusters/local/ssl-server.xml.example
@@ -0,0 +1,77 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+<property>
+ <name>ssl.server.truststore.location</name>
+ <value></value>
+ <description>Truststore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.password</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+</property>
+
+<property>
+ <name>ssl.server.keystore.location</name>
+ <value></value>
+ <description>Keystore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.password</name>
+ <value></value>
+ <description>Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.keypassword</name>
+ <value></value>
+ <description>Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+</configuration>
diff --git a/src/test/clusters/local/yarn-env.cmd b/src/test/clusters/local/yarn-env.cmd
new file mode 100644
index 0000000..3329f8f
--- /dev/null
+++ b/src/test/clusters/local/yarn-env.cmd
@@ -0,0 +1,60 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem User for YARN daemons
+if not defined HADOOP_YARN_USER (
+ set HADOOP_YARN_USER=%yarn%
+)
+
+if not defined YARN_CONF_DIR (
+ set YARN_CONF_DIR=%HADOOP_YARN_HOME%\conf
+)
+
+if defined YARN_HEAPSIZE (
+ @rem echo run with Java heapsize %YARN_HEAPSIZE%
+ set JAVA_HEAP_MAX=-Xmx%YARN_HEAPSIZE%m
+)
+
+if not defined YARN_LOG_DIR (
+ set YARN_LOG_DIR=%HADOOP_YARN_HOME%\logs
+)
+
+if not defined YARN_LOGFILE (
+ set YARN_LOGFILE=yarn.log
+)
+
+@rem default policy file for service-level authorization
+if not defined YARN_POLICYFILE (
+ set YARN_POLICYFILE=hadoop-policy.xml
+)
+
+if not defined YARN_ROOT_LOGGER (
+ set YARN_ROOT_LOGGER=INFO,console
+)
+
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.dir=%YARN_LOG_DIR%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.log.dir=%YARN_LOG_DIR%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.file=%YARN_LOGFILE%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.log.file=%YARN_LOGFILE%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.home.dir=%HADOOP_YARN_HOME%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.id.str=%YARN_IDENT_STRING%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.home.dir=%HADOOP_YARN_HOME%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.root.logger=%YARN_ROOT_LOGGER%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.root.logger=%YARN_ROOT_LOGGER%
+if defined JAVA_LIBRARY_PATH (
+ set YARN_OPTS=%YARN_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
+)
+set YARN_OPTS=%YARN_OPTS% -Dyarn.policy.file=%YARN_POLICYFILE%
\ No newline at end of file
diff --git a/src/test/clusters/local/yarn-env.sh b/src/test/clusters/local/yarn-env.sh
new file mode 100644
index 0000000..cfce28d
--- /dev/null
+++ b/src/test/clusters/local/yarn-env.sh
@@ -0,0 +1,112 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# User for YARN daemons
+export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+
+# resolve links - $0 may be a softlink
+export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+
+# some Java parameters
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+if [ "$JAVA_HOME" != "" ]; then
+ #echo "run java in $JAVA_HOME"
+ JAVA_HOME=$JAVA_HOME
+fi
+
+if [ "$JAVA_HOME" = "" ]; then
+ echo "Error: JAVA_HOME is not set."
+ exit 1
+fi
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m
+
+# For setting YARN specific HEAP sizes please use this
+# Parameter and set appropriately
+# YARN_HEAPSIZE=1000
+
+# check envvars which might override default args
+if [ "$YARN_HEAPSIZE" != "" ]; then
+ JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+fi
+
+# Resource Manager specific parameters
+
+# Specify the max Heapsize for the ResourceManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_RESOURCEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#export YARN_RESOURCEMANAGER_HEAPSIZE=1000
+
+# Specify the JVM options to be used when starting the ResourceManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_RESOURCEMANAGER_OPTS=
+
+# Node Manager specific parameters
+
+# Specify the max Heapsize for the NodeManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_NODEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#export YARN_NODEMANAGER_HEAPSIZE=1000
+
+# Specify the JVM options to be used when starting the NodeManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_NODEMANAGER_OPTS=
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+
+# default log directory & file
+if [ "$YARN_LOG_DIR" = "" ]; then
+ YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
+fi
+if [ "$YARN_LOGFILE" = "" ]; then
+ YARN_LOGFILE='yarn.log'
+fi
+
+# default policy file for service-level authorization
+if [ "$YARN_POLICYFILE" = "" ]; then
+ YARN_POLICYFILE="hadoop-policy.xml"
+fi
+
+# restore ordinary behaviour
+unset IFS
+
+
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
+YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
+YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+ YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi
+YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+
+
diff --git a/src/test/clusters/local/yarn-site.xml b/src/test/clusters/local/yarn-site.xml
new file mode 100644
index 0000000..553b632
--- /dev/null
+++ b/src/test/clusters/local/yarn-site.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+
+<!-- Site specific YARN configuration properties -->
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>localhost:8032</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.webapp.address</name>
+ <value>localhost:9081</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>localhost:8031</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>16</value>
+ </property>
+</configuration>
diff --git a/src/test/clusters/offline/accumulo/accumulo-env.sh b/src/test/clusters/offline/accumulo/accumulo-env.sh
new file mode 100755
index 0000000..11e5ccc
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/accumulo-env.sh
@@ -0,0 +1,56 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###
+### Configure these environment variables to point to your local installations.
+###
+### The functional tests require conditional values, so keep this style:
+###
+### test -z "$JAVA_HOME" && export JAVA_HOME=/usr/local/lib/jdk-1.6.0
+###
+###
+### Note that the -Xmx -Xms settings below require substantial free memory:
+### you may want to use smaller values, especially when running everything
+### on a single machine.
+###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
+
+test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
+test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
+test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
+then
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
+fi
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m "
+test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
+test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx64m -Xms64m"
+test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
+test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Dorg.apache.hoya.container=${CONTAINER_ID}"
+test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx128m -Xms64m"
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/monitor ; echo localhost ) 2>/dev/null | head -1`
+# what do when the JVM runs out of heap memory
+export ACCUMULO_KILL_CMD='kill -9 %p'
diff --git a/src/test/clusters/offline/accumulo/accumulo-metrics.xml b/src/test/clusters/offline/accumulo/accumulo-metrics.xml
new file mode 100644
index 0000000..60f9f8d
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/accumulo-metrics.xml
@@ -0,0 +1,60 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ This file follows the conventions for XMLConfiguration files specified in the Apache Commons Configuration 1.5 Library. Changes to this file will be noticed
+ at runtime (see the FileChangedReloadingStrategy class in Commons Configuration).
+-->
+<config>
+<!--
+ Metrics log directory
+-->
+ <logging>
+ <dir>${ACCUMULO_HOME}/metrics</dir>
+ </logging>
+<!--
+ Enable/Disable metrics accumulation on the different servers and their components
+ NOTE: Turning on logging can be expensive because it will use several more file handles and will create a lot of short lived objects.
+-->
+ <master>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </master>
+ <logger>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </logger>
+ <tserver>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ <update>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </update>
+ <scan>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </scan>
+ <minc>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </minc>
+ </tserver>
+ <thrift>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </thrift>
+</config>
diff --git a/src/test/clusters/offline/accumulo/accumulo-site.xml b/src/test/clusters/offline/accumulo/accumulo-site.xml
new file mode 100644
index 0000000..72e68a1
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/accumulo-site.xml
@@ -0,0 +1,143 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- Put your site-specific accumulo configurations here. The available configuration values along with their defaults are documented in docs/config.html Unless
+ you are simply testing at your workstation, you will most definitely need to change the three entries below. -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The property only needs to be set if upgrading from 1.4 which used to store write-ahead logs on the local
+ filesystem. In 1.5 write-ahead logs are stored in DFS. When 1.5 is started for the first time it will copy any 1.4
+ write ahead logs into DFS. It is possible to specify a comma-separated list of directories.
+ </description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers
+ must know in order to communicate with one another.
+ Change it before initialization. To
+ change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.native.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.token.property.password</name>
+ <!-- change this to the root user's password, and/or change the user below -->
+ <value>secret</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <!--
+ Add the following for Hadoop2, actual needs depend on Hadoop installation details.
+ This list may be excessive, but this should cause no issues. Append these values
+ after the $HADOOP_PREFIX entries
+
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+ /usr/lib/hadoop/.*.jar,
+ /usr/lib/hadoop/lib/.*.jar,
+ /usr/lib/hadoop-hdfs/.*.jar,
+ /usr/lib/hadoop-mapreduce/.*.jar,
+ /usr/lib/hadoop-yarn/.*.jar,
+ -->
+ <value>
+ $ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-server.jar,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-core.jar,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-start.jar,
+ $ACCUMULO_HOME/fate/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-fate.jar,
+ $ACCUMULO_HOME/proxy/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-proxy.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+ /usr/lib/hadoop/.*.jar,
+ /usr/lib/hadoop/lib/.*.jar,
+ /usr/lib/hadoop-hdfs/.*.jar,
+ /usr/lib/hadoop-mapreduce/.*.jar,
+ /usr/lib/hadoop-yarn/.*.jar,
+ /etc/hadoop/conf
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.
+ When using the Security Manager, please remove the ".../target/classes/"
+ values.
+ </description>
+ </property>
+</configuration>
diff --git a/src/test/clusters/offline/accumulo/accumulo.policy.example b/src/test/clusters/offline/accumulo/accumulo.policy.example
new file mode 100644
index 0000000..2964f06
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/accumulo.policy.example
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grant codeBase "file:${java.home}/lib/ext/*" {
+ permission java.security.AllPermission;
+};
+
+// These should all be empty in a fielded system
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/server/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/core/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/start/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/examples/target/classes/" {
+ permission java.security.AllPermission;
+};
+
+grant codebase "file:${hadoop.home.dir}/*" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "shutdownHooks"; // hadoop libs use executables to discover usernames, groups, etc.
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.io.FilePermission "<<ALL FILES>>", "read, execute";
+ permission java.io.FilePermission "/tmp", "write, delete";
+ permission java.io.FilePermission "/tmp/-", "write, delete";
+ permission java.io.FilePermission "/", "write";
+ permission java.net.SocketPermission "*", "connect, resolve";
+ permission java.util.PropertyPermission "java.library.path", "read";
+ permission java.util.PropertyPermission "user.dir", "read";
+ permission java.util.PropertyPermission "org.apache.commons.logging.*", "read";
+ permission java.util.PropertyPermission "entityExpansionLimit", "read";
+ permission java.util.PropertyPermission "maxOccurLimit", "read";
+ permission java.util.PropertyPermission "os.name", "read";
+};
+
+grant codebase "file:${hadoop.home.dir}/lib/*" {
+ // monitor's jetty web service
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ // need to accept web requests, and talk to job tracker, name node, etc.
+ permission java.net.SocketPermission "*", "accept, listen, resolve, connect, resolve";
+ permission java.lang.RuntimePermission "getenv.*";
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.util.PropertyPermission "org.mortbay.*", "read";
+ permission java.util.PropertyPermission "VERBOSE", "read";
+ permission java.util.PropertyPermission "IGNORED", "read";
+ permission java.util.PropertyPermission "ISO_8859_1", "read";
+ permission java.util.PropertyPermission "org.apache.commons.logging.*", "read";
+ permission java.util.PropertyPermission "accumulo.*", "read";
+ permission java.util.PropertyPermission "org.jfree.*", "read";
+ permission java.util.PropertyPermission "elementAttributeLimit", "read";
+ permission java.util.PropertyPermission "entityExpansionLimit", "read";
+ permission java.util.PropertyPermission "maxOccurLimit", "read";
+ // some resources come out of accumulo jars
+ permission java.lang.RuntimePermission "getClassLoader";
+ permission java.io.FilePermission "${org.apache.accumulo.core.home.dir}/lib/*", "read";
+ permission java.io.FilePermission "${org.apache.accumulo.core.home.dir}/src/-", "read";
+ permission java.io.FilePermission "${hadoop.home.dir}/lib/*", "read";
+ // images are cached in /tmp
+ permission java.io.FilePermission "/tmp/*", "read, write";
+ permission java.io.FilePermission "/", "write";
+};
+
+grant codebase "file:${zookeeper.home.dir}/*" {
+ permission java.net.SocketPermission "*", "connect, resolve";
+ permission java.util.PropertyPermission "user.*", "read";
+ permission java.util.PropertyPermission "java.*", "read";
+ permission java.util.PropertyPermission "zookeeper.*", "read";
+ permission java.util.PropertyPermission "jute.*", "read";
+ permission java.util.PropertyPermission "os.*", "read";
+ // accumulo properties read in callbacks
+ permission java.util.PropertyPermission "accumulo.*", "read";
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "exitVM";
+};
+
+grant codebase "file:${org.apache.accumulo.core.home.dir}/lib/ext/*" {
+};
+
+grant codebase "file:${org.apache.accumulo.core.home.dir}/lib/*" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ // logging, configuration and getting user id
+ permission java.io.FilePermission "<<ALL FILES>>", "read, write, execute, delete";
+ permission java.util.PropertyPermission "*", "read, write";
+ permission java.lang.RuntimePermission "getenv.*";
+ permission java.lang.RuntimePermission "getClassLoader";
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.lang.RuntimePermission "accessDeclaredMembers";
+ permission java.lang.RuntimePermission "selectorProvider";
+ permission java.lang.RuntimePermission "accessClassInPackage.*";
+ permission java.lang.RuntimePermission "readFileDescriptor";
+ permission java.lang.RuntimePermission "writeFileDescriptor";
+ permission java.lang.RuntimePermission "modifyThread";
+ permission java.lang.RuntimePermission "modifyThreadGroup";
+ permission java.lang.RuntimePermission "createClassLoader";
+ permission java.lang.RuntimePermission "setContextClassLoader";
+ permission java.lang.RuntimePermission "exitVM";
+ permission java.lang.RuntimePermission "shutdownHooks";
+ permission java.security.SecurityPermission "getPolicy";
+ permission java.security.SecurityPermission "getProperty.*";
+ permission java.security.SecurityPermission "putProviderProperty.*";
+ permission java.security.SecurityPermission "setSystemScope";
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.util.logging.LoggingPermission "control";
+ permission java.net.NetPermission "getProxySelector";
+ permission javax.management.MBeanServerPermission "createMBeanServer";
+ permission javax.management.MBeanTrustPermission "register";
+ permission javax.management.MBeanPermission "*", "registerMBean";
+ permission java.net.SocketPermission "*", "accept, connect, listen, resolve";
+};
diff --git a/src/test/clusters/offline/accumulo/auditLog.xml b/src/test/clusters/offline/accumulo/auditLog.xml
new file mode 100644
index 0000000..0d81dbe
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/auditLog.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+
+
+ <!-- Write out Audit info to an Audit file -->
+ <appender name="Audit" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.audit"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="DatePattern" value="'.'yyyy-MM-dd"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss,SSS/Z} [%c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+ <logger name="Audit" additivity="false">
+ <appender-ref ref="Audit" />
+ <level value="OFF"/>
+ </logger>
+
+
+
+
+
+</log4j:configuration>
diff --git a/src/test/clusters/offline/accumulo/gc b/src/test/clusters/offline/accumulo/gc
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/gc
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/offline/accumulo/generic_logger.xml b/src/test/clusters/offline/accumulo/generic_logger.xml
new file mode 100644
index 0000000..9fac0bc
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/generic_logger.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.debug.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Send all logging data to a centralized logger -->
+ <appender name="N1" class="org.apache.log4j.net.SocketAppender">
+ <param name="remoteHost" value="${org.apache.accumulo.core.host.log}"/>
+ <param name="port" value="4560"/>
+ <param name="application" value="${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname}"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- If the centralized logger is down, buffer the log events, but drop them if it stays down -->
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="N1" />
+ </appender>
+
+ <!-- Log accumulo events to the debug, normal and remote logs. -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="ASYNC" />
+ </logger>
+
+ <logger name="org.apache.accumulo.core.file.rfile.bcfile">
+ <level value="INFO"/>
+ </logger>
+
+ <logger name="org.mortbay.log">
+ <level value="WARN"/>
+ </logger>
+
+ <logger name="org.apache.zookeeper">
+ <level value="ERROR"/>
+ </logger>
+
+ <!-- Log non-accumulo events to the debug and normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
diff --git a/src/test/clusters/offline/accumulo/log4j.properties b/src/test/clusters/offline/accumulo/log4j.properties
new file mode 100644
index 0000000..a4bcb2e
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/log4j.properties
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# default logging properties:
+# by default, log everything at INFO or higher to the console
+log4j.rootLogger=INFO,A1
+
+# hide Jetty junk
+log4j.logger.org.mortbay.log=WARN,A1
+
+# hide "Got brand-new compresssor" messages
+log4j.logger.org.apache.hadoop.io.compress=WARN,A1
+
+# hide junk from TestRandomDeletes
+log4j.logger.org.apache.accumulo.test.TestRandomDeletes=WARN,A1
+
+# hide junk from VFS
+log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN,A1
+
+# hide almost everything from zookeeper
+log4j.logger.org.apache.zookeeper=ERROR,A1
+
+# hide AUDIT messages in the shell, alternatively you could send them to a different logger
+log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=WARN,A1
+
+# Send most things to the console
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.appender.A1.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
diff --git a/src/test/clusters/offline/accumulo/masters b/src/test/clusters/offline/accumulo/masters
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/masters
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/offline/accumulo/monitor b/src/test/clusters/offline/accumulo/monitor
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/monitor
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/offline/accumulo/monitor_logger.xml b/src/test/clusters/offline/accumulo/monitor_logger.xml
new file mode 100644
index 0000000..b344c23
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/monitor_logger.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.debug.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Keep the last few log messages for display to the user -->
+ <appender name="GUI" class="org.apache.accumulo.server.monitor.LogService">
+ <param name="keep" value="40"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- Log accumulo messages to debug, normal and GUI -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="GUI" />
+ </logger>
+
+ <!-- Log non-accumulo messages to debug, normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
diff --git a/src/test/clusters/offline/accumulo/slaves b/src/test/clusters/offline/accumulo/slaves
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/slaves
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/offline/accumulo/tracers b/src/test/clusters/offline/accumulo/tracers
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/offline/accumulo/tracers
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/offline/hbase/hadoop-metrics.properties b/src/test/clusters/offline/hbase/hadoop-metrics.properties
new file mode 100644
index 0000000..e31c059
--- /dev/null
+++ b/src/test/clusters/offline/hbase/hadoop-metrics.properties
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See http://wiki.apache.org/hadoop/GangliaMetrics
+# Make sure you know whether you are using ganglia 3.0 or 3.1.
+# If 3.1, you will have to patch your hadoop instance with HADOOP-4675
+# And, yes, this file is named hadoop-metrics.properties rather than
+# hbase-metrics.properties because we're leveraging the hadoop metrics
+# package and hadoop-metrics.properties is an hardcoded-name, at least
+# for the moment.
+#
+# See also http://hadoop.apache.org/hbase/docs/current/metrics.html
+# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia
+# meta daemon (gmetad) service is running
+
+# Configuration of the "hbase" context for NullContextWithUpdateThread
+# NullContextWithUpdateThread is a null context which has a thread calling
+# periodically when monitoring is started. This keeps the data sampled
+# correctly.
+hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+hbase.period=10
+
+# Configuration of the "hbase" context for file
+# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# hbase.fileName=/tmp/metrics_hbase.log
+
+# HBase-specific configuration to reset long-running stats (e.g. compactions)
+# If this variable is left out, then the default is no expiration.
+hbase.extendedperiod = 3600
+
+# Configuration of the "hbase" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# hbase.period=10
+# hbase.servers=GMETADHOST_IP:8649
+
+# Configuration of the "jvm" context for null
+jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+jvm.period=10
+
+# Configuration of the "jvm" context for file
+# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# jvm.fileName=/tmp/metrics_jvm.log
+
+# Configuration of the "jvm" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# jvm.period=10
+# jvm.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rpc" context for null
+rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+rpc.period=10
+
+# Configuration of the "rpc" context for file
+# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# rpc.fileName=/tmp/metrics_rpc.log
+
+# Configuration of the "rpc" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rpc.period=10
+# rpc.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rest" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rest.period=10
+# rest.servers=GMETADHOST_IP:8649
diff --git a/src/test/clusters/offline/hbase/hbase-env.sh b/src/test/clusters/offline/hbase/hbase-env.sh
new file mode 100644
index 0000000..da53a27
--- /dev/null
+++ b/src/test/clusters/offline/hbase/hbase-env.sh
@@ -0,0 +1,106 @@
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Set environment variables here.
+
+# This script sets variables multiple times over the course of starting an hbase process,
+# so try to keep things idempotent unless you want to take an even deeper look
+# into the startup scripts (bin/hbase, etc.)
+
+# The java implementation to use. Java 1.6 required.
+# export JAVA_HOME=/usr/java/jdk1.6.0/
+
+# Extra Java CLASSPATH elements. Optional.
+# export HBASE_CLASSPATH=
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+
+# Uncomment below to enable java garbage collection logging for the server-side processes
+# this enables basic gc logging for the server processes to the .out file
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M $HBASE_GC_OPTS"
+
+# Uncomment below to enable java garbage collection logging for the client processes in the .out file.
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS).
+# This applies to both the server and client GC options above
+# export HBASE_USE_GC_LOGFILE=true
+
+
+# Uncomment below if you intend to use the EXPERIMENTAL off heap cache.
+# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize="
+# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value.
+
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers
+
+# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default.
+# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored. $HBASE_HOME/logs by default.
+# export HBASE_LOG_DIR=${HBASE_HOME}/logs
+
+# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073"
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+# export HBASE_PID_DIR=/var/hadoop/pids
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+# export HBASE_MANAGES_ZK=true
diff --git a/src/test/clusters/offline/hbase/hbase-policy.xml b/src/test/clusters/offline/hbase/hbase-policy.xml
new file mode 100644
index 0000000..e45f23c
--- /dev/null
+++ b/src/test/clusters/offline/hbase/hbase-policy.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HRegionInterface protocol implementations (ie.
+ clients talking to HRegionServers)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterInterface protocol implementation (ie.
+ clients talking to HMaster for admin operations).
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterRegionInterface protocol implementations
+ (for HRegionServers communicating with HMaster)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+</configuration>
diff --git a/src/test/clusters/offline/hbase/hbase-site.xml b/src/test/clusters/offline/hbase/hbase-site.xml
new file mode 100644
index 0000000..1e58691
--- /dev/null
+++ b/src/test/clusters/offline/hbase/hbase-site.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+ <!--
+ This is a template HBase site file that
+ does not include all the properties
+ required for a valid configuration -
+ the remainder are injected during
+ conversion from a template to
+ actual file
+ -->
+<configuration>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>./hbase-tmp</value>
+ </property>
+ <property>
+ <name>hbase.regionserver.hlog.tolerable.lowreplication</name>
+ <value>1</value>
+
+ </property>
+ <!--just here to verify propagation of properties all the way to the AM-->
+ <property>
+ <name>hoya.unused.option</name>
+ <value>1</value>
+ </property>
+
+</configuration>
diff --git a/src/test/clusters/offline/hbase/log4j.properties b/src/test/clusters/offline/hbase/log4j.properties
new file mode 100644
index 0000000..81233d4
--- /dev/null
+++ b/src/test/clusters/offline/hbase/log4j.properties
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.security.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+#
+# Security audit appender
+#
+hbase.security.log.file=SecurityAuth.audit
+log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file}
+log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of .META. messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
diff --git a/src/test/clusters/offline/hbase/regionservers b/src/test/clusters/offline/hbase/regionservers
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/test/clusters/offline/hbase/regionservers
diff --git a/src/test/clusters/offline/readme.md b/src/test/clusters/offline/readme.md
new file mode 100644
index 0000000..23d9d80
--- /dev/null
+++ b/src/test/clusters/offline/readme.md
@@ -0,0 +1,22 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Offline configuration
+
+This is a configuration for *not* running the functional tests,
+apart from a few that work with the `bin/slider`binary to verify
+that the assembly and scripts are valid.
+
+
+
\ No newline at end of file
diff --git a/src/test/clusters/offline/slider/log4j.properties b/src/test/clusters/offline/slider/log4j.properties
new file mode 100644
index 0000000..6211771
--- /dev/null
+++ b/src/test/clusters/offline/slider/log4j.properties
@@ -0,0 +1,83 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+log4j.logger.org.apache.slider=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+
+
diff --git a/src/test/clusters/offline/slider/slider-client.xml b/src/test/clusters/offline/slider/slider-client.xml
new file mode 100644
index 0000000..4f39945
--- /dev/null
+++ b/src/test/clusters/offline/slider/slider-client.xml
@@ -0,0 +1,119 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License. See accompanying LICENSE file.
+ -->
+<!--
+ Properties set here are picked up in the client.
+ They are not passed to the AM
+-->
+<configuration>
+ <property>
+ <name>slider.client.resource.origin</name>
+ <value>configs/sandbox/hoya</value>
+ <description>This is just for diagnostics</description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>sandbox.hortonworks.com:8050</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>file:///</value>
+ </property>
+
+ <property>
+ <name>hoya.funtest.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>slider.security.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>
+ /etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
+ </value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <description>Path to the HBase Tar file in HDFS</description>
+ <value></value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.appconf</name>
+ <description>Path to the directory containing the HBase application config</description>
+ <value></value>
+ </property>
+
+ <property>
+ <name>slider.test.zkhosts</name>
+ <description>list of the zookeeper hosts</description>
+ <value></value>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.enabled</name>
+ <description>Flag to enable/disable Accumulo tests</description>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.enabled</name>
+ <description>Flag to enable/disable HBase tests</description>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.tar</name>
+ <description>Path to the Accumulo Tar file in HDFS</description>
+ <value></value>
+ </property>
+
+
+ <property>
+ <name>slider.test.am.restart.time</name>
+ <description>Time in millis to await an AM restart</description>
+ <value></value>
+ </property>
+
+
+ <property>
+ <name>slider.test.accumulo.appconf</name>
+ <description>Path to the directory containing the Accumulo application
+ config
+ </description>
+ <value>file://${user.dir}/src/test/configs/sandbox/accumulo</value>
+ </property>
+
+
+ <property>
+ <name>zk.home</name>
+ <value>/usr/lib/zookeeper</value>
+ <description>Zookeeper home dir on target systems</description>
+ </property>
+
+ <property>
+ <name>hadoop.home</name>
+ <value>/usr/lib/hadoop</value>
+ <description>Hadoop home dir on target systems</description>
+ </property>
+
+</configuration>
diff --git a/src/test/clusters/sandbox/README.md b/src/test/clusters/sandbox/README.md
new file mode 100644
index 0000000..34e3342
--- /dev/null
+++ b/src/test/clusters/sandbox/README.md
@@ -0,0 +1,49 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+ # README
+
+This is a set of configurations for a single-node YARN cluster built
+from the hortonworks sandbox, *with a DNS entry "sandbox" to match*
+
+
+
+ * Namenode [http:///ubuntu:50070/dfshealth.jsp](http://ubuntu:50070/dfshealth.jsp)
+ * YARN RM [http://ubuntu:9081/cluster](http://ubuntu:9081/cluster)
+
+ # Core settings
+
+ <configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://sandbox:9090</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>sandbox:8032</value>
+ </property>
+
+ <property>
+ <name>slider.zookeeper.quorum</name>
+ <value>sandbox:2181</value>
+ </property>
+
+ </configuration>
+
+ For the hoya command line
+
+ --manager sandbox:8032 --filesystem hdfs://sandbox:9090
+
+
\ No newline at end of file
diff --git a/src/test/clusters/sandbox/accumulo/accumulo-env.sh b/src/test/clusters/sandbox/accumulo/accumulo-env.sh
new file mode 100755
index 0000000..11e5ccc
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/accumulo-env.sh
@@ -0,0 +1,56 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###
+### Configure these environment variables to point to your local installations.
+###
+### The functional tests require conditional values, so keep this style:
+###
+### test -z "$JAVA_HOME" && export JAVA_HOME=/usr/local/lib/jdk-1.6.0
+###
+###
+### Note that the -Xmx -Xms settings below require substantial free memory:
+### you may want to use smaller values, especially when running everything
+### on a single machine.
+###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
+
+test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
+test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
+test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
+then
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
+fi
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m "
+test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
+test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx64m -Xms64m"
+test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
+test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Dorg.apache.hoya.container=${CONTAINER_ID}"
+test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx128m -Xms64m"
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/monitor ; echo localhost ) 2>/dev/null | head -1`
+# what do when the JVM runs out of heap memory
+export ACCUMULO_KILL_CMD='kill -9 %p'
diff --git a/src/test/clusters/sandbox/accumulo/accumulo-metrics.xml b/src/test/clusters/sandbox/accumulo/accumulo-metrics.xml
new file mode 100644
index 0000000..60f9f8d
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/accumulo-metrics.xml
@@ -0,0 +1,60 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ This file follows the conventions for XMLConfiguration files specified in the Apache Commons Configuration 1.5 Library. Changes to this file will be noticed
+ at runtime (see the FileChangedReloadingStrategy class in Commons Configuration).
+-->
+<config>
+<!--
+ Metrics log directory
+-->
+ <logging>
+ <dir>${ACCUMULO_HOME}/metrics</dir>
+ </logging>
+<!--
+ Enable/Disable metrics accumulation on the different servers and their components
+ NOTE: Turning on logging can be expensive because it will use several more file handles and will create a lot of short lived objects.
+-->
+ <master>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </master>
+ <logger>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </logger>
+ <tserver>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ <update>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </update>
+ <scan>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </scan>
+ <minc>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </minc>
+ </tserver>
+ <thrift>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </thrift>
+</config>
diff --git a/src/test/clusters/sandbox/accumulo/accumulo-site.xml b/src/test/clusters/sandbox/accumulo/accumulo-site.xml
new file mode 100644
index 0000000..72e68a1
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/accumulo-site.xml
@@ -0,0 +1,143 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- Put your site-specific accumulo configurations here. The available configuration values along with their defaults are documented in docs/config.html Unless
+ you are simply testing at your workstation, you will most definitely need to change the three entries below. -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The property only needs to be set if upgrading from 1.4 which used to store write-ahead logs on the local
+ filesystem. In 1.5 write-ahead logs are stored in DFS. When 1.5 is started for the first time it will copy any 1.4
+ write ahead logs into DFS. It is possible to specify a comma-separated list of directories.
+ </description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers
+ must know in order to communicate with one another.
+ Change it before initialization. To
+ change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.native.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.token.property.password</name>
+ <!-- change this to the root user's password, and/or change the user below -->
+ <value>secret</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <!--
+ Add the following for Hadoop2, actual needs depend on Hadoop installation details.
+ This list may be excessive, but this should cause no issues. Append these values
+ after the $HADOOP_PREFIX entries
+
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+ /usr/lib/hadoop/.*.jar,
+ /usr/lib/hadoop/lib/.*.jar,
+ /usr/lib/hadoop-hdfs/.*.jar,
+ /usr/lib/hadoop-mapreduce/.*.jar,
+ /usr/lib/hadoop-yarn/.*.jar,
+ -->
+ <value>
+ $ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-server.jar,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-core.jar,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-start.jar,
+ $ACCUMULO_HOME/fate/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-fate.jar,
+ $ACCUMULO_HOME/proxy/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-proxy.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+ /usr/lib/hadoop/.*.jar,
+ /usr/lib/hadoop/lib/.*.jar,
+ /usr/lib/hadoop-hdfs/.*.jar,
+ /usr/lib/hadoop-mapreduce/.*.jar,
+ /usr/lib/hadoop-yarn/.*.jar,
+ /etc/hadoop/conf
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.
+ When using the Security Manager, please remove the ".../target/classes/"
+ values.
+ </description>
+ </property>
+</configuration>
diff --git a/src/test/clusters/sandbox/accumulo/accumulo.policy.example b/src/test/clusters/sandbox/accumulo/accumulo.policy.example
new file mode 100644
index 0000000..2964f06
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/accumulo.policy.example
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grant codeBase "file:${java.home}/lib/ext/*" {
+ permission java.security.AllPermission;
+};
+
+// These should all be empty in a fielded system
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/server/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/core/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/start/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/examples/target/classes/" {
+ permission java.security.AllPermission;
+};
+
+grant codebase "file:${hadoop.home.dir}/*" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "shutdownHooks"; // hadoop libs use executables to discover usernames, groups, etc.
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.io.FilePermission "<<ALL FILES>>", "read, execute";
+ permission java.io.FilePermission "/tmp", "write, delete";
+ permission java.io.FilePermission "/tmp/-", "write, delete";
+ permission java.io.FilePermission "/", "write";
+ permission java.net.SocketPermission "*", "connect, resolve";
+ permission java.util.PropertyPermission "java.library.path", "read";
+ permission java.util.PropertyPermission "user.dir", "read";
+ permission java.util.PropertyPermission "org.apache.commons.logging.*", "read";
+ permission java.util.PropertyPermission "entityExpansionLimit", "read";
+ permission java.util.PropertyPermission "maxOccurLimit", "read";
+ permission java.util.PropertyPermission "os.name", "read";
+};
+
+grant codebase "file:${hadoop.home.dir}/lib/*" {
+ // monitor's jetty web service
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ // need to accept web requests, and talk to job tracker, name node, etc.
+ permission java.net.SocketPermission "*", "accept, listen, resolve, connect, resolve";
+ permission java.lang.RuntimePermission "getenv.*";
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.util.PropertyPermission "org.mortbay.*", "read";
+ permission java.util.PropertyPermission "VERBOSE", "read";
+ permission java.util.PropertyPermission "IGNORED", "read";
+ permission java.util.PropertyPermission "ISO_8859_1", "read";
+ permission java.util.PropertyPermission "org.apache.commons.logging.*", "read";
+ permission java.util.PropertyPermission "accumulo.*", "read";
+ permission java.util.PropertyPermission "org.jfree.*", "read";
+ permission java.util.PropertyPermission "elementAttributeLimit", "read";
+ permission java.util.PropertyPermission "entityExpansionLimit", "read";
+ permission java.util.PropertyPermission "maxOccurLimit", "read";
+ // some resources come out of accumulo jars
+ permission java.lang.RuntimePermission "getClassLoader";
+ permission java.io.FilePermission "${org.apache.accumulo.core.home.dir}/lib/*", "read";
+ permission java.io.FilePermission "${org.apache.accumulo.core.home.dir}/src/-", "read";
+ permission java.io.FilePermission "${hadoop.home.dir}/lib/*", "read";
+ // images are cached in /tmp
+ permission java.io.FilePermission "/tmp/*", "read, write";
+ permission java.io.FilePermission "/", "write";
+};
+
+grant codebase "file:${zookeeper.home.dir}/*" {
+ permission java.net.SocketPermission "*", "connect, resolve";
+ permission java.util.PropertyPermission "user.*", "read";
+ permission java.util.PropertyPermission "java.*", "read";
+ permission java.util.PropertyPermission "zookeeper.*", "read";
+ permission java.util.PropertyPermission "jute.*", "read";
+ permission java.util.PropertyPermission "os.*", "read";
+ // accumulo properties read in callbacks
+ permission java.util.PropertyPermission "accumulo.*", "read";
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "exitVM";
+};
+
+grant codebase "file:${org.apache.accumulo.core.home.dir}/lib/ext/*" {
+};
+
+grant codebase "file:${org.apache.accumulo.core.home.dir}/lib/*" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ // logging, configuration and getting user id
+ permission java.io.FilePermission "<<ALL FILES>>", "read, write, execute, delete";
+ permission java.util.PropertyPermission "*", "read, write";
+ permission java.lang.RuntimePermission "getenv.*";
+ permission java.lang.RuntimePermission "getClassLoader";
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.lang.RuntimePermission "accessDeclaredMembers";
+ permission java.lang.RuntimePermission "selectorProvider";
+ permission java.lang.RuntimePermission "accessClassInPackage.*";
+ permission java.lang.RuntimePermission "readFileDescriptor";
+ permission java.lang.RuntimePermission "writeFileDescriptor";
+ permission java.lang.RuntimePermission "modifyThread";
+ permission java.lang.RuntimePermission "modifyThreadGroup";
+ permission java.lang.RuntimePermission "createClassLoader";
+ permission java.lang.RuntimePermission "setContextClassLoader";
+ permission java.lang.RuntimePermission "exitVM";
+ permission java.lang.RuntimePermission "shutdownHooks";
+ permission java.security.SecurityPermission "getPolicy";
+ permission java.security.SecurityPermission "getProperty.*";
+ permission java.security.SecurityPermission "putProviderProperty.*";
+ permission java.security.SecurityPermission "setSystemScope";
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.util.logging.LoggingPermission "control";
+ permission java.net.NetPermission "getProxySelector";
+ permission javax.management.MBeanServerPermission "createMBeanServer";
+ permission javax.management.MBeanTrustPermission "register";
+ permission javax.management.MBeanPermission "*", "registerMBean";
+ permission java.net.SocketPermission "*", "accept, connect, listen, resolve";
+};
diff --git a/src/test/clusters/sandbox/accumulo/auditLog.xml b/src/test/clusters/sandbox/accumulo/auditLog.xml
new file mode 100644
index 0000000..0d81dbe
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/auditLog.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+
+
+ <!-- Write out Audit info to an Audit file -->
+ <appender name="Audit" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.audit"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="DatePattern" value="'.'yyyy-MM-dd"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss,SSS/Z} [%c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+ <logger name="Audit" additivity="false">
+ <appender-ref ref="Audit" />
+ <level value="OFF"/>
+ </logger>
+
+
+
+
+
+</log4j:configuration>
diff --git a/src/test/clusters/sandbox/accumulo/gc b/src/test/clusters/sandbox/accumulo/gc
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/gc
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/sandbox/accumulo/generic_logger.xml b/src/test/clusters/sandbox/accumulo/generic_logger.xml
new file mode 100644
index 0000000..9fac0bc
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/generic_logger.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.debug.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Send all logging data to a centralized logger -->
+ <appender name="N1" class="org.apache.log4j.net.SocketAppender">
+ <param name="remoteHost" value="${org.apache.accumulo.core.host.log}"/>
+ <param name="port" value="4560"/>
+ <param name="application" value="${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname}"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- If the centralized logger is down, buffer the log events, but drop them if it stays down -->
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="N1" />
+ </appender>
+
+ <!-- Log accumulo events to the debug, normal and remote logs. -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="ASYNC" />
+ </logger>
+
+ <logger name="org.apache.accumulo.core.file.rfile.bcfile">
+ <level value="INFO"/>
+ </logger>
+
+ <logger name="org.mortbay.log">
+ <level value="WARN"/>
+ </logger>
+
+ <logger name="org.apache.zookeeper">
+ <level value="ERROR"/>
+ </logger>
+
+ <!-- Log non-accumulo events to the debug and normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
diff --git a/src/test/clusters/sandbox/accumulo/log4j.properties b/src/test/clusters/sandbox/accumulo/log4j.properties
new file mode 100644
index 0000000..a4bcb2e
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/log4j.properties
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# default logging properties:
+# by default, log everything at INFO or higher to the console
+log4j.rootLogger=INFO,A1
+
+# hide Jetty junk
+log4j.logger.org.mortbay.log=WARN,A1
+
+# hide "Got brand-new compresssor" messages
+log4j.logger.org.apache.hadoop.io.compress=WARN,A1
+
+# hide junk from TestRandomDeletes
+log4j.logger.org.apache.accumulo.test.TestRandomDeletes=WARN,A1
+
+# hide junk from VFS
+log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN,A1
+
+# hide almost everything from zookeeper
+log4j.logger.org.apache.zookeeper=ERROR,A1
+
+# hide AUDIT messages in the shell, alternatively you could send them to a different logger
+log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=WARN,A1
+
+# Send most things to the console
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.appender.A1.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
diff --git a/src/test/clusters/sandbox/accumulo/masters b/src/test/clusters/sandbox/accumulo/masters
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/masters
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/sandbox/accumulo/monitor b/src/test/clusters/sandbox/accumulo/monitor
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/monitor
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/sandbox/accumulo/monitor_logger.xml b/src/test/clusters/sandbox/accumulo/monitor_logger.xml
new file mode 100644
index 0000000..b344c23
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/monitor_logger.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.debug.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Keep the last few log messages for display to the user -->
+ <appender name="GUI" class="org.apache.accumulo.server.monitor.LogService">
+ <param name="keep" value="40"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- Log accumulo messages to debug, normal and GUI -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="GUI" />
+ </logger>
+
+ <!-- Log non-accumulo messages to debug, normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
diff --git a/src/test/clusters/sandbox/accumulo/slaves b/src/test/clusters/sandbox/accumulo/slaves
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/slaves
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/sandbox/accumulo/tracers b/src/test/clusters/sandbox/accumulo/tracers
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/sandbox/accumulo/tracers
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/sandbox/actions.json b/src/test/clusters/sandbox/actions.json
new file mode 100644
index 0000000..5690693
--- /dev/null
+++ b/src/test/clusters/sandbox/actions.json
@@ -0,0 +1,11 @@
+{
+ "confdir": {
+ "action":"mdkir",
+ "dest":"{{dest-conf-dir}}"
+ },
+ "site": {
+ "action":"expand",
+ "source":"hbase-site.xml.j2",
+ "dest":"{{dest-conf-dir}}/hbase-site.xml"
+ }
+}
\ No newline at end of file
diff --git a/src/test/clusters/sandbox/conf/capacity-scheduler.xml b/src/test/clusters/sandbox/conf/capacity-scheduler.xml
new file mode 100644
index 0000000..9a1a537
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/capacity-scheduler.xml
@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--Fri Oct 25 05:21:20 2013-->
+ <configuration>
+ <property>
+ <name>yarn.scheduler.capacity.root.acl_administer_queues</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.root.default.state</name>
+ <value>RUNNING</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.root.unfunded.capacity</name>
+ <value>50</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.root.capacity</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+ <value>0.2</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_submit_jobs</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.maximum-applications</name>
+ <value>10000</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.root.default.capacity</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ <value>default</value>
+ </property>
+ </configuration>
\ No newline at end of file
diff --git a/src/test/clusters/sandbox/conf/commons-logging.properties b/src/test/clusters/sandbox/conf/commons-logging.properties
new file mode 100644
index 0000000..77e458f
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/commons-logging.properties
@@ -0,0 +1,25 @@
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+#Logging Implementation
+
+#Log4J
+org.apache.commons.logging.Log=org.apache.commons.logging.impl.Log4JLogger
+
+#JDK Logger
+#org.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger
diff --git a/src/test/clusters/sandbox/conf/configuration.xsl b/src/test/clusters/sandbox/conf/configuration.xsl
new file mode 100644
index 0000000..d50d80b
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/configuration.xsl
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="html"/>
+<xsl:template match="configuration">
+<html>
+<body>
+<table border="1">
+<tr>
+ <td>name</td>
+ <td>value</td>
+ <td>description</td>
+</tr>
+<xsl:for-each select="property">
+<tr>
+ <td><a name="{name}"><xsl:value-of select="name"/></a></td>
+ <td><xsl:value-of select="value"/></td>
+ <td><xsl:value-of select="description"/></td>
+</tr>
+</xsl:for-each>
+</table>
+</body>
+</html>
+</xsl:template>
+</xsl:stylesheet>
diff --git a/src/test/clusters/sandbox/conf/container-executor.cfg b/src/test/clusters/sandbox/conf/container-executor.cfg
new file mode 100644
index 0000000..435c89b
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/container-executor.cfg
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodemanager.linux-container-executor.group
+banned.users=#comma separated list of users who can not run applications
+min.user.id=1000#Prevent other super-users
+allowed.system.users=##comma separated list of system users who CAN run applications
diff --git a/src/test/clusters/sandbox/conf/core-site.xml b/src/test/clusters/sandbox/conf/core-site.xml
new file mode 100644
index 0000000..af8936d
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/core-site.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--Fri Oct 25 05:21:20 2013-->
+ <configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://sandbox.hortonworks.com:8020</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.hcat.groups</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.hive.groups</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>fs.trash.interval</name>
+ <value>360</value>
+ </property>
+ <property>
+ <name>mapreduce.jobtracker.webinterface.trusted</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>io.serializations</name>
+ <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
+ </property>
+ <property>
+ <name>fs.checkpoint.size</name>
+ <value>0.5</value>
+ </property>
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>
+ RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/
+ RULE:[2:$1@$0](jhs@.*)s/.*/mapred/
+ RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/
+ RULE:[2:$1@$0](hm@.*)s/.*/hbase/
+ RULE:[2:$1@$0](rs@.*)s/.*/hbase/
+ DEFAULT</value>
+ </property>
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.hue.groups</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.hue.hosts</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>ipc.client.connection.maxidletime</name>
+ <value>30000</value>
+ </property>
+ <property>
+ <name>io.file.buffer.size</name>
+ <value>131072</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.oozie.hosts</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>io.compression.codecs</name>
+ <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec</value>
+ </property>
+ <property>
+ <name>ipc.client.idlethreshold</name>
+ <value>8000</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.hive.hosts</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.oozie.groups</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>ipc.client.connect.max.retries</name>
+ <value>50</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.hcat.hosts</name>
+ <value>*</value>
+ </property>
+ </configuration>
\ No newline at end of file
diff --git a/src/test/clusters/sandbox/conf/dfs.exclude b/src/test/clusters/sandbox/conf/dfs.exclude
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/dfs.exclude
diff --git a/src/test/clusters/sandbox/conf/hadoop-env.cmd b/src/test/clusters/sandbox/conf/hadoop-env.cmd
new file mode 100644
index 0000000..05badc2
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/hadoop-env.cmd
@@ -0,0 +1,81 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Set Hadoop-specific environment variables here.
+
+@rem The only required environment variable is JAVA_HOME. All others are
+@rem optional. When running a distributed configuration it is best to
+@rem set JAVA_HOME in this file, so that it is correctly defined on
+@rem remote nodes.
+
+@rem The java implementation to use. Required.
+set JAVA_HOME=%JAVA_HOME%
+
+@rem The jsvc implementation to use. Jsvc is required to run secure datanodes.
+@rem set JSVC_HOME=%JSVC_HOME%
+
+@rem set HADOOP_CONF_DIR=
+
+@rem Extra Java CLASSPATH elements. Automatically insert capacity-scheduler.
+if exist %HADOOP_HOME%\contrib\capacity-scheduler (
+ if not defined HADOOP_CLASSPATH (
+ set HADOOP_CLASSPATH=%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
+ ) else (
+ set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
+ )
+)
+
+@rem The maximum amount of heap to use, in MB. Default is 1000.
+@rem set HADOOP_HEAPSIZE=
+@rem set HADOOP_NAMENODE_INIT_HEAPSIZE=""
+
+@rem Extra Java runtime options. Empty by default.
+@rem set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true
+
+@rem Command specific options appended to HADOOP_OPTS when specified
+if not defined HADOOP_SECURITY_LOGGER (
+ set HADOOP_SECURITY_LOGGER=INFO,RFAS
+)
+if not defined HDFS_AUDIT_LOGGER (
+ set HDFS_AUDIT_LOGGER=INFO,NullAppender
+)
+
+set HADOOP_NAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_NAMENODE_OPTS%
+set HADOOP_DATANODE_OPTS=-Dhadoop.security.logger=ERROR,RFAS %HADOOP_DATANODE_OPTS%
+set HADOOP_SECONDARYNAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_SECONDARYNAMENODE_OPTS%
+
+@rem The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+set HADOOP_CLIENT_OPTS=-Xmx128m %HADOOP_CLIENT_OPTS%
+@rem set HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData %HADOOP_JAVA_PLATFORM_OPTS%"
+
+@rem On secure datanodes, user to run the datanode as after dropping privileges
+set HADOOP_SECURE_DN_USER=%HADOOP_SECURE_DN_USER%
+
+@rem Where log files are stored. %HADOOP_HOME%/logs by default.
+@rem set HADOOP_LOG_DIR=%HADOOP_LOG_DIR%\%USERNAME%
+
+@rem Where log files are stored in the secure data environment.
+set HADOOP_SECURE_DN_LOG_DIR=%HADOOP_LOG_DIR%\%HADOOP_HDFS_USER%
+
+@rem The directory where pid files are stored. /tmp by default.
+@rem NOTE: this should be set to a directory that can only be written to by
+@rem the user that will run the hadoop daemons. Otherwise there is the
+@rem potential for a symlink attack.
+set HADOOP_PID_DIR=%HADOOP_PID_DIR%
+set HADOOP_SECURE_DN_PID_DIR=%HADOOP_PID_DIR%
+
+@rem A string representing this instance of hadoop. %USERNAME% by default.
+set HADOOP_IDENT_STRING=%USERNAME%
diff --git a/src/test/clusters/sandbox/conf/hadoop-env.sh b/src/test/clusters/sandbox/conf/hadoop-env.sh
new file mode 100755
index 0000000..945a917
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/hadoop-env.sh
@@ -0,0 +1,122 @@
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME. All others are
+# optional. When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use. Required.
+export JAVA_HOME=/usr/jdk64/jdk1.6.0_31
+export HADOOP_HOME_WARN_SUPPRESS=1
+
+# Hadoop Configuration Directory
+#TODO: if env var set that can cause problems
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
+
+
+# Path to jsvc required by secure HDP 2.0 datanode
+export JSVC_HOME=/usr/libexec/bigtop-utils
+
+
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+export HADOOP_HEAPSIZE="1024"
+
+export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms1024m"
+
+# Extra Java runtime options. Empty by default.
+export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}"
+
+# Command specific options appended to HADOOP_OPTS when specified
+export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=640m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms1024m -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}"
+HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
+
+HADOOP_TASKTRACKER_OPTS="-server -Xmx1024m -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}"
+HADOOP_DATANODE_OPTS="-Xmx1024m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}"
+HADOOP_BALANCER_OPTS="-server -Xmx1024m ${HADOOP_BALANCER_OPTS}"
+
+export HADOOP_SECONDARYNAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=640m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}"
+
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS"
+# On secure datanodes, user to run the datanode as after dropping privileges
+export HADOOP_SECURE_DN_USER=hdfs
+
+# Extra ssh options. Empty by default.
+export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR"
+
+# Where log files are stored. $HADOOP_HOME/logs by default.
+export HADOOP_LOG_DIR=/var/log/hadoop/$USER
+
+# History server logs
+export HADOOP_MAPRED_LOG_DIR=/var/log/hadoop-mapreduce/$USER
+
+# Where log files are stored in the secure data environment.
+export HADOOP_SECURE_DN_LOG_DIR=/var/log/hadoop/$HADOOP_SECURE_DN_USER
+
+# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.
+# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves
+
+# host:path where hadoop code should be rsync'd from. Unset by default.
+# export HADOOP_MASTER=master:/home/$USER/src/hadoop
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HADOOP_SLAVE_SLEEP=0.1
+
+# The directory where pid files are stored. /tmp by default.
+export HADOOP_PID_DIR=/var/run/hadoop/$USER
+export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop/$HADOOP_SECURE_DN_USER
+
+# History server pid
+export HADOOP_MAPRED_PID_DIR=/var/run/hadoop-mapreduce/$USER
+
+YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY"
+
+# A string representing this instance of hadoop. $USER by default.
+export HADOOP_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+
+# export HADOOP_NICENESS=10
+
+# Use libraries from standard classpath
+JAVA_JDBC_LIBS=""
+#Add libraries required by mysql connector
+for jarFile in `ls /usr/share/java/*mysql* 2>/dev/null`
+do
+ JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
+done
+#Add libraries required by oracle connector
+for jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`
+do
+ JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
+done
+#Add libraries required by nodemanager
+MAPREDUCE_LIBS=/usr/lib/hadoop-mapreduce/*
+export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}
+
+# Setting path to hdfs command line
+export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec
+
+#Mostly required for hadoop 2.0
+export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64
diff --git a/src/test/clusters/sandbox/conf/hadoop-metrics.properties b/src/test/clusters/sandbox/conf/hadoop-metrics.properties
new file mode 100644
index 0000000..bb4393b
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/hadoop-metrics.properties
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configuration of the "dfs" context for null
+dfs.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "dfs" context for file
+#dfs.class=org.apache.hadoop.metrics.file.FileContext
+#dfs.period=10
+#dfs.fileName=/tmp/dfsmetrics.log
+
+# Configuration of the "dfs" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# dfs.period=10
+# dfs.servers=localhost:8649
+
+
+# Configuration of the "mapred" context for null
+mapred.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "mapred" context for file
+#mapred.class=org.apache.hadoop.metrics.file.FileContext
+#mapred.period=10
+#mapred.fileName=/tmp/mrmetrics.log
+
+# Configuration of the "mapred" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# mapred.period=10
+# mapred.servers=localhost:8649
+
+
+# Configuration of the "jvm" context for null
+#jvm.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "jvm" context for file
+#jvm.class=org.apache.hadoop.metrics.file.FileContext
+#jvm.period=10
+#jvm.fileName=/tmp/jvmmetrics.log
+
+# Configuration of the "jvm" context for ganglia
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# jvm.period=10
+# jvm.servers=localhost:8649
+
+# Configuration of the "rpc" context for null
+rpc.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "rpc" context for file
+#rpc.class=org.apache.hadoop.metrics.file.FileContext
+#rpc.period=10
+#rpc.fileName=/tmp/rpcmetrics.log
+
+# Configuration of the "rpc" context for ganglia
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rpc.period=10
+# rpc.servers=localhost:8649
+
+
+# Configuration of the "ugi" context for null
+ugi.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "ugi" context for file
+#ugi.class=org.apache.hadoop.metrics.file.FileContext
+#ugi.period=10
+#ugi.fileName=/tmp/ugimetrics.log
+
+# Configuration of the "ugi" context for ganglia
+# ugi.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# ugi.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# ugi.period=10
+# ugi.servers=localhost:8649
+
diff --git a/src/test/clusters/sandbox/conf/hadoop-metrics2.properties b/src/test/clusters/sandbox/conf/hadoop-metrics2.properties
new file mode 100644
index 0000000..0071b28
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/hadoop-metrics2.properties
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# syntax: [prefix].[source|sink|jmx].[instance].[options]
+# See package.html for org.apache.hadoop.metrics2 for details
+
+
+*.period=60
+
+*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
+*.sink.ganglia.period=10
+
+# default for supportsparse is false
+*.sink.ganglia.supportsparse=true
+
+.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
+.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
+
+# Hook up to the server
+namenode.sink.ganglia.servers=sandbox.hortonworks.com:8661
+datanode.sink.ganglia.servers=sandbox.hortonworks.com:8660
+jobtracker.sink.ganglia.servers=sandbox.hortonworks.com:8662
+tasktracker.sink.ganglia.servers=sandbox.hortonworks.com:8660
+maptask.sink.ganglia.servers=sandbox.hortonworks.com:8660
+reducetask.sink.ganglia.servers=sandbox.hortonworks.com:8660
+resourcemanager.sink.ganglia.servers=sandbox.hortonworks.com:8664
+nodemanager.sink.ganglia.servers=sandbox.hortonworks.com:8660
+historyserver.sink.ganglia.servers=sandbox.hortonworks.com:8666
+journalnode.sink.ganglia.servers=sandbox.hortonworks.com:8660
+
+resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
+
+
diff --git a/src/test/clusters/sandbox/conf/hadoop-policy.xml b/src/test/clusters/sandbox/conf/hadoop-policy.xml
new file mode 100644
index 0000000..491dbe7
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/hadoop-policy.xml
@@ -0,0 +1,219 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+
+ Copyright 2011 The Apache Software Foundation
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ClientProtocol, which is used by user code
+ via the DistributedFileSystem.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.client.datanode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
+ for block recovery.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.datanode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for DatanodeProtocol, which is used by datanodes to
+ communicate with the namenode.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.inter.datanode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for InterDatanodeProtocol, the inter-datanode protocol
+ for updating generation timestamp.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.namenode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for NamenodeProtocol, the protocol used by the secondary
+ namenode to communicate with the namenode.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.operations.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for AdminOperationsProtocol. Used for admin commands.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.refresh.usertogroups.mappings.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for RefreshUserMappingsProtocol. Used to refresh
+ users mappings. The ACL is a comma-separated list of user and
+ group names. The user and group list is separated by a blank. For
+ e.g. "alice,bob users,wheel". A special value of "*" means all
+ users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.refresh.policy.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
+ dfsadmin and mradmin commands to refresh the security policy in-effect.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.ha.service.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HAService protocol used by HAAdmin to manage the
+ active and stand-by states of namenode.</description>
+ </property>
+
+ <property>
+ <name>security.zkfc.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for access to the ZK Failover Controller
+ </description>
+ </property>
+
+ <property>
+ <name>security.qjournal.service.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for QJournalProtocol, used by the NN to communicate with
+ JNs when using the QuorumJournalManager for edit logs.</description>
+ </property>
+
+ <property>
+ <name>security.mrhs.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HSClientProtocol, used by job clients to
+ communciate with the MR History Server job status etc.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <!-- YARN Protocols -->
+
+ <property>
+ <name>security.resourcetracker.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ResourceTrackerProtocol, used by the
+ ResourceManager and NodeManager to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.resourcemanager-administration.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ResourceManagerAdministrationProtocol, for admin commands.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.applicationclient.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ApplicationClientProtocol, used by the ResourceManager
+ and applications submission clients to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.applicationmaster.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ApplicationMasterProtocol, used by the ResourceManager
+ and ApplicationMasters to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.containermanagement.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ContainerManagementProtocol protocol, used by the NodeManager
+ and ApplicationMasters to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.resourcelocalizer.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ResourceLocalizer protocol, used by the NodeManager
+ and ResourceLocalizer to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.job.task.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
+ tasks to communicate with the parent tasktracker.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.job.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for MRClientProtocol, used by job clients to
+ communciate with the MR ApplicationMaster to query job status etc.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+</configuration>
diff --git a/src/test/clusters/sandbox/conf/hdfs-site.xml b/src/test/clusters/sandbox/conf/hdfs-site.xml
new file mode 100644
index 0000000..9e0d6ea
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/hdfs-site.xml
@@ -0,0 +1,192 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--Fri Oct 25 05:21:20 2013-->
+ <configuration>
+ <property>
+ <name>dfs.namenode.safemode.threshold-pct</name>
+ <value>1.0f</value>
+ </property>
+ <property>
+ <name>dfs.datanode.du.reserved</name>
+ <value>1073741824</value>
+ </property>
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>1024</value>
+ </property>
+ <property>
+ <name>dfs.namenode.stale.datanode.interval</name>
+ <value>30000</value>
+ </property>
+ <property>
+ <name>dfs.https.port</name>
+ <value>50470</value>
+ </property>
+ <property>
+ <name>dfs.cluster.administrators</name>
+ <value> hdfs</value>
+ </property>
+ <property>
+ <name>dfs.blockreport.initialDelay</name>
+ <value>120</value>
+ </property>
+ <property>
+ <name>dfs.journalnode.http-address</name>
+ <value>0.0.0.0:8480</value>
+ </property>
+ <property>
+ <name>dfs.namenode.accesstime.precision</name>
+ <value>0</value>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dfs.replication</name>
+ <value>3</value>
+ </property>
+ <property>
+ <name>dfs.namenode.avoid.write.stale.datanode</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dfs.permissions.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>dfs.replication.max</name>
+ <value>50</value>
+ </property>
+ <property>
+ <name>fs.permissions.umask-mode</name>
+ <value>022</value>
+ </property>
+ <property>
+ <name>dfs.datanode.address</name>
+ <value>0.0.0.0:50010</value>
+ </property>
+ <property>
+ <name>dfs.namenode.checkpoint.period</name>
+ <value>21600</value>
+ </property>
+ <property>
+ <name>dfs.block.access.token.enable</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dfs.hosts.exclude</name>
+ <value>/etc/hadoop/conf/dfs.exclude</value>
+ </property>
+ <property>
+ <name>dfs.namenode.checkpoint.dir</name>
+ <value>/hadoop/hdfs/namesecondary</value>
+ </property>
+ <property>
+ <name>dfs.namenode.checkpoint.edits.dir</name>
+ <value>${dfs.namenode.checkpoint.dir}</value>
+ </property>
+ <property>
+ <name>dfs.blocksize</name>
+ <value>134217728</value>
+ </property>
+ <property>
+ <name>dfs.namenode.http-address</name>
+ <value>sandbox.hortonworks.com:50070</value>
+ </property>
+ <property>
+ <name>dfs.namenode.secondary.http-address</name>
+ <value>sandbox.hortonworks.com:50090</value>
+ </property>
+ <property>
+ <name>dfs.client.read.shortcircuit.streams.cache.size</name>
+ <value>4096</value>
+ </property>
+ <property>
+ <name>dfs.datanode.http.address</name>
+ <value>0.0.0.0:50075</value>
+ </property>
+ <property>
+ <name>dfs.namenode.write.stale.datanode.ratio</name>
+ <value>1.0f</value>
+ </property>
+ <property>
+ <name>dfs.support.append</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dfs.datanode.ipc.address</name>
+ <value>0.0.0.0:8010</value>
+ </property>
+ <property>
+ <name>dfs.datanode.balance.bandwidthPerSec</name>
+ <value>6250000</value>
+ </property>
+ <property>
+ <name>dfs.heartbeat.interval</name>
+ <value>3</value>
+ </property>
+ <property>
+ <name>dfs.datanode.failed.volumes.tolerated</name>
+ <value>0</value>
+ </property>
+ <property>
+ <name>dfs.permissions.superusergroup</name>
+ <value>hdfs</value>
+ </property>
+ <property>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ </property>
+ <property>
+ <name>dfs.namenode.avoid.read.stale.datanode</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dfs.journalnode.edits.dir</name>
+ <value>/grid/0/hdfs/journal</value>
+ </property>
+ <property>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dfs.client.block.write.replace-datanode-on-failure.policy</name>
+ <value>NEVER</value>
+ </property>
+ <property>
+ <name>dfs.webhdfs.enabled</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dfs.datanode.data.dir.perm</name>
+ <value>750</value>
+ </property>
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/hadoop/hdfs/data</value>
+ </property>
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>/hadoop/hdfs/namenode</value>
+ </property>
+ <property>
+ <name>dfs.namenode.https-address</name>
+ <value>sandbox.hortonworks.com:50470</value>
+ </property>
+ </configuration>
\ No newline at end of file
diff --git a/src/test/clusters/sandbox/conf/health_check b/src/test/clusters/sandbox/conf/health_check
new file mode 100644
index 0000000..a7142ae
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/health_check
@@ -0,0 +1,118 @@
+#!/bin/bash
+#
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+err=0;
+
+function check_disks {
+
+ for m in `awk '$3~/ext3/ {printf" %s ",$2}' /etc/fstab` ; do
+ fsdev=""
+ fsdev=`awk -v m=$m '$2==m {print $1}' /proc/mounts`;
+ if [ -z "$fsdev" ] ; then
+ msg_="$msg_ $m(u)"
+ else
+ msg_="$msg_`awk -v m=$m '$2==m { if ( $4 ~ /^ro,/ ) {printf"%s(ro)",$2 } ; }' /proc/mounts`"
+ fi
+ done
+
+ if [ -z "$msg_" ] ; then
+ echo "disks ok" ; exit 0
+ else
+ echo "$msg_" ; exit 2
+ fi
+
+}
+
+function check_taskcontroller {
+ if [ "false" == "true" ]; then
+ perm=`stat -c %a:%U:%G 2>/dev/null`
+ if [ $? -eq 0 ] && [ "$perm" == "6050:root:hadoop" ] ; then
+ echo "taskcontroller ok"
+ else
+ echo 'check taskcontroller' ; exit 1
+ fi
+ fi
+}
+
+function check_jetty {
+ hname=`hostname`
+ jmx=`curl -s -S -m 5 "http://$hname:50060/jmx?qry=Hadoop:service=TaskTracker,name=ShuffleServerMetrics" 2>/dev/null` ;
+ if [ $? -eq 0 ] ; then
+ e=`echo $jmx | awk '/shuffle_exceptions_caught/ {printf"%d",$2}'` ;
+ e=${e:-0} # no jmx servlet ?
+ if [ $e -gt 10 ] ; then
+ echo "check jetty: shuffle_exceptions=$e" ; exit 1
+ else
+ echo "jetty ok"
+ fi
+ else
+ echo "check jetty: ping failed" ; exit 1
+ fi
+}
+
+function check_link {
+ snmp=/usr/bin/snmpwalk
+ if [ -e $snmp ] ; then
+ $snmp -t 5 -Oe -Oq -Os -v 1 -c public localhost if | \
+ awk ' {
+ split($1,a,".") ;
+ if ( a[1] == "ifIndex" ) { ifIndex[a[2]] = $2 }
+ if ( a[1] == "ifDescr" ) { ifDescr[a[2]] = $2 }
+ if ( a[1] == "ifType" ) { ifType[a[2]] = $2 }
+ if ( a[1] == "ifSpeed" ) { ifSpeed[a[2]] = $2 }
+ if ( a[1] == "ifAdminStatus" ) { ifAdminStatus[a[2]] = $2 }
+ if ( a[1] == "ifOperStatus" ) { ifOperStatus[a[2]] = $2 }
+ }
+ END {
+ up=0;
+ for (i in ifIndex ) {
+ if ( ifType[i] == 6 && ifAdminStatus[i] == 1 && ifOperStatus[i] == 1 && ifSpeed[i] == 1000000000 ) {
+ up=i;
+ }
+ }
+ if ( up == 0 ) { print "check link" ; exit 2 }
+ else { print ifDescr[up],"ok" }
+ }'
+ exit $? ;
+ fi
+}
+
+# Run all checks
+# Disabled 'check_link' for now...
+for check in disks taskcontroller jetty; do
+ msg=`check_${check}` ;
+ if [ $? -eq 0 ] ; then
+ ok_msg="$ok_msg$msg,"
+ else
+ err_msg="$err_msg$msg,"
+ fi
+done
+
+if [ ! -z "$err_msg" ] ; then
+ echo -n "ERROR $err_msg "
+fi
+if [ ! -z "$ok_msg" ] ; then
+ echo -n "OK: $ok_msg"
+fi
+
+echo
+
+# Success!
+exit 0
diff --git a/src/test/clusters/sandbox/conf/log4j.properties b/src/test/clusters/sandbox/conf/log4j.properties
new file mode 100644
index 0000000..77d4642
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/log4j.properties
@@ -0,0 +1,219 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+hadoop.root.logger=INFO,console
+hadoop.log.dir=.
+hadoop.log.file=hadoop.log
+
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+#
+# Daily Rolling File Appender
+#
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+#
+# TaskLog Appender
+#
+
+#Default values
+hadoop.tasklog.taskid=null
+hadoop.tasklog.iscleanup=false
+hadoop.tasklog.noKeepSplits=4
+hadoop.tasklog.totalLogFileSize=100
+hadoop.tasklog.purgeLogSplits=true
+hadoop.tasklog.logsRetainHours=12
+
+log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
+log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
+
+log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+#Security audit appender
+#
+hadoop.security.logger=INFO,console
+hadoop.security.log.maxfilesize=256MB
+hadoop.security.log.maxbackupindex=20
+log4j.category.SecurityLogger=${hadoop.security.logger}
+hadoop.security.log.file=SecurityAuth.audit
+log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
+log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
+
+log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
+log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
+log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
+log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
+
+#
+# hdfs audit logging
+#
+hdfs.audit.logger=INFO,console
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
+log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
+log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
+log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd
+
+#
+# mapred audit logging
+#
+mapred.audit.logger=INFO,console
+log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
+log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
+log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
+log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd
+
+#
+# Rolling File Appender
+#
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Logfile size and and 30-day backups
+log4j.appender.RFA.MaxFileSize=256MB
+log4j.appender.RFA.MaxBackupIndex=10
+
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+# Custom Logging levels
+
+hadoop.metrics.log.level=INFO
+#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
+#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}
+
+# Jets3t library
+log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+
+#
+# Null Appender
+# Trap security logger on the hadoop client side
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# Event Counter Appender
+# Sends counts of logging messages at different severity levels to Hadoop Metrics.
+#
+log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
+
+
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file rolled daily:
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
+
+# Set the ResourceManager summary log filename
+yarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log
+# Set the ResourceManager summary log level and appender
+yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
+#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
+
+# To enable AppSummaryLogging for the RM,
+# set yarn.server.resourcemanager.appsummary.logger to
+# <LEVEL>,RMSUMMARY in hadoop-env.sh
+
+# Appender for ResourceManager Application Summary Log
+# Requires the following properties to be set
+# - hadoop.log.dir (Hadoop Log directory)
+# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
+# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
+log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
+log4j.appender.RMSUMMARY.File=/var/log/hadoop-yarn/yarn/${yarn.server.resourcemanager.appsummary.log.file}
+log4j.appender.RMSUMMARY.MaxFileSize=256MB
+log4j.appender.RMSUMMARY.MaxBackupIndex=20
+log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
+log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+log4j.appender.JSA.DatePattern=.yyyy-MM-dd
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
+log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
+
+###ambari.jobhistory.database=jdbc:postgresql://sandbox.hortonworks.com/ambarirca
+###ambari.jobhistory.driver=org.postgresql.Driver
+###ambari.jobhistory.user=mapred
+###ambari.jobhistory.password=mapred
+###ambari.jobhistory.logger=DEBUG,JHA
+
+###log4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender
+###log4j.appender.JHA.database=${ambari.jobhistory.database}
+###log4j.appender.JHA.driver=${ambari.jobhistory.driver}
+###log4j.appender.JHA.user=${ambari.jobhistory.user}
+###log4j.appender.JHA.password=${ambari.jobhistory.password}
+
+###log4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=${ambari.jobhistory.logger}
+###log4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true
diff --git a/src/test/clusters/sandbox/conf/mapred-env.cmd b/src/test/clusters/sandbox/conf/mapred-env.cmd
new file mode 100644
index 0000000..610d593
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/mapred-env.cmd
@@ -0,0 +1,20 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+set HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
+
+set HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+
diff --git a/src/test/clusters/sandbox/conf/mapred-env.sh b/src/test/clusters/sandbox/conf/mapred-env.sh
new file mode 100644
index 0000000..6be1e27
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/mapred-env.sh
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+
+export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
+
+export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+
+#export HADOOP_JOB_HISTORYSERVER_OPTS=
+#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.
+#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.
+#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.
+#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default
+#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.
diff --git a/src/test/clusters/sandbox/conf/mapred-queues.xml.template b/src/test/clusters/sandbox/conf/mapred-queues.xml.template
new file mode 100644
index 0000000..ce6cd20
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/mapred-queues.xml.template
@@ -0,0 +1,92 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- This is the template for queue configuration. The format supports nesting of
+ queues within queues - a feature called hierarchical queues. All queues are
+ defined within the 'queues' tag which is the top level element for this
+ XML document. The queue acls configured here for different queues are
+ checked for authorization only if the configuration property
+ mapreduce.cluster.acls.enabled is set to true. -->
+<queues>
+
+ <!-- Configuration for a queue is specified by defining a 'queue' element. -->
+ <queue>
+
+ <!-- Name of a queue. Queue name cannot contain a ':' -->
+ <name>default</name>
+
+ <!-- properties for a queue, typically used by schedulers,
+ can be defined here -->
+ <properties>
+ </properties>
+
+ <!-- State of the queue. If running, the queue will accept new jobs.
+ If stopped, the queue will not accept new jobs. -->
+ <state>running</state>
+
+ <!-- Specifies the ACLs to check for submitting jobs to this queue.
+ If set to '*', it allows all users to submit jobs to the queue.
+ If set to ' '(i.e. space), no user will be allowed to do this
+ operation. The default value for any queue acl is ' '.
+ For specifying a list of users and groups the format to use is
+ user1,user2 group1,group2
+
+ It is only used if authorization is enabled in Map/Reduce by setting
+ the configuration property mapreduce.cluster.acls.enabled to true.
+
+ Irrespective of this ACL configuration, the user who started the
+ cluster and cluster administrators configured via
+ mapreduce.cluster.administrators can do this operation. -->
+ <acl-submit-job> </acl-submit-job>
+
+ <!-- Specifies the ACLs to check for viewing and modifying jobs in this
+ queue. Modifications include killing jobs, tasks of jobs or changing
+ priorities.
+ If set to '*', it allows all users to view, modify jobs of the queue.
+ If set to ' '(i.e. space), no user will be allowed to do this
+ operation.
+ For specifying a list of users and groups the format to use is
+ user1,user2 group1,group2
+
+ It is only used if authorization is enabled in Map/Reduce by setting
+ the configuration property mapreduce.cluster.acls.enabled to true.
+
+ Irrespective of this ACL configuration, the user who started the
+ cluster and cluster administrators configured via
+ mapreduce.cluster.administrators can do the above operations on all
+ the jobs in all the queues. The job owner can do all the above
+ operations on his/her job irrespective of this ACL configuration. -->
+ <acl-administer-jobs> </acl-administer-jobs>
+ </queue>
+
+ <!-- Here is a sample of a hierarchical queue configuration
+ where q2 is a child of q1. In this example, q2 is a leaf level
+ queue as it has no queues configured within it. Currently, ACLs
+ and state are only supported for the leaf level queues.
+ Note also the usage of properties for the queue q2.
+ <queue>
+ <name>q1</name>
+ <queue>
+ <name>q2</name>
+ <properties>
+ <property key="capacity" value="20"/>
+ <property key="user-limit" value="30"/>
+ </properties>
+ </queue>
+ </queue>
+ -->
+</queues>
diff --git a/src/test/clusters/sandbox/conf/mapred-site.xml b/src/test/clusters/sandbox/conf/mapred-site.xml
new file mode 100644
index 0000000..b7f05f9
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/mapred-site.xml
@@ -0,0 +1,168 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--Fri Oct 25 05:21:20 2013-->
+ <configuration>
+ <property>
+ <name>yarn.app.mapreduce.am.resource.mb</name>
+ <value>250</value>
+ </property>
+ <property>
+ <name>mapreduce.cluster.administrators</name>
+ <value> hadoop</value>
+ </property>
+ <property>
+ <name>mapreduce.map.java.opts</name>
+ <value>-Xmx512m</value>
+ </property>
+ <property>
+ <name>mapreduce.reduce.shuffle.parallelcopies</name>
+ <value>30</value>
+ </property>
+ <property>
+ <name>mapreduce.task.io.sort.factor</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.admin-command-opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
+ </property>
+ <property>
+ <name>mapreduce.admin.reduce.child.java.opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
+ </property>
+ <property>
+ <name>mapreduce.application.classpath</name>
+ <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.log.level</name>
+ <value>INFO</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>sandbox.hortonworks.com:19888</value>
+ </property>
+ <property>
+ <name>mapreduce.reduce.input.buffer.percent</name>
+ <value>0.0</value>
+ </property>
+ <property>
+ <name>mapreduce.reduce.java.opts</name>
+ <value>-Xmx512m</value>
+ </property>
+ <property>
+ <name>mapreduce.admin.map.child.java.opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.command-opts</name>
+ <value>-Xmx312m</value>
+ </property>
+ <property>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>512</value>
+ </property>
+ <property>
+ <name>mapreduce.task.io.sort.mb</name>
+ <value>200</value>
+ </property>
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.address</name>
+ <value>sandbox.hortonworks.com:10020</value>
+ </property>
+ <property>
+ <name>mapreduce.reduce.log.level</name>
+ <value>INFO</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.done-dir</name>
+ <value>/mr-history/done</value>
+ </property>
+ <property>
+ <name>mapreduce.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:/usr/lib/hadoop/lib/native/`$JAVA_HOME/bin/java -d32 -version &> /dev/null;if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`</value>
+ </property>
+ <property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>512</value>
+ </property>
+ <property>
+ <name>mapreduce.reduce.speculative</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
+ <value>0.7</value>
+ </property>
+ <property>
+ <name>mapreduce.am.max-attempts</name>
+ <value>2</value>
+ </property>
+ <property>
+ <name>mapreduce.map.output.compress</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>mapreduce.reduce.shuffle.merge.percent</name>
+ <value>0.66</value>
+ </property>
+ <property>
+ <name>mapreduce.map.log.level</name>
+ <value>INFO</value>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <value>/user</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.intermediate-done-dir</name>
+ <value>/mr-history/tmp</value>
+ </property>
+ <property>
+ <name>mapreduce.map.speculative</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>mapreduce.shuffle.port</name>
+ <value>13562</value>
+ </property>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+ <property>
+ <name>mapreduce.job.reduce.slowstart.completedmaps</name>
+ <value>0.05</value>
+ </property>
+ <property>
+ <name>mapreduce.map.sort.spill.percent</name>
+ <value>0.7</value>
+ </property>
+ <property>
+ <name>mapreduce.task.timeout</name>
+ <value>300000</value>
+ </property>
+ </configuration>
\ No newline at end of file
diff --git a/src/test/clusters/sandbox/conf/mapred-site.xml.rpmnew b/src/test/clusters/sandbox/conf/mapred-site.xml.rpmnew
new file mode 100644
index 0000000..3fc8f34
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/mapred-site.xml.rpmnew
@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+</configuration>
diff --git a/src/test/clusters/sandbox/conf/mapred-site.xml.template b/src/test/clusters/sandbox/conf/mapred-site.xml.template
new file mode 100644
index 0000000..761c352
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/mapred-site.xml.template
@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>
diff --git a/src/test/clusters/sandbox/conf/slaves b/src/test/clusters/sandbox/conf/slaves
new file mode 100644
index 0000000..25c0dd8
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/slaves
@@ -0,0 +1 @@
+sandbox.hortonworks.com
diff --git a/src/test/clusters/sandbox/conf/ssl-client.xml.example b/src/test/clusters/sandbox/conf/ssl-client.xml.example
new file mode 100644
index 0000000..a50dce4
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/ssl-client.xml.example
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+<property>
+ <name>ssl.client.truststore.location</name>
+ <value></value>
+ <description>Truststore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.password</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.location</name>
+ <value></value>
+ <description>Keystore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.password</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.keypassword</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+</configuration>
diff --git a/src/test/clusters/sandbox/conf/ssl-server.xml.example b/src/test/clusters/sandbox/conf/ssl-server.xml.example
new file mode 100644
index 0000000..02d300c
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/ssl-server.xml.example
@@ -0,0 +1,78 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+<property>
+ <name>ssl.server.truststore.location</name>
+ <value></value>
+ <description>Truststore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.password</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.location</name>
+ <value></value>
+ <description>Keystore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.password</name>
+ <value></value>
+ <description>Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.keypassword</name>
+ <value></value>
+ <description>Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+</configuration>
diff --git a/src/test/clusters/sandbox/conf/task-log4j.properties b/src/test/clusters/sandbox/conf/task-log4j.properties
new file mode 100644
index 0000000..c8939fc
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/task-log4j.properties
@@ -0,0 +1,132 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+
+# Define some default values that can be overridden by system properties
+hadoop.root.logger=INFO,console
+hadoop.log.dir=.
+hadoop.log.file=hadoop.log
+
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file rolled daily:
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+#
+# Daily Rolling File Appender
+#
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+#
+# TaskLog Appender
+#
+
+#Default values
+hadoop.tasklog.taskid=null
+hadoop.tasklog.iscleanup=false
+hadoop.tasklog.noKeepSplits=4
+hadoop.tasklog.totalLogFileSize=100
+hadoop.tasklog.purgeLogSplits=true
+hadoop.tasklog.logsRetainHours=12
+
+log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
+log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
+
+log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+# Rolling File Appender
+#
+
+#log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Logfile size and and 30-day backups
+#log4j.appender.RFA.MaxFileSize=1MB
+#log4j.appender.RFA.MaxBackupIndex=30
+
+#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+# Custom Logging levels
+
+hadoop.metrics.log.level=INFO
+#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
+#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}
+
+# Jets3t library
+log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+
+#
+# Null Appender
+# Trap security logger on the hadoop client side
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# Event Counter Appender
+# Sends counts of logging messages at different severity levels to Hadoop Metrics.
+#
+log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
+
diff --git a/src/test/clusters/sandbox/conf/taskcontroller.cfg b/src/test/clusters/sandbox/conf/taskcontroller.cfg
new file mode 100644
index 0000000..4f953a1
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/taskcontroller.cfg
@@ -0,0 +1,20 @@
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+mapred.local.dir=/tmp/hadoop-mapred/mapred/local
+mapreduce.tasktracker.group=hadoop
+hadoop.log.dir=/var/log/hadoop/mapred
diff --git a/src/test/clusters/sandbox/conf/yarn-env.cmd b/src/test/clusters/sandbox/conf/yarn-env.cmd
new file mode 100644
index 0000000..3329f8f
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/yarn-env.cmd
@@ -0,0 +1,60 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem User for YARN daemons
+if not defined HADOOP_YARN_USER (
+ set HADOOP_YARN_USER=%yarn%
+)
+
+if not defined YARN_CONF_DIR (
+ set YARN_CONF_DIR=%HADOOP_YARN_HOME%\conf
+)
+
+if defined YARN_HEAPSIZE (
+ @rem echo run with Java heapsize %YARN_HEAPSIZE%
+ set JAVA_HEAP_MAX=-Xmx%YARN_HEAPSIZE%m
+)
+
+if not defined YARN_LOG_DIR (
+ set YARN_LOG_DIR=%HADOOP_YARN_HOME%\logs
+)
+
+if not defined YARN_LOGFILE (
+ set YARN_LOGFILE=yarn.log
+)
+
+@rem default policy file for service-level authorization
+if not defined YARN_POLICYFILE (
+ set YARN_POLICYFILE=hadoop-policy.xml
+)
+
+if not defined YARN_ROOT_LOGGER (
+ set YARN_ROOT_LOGGER=INFO,console
+)
+
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.dir=%YARN_LOG_DIR%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.log.dir=%YARN_LOG_DIR%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.file=%YARN_LOGFILE%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.log.file=%YARN_LOGFILE%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.home.dir=%HADOOP_YARN_HOME%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.id.str=%YARN_IDENT_STRING%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.home.dir=%HADOOP_YARN_HOME%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.root.logger=%YARN_ROOT_LOGGER%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.root.logger=%YARN_ROOT_LOGGER%
+if defined JAVA_LIBRARY_PATH (
+ set YARN_OPTS=%YARN_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
+)
+set YARN_OPTS=%YARN_OPTS% -Dyarn.policy.file=%YARN_POLICYFILE%
\ No newline at end of file
diff --git a/src/test/clusters/sandbox/conf/yarn-env.sh b/src/test/clusters/sandbox/conf/yarn-env.sh
new file mode 100755
index 0000000..8928f66
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/yarn-env.sh
@@ -0,0 +1,119 @@
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+export HADOOP_YARN_HOME=/usr/lib/hadoop-yarn
+export YARN_LOG_DIR=/var/log/hadoop-yarn/$USER
+export YARN_PID_DIR=/var/run/hadoop-yarn/$USER
+export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec
+export JAVA_HOME=/usr/jdk64/jdk1.6.0_31
+
+# User for YARN daemons
+export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+
+# resolve links - $0 may be a softlink
+export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+
+# some Java parameters
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+if [ "$JAVA_HOME" != "" ]; then
+ #echo "run java in $JAVA_HOME"
+ JAVA_HOME=$JAVA_HOME
+fi
+
+if [ "$JAVA_HOME" = "" ]; then
+ echo "Error: JAVA_HOME is not set."
+ exit 1
+fi
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m
+
+# For setting YARN specific HEAP sizes please use this
+# Parameter and set appropriately
+YARN_HEAPSIZE=1024
+
+# check envvars which might override default args
+if [ "$YARN_HEAPSIZE" != "" ]; then
+ JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+fi
+
+# Resource Manager specific parameters
+
+# Specify the max Heapsize for the ResourceManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_RESOURCEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_RESOURCEMANAGER_HEAPSIZE=768
+
+# Specify the JVM options to be used when starting the ResourceManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_RESOURCEMANAGER_OPTS=
+
+# Node Manager specific parameters
+
+# Specify the max Heapsize for the NodeManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_NODEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_NODEMANAGER_HEAPSIZE=768
+
+# Specify the JVM options to be used when starting the NodeManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_NODEMANAGER_OPTS=
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+
+# default log directory & file
+if [ "$YARN_LOG_DIR" = "" ]; then
+ YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
+fi
+if [ "$YARN_LOGFILE" = "" ]; then
+ YARN_LOGFILE='yarn.log'
+fi
+
+# default policy file for service-level authorization
+if [ "$YARN_POLICYFILE" = "" ]; then
+ YARN_POLICYFILE="hadoop-policy.xml"
+fi
+
+# restore ordinary behaviour
+unset IFS
+
+
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
+YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
+YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+ YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi
+YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
diff --git a/src/test/clusters/sandbox/conf/yarn-site.xml b/src/test/clusters/sandbox/conf/yarn-site.xml
new file mode 100644
index 0000000..9026478
--- /dev/null
+++ b/src/test/clusters/sandbox/conf/yarn-site.xml
@@ -0,0 +1,177 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--Fri Oct 25 05:20:08 2013-->
+ <configuration>
+ <property>
+ <name>yarn.nodemanager.remote-app-log-dir</name>
+ <value>/app-logs</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value>/hadoop/yarn/local</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.health-checker.interval-ms</name>
+ <value>135000</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.admin-env</name>
+ <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
+ </property>
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>hadoop</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>sandbox.hortonworks.com:8025</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.admin.address</name>
+ <value>sandbox.hortonworks.com:8141</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
+ <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.am.max-attempts</name>
+ <value>2</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.address</name>
+ <value>0.0.0.0:45454</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>0</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.hostname</name>
+ <value>sandbox.hortonworks.com</value>
+ </property>
+ <property>
+ <name>yarn.acl.enable</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>sandbox.hortonworks.com:8030</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
+ <value>logs</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>64</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-dirs</name>
+ <value>/hadoop/yarn/log</value>
+ </property>
+ <property>
+ <name>yarn.log-aggregation.retain-seconds</name>
+ <value>2592000</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log.retain-second</name>
+ <value>604800</value>
+ </property>
+ <property>
+ <name>yarn.log.server.url</name>
+ <value>http://sandbox.hortonworks.com:19888/jobhistory/logs</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
+ <value>0.25</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
+ <value>60000</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>2048</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.webapp.address</name>
+ <value>sandbox.hortonworks.com:8088</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>2250</value>
+ </property>
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.container-monitor.interval-ms</name>
+ <value>3000</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>sandbox.hortonworks.com:8050</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.compression-type</name>
+ <value>gz</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.vmem-pmem-ratio</name>
+ <value>10</value>
+ </property>
+ <property>
+ <name>yarn.admin.acl</name>
+ <value>*</value>
+ </property>
+
+ <!-- 10 minutes after a failure to see what is left in the directory-->
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>600</value>
+ </property>
+
+ <!--ten seconds before the process gets a -9 -->
+ <property>
+ <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
+ <value>30000</value>
+ </property>
+
+ </configuration>
\ No newline at end of file
diff --git a/src/test/clusters/sandbox/hbase/hadoop-metrics.properties b/src/test/clusters/sandbox/hbase/hadoop-metrics.properties
new file mode 100644
index 0000000..e31c059
--- /dev/null
+++ b/src/test/clusters/sandbox/hbase/hadoop-metrics.properties
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See http://wiki.apache.org/hadoop/GangliaMetrics
+# Make sure you know whether you are using ganglia 3.0 or 3.1.
+# If 3.1, you will have to patch your hadoop instance with HADOOP-4675
+# And, yes, this file is named hadoop-metrics.properties rather than
+# hbase-metrics.properties because we're leveraging the hadoop metrics
+# package and hadoop-metrics.properties is an hardcoded-name, at least
+# for the moment.
+#
+# See also http://hadoop.apache.org/hbase/docs/current/metrics.html
+# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia
+# meta daemon (gmetad) service is running
+
+# Configuration of the "hbase" context for NullContextWithUpdateThread
+# NullContextWithUpdateThread is a null context which has a thread calling
+# periodically when monitoring is started. This keeps the data sampled
+# correctly.
+hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+hbase.period=10
+
+# Configuration of the "hbase" context for file
+# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# hbase.fileName=/tmp/metrics_hbase.log
+
+# HBase-specific configuration to reset long-running stats (e.g. compactions)
+# If this variable is left out, then the default is no expiration.
+hbase.extendedperiod = 3600
+
+# Configuration of the "hbase" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# hbase.period=10
+# hbase.servers=GMETADHOST_IP:8649
+
+# Configuration of the "jvm" context for null
+jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+jvm.period=10
+
+# Configuration of the "jvm" context for file
+# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# jvm.fileName=/tmp/metrics_jvm.log
+
+# Configuration of the "jvm" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# jvm.period=10
+# jvm.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rpc" context for null
+rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+rpc.period=10
+
+# Configuration of the "rpc" context for file
+# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# rpc.fileName=/tmp/metrics_rpc.log
+
+# Configuration of the "rpc" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rpc.period=10
+# rpc.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rest" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rest.period=10
+# rest.servers=GMETADHOST_IP:8649
diff --git a/src/test/clusters/sandbox/hbase/hbase-env.sh b/src/test/clusters/sandbox/hbase/hbase-env.sh
new file mode 100644
index 0000000..da53a27
--- /dev/null
+++ b/src/test/clusters/sandbox/hbase/hbase-env.sh
@@ -0,0 +1,106 @@
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Set environment variables here.
+
+# This script sets variables multiple times over the course of starting an hbase process,
+# so try to keep things idempotent unless you want to take an even deeper look
+# into the startup scripts (bin/hbase, etc.)
+
+# The java implementation to use. Java 1.6 required.
+# export JAVA_HOME=/usr/java/jdk1.6.0/
+
+# Extra Java CLASSPATH elements. Optional.
+# export HBASE_CLASSPATH=
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+
+# Uncomment below to enable java garbage collection logging for the server-side processes
+# this enables basic gc logging for the server processes to the .out file
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M $HBASE_GC_OPTS"
+
+# Uncomment below to enable java garbage collection logging for the client processes in the .out file.
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS).
+# This applies to both the server and client GC options above
+# export HBASE_USE_GC_LOGFILE=true
+
+
+# Uncomment below if you intend to use the EXPERIMENTAL off heap cache.
+# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize="
+# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value.
+
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers
+
+# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default.
+# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored. $HBASE_HOME/logs by default.
+# export HBASE_LOG_DIR=${HBASE_HOME}/logs
+
+# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073"
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+# export HBASE_PID_DIR=/var/hadoop/pids
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+# export HBASE_MANAGES_ZK=true
diff --git a/src/test/clusters/sandbox/hbase/hbase-policy.xml b/src/test/clusters/sandbox/hbase/hbase-policy.xml
new file mode 100644
index 0000000..e45f23c
--- /dev/null
+++ b/src/test/clusters/sandbox/hbase/hbase-policy.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HRegionInterface protocol implementations (ie.
+ clients talking to HRegionServers)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterInterface protocol implementation (ie.
+ clients talking to HMaster for admin operations).
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterRegionInterface protocol implementations
+ (for HRegionServers communicating with HMaster)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+</configuration>
diff --git a/src/test/clusters/sandbox/hbase/hbase-site.xml b/src/test/clusters/sandbox/hbase/hbase-site.xml
new file mode 100644
index 0000000..1e58691
--- /dev/null
+++ b/src/test/clusters/sandbox/hbase/hbase-site.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+ <!--
+ This is a template HBase site file that
+ does not include all the properties
+ required for a valid configuration -
+ the remainder are injected during
+ conversion from a template to
+ actual file
+ -->
+<configuration>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>./hbase-tmp</value>
+ </property>
+ <property>
+ <name>hbase.regionserver.hlog.tolerable.lowreplication</name>
+ <value>1</value>
+
+ </property>
+ <!--just here to verify propagation of properties all the way to the AM-->
+ <property>
+ <name>hoya.unused.option</name>
+ <value>1</value>
+ </property>
+
+</configuration>
diff --git a/src/test/clusters/sandbox/hbase/log4j.properties b/src/test/clusters/sandbox/hbase/log4j.properties
new file mode 100644
index 0000000..81233d4
--- /dev/null
+++ b/src/test/clusters/sandbox/hbase/log4j.properties
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.security.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+#
+# Security audit appender
+#
+hbase.security.log.file=SecurityAuth.audit
+log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file}
+log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of .META. messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
diff --git a/src/test/clusters/sandbox/hbase/regionservers b/src/test/clusters/sandbox/hbase/regionservers
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/test/clusters/sandbox/hbase/regionservers
diff --git a/src/test/clusters/sandbox/operations.md b/src/test/clusters/sandbox/operations.md
new file mode 100644
index 0000000..245100d
--- /dev/null
+++ b/src/test/clusters/sandbox/operations.md
@@ -0,0 +1,184 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Just some operations for manual runs against steve's secure VM
+
+
+ export HOYA_CONF_DIR=/Users/stevel/Projects/Hortonworks/Projects/slider/slider-core/src/test/configs/sandbox/hoya
+
+## Local manual tests
+
+
+
+ slider create cluster1 \
+ --component worker 4\
+ --zkhosts sandbox:2181 \
+ --provider hbase \
+ --image hdfs://sandbox.hortonworks.com:8020/user/hoya/hbase.tar.gz \
+ --appconf file:////Users/hoya/Hadoop/configs/master/hbase \
+ --compopt master jvm.heap 128 \
+ --compopt worker jvm.heap 128
+
+
+### bypassing /etc/krb.conf via the -S argument
+
+ bin/slider create cl1 \
+ --manager sandbox:8032 --filesystem hdfs://sandbox.hortonworks.com:8020 \
+ --component worker 1\
+ --component master 0\
+ --zkhosts sandbox:2181 \
+ --provider hbase \
+ --image hdfs://sandbox.hortonworks.com:8020/user/hoya/hbase.tar.gz \
+ --appconf file:///Users/stevel/Projects/slider/slider-core/src/test/configs/sandbox/hbase \
+ --compopt master jvm.heap 128 \
+ --compopt master env.MALLOC_ARENA_MAX 4 \
+ --compopt worker jvm.heap 128
+
+
+
+ bin/slider create cl1 \
+ --component master 0 \
+ --zkhosts sandbox:2181 \
+ --image hdfs://sandbox.hortonworks.com:8020/user/hoya/hbase.tar.gz \
+ --appconf file:///Users/stevel/Projects/slider/slider-core/src/test/configs/sandbox/hbase \
+ --compopt master jvm.heap 128 \
+ --compopt master env.MALLOC_ARENA_MAX 4
+
+
+
+ bin/slider status clu1 \
+ --manager sandbox:8032 --filesystem hdfs://sandbox.hortonworks.com:8020 \
+
+ bin/slider list \
+ --manager sandbox:8032 \
+
+
+
+# single master & workre
+
+ bin/slider create cluster3 \
+ --zkhosts sandbox:2181 \
+ --provider hbase \
+ --image hdfs://sandbox.hortonworks.com:8020/user/hoya/hbase.tar.gz \
+ --appconf file:///Users/stevel/Projects/slider/slider-core/src/test/configs/sandbox/hbase \
+ --component master 1 \
+ --component worker 1
+
+
+# one master
+
+ bin/slider create cl1 \
+ --zkhosts sandbox:2181 \
+ --provider hbase \
+ --image hdfs://sandbox.hortonworks.com:8020/user/hoya/hbase.tar.gz \
+ --appconf file:///Users/stevel/Projects/slider/slider-core/src/test/configs/sandbox/hbase \
+ --component master 1
+
+# one master env set up
+
+ bin/slider create cl1 \
+ --zkhosts sandbox:2181 \
+ --provider hbase \
+ --image hdfs://sandbox.hortonworks.com:8020/user/hoya/hbase.tar.gz \
+ --appconf file:///Users/stevel/Projects/slider/slider-core/src/test/configs/sandbox/hbase \
+ --component master 1 \
+ --component worker 1
+
+# build but don't deploy single master
+
+ bin/slider build cl1 \
+ --zkhosts sandbox:2181 \
+ --provider hbase \
+ --image hdfs://sandbox.hortonworks.com:8020/user/hoya/hbase.tar.gz \
+ --appconf file:///Users/stevel/Projects/slider/slider-core/src/test/configs/sandbox/hbase \
+ --component master 1
+
+
+
+ bin/slider status cl1
+
+
+ bin/slider status cl1
+
+
+
+
+
+ bin/slider thaw cl1
+
+ bin/slider freeze cl1
+ bin/slider freeze cluster3
+ bin/slider destroy cl1
+
+
+
+
+ bin/slider emergency-force-kill all
+
+
+## All configured
+
+
+ bin/slider create cl1 \
+ --component worker 1\
+ --component master 2\
+ --zkhosts sandbox:2181 \
+ --provider hbase \
+ --image hdfs://sandbox.hortonworks.com:8020/user/hoya/hbase.tar.gz \
+ --appconf file:///Users/stevel/Projects/slider/slider-core/src/test/configs/sandbox/hbase \
+ --compopt master env.MALLOC_ARENA_MAX 4 \
+ --compopt worker app.infoport 0 \
+
+### flex the cluster
+
+ bin/slider flex cl1 \
+ --component master 1 \
+ --component worker 2
+
+### freeze
+
+ bin/slider freeze cl1
+
+ bin/slider freeze cl1 --force
+
+### thaw
+
+ bin/slider thaw cl1 -D slider.yarn.queue.priority=5 -D slider.yarn.queue=default
+
+
+### thaw with bad queue: _MUST_ fail
+
+ bin/slider thaw cl1 -D slider.yarn.queue=unknown
+
+### monitor
+
+ bin/slider monitor cl1
+
+### list all
+
+ bin/slider list
+
+### list
+
+ bin/slider list cl1
+
+### status
+
+ bin/slider status cl1
+
+### destroy
+
+ bin/slider destroy cl1
+
+
\ No newline at end of file
diff --git a/src/test/clusters/sandbox/slider/log4j.properties b/src/test/clusters/sandbox/slider/log4j.properties
new file mode 100644
index 0000000..6211771
--- /dev/null
+++ b/src/test/clusters/sandbox/slider/log4j.properties
@@ -0,0 +1,83 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+log4j.logger.org.apache.slider=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+
+
diff --git a/src/test/clusters/sandbox/slider/slider-client.xml b/src/test/clusters/sandbox/slider/slider-client.xml
new file mode 100644
index 0000000..a5d0b6e
--- /dev/null
+++ b/src/test/clusters/sandbox/slider/slider-client.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License. See accompanying LICENSE file.
+ -->
+<!--
+ Properties set here are picked up in the client.
+ They are not passed to the AM
+-->
+<configuration>
+ <property>
+ <name>slider.client.resource.origin</name>
+ <value>configs/sandbox/hoya</value>
+ <description>This is just for diagnostics</description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>sandbox.hortonworks.com:8050</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://sandbox.hortonworks.com:8020</value>
+ </property>
+
+ <property>
+ <name>slider.security.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>slider.zookeeper.quorum</name>
+ <value>sandbox:2181</value>
+ </property>
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>
+ /etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
+ </value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <description>Path to the HBase Tar file in HDFS</description>
+ <value>hdfs://sandbox.hortonworks.com:8020/hbase-0.98.1-bin.tar.gz</value>
+ </property>
+
+
+ <property>
+ <name>slider.test.hbase.appconf</name>
+ <description>Path to the directory containing the HBase application config
+ </description>
+ <value>file://${slider.test.conf.dir}/../hbase</value>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.enabled</name>
+ <description>Flag to enable/disable Accumulo tests</description>
+ <value>true</value>
+ </property>
+
+
+ <property>
+ <name>slider.test.accumulo.tar</name>
+ <description>Path to the Accumulo Tar file in HDFS</description>
+ <value>hdfs://sandbox.hortonworks.com:8020/user/hoya/accumulo.tar</value>
+ </property>
+
+
+ <property>
+ <name>slider.test.am.restart.time</name>
+ <description>Time in millis to await an AM restart</description>
+ <value>60000</value>
+ </property>
+
+
+ <property>
+ <name>slider.test.accumulo.appconf</name>
+ <description>Path to the directory containing the Accumulo application
+ config
+ </description>
+ <value>file:///Users/stevel/Projects/Hortonworks/Projects/slider/test-configs/sandbox/accumulo</value>
+ </property>
+
+
+ <property>
+ <name>zk.home</name>
+ <value>/usr/lib/zookeeper</value>
+ <description>Zookeeper home dir on target systems</description>
+ </property>
+
+ <property>
+ <name>hadoop.home</name>
+ <value>/usr/lib/hadoop</value>
+ <description>Hadoop home dir on target systems</description>
+ </property>
+
+</configuration>
diff --git a/src/test/clusters/script.md b/src/test/clusters/script.md
new file mode 100644
index 0000000..7dc9054
--- /dev/null
+++ b/src/test/clusters/script.md
@@ -0,0 +1,54 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+ # script
+
+
+ # starting
+
+
+
+## env
+
+ export HADOOP_CONF_DIR=/home/stevel/conf
+ export PATH=/home/stevel/hadoop/bin:/home/stevel/hadoop/sbin:$PATH
+
+
+## filesystem
+
+ hdfs namenode -format vm
+
+
+
+## start all the services
+
+ hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start namenode
+ hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start datanode
+ yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
+ yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
+ zookeeper-3.4.5/bin/zkServer.sh start
+
+
+
+* NN up on (http://localhost:50070/dfshealth.jsp)[http://localhost:50070/dfshealth.jsp]
+
+
+
+## shutdown
+
+./zookeeper-3.4.5/bin/zkServer.sh stop
+
+
+
+
\ No newline at end of file
diff --git a/src/test/clusters/ubuntu-secure/README.md b/src/test/clusters/ubuntu-secure/README.md
new file mode 100644
index 0000000..4d3ed90
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/README.md
@@ -0,0 +1,45 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+ # README
+
+This is a set of configurations for a single-node YARN cluster running in
+a VM, with HDFS and YARN brought up on the external network, not localhost.
+Assuming the physical host has a DNS entry for the machine, *ubuntu*,
+a Hoya Client running on the host can upload Hoya and deploy HBase onto
+the single-node cluster even though neither is installed on the VM.
+
+
+ * Namenode [http:///ubuntu:50070/dfshealth.jsp](http://ubuntu:50070/dfshealth.jsp)
+ * YARN RM [http://ubuntu:9081/cluster](http://ubuntu:9081/cluster)
+
+ # Core settings
+
+ <configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://ubuntu:9090</value>
+ </property>
+ </configuration>
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>ubuntu:8032</value>
+ </property>
+
+
+ For the hoya command line
+
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 --zkhosts localhost
+
+
\ No newline at end of file
diff --git a/src/test/clusters/ubuntu-secure/accumulo/accumulo-env.sh b/src/test/clusters/ubuntu-secure/accumulo/accumulo-env.sh
new file mode 100755
index 0000000..11e5ccc
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/accumulo-env.sh
@@ -0,0 +1,56 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###
+### Configure these environment variables to point to your local installations.
+###
+### The functional tests require conditional values, so keep this style:
+###
+### test -z "$JAVA_HOME" && export JAVA_HOME=/usr/local/lib/jdk-1.6.0
+###
+###
+### Note that the -Xmx -Xms settings below require substantial free memory:
+### you may want to use smaller values, especially when running everything
+### on a single machine.
+###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
+
+test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
+test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
+test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
+if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
+then
+ POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy"
+fi
+test -z "$ACCUMULO_TSERVER_OPTS" && export ACCUMULO_TSERVER_OPTS="${POLICY} -Xmx128m -Xms128m "
+test -z "$ACCUMULO_MASTER_OPTS" && export ACCUMULO_MASTER_OPTS="${POLICY} -Xmx128m -Xms128m"
+test -z "$ACCUMULO_MONITOR_OPTS" && export ACCUMULO_MONITOR_OPTS="${POLICY} -Xmx64m -Xms64m"
+test -z "$ACCUMULO_GC_OPTS" && export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
+test -z "$ACCUMULO_GENERAL_OPTS" && export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Dorg.apache.hoya.container=${CONTAINER_ID}"
+test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="-Xmx128m -Xms64m"
+export ACCUMULO_LOG_HOST=`(grep -v '^#' $ACCUMULO_HOME/conf/monitor ; echo localhost ) 2>/dev/null | head -1`
+# what do when the JVM runs out of heap memory
+export ACCUMULO_KILL_CMD='kill -9 %p'
diff --git a/src/test/clusters/ubuntu-secure/accumulo/accumulo-metrics.xml b/src/test/clusters/ubuntu-secure/accumulo/accumulo-metrics.xml
new file mode 100644
index 0000000..60f9f8d
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/accumulo-metrics.xml
@@ -0,0 +1,60 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ This file follows the conventions for XMLConfiguration files specified in the Apache Commons Configuration 1.5 Library. Changes to this file will be noticed
+ at runtime (see the FileChangedReloadingStrategy class in Commons Configuration).
+-->
+<config>
+<!--
+ Metrics log directory
+-->
+ <logging>
+ <dir>${ACCUMULO_HOME}/metrics</dir>
+ </logging>
+<!--
+ Enable/Disable metrics accumulation on the different servers and their components
+ NOTE: Turning on logging can be expensive because it will use several more file handles and will create a lot of short lived objects.
+-->
+ <master>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </master>
+ <logger>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </logger>
+ <tserver>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ <update>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </update>
+ <scan>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </scan>
+ <minc>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </minc>
+ </tserver>
+ <thrift>
+ <enabled type="boolean">false</enabled>
+ <logging type="boolean">false</logging>
+ </thrift>
+</config>
diff --git a/src/test/clusters/ubuntu-secure/accumulo/accumulo-site.xml b/src/test/clusters/ubuntu-secure/accumulo/accumulo-site.xml
new file mode 100644
index 0000000..53a39b3
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/accumulo-site.xml
@@ -0,0 +1,142 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- Put your site-specific accumulo configurations here. The available configuration values along with their defaults are documented in docs/config.html Unless
+ you are simply testing at your workstation, you will most definitely need to change the three entries below. -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The property only needs to be set if upgrading from 1.4 which used to store write-ahead logs on the local
+ filesystem. In 1.5 write-ahead logs are stored in DFS. When 1.5 is started for the first time it will copy any 1.4
+ write ahead logs into DFS. It is possible to specify a comma-separated list of directories.
+ </description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers
+ must know in order to communicate with one another.
+ Change it before initialization. To
+ change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret --old [oldpasswd] --new [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.native.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.token.property.password</name>
+ <!-- change this to the root user's password, and/or change the user below -->
+ <value>secret</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <!--
+ Add the following for Hadoop2, actual needs depend on Hadoop installation details.
+ This list may be excessive, but this should cause no issues. Append these values
+ after the $HADOOP_PREFIX entries
+
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+ /usr/lib/hadoop/.*.jar,
+ /usr/lib/hadoop/lib/.*.jar,
+ /usr/lib/hadoop-hdfs/.*.jar,
+ /usr/lib/hadoop-mapreduce/.*.jar,
+ /usr/lib/hadoop-yarn/.*.jar,
+ -->
+ <value>
+ $ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-server.jar,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-core.jar,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-start.jar,
+ $ACCUMULO_HOME/fate/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-fate.jar,
+ $ACCUMULO_HOME/proxy/target/classes/,
+ $ACCUMULO_HOME/lib/accumulo-proxy.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/yarn/.*.jar,
+ /usr/lib/hadoop/.*.jar,
+ /usr/lib/hadoop/lib/.*.jar,
+ /usr/lib/hadoop-hdfs/.*.jar,
+ /usr/lib/hadoop-mapreduce/.*.jar,
+ /usr/lib/hadoop-yarn/.*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.
+ When using the Security Manager, please remove the ".../target/classes/"
+ values.
+ </description>
+ </property>
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/accumulo/accumulo.policy.example b/src/test/clusters/ubuntu-secure/accumulo/accumulo.policy.example
new file mode 100644
index 0000000..2964f06
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/accumulo.policy.example
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grant codeBase "file:${java.home}/lib/ext/*" {
+ permission java.security.AllPermission;
+};
+
+// These should all be empty in a fielded system
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/server/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/core/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/start/target/classes/" {
+ permission java.security.AllPermission;
+};
+grant codeBase "file:${org.apache.accumulo.core.home.dir}/src/examples/target/classes/" {
+ permission java.security.AllPermission;
+};
+
+grant codebase "file:${hadoop.home.dir}/*" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "shutdownHooks"; // hadoop libs use executables to discover usernames, groups, etc.
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.io.FilePermission "<<ALL FILES>>", "read, execute";
+ permission java.io.FilePermission "/tmp", "write, delete";
+ permission java.io.FilePermission "/tmp/-", "write, delete";
+ permission java.io.FilePermission "/", "write";
+ permission java.net.SocketPermission "*", "connect, resolve";
+ permission java.util.PropertyPermission "java.library.path", "read";
+ permission java.util.PropertyPermission "user.dir", "read";
+ permission java.util.PropertyPermission "org.apache.commons.logging.*", "read";
+ permission java.util.PropertyPermission "entityExpansionLimit", "read";
+ permission java.util.PropertyPermission "maxOccurLimit", "read";
+ permission java.util.PropertyPermission "os.name", "read";
+};
+
+grant codebase "file:${hadoop.home.dir}/lib/*" {
+ // monitor's jetty web service
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ // need to accept web requests, and talk to job tracker, name node, etc.
+ permission java.net.SocketPermission "*", "accept, listen, resolve, connect, resolve";
+ permission java.lang.RuntimePermission "getenv.*";
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.util.PropertyPermission "org.mortbay.*", "read";
+ permission java.util.PropertyPermission "VERBOSE", "read";
+ permission java.util.PropertyPermission "IGNORED", "read";
+ permission java.util.PropertyPermission "ISO_8859_1", "read";
+ permission java.util.PropertyPermission "org.apache.commons.logging.*", "read";
+ permission java.util.PropertyPermission "accumulo.*", "read";
+ permission java.util.PropertyPermission "org.jfree.*", "read";
+ permission java.util.PropertyPermission "elementAttributeLimit", "read";
+ permission java.util.PropertyPermission "entityExpansionLimit", "read";
+ permission java.util.PropertyPermission "maxOccurLimit", "read";
+ // some resources come out of accumulo jars
+ permission java.lang.RuntimePermission "getClassLoader";
+ permission java.io.FilePermission "${org.apache.accumulo.core.home.dir}/lib/*", "read";
+ permission java.io.FilePermission "${org.apache.accumulo.core.home.dir}/src/-", "read";
+ permission java.io.FilePermission "${hadoop.home.dir}/lib/*", "read";
+ // images are cached in /tmp
+ permission java.io.FilePermission "/tmp/*", "read, write";
+ permission java.io.FilePermission "/", "write";
+};
+
+grant codebase "file:${zookeeper.home.dir}/*" {
+ permission java.net.SocketPermission "*", "connect, resolve";
+ permission java.util.PropertyPermission "user.*", "read";
+ permission java.util.PropertyPermission "java.*", "read";
+ permission java.util.PropertyPermission "zookeeper.*", "read";
+ permission java.util.PropertyPermission "jute.*", "read";
+ permission java.util.PropertyPermission "os.*", "read";
+ // accumulo properties read in callbacks
+ permission java.util.PropertyPermission "accumulo.*", "read";
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "exitVM";
+};
+
+grant codebase "file:${org.apache.accumulo.core.home.dir}/lib/ext/*" {
+};
+
+grant codebase "file:${org.apache.accumulo.core.home.dir}/lib/*" {
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ // logging, configuration and getting user id
+ permission java.io.FilePermission "<<ALL FILES>>", "read, write, execute, delete";
+ permission java.util.PropertyPermission "*", "read, write";
+ permission java.lang.RuntimePermission "getenv.*";
+ permission java.lang.RuntimePermission "getClassLoader";
+ permission java.lang.RuntimePermission "loadLibrary.*";
+ permission java.lang.RuntimePermission "accessDeclaredMembers";
+ permission java.lang.RuntimePermission "selectorProvider";
+ permission java.lang.RuntimePermission "accessClassInPackage.*";
+ permission java.lang.RuntimePermission "readFileDescriptor";
+ permission java.lang.RuntimePermission "writeFileDescriptor";
+ permission java.lang.RuntimePermission "modifyThread";
+ permission java.lang.RuntimePermission "modifyThreadGroup";
+ permission java.lang.RuntimePermission "createClassLoader";
+ permission java.lang.RuntimePermission "setContextClassLoader";
+ permission java.lang.RuntimePermission "exitVM";
+ permission java.lang.RuntimePermission "shutdownHooks";
+ permission java.security.SecurityPermission "getPolicy";
+ permission java.security.SecurityPermission "getProperty.*";
+ permission java.security.SecurityPermission "putProviderProperty.*";
+ permission java.security.SecurityPermission "setSystemScope";
+ permission java.security.SecurityPermission "configurationPermission";
+ permission java.security.SecurityPermission "tablesPermission";
+ permission java.security.SecurityPermission "zookeeperWriterPermission";
+ permission java.security.SecurityPermission "tableManagerPermission";
+ permission java.security.SecurityPermission "transportPoolPermission";
+ permission java.security.SecurityPermission "systemCredentialsPermission";
+ permission java.util.logging.LoggingPermission "control";
+ permission java.net.NetPermission "getProxySelector";
+ permission javax.management.MBeanServerPermission "createMBeanServer";
+ permission javax.management.MBeanTrustPermission "register";
+ permission javax.management.MBeanPermission "*", "registerMBean";
+ permission java.net.SocketPermission "*", "accept, connect, listen, resolve";
+};
diff --git a/src/test/clusters/ubuntu-secure/accumulo/auditLog.xml b/src/test/clusters/ubuntu-secure/accumulo/auditLog.xml
new file mode 100644
index 0000000..0d81dbe
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/auditLog.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+
+
+ <!-- Write out Audit info to an Audit file -->
+ <appender name="Audit" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.audit"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="DatePattern" value="'.'yyyy-MM-dd"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss,SSS/Z} [%c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+ <logger name="Audit" additivity="false">
+ <appender-ref ref="Audit" />
+ <level value="OFF"/>
+ </logger>
+
+
+
+
+
+</log4j:configuration>
diff --git a/src/test/clusters/ubuntu-secure/accumulo/gc b/src/test/clusters/ubuntu-secure/accumulo/gc
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/gc
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/ubuntu-secure/accumulo/generic_logger.xml b/src/test/clusters/ubuntu-secure/accumulo/generic_logger.xml
new file mode 100644
index 0000000..9fac0bc
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/generic_logger.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.debug.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Send all logging data to a centralized logger -->
+ <appender name="N1" class="org.apache.log4j.net.SocketAppender">
+ <param name="remoteHost" value="${org.apache.accumulo.core.host.log}"/>
+ <param name="port" value="4560"/>
+ <param name="application" value="${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname}"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- If the centralized logger is down, buffer the log events, but drop them if it stays down -->
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="N1" />
+ </appender>
+
+ <!-- Log accumulo events to the debug, normal and remote logs. -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="ASYNC" />
+ </logger>
+
+ <logger name="org.apache.accumulo.core.file.rfile.bcfile">
+ <level value="INFO"/>
+ </logger>
+
+ <logger name="org.mortbay.log">
+ <level value="WARN"/>
+ </logger>
+
+ <logger name="org.apache.zookeeper">
+ <level value="ERROR"/>
+ </logger>
+
+ <!-- Log non-accumulo events to the debug and normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
diff --git a/src/test/clusters/ubuntu-secure/accumulo/log4j.properties b/src/test/clusters/ubuntu-secure/accumulo/log4j.properties
new file mode 100644
index 0000000..a4bcb2e
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/log4j.properties
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# default logging properties:
+# by default, log everything at INFO or higher to the console
+log4j.rootLogger=INFO,A1
+
+# hide Jetty junk
+log4j.logger.org.mortbay.log=WARN,A1
+
+# hide "Got brand-new compresssor" messages
+log4j.logger.org.apache.hadoop.io.compress=WARN,A1
+
+# hide junk from TestRandomDeletes
+log4j.logger.org.apache.accumulo.test.TestRandomDeletes=WARN,A1
+
+# hide junk from VFS
+log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN,A1
+
+# hide almost everything from zookeeper
+log4j.logger.org.apache.zookeeper=ERROR,A1
+
+# hide AUDIT messages in the shell, alternatively you could send them to a different logger
+log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=WARN,A1
+
+# Send most things to the console
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.appender.A1.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
diff --git a/src/test/clusters/ubuntu-secure/accumulo/masters b/src/test/clusters/ubuntu-secure/accumulo/masters
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/masters
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/ubuntu-secure/accumulo/monitor b/src/test/clusters/ubuntu-secure/accumulo/monitor
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/monitor
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/ubuntu-secure/accumulo/monitor_logger.xml b/src/test/clusters/ubuntu-secure/accumulo/monitor_logger.xml
new file mode 100644
index 0000000..b344c23
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/monitor_logger.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.debug.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_${org.apache.hoya.container}.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Keep the last few log messages for display to the user -->
+ <appender name="GUI" class="org.apache.accumulo.server.monitor.LogService">
+ <param name="keep" value="40"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- Log accumulo messages to debug, normal and GUI -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="GUI" />
+ </logger>
+
+ <!-- Log non-accumulo messages to debug, normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
diff --git a/src/test/clusters/ubuntu-secure/accumulo/slaves b/src/test/clusters/ubuntu-secure/accumulo/slaves
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/slaves
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/ubuntu-secure/accumulo/tracers b/src/test/clusters/ubuntu-secure/accumulo/tracers
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/accumulo/tracers
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
diff --git a/src/test/clusters/ubuntu-secure/capacity-scheduler.xml b/src/test/clusters/ubuntu-secure/capacity-scheduler.xml
new file mode 100644
index 0000000..80a9fec
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/capacity-scheduler.xml
@@ -0,0 +1,111 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-applications</name>
+ <value>10000</value>
+ <description>
+ Maximum number of applications that can be pending and running.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+ <value>0.1</value>
+ <description>
+ Maximum percent of resources in the cluster which can be used to run
+ application masters i.e. controls number of concurrent running
+ applications.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.resource-calculator</name>
+ <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
+ <description>
+ The ResourceCalculator implementation to be used to compare
+ Resources in the scheduler.
+ The default i.e. DefaultResourceCalculator only uses Memory while
+ DominantResourceCalculator uses dominant-resource to compare
+ multi-dimensional resources such as Memory, CPU etc.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ <value>default</value>
+ <description>
+ The queues at the this level (root is the root queue).
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.capacity</name>
+ <value>100</value>
+ <description>Default queue target capacity.</description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
+ <value>1</value>
+ <description>
+ Default queue user limit a percentage from 0.0 to 1.0.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
+ <value>100</value>
+ <description>
+ The maximum capacity of the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.state</name>
+ <value>RUNNING</value>
+ <description>
+ The state of the default queue. State can be one of RUNNING or STOPPED.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
+ <value>*</value>
+ <description>
+ The ACL of who can submit jobs to the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
+ <value>*</value>
+ <description>
+ The ACL of who can administer jobs on the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.node-locality-delay</name>
+ <value>-1</value>
+ <description>
+ Number of missed scheduling opportunities after which the CapacityScheduler
+ attempts to schedule rack-local containers.
+ Typically this should be set to number of racks in the cluster, this
+ feature is disabled by default, set to -1.
+ </description>
+ </property>
+
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/configuration.xsl b/src/test/clusters/ubuntu-secure/configuration.xsl
new file mode 100644
index 0000000..d50d80b
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/configuration.xsl
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="html"/>
+<xsl:template match="configuration">
+<html>
+<body>
+<table border="1">
+<tr>
+ <td>name</td>
+ <td>value</td>
+ <td>description</td>
+</tr>
+<xsl:for-each select="property">
+<tr>
+ <td><a name="{name}"><xsl:value-of select="name"/></a></td>
+ <td><xsl:value-of select="value"/></td>
+ <td><xsl:value-of select="description"/></td>
+</tr>
+</xsl:for-each>
+</table>
+</body>
+</html>
+</xsl:template>
+</xsl:stylesheet>
diff --git a/src/test/clusters/ubuntu-secure/container-executor.cfg b/src/test/clusters/ubuntu-secure/container-executor.cfg
new file mode 100644
index 0000000..53baf87
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/container-executor.cfg
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodemanager.linux-container-executor.group
+banned.users=#comma separated list of users who can not run applications
+min.user.id=1000#Prevent other super-users
diff --git a/src/test/clusters/ubuntu-secure/core-site.xml b/src/test/clusters/ubuntu-secure/core-site.xml
new file mode 100644
index 0000000..2c8ac92
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/core-site.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://ubuntu:9090</value>
+ </property>
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>hdfs/ubuntu@COTHAM</value>
+ </property>
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>ignore.secure.ports.for.testing</name>
+ <value>true</value>
+ </property>
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/hadoop-env.cmd b/src/test/clusters/ubuntu-secure/hadoop-env.cmd
new file mode 100644
index 0000000..05badc2
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hadoop-env.cmd
@@ -0,0 +1,81 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Set Hadoop-specific environment variables here.
+
+@rem The only required environment variable is JAVA_HOME. All others are
+@rem optional. When running a distributed configuration it is best to
+@rem set JAVA_HOME in this file, so that it is correctly defined on
+@rem remote nodes.
+
+@rem The java implementation to use. Required.
+set JAVA_HOME=%JAVA_HOME%
+
+@rem The jsvc implementation to use. Jsvc is required to run secure datanodes.
+@rem set JSVC_HOME=%JSVC_HOME%
+
+@rem set HADOOP_CONF_DIR=
+
+@rem Extra Java CLASSPATH elements. Automatically insert capacity-scheduler.
+if exist %HADOOP_HOME%\contrib\capacity-scheduler (
+ if not defined HADOOP_CLASSPATH (
+ set HADOOP_CLASSPATH=%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
+ ) else (
+ set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
+ )
+)
+
+@rem The maximum amount of heap to use, in MB. Default is 1000.
+@rem set HADOOP_HEAPSIZE=
+@rem set HADOOP_NAMENODE_INIT_HEAPSIZE=""
+
+@rem Extra Java runtime options. Empty by default.
+@rem set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true
+
+@rem Command specific options appended to HADOOP_OPTS when specified
+if not defined HADOOP_SECURITY_LOGGER (
+ set HADOOP_SECURITY_LOGGER=INFO,RFAS
+)
+if not defined HDFS_AUDIT_LOGGER (
+ set HDFS_AUDIT_LOGGER=INFO,NullAppender
+)
+
+set HADOOP_NAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_NAMENODE_OPTS%
+set HADOOP_DATANODE_OPTS=-Dhadoop.security.logger=ERROR,RFAS %HADOOP_DATANODE_OPTS%
+set HADOOP_SECONDARYNAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_SECONDARYNAMENODE_OPTS%
+
+@rem The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+set HADOOP_CLIENT_OPTS=-Xmx128m %HADOOP_CLIENT_OPTS%
+@rem set HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData %HADOOP_JAVA_PLATFORM_OPTS%"
+
+@rem On secure datanodes, user to run the datanode as after dropping privileges
+set HADOOP_SECURE_DN_USER=%HADOOP_SECURE_DN_USER%
+
+@rem Where log files are stored. %HADOOP_HOME%/logs by default.
+@rem set HADOOP_LOG_DIR=%HADOOP_LOG_DIR%\%USERNAME%
+
+@rem Where log files are stored in the secure data environment.
+set HADOOP_SECURE_DN_LOG_DIR=%HADOOP_LOG_DIR%\%HADOOP_HDFS_USER%
+
+@rem The directory where pid files are stored. /tmp by default.
+@rem NOTE: this should be set to a directory that can only be written to by
+@rem the user that will run the hadoop daemons. Otherwise there is the
+@rem potential for a symlink attack.
+set HADOOP_PID_DIR=%HADOOP_PID_DIR%
+set HADOOP_SECURE_DN_PID_DIR=%HADOOP_PID_DIR%
+
+@rem A string representing this instance of hadoop. %USERNAME% by default.
+set HADOOP_IDENT_STRING=%USERNAME%
diff --git a/src/test/clusters/ubuntu-secure/hadoop-env.sh b/src/test/clusters/ubuntu-secure/hadoop-env.sh
new file mode 100644
index 0000000..b852a02
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hadoop-env.sh
@@ -0,0 +1,77 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME. All others are
+# optional. When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use.
+export JAVA_HOME=${JAVA_HOME}
+
+# The jsvc implementation to use. Jsvc is required to run secure datanodes.
+#export JSVC_HOME=${JSVC_HOME}
+
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
+
+# Extra Java CLASSPATH elements. Automatically insert capacity-scheduler.
+for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
+ if [ "$HADOOP_CLASSPATH" ]; then
+ export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
+ else
+ export HADOOP_CLASSPATH=$f
+ fi
+done
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+export HADOOP_HEAPSIZE="256"
+#export HADOOP_NAMENODE_INIT_HEAPSIZE=""
+
+# Extra Java runtime options. Empty by default.
+export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
+
+# Command specific options appended to HADOOP_OPTS when specified
+export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
+export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
+
+export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
+
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"
+#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
+
+# On secure datanodes, user to run the datanode as after dropping privileges
+export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
+
+# Where log files are stored. $HADOOP_HOME/logs by default.
+#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
+
+# Where log files are stored in the secure data environment.
+export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
+
+# The directory where pid files are stored. /tmp by default.
+# NOTE: this should be set to a directory that can only be written to by
+# the user that will run the hadoop daemons. Otherwise there is the
+# potential for a symlink attack.
+export HADOOP_PID_DIR=${HADOOP_PID_DIR}
+export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
+
+# A string representing this instance of hadoop. $USER by default.
+export HADOOP_IDENT_STRING=$USER
diff --git a/src/test/clusters/ubuntu-secure/hadoop-metrics.properties b/src/test/clusters/ubuntu-secure/hadoop-metrics.properties
new file mode 100644
index 0000000..1ca67ce
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hadoop-metrics.properties
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configuration of the "dfs" context for null
+dfs.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "dfs" context for file
+#dfs.class=org.apache.hadoop.metrics.file.FileContext
+#dfs.period=10
+#dfs.fileName=/tmp/dfsmetrics.log
+
+# Configuration of the "dfs" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# dfs.period=10
+# dfs.servers=ubuntu:8649
+
+
+# Configuration of the "mapred" context for null
+mapred.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "mapred" context for file
+#mapred.class=org.apache.hadoop.metrics.file.FileContext
+#mapred.period=10
+#mapred.fileName=/tmp/mrmetrics.log
+
+# Configuration of the "mapred" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# mapred.period=10
+# mapred.servers=ubuntu:8649
+
+
+# Configuration of the "jvm" context for null
+#jvm.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "jvm" context for file
+#jvm.class=org.apache.hadoop.metrics.file.FileContext
+#jvm.period=10
+#jvm.fileName=/tmp/jvmmetrics.log
+
+# Configuration of the "jvm" context for ganglia
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# jvm.period=10
+# jvm.servers=ubuntu:8649
+
+# Configuration of the "rpc" context for null
+rpc.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "rpc" context for file
+#rpc.class=org.apache.hadoop.metrics.file.FileContext
+#rpc.period=10
+#rpc.fileName=/tmp/rpcmetrics.log
+
+# Configuration of the "rpc" context for ganglia
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rpc.period=10
+# rpc.servers=ubuntu:8649
+
+
+# Configuration of the "ugi" context for null
+ugi.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "ugi" context for file
+#ugi.class=org.apache.hadoop.metrics.file.FileContext
+#ugi.period=10
+#ugi.fileName=/tmp/ugimetrics.log
+
+# Configuration of the "ugi" context for ganglia
+# ugi.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# ugi.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# ugi.period=10
+# ugi.servers=ubuntu:8649
+
diff --git a/src/test/clusters/ubuntu-secure/hadoop-metrics2.properties b/src/test/clusters/ubuntu-secure/hadoop-metrics2.properties
new file mode 100644
index 0000000..c3ffe31
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hadoop-metrics2.properties
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# syntax: [prefix].[source|sink].[instance].[options]
+# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
+
+*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink
+# default sampling period, in seconds
+*.period=10
+
+# The namenode-metrics.out will contain metrics from all context
+#namenode.sink.file.filename=namenode-metrics.out
+# Specifying a special sampling period for namenode:
+#namenode.sink.*.period=8
+
+#datanode.sink.file.filename=datanode-metrics.out
+
+# the following example split metrics of different
+# context to different sinks (in this case files)
+#jobtracker.sink.file_jvm.context=jvm
+#jobtracker.sink.file_jvm.filename=jobtracker-jvm-metrics.out
+#jobtracker.sink.file_mapred.context=mapred
+#jobtracker.sink.file_mapred.filename=jobtracker-mapred-metrics.out
+
+#tasktracker.sink.file.filename=tasktracker-metrics.out
+
+#maptask.sink.file.filename=maptask-metrics.out
+
+#reducetask.sink.file.filename=reducetask-metrics.out
+
diff --git a/src/test/clusters/ubuntu-secure/hadoop-policy.xml b/src/test/clusters/ubuntu-secure/hadoop-policy.xml
new file mode 100644
index 0000000..491dbe7
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hadoop-policy.xml
@@ -0,0 +1,219 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+
+ Copyright 2011 The Apache Software Foundation
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ClientProtocol, which is used by user code
+ via the DistributedFileSystem.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.client.datanode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
+ for block recovery.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.datanode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for DatanodeProtocol, which is used by datanodes to
+ communicate with the namenode.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.inter.datanode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for InterDatanodeProtocol, the inter-datanode protocol
+ for updating generation timestamp.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.namenode.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for NamenodeProtocol, the protocol used by the secondary
+ namenode to communicate with the namenode.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.operations.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for AdminOperationsProtocol. Used for admin commands.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.refresh.usertogroups.mappings.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for RefreshUserMappingsProtocol. Used to refresh
+ users mappings. The ACL is a comma-separated list of user and
+ group names. The user and group list is separated by a blank. For
+ e.g. "alice,bob users,wheel". A special value of "*" means all
+ users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.refresh.policy.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
+ dfsadmin and mradmin commands to refresh the security policy in-effect.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.ha.service.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HAService protocol used by HAAdmin to manage the
+ active and stand-by states of namenode.</description>
+ </property>
+
+ <property>
+ <name>security.zkfc.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for access to the ZK Failover Controller
+ </description>
+ </property>
+
+ <property>
+ <name>security.qjournal.service.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for QJournalProtocol, used by the NN to communicate with
+ JNs when using the QuorumJournalManager for edit logs.</description>
+ </property>
+
+ <property>
+ <name>security.mrhs.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HSClientProtocol, used by job clients to
+ communciate with the MR History Server job status etc.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <!-- YARN Protocols -->
+
+ <property>
+ <name>security.resourcetracker.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ResourceTrackerProtocol, used by the
+ ResourceManager and NodeManager to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.resourcemanager-administration.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ResourceManagerAdministrationProtocol, for admin commands.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.applicationclient.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ApplicationClientProtocol, used by the ResourceManager
+ and applications submission clients to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.applicationmaster.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ApplicationMasterProtocol, used by the ResourceManager
+ and ApplicationMasters to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.containermanagement.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ContainerManagementProtocol protocol, used by the NodeManager
+ and ApplicationMasters to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.resourcelocalizer.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for ResourceLocalizer protocol, used by the NodeManager
+ and ResourceLocalizer to communicate with each other.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.job.task.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
+ tasks to communicate with the parent tasktracker.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.job.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for MRClientProtocol, used by job clients to
+ communciate with the MR ApplicationMaster to query job status etc.
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/hbase/hadoop-metrics.properties b/src/test/clusters/ubuntu-secure/hbase/hadoop-metrics.properties
new file mode 100644
index 0000000..e31c059
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hbase/hadoop-metrics.properties
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See http://wiki.apache.org/hadoop/GangliaMetrics
+# Make sure you know whether you are using ganglia 3.0 or 3.1.
+# If 3.1, you will have to patch your hadoop instance with HADOOP-4675
+# And, yes, this file is named hadoop-metrics.properties rather than
+# hbase-metrics.properties because we're leveraging the hadoop metrics
+# package and hadoop-metrics.properties is an hardcoded-name, at least
+# for the moment.
+#
+# See also http://hadoop.apache.org/hbase/docs/current/metrics.html
+# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia
+# meta daemon (gmetad) service is running
+
+# Configuration of the "hbase" context for NullContextWithUpdateThread
+# NullContextWithUpdateThread is a null context which has a thread calling
+# periodically when monitoring is started. This keeps the data sampled
+# correctly.
+hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+hbase.period=10
+
+# Configuration of the "hbase" context for file
+# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# hbase.fileName=/tmp/metrics_hbase.log
+
+# HBase-specific configuration to reset long-running stats (e.g. compactions)
+# If this variable is left out, then the default is no expiration.
+hbase.extendedperiod = 3600
+
+# Configuration of the "hbase" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# hbase.period=10
+# hbase.servers=GMETADHOST_IP:8649
+
+# Configuration of the "jvm" context for null
+jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+jvm.period=10
+
+# Configuration of the "jvm" context for file
+# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# jvm.fileName=/tmp/metrics_jvm.log
+
+# Configuration of the "jvm" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# jvm.period=10
+# jvm.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rpc" context for null
+rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread
+rpc.period=10
+
+# Configuration of the "rpc" context for file
+# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext
+# rpc.fileName=/tmp/metrics_rpc.log
+
+# Configuration of the "rpc" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rpc.period=10
+# rpc.servers=GMETADHOST_IP:8649
+
+# Configuration of the "rest" context for ganglia
+# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext
+# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+# rest.period=10
+# rest.servers=GMETADHOST_IP:8649
diff --git a/src/test/clusters/ubuntu-secure/hbase/hbase-env.sh b/src/test/clusters/ubuntu-secure/hbase/hbase-env.sh
new file mode 100644
index 0000000..da53a27
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hbase/hbase-env.sh
@@ -0,0 +1,106 @@
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Set environment variables here.
+
+# This script sets variables multiple times over the course of starting an hbase process,
+# so try to keep things idempotent unless you want to take an even deeper look
+# into the startup scripts (bin/hbase, etc.)
+
+# The java implementation to use. Java 1.6 required.
+# export JAVA_HOME=/usr/java/jdk1.6.0/
+
+# Extra Java CLASSPATH elements. Optional.
+# export HBASE_CLASSPATH=
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+
+# Uncomment below to enable java garbage collection logging for the server-side processes
+# this enables basic gc logging for the server processes to the .out file
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above
+# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M $HBASE_GC_OPTS"
+
+# Uncomment below to enable java garbage collection logging for the client processes in the .out file.
+# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS"
+
+# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS).
+# This applies to both the server and client GC options above
+# export HBASE_USE_GC_LOGFILE=true
+
+
+# Uncomment below if you intend to use the EXPERIMENTAL off heap cache.
+# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize="
+# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value.
+
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers
+
+# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default.
+# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored. $HBASE_HOME/logs by default.
+# export HBASE_LOG_DIR=${HBASE_HOME}/logs
+
+# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073"
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+# export HBASE_PID_DIR=/var/hadoop/pids
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+# export HBASE_MANAGES_ZK=true
diff --git a/src/test/clusters/ubuntu-secure/hbase/hbase-policy.xml b/src/test/clusters/ubuntu-secure/hbase/hbase-policy.xml
new file mode 100644
index 0000000..e45f23c
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hbase/hbase-policy.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HRegionInterface protocol implementations (ie.
+ clients talking to HRegionServers)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterInterface protocol implementation (ie.
+ clients talking to HMaster for admin operations).
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterRegionInterface protocol implementations
+ (for HRegionServers communicating with HMaster)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/hbase/hbase-site.xml b/src/test/clusters/ubuntu-secure/hbase/hbase-site.xml
new file mode 100644
index 0000000..cc62b01
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hbase/hbase-site.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+ <!--
+ This is a template HBase site file that
+ does not include all the properties
+ required for a valid configuration -
+ the remainder are injected during
+ conversion from a template to
+ actual file
+ -->
+<configuration>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>./hbase-tmp</value>
+ </property>
+ <property>
+ <name>hbase.regionserver.hlog.tolerable.lowreplication</name>
+ <value>1</value>
+
+ </property>
+
+ <property>
+ <name>hbase.regionserver.kerberos.principal</name>
+ <value>stevel/ubuntu@COTHAM</value>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.keytab.file</name>
+ <value>/home/stevel/conf/stevel.keytab</value>
+ </property>
+
+ <property>
+ <name>hbase.master.kerberos.principal</name>
+ <value>stevel/ubuntu@COTHAM</value>
+ </property>
+ <property>
+ <name>hbase.master.keytab.file</name>
+ <value>/home/stevel/conf/stevel.keytab</value>
+ </property>
+
+
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/hbase/log4j.properties b/src/test/clusters/ubuntu-secure/hbase/log4j.properties
new file mode 100644
index 0000000..81233d4
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hbase/log4j.properties
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.security.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+#
+# Security audit appender
+#
+hbase.security.log.file=SecurityAuth.audit
+log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file}
+log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of .META. messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
diff --git a/src/test/clusters/ubuntu-secure/hbase/regionservers b/src/test/clusters/ubuntu-secure/hbase/regionservers
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hbase/regionservers
diff --git a/src/test/clusters/ubuntu-secure/hdfs-site.xml b/src/test/clusters/ubuntu-secure/hdfs-site.xml
new file mode 100644
index 0000000..2878c45
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/hdfs-site.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>file:///home/stevel/hdfs/namenode</value>
+ </property>
+
+
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>file:///home/stevel/hdfs/datanode</value>
+ </property>
+
+ <!-- General HDFS security config -->
+ <property>
+ <name>dfs.block.access.token.enable</name>
+ <value>true</value>
+ </property>
+
+ <!-- NN -->
+ <property>
+ <name>dfs.namenode.keytab.file</name>
+ <value>/home/stevel/conf/hdfs.keytab</value>
+ </property>
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>hdfs/ubuntu@COTHAM</value>
+ </property>
+ <property>
+ <name>dfs.web.authentication.kerberos.principal</name>
+ <value>HTTP/ubuntu@COTHAM</value>
+ </property>
+
+
+ <property>
+ <name>dfs.webhdfs.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.kerberos.internal.spnego.principal</name>
+ <value>HTTP/ubuntu@COTHAM</value>
+ </property>
+
+ <!-- DataNode security config -->
+ <property>
+ <name>dfs.datanode.data.dir.perm</name>
+ <value>700</value>
+ </property>
+ <property>
+ <name>dfs.datanode.keytab.file</name>
+ <value>/home/stevel/conf/hdfs.keytab</value>
+ </property>
+ <property>
+ <name>dfs.datanode.kerberos.principal</name>
+ <value>hdfs/ubuntu@COTHAM</value>
+ </property>
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/httpfs-env.sh b/src/test/clusters/ubuntu-secure/httpfs-env.sh
new file mode 100644
index 0000000..84c67b7
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/httpfs-env.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# Set httpfs specific environment variables here.
+
+# Settings for the Embedded Tomcat that runs HttpFS
+# Java System properties for HttpFS should be specified in this variable
+#
+# export CATALINA_OPTS=
+
+# HttpFS logs directory
+#
+# export HTTPFS_LOG=${HTTPFS_HOME}/logs
+
+# HttpFS temporary directory
+#
+# export HTTPFS_TEMP=${HTTPFS_HOME}/temp
+
+# The HTTP port used by HttpFS
+#
+# export HTTPFS_HTTP_PORT=14000
+
+# The Admin port used by HttpFS
+#
+# export HTTPFS_ADMIN_PORT=`expr ${HTTPFS_HTTP_PORT} + 1`
+
+# The hostname HttpFS server runs on
+#
+# export HTTPFS_HTTP_HOSTNAME=`hostname -f`
diff --git a/src/test/clusters/ubuntu-secure/httpfs-log4j.properties b/src/test/clusters/ubuntu-secure/httpfs-log4j.properties
new file mode 100644
index 0000000..284a819
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/httpfs-log4j.properties
@@ -0,0 +1,35 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# If the Java System property 'httpfs.log.dir' is not defined at HttpFSServer start up time
+# Setup sets its value to '${httpfs.home}/logs'
+
+log4j.appender.httpfs=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.httpfs.DatePattern='.'yyyy-MM-dd
+log4j.appender.httpfs.File=${httpfs.log.dir}/httpfs.log
+log4j.appender.httpfs.Append=true
+log4j.appender.httpfs.layout=org.apache.log4j.PatternLayout
+log4j.appender.httpfs.layout.ConversionPattern=%d{ISO8601} %5p %c{1} [%X{hostname}][%X{user}:%X{doAs}] %X{op} %m%n
+
+log4j.appender.httpfsaudit=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.httpfsaudit.DatePattern='.'yyyy-MM-dd
+log4j.appender.httpfsaudit.File=${httpfs.log.dir}/httpfs-audit.log
+log4j.appender.httpfsaudit.Append=true
+log4j.appender.httpfsaudit.layout=org.apache.log4j.PatternLayout
+log4j.appender.httpfsaudit.layout.ConversionPattern=%d{ISO8601} %5p [%X{hostname}][%X{user}:%X{doAs}] %X{op} %m%n
+
+log4j.logger.httpfsaudit=INFO, httpfsaudit
+
+log4j.logger.org.apache.hadoop.fs.http.server=INFO, httpfs
+log4j.logger.org.apache.hadoop.lib=INFO, httpfs
diff --git a/src/test/clusters/ubuntu-secure/httpfs-signature.secret b/src/test/clusters/ubuntu-secure/httpfs-signature.secret
new file mode 100644
index 0000000..56466e9
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/httpfs-signature.secret
@@ -0,0 +1 @@
+hadoop httpfs secret
diff --git a/src/test/clusters/ubuntu-secure/httpfs-site.xml b/src/test/clusters/ubuntu-secure/httpfs-site.xml
new file mode 100644
index 0000000..4a718e1
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/httpfs-site.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/log4j.properties b/src/test/clusters/ubuntu-secure/log4j.properties
new file mode 100644
index 0000000..2730884
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/log4j.properties
@@ -0,0 +1,227 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+hadoop.root.logger=INFO,console
+hadoop.log.dir=.
+hadoop.log.file=hadoop.log
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshold=ALL
+
+# Null Appender
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# Rolling File Appender - cap space usage at 5gb.
+#
+hadoop.log.maxfilesize=256MB
+hadoop.log.maxbackupindex=20
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize}
+log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex}
+
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# Daily Rolling File Appender
+#
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+#
+# TaskLog Appender
+#
+
+#Default values
+hadoop.tasklog.taskid=null
+hadoop.tasklog.iscleanup=false
+hadoop.tasklog.noKeepSplits=4
+hadoop.tasklog.totalLogFileSize=100
+hadoop.tasklog.purgeLogSplits=true
+hadoop.tasklog.logsRetainHours=12
+
+log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
+log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
+
+log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+# HDFS block state change log from block manager
+#
+# Uncomment the following to suppress normal block state change
+# messages from BlockManager in NameNode.
+#log4j.logger.BlockStateChange=WARN
+
+#
+#Security appender
+#
+hadoop.security.logger=INFO,NullAppender
+hadoop.security.log.maxfilesize=256MB
+hadoop.security.log.maxbackupindex=20
+log4j.category.SecurityLogger=${hadoop.security.logger}
+hadoop.security.log.file=SecurityAuth-${user.name}.audit
+log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
+log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
+log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
+log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
+
+#
+# Daily Rolling Security appender
+#
+log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
+log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
+
+#
+# hdfs audit logging
+#
+hdfs.audit.logger=INFO,NullAppender
+hdfs.audit.log.maxfilesize=256MB
+hdfs.audit.log.maxbackupindex=20
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
+log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
+log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender
+log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
+log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize}
+log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex}
+
+#
+# mapred audit logging
+#
+mapred.audit.logger=INFO,NullAppender
+mapred.audit.log.maxfilesize=256MB
+mapred.audit.log.maxbackupindex=20
+log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
+log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
+log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender
+log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
+log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize}
+log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
+
+# Custom Logging levels
+
+#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
+#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
+#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG
+
+# for debugging placement "issues"
+log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy=DEBUG
+
+# Jets3t library
+log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+
+#
+# Event Counter Appender
+# Sends counts of logging messages at different severity levels to Hadoop Metrics.
+#
+log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
+
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file :
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+hadoop.mapreduce.jobsummary.log.maxfilesize=256MB
+hadoop.mapreduce.jobsummary.log.maxbackupindex=20
+log4j.appender.JSA=org.apache.log4j.RollingFileAppender
+log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
+log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize}
+log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex}
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
+log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
+
+#
+# Yarn ResourceManager Application Summary Log
+#
+# Set the ResourceManager summary log filename
+yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log
+# Set the ResourceManager summary log level and appender
+yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
+#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
+
+# To enable AppSummaryLogging for the RM,
+# set yarn.server.resourcemanager.appsummary.logger to
+# <LEVEL>,RMSUMMARY in hadoop-env.sh
+
+# Appender for ResourceManager Application Summary Log
+# Requires the following properties to be set
+# - hadoop.log.dir (Hadoop Log directory)
+# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
+# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
+
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
+log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
+log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
+log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
+log4j.appender.RMSUMMARY.MaxFileSize=256MB
+log4j.appender.RMSUMMARY.MaxBackupIndex=20
+log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
+log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
diff --git a/src/test/clusters/ubuntu-secure/mapred-env.cmd b/src/test/clusters/ubuntu-secure/mapred-env.cmd
new file mode 100644
index 0000000..610d593
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/mapred-env.cmd
@@ -0,0 +1,20 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+set HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
+
+set HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+
diff --git a/src/test/clusters/ubuntu-secure/mapred-env.sh b/src/test/clusters/ubuntu-secure/mapred-env.sh
new file mode 100644
index 0000000..1f7f5a6
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/mapred-env.sh
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+
+export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=256
+
+export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+
+#export HADOOP_JOB_HISTORYSERVER_OPTS=
+#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.
+#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.
+#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.
+#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default
+#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.
diff --git a/src/test/clusters/ubuntu-secure/mapred-queues.xml.template b/src/test/clusters/ubuntu-secure/mapred-queues.xml.template
new file mode 100644
index 0000000..ce6cd20
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/mapred-queues.xml.template
@@ -0,0 +1,92 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- This is the template for queue configuration. The format supports nesting of
+ queues within queues - a feature called hierarchical queues. All queues are
+ defined within the 'queues' tag which is the top level element for this
+ XML document. The queue acls configured here for different queues are
+ checked for authorization only if the configuration property
+ mapreduce.cluster.acls.enabled is set to true. -->
+<queues>
+
+ <!-- Configuration for a queue is specified by defining a 'queue' element. -->
+ <queue>
+
+ <!-- Name of a queue. Queue name cannot contain a ':' -->
+ <name>default</name>
+
+ <!-- properties for a queue, typically used by schedulers,
+ can be defined here -->
+ <properties>
+ </properties>
+
+ <!-- State of the queue. If running, the queue will accept new jobs.
+ If stopped, the queue will not accept new jobs. -->
+ <state>running</state>
+
+ <!-- Specifies the ACLs to check for submitting jobs to this queue.
+ If set to '*', it allows all users to submit jobs to the queue.
+ If set to ' '(i.e. space), no user will be allowed to do this
+ operation. The default value for any queue acl is ' '.
+ For specifying a list of users and groups the format to use is
+ user1,user2 group1,group2
+
+ It is only used if authorization is enabled in Map/Reduce by setting
+ the configuration property mapreduce.cluster.acls.enabled to true.
+
+ Irrespective of this ACL configuration, the user who started the
+ cluster and cluster administrators configured via
+ mapreduce.cluster.administrators can do this operation. -->
+ <acl-submit-job> </acl-submit-job>
+
+ <!-- Specifies the ACLs to check for viewing and modifying jobs in this
+ queue. Modifications include killing jobs, tasks of jobs or changing
+ priorities.
+ If set to '*', it allows all users to view, modify jobs of the queue.
+ If set to ' '(i.e. space), no user will be allowed to do this
+ operation.
+ For specifying a list of users and groups the format to use is
+ user1,user2 group1,group2
+
+ It is only used if authorization is enabled in Map/Reduce by setting
+ the configuration property mapreduce.cluster.acls.enabled to true.
+
+ Irrespective of this ACL configuration, the user who started the
+ cluster and cluster administrators configured via
+ mapreduce.cluster.administrators can do the above operations on all
+ the jobs in all the queues. The job owner can do all the above
+ operations on his/her job irrespective of this ACL configuration. -->
+ <acl-administer-jobs> </acl-administer-jobs>
+ </queue>
+
+ <!-- Here is a sample of a hierarchical queue configuration
+ where q2 is a child of q1. In this example, q2 is a leaf level
+ queue as it has no queues configured within it. Currently, ACLs
+ and state are only supported for the leaf level queues.
+ Note also the usage of properties for the queue q2.
+ <queue>
+ <name>q1</name>
+ <queue>
+ <name>q2</name>
+ <properties>
+ <property key="capacity" value="20"/>
+ <property key="user-limit" value="30"/>
+ </properties>
+ </queue>
+ </queue>
+ -->
+</queues>
diff --git a/src/test/clusters/ubuntu-secure/mapred-site.xml b/src/test/clusters/ubuntu-secure/mapred-site.xml
new file mode 100644
index 0000000..b4471d7
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/mapred-site.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>mapred.job.tracker</name>
+ <value>ubuntu:9001</value>
+ </property>
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/operations.md b/src/test/clusters/ubuntu-secure/operations.md
new file mode 100644
index 0000000..35c1d49
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/operations.md
@@ -0,0 +1,263 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Just some operations for manual runs against steve's secure VM
+
+ export SLIDER_JVM_OPTS="-Djava.security.krb5.realm=COTHAM -Djava.security.krb5.kdc=ubuntu -Djava.net.preferIPv4Stack=true"
+
+
+## Local manual tests
+
+
+
+
+ slider-assembly/target/slider-assembly-0.5.1-SNAPSHOT-bin/bin/slider \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 list -D slider.security.enabled=true
+
+ slider create cluster1 \
+ --provider hbase \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ --role workers 4\
+ --zkhosts ubuntu --zkport 2121 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM \
+ -S java.security.krb5.kdc=ubuntu \
+ --image hdfs://ubuntu:9090/hbase.tar \
+ --appconf file:////Users/slider/Hadoop/configs/master/hbase \
+ --roleopt master jvm.heap 128 \
+ --roleopt master env.MALLOC_ARENA_MAX 4 \
+ --roleopt worker jvm.heap 128
+
+
+### bypassing /etc/krb.conf via the -S argument
+
+ bin/slider create cl1 \
+ --provider hbase \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM \
+ -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ --role worker 1\
+ --role master 0\
+ --zkhosts ubuntu --zkport 2121 \
+ --image hdfs://ubuntu:9090/hbase.tar \
+ --appconf file:///Users/stevel/Projects/Hortonworks/Projects/slider/slider-funtest/src/test/configs/ubuntu-secure/hbase \
+ --roleopt master jvm.heap 128 \
+ --roleopt master env.MALLOC_ARENA_MAX 4 \
+ --roleopt worker jvm.heap 128
+
+
+
+ bin/slider create cl1 \
+ --provider hbase \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM \
+ -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM \
+ --role master 0 \
+ --zkhosts ubuntu --zkport 2121 \
+ --image hdfs://ubuntu:9090/hbase.tar \
+ --appconf file:///Users/stevel/Projects/Hortonworks/Projects/slider/slider-funtest/src/test/configs/ubuntu-secure/hbase \
+ --roleopt master jvm.heap 128 \
+ --roleopt master env.MALLOC_ARENA_MAX 4
+
+
+
+ bin/slider status clu1 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM \
+ -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
+
+ bin/slider list \
+ --manager ubuntu:8032 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
+
+
+
+
+# single master & workre
+
+ bin/slider create cluster3 \
+ --provider hbase \
+ --zkhosts ubuntu --zkport 2121 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM \
+ --image hdfs://ubuntu:9090/hbase.tar \
+ --appconf file:///Users/stevel/Projects/Hortonworks/Projects/slider/slider-funtest/src/test/configs/ubuntu-secure/hbase \
+ --roleopt master app.infoport 8080 \
+ --role master 1 \
+ --role worker 1
+
+
+# one master
+
+ bin/slider create cl1 \
+ --provider hbase \
+ --zkhosts ubuntu --zkport 2121 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM \
+ --image hdfs://ubuntu:9090/hbase.tar \
+ --appconf file:///Users/stevel/Projects/Hortonworks/Projects/slider/slider-funtest/src/test/configs/ubuntu-secure/hbase \
+ --role master 1
+
+# one master env set up
+
+ bin/slider create cl1 \
+ --provider hbase \
+ --zkhosts ubuntu --zkport 2121 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM \
+ --image hdfs://ubuntu:9090/hbase.tar \
+ --appconf file:///Users/stevel/Projects/Hortonworks/Projects/slider/slider-funtest/src/test/configs/ubuntu-secure/hbase \
+ --role master 1 \
+ --role worker 1
+
+# build but don't deploy single master
+
+ bin/slider build cl1 \
+ --provider hbase \
+ --zkhosts ubuntu \
+ --zkport 2121 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM \
+ --image hdfs://ubuntu:9090/hbase.tar \
+ --appconf file:///Users/stevel/Projects/Hortonworks/Projects/slider/slider-funtest/src/test/configs/ubuntu-secure/hbase \
+ --role master 1
+
+
+ bin/slider status cl1 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
+
+
+ bin/slider status cl1 -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu
+
+
+ bin/slider status cl1 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
+
+ bin/slider status cluster3 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
+
+
+
+ bin/slider thaw cl1 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true \
+ -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
+
+ bin/slider freeze cl1 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true \
+ -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
+
+ bin/slider freeze cluster3 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true \
+ -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
+
+ bin/slider destroy cl1 \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true \
+ -S java.security.krb5.realm=COTHAM -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM \
+
+
+
+
+ bin/slider emergency-force-kill all \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ -D slider.security.enabled=true -S java.security.krb5.realm=COTHAM \
+ -S java.security.krb5.kdc=ubuntu \
+ -D yarn.resourcemanager.principal=yarn/ubuntu@COTHAM \
+ -D dfs.namenode.kerberos.principal=hdfs/ubuntu@COTHAM
+
+
+## All configured
+
+
+ bin/slider create cl1 \
+ -S java.security.krb5.realm=COTHAM \
+ -S java.security.krb5.kdc=ubuntu \
+ --manager ubuntu:8032 --filesystem hdfs://ubuntu:9090 \
+ --role worker 1\
+ --role master 2\
+ --zkhosts ubuntu \
+ --zkport 2121 \
+ --image hdfs://ubuntu:9090/hbase.tar \
+ --appconf file:///Users/stevel/Projects/Hortonworks/Projects/hoya/hoya-funtest/src/test/configs/ubuntu-secure/hbase \
+ --roleopt master env.MALLOC_ARENA_MAX 4 \
+ --roleopt worker app.infoport 0 \
+
+# flex the cluster
+
+ bin/slider flex cl1 \
+ --role master 1 \
+ --role worker 2
+
+# freeze
+
+ bin/slider freeze cl1
+
+# thaw
+
+ bin/slider thaw cl1
+
+# monitor
+
+ bin/slider monitor cl1
+
+# list all
+
+ bin/slider list
+
+# list
+
+ bin/slider list cl1
+
+# status
+
+ bin/slider status cl1
+
+# destroy
+
+ bin/slider destroy cl1
+
diff --git a/src/test/clusters/ubuntu-secure/slaves b/src/test/clusters/ubuntu-secure/slaves
new file mode 100644
index 0000000..e9e5f7c
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/slaves
@@ -0,0 +1 @@
+ubuntu
diff --git a/src/test/clusters/ubuntu-secure/slider/log4j.properties b/src/test/clusters/ubuntu-secure/slider/log4j.properties
new file mode 100644
index 0000000..c99d4a3
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/slider/log4j.properties
@@ -0,0 +1,84 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+
+log4j.logger.org.apache.slider=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+#log4j.logger.org.apache.hadoop.security.SaslRpcClient=DEBUG
+
diff --git a/src/test/clusters/ubuntu-secure/slider/slider-client.xml b/src/test/clusters/ubuntu-secure/slider/slider-client.xml
new file mode 100644
index 0000000..062bc24
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/slider/slider-client.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License. See accompanying LICENSE file.
+ -->
+
+<!--
+ Properties set here are picked up in the client.
+ They are not passed to the AM
+-->
+<configuration>
+ <property>
+ <name>slider.client.resource.origin</name>
+ <value>ubuntu-secure/conf/slider-client.xml</value>
+ <description>This is just for diagnostics</description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>ubuntu:8032</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://ubuntu:9090/</value>
+ </property>
+
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>yarn/ubuntu@COTHAM</value>
+ </property>
+
+ <property>
+ <name>slider.zookeeper.quorum</name>
+ <value>ubuntu:2181</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>hdfs/ubuntu@COTHAM</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <description>Path to the HBase Tar file in HDFS</description>
+ <value>hdfs://ubuntu:9090/hbase-0.98.1-bin.tar.gz</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.appconf</name>
+ <description>Path to the directory containing the HBase application config
+ </description>
+ <value>file://${slider.test.conf.dir}/../hbase</value>
+ </property>
+
+ <property>
+ <name>zk.home</name>
+ <description>zookeeper home dir -needed for accumulo</description>
+ <value>/home/stevel/zookeeper</value>
+ </property>
+
+ <property>
+ <name>hadoop.home</name>
+ <description>hadoop home dir -needed for accumulo</description>
+ <value>/home/stevel/hadoop</value>
+ </property>
+
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/ssl-client.xml.example b/src/test/clusters/ubuntu-secure/ssl-client.xml.example
new file mode 100644
index 0000000..a50dce4
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/ssl-client.xml.example
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+<property>
+ <name>ssl.client.truststore.location</name>
+ <value></value>
+ <description>Truststore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.password</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.location</name>
+ <value></value>
+ <description>Keystore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.password</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.keypassword</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/ssl-server.xml.example b/src/test/clusters/ubuntu-secure/ssl-server.xml.example
new file mode 100644
index 0000000..4b363ff
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/ssl-server.xml.example
@@ -0,0 +1,77 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+<property>
+ <name>ssl.server.truststore.location</name>
+ <value></value>
+ <description>Truststore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.password</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+</property>
+
+<property>
+ <name>ssl.server.keystore.location</name>
+ <value></value>
+ <description>Keystore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.password</name>
+ <value></value>
+ <description>Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.keypassword</name>
+ <value></value>
+ <description>Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+</configuration>
diff --git a/src/test/clusters/ubuntu-secure/yarn-env.cmd b/src/test/clusters/ubuntu-secure/yarn-env.cmd
new file mode 100644
index 0000000..3329f8f
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/yarn-env.cmd
@@ -0,0 +1,60 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem User for YARN daemons
+if not defined HADOOP_YARN_USER (
+ set HADOOP_YARN_USER=%yarn%
+)
+
+if not defined YARN_CONF_DIR (
+ set YARN_CONF_DIR=%HADOOP_YARN_HOME%\conf
+)
+
+if defined YARN_HEAPSIZE (
+ @rem echo run with Java heapsize %YARN_HEAPSIZE%
+ set JAVA_HEAP_MAX=-Xmx%YARN_HEAPSIZE%m
+)
+
+if not defined YARN_LOG_DIR (
+ set YARN_LOG_DIR=%HADOOP_YARN_HOME%\logs
+)
+
+if not defined YARN_LOGFILE (
+ set YARN_LOGFILE=yarn.log
+)
+
+@rem default policy file for service-level authorization
+if not defined YARN_POLICYFILE (
+ set YARN_POLICYFILE=hadoop-policy.xml
+)
+
+if not defined YARN_ROOT_LOGGER (
+ set YARN_ROOT_LOGGER=INFO,console
+)
+
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.dir=%YARN_LOG_DIR%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.log.dir=%YARN_LOG_DIR%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.file=%YARN_LOGFILE%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.log.file=%YARN_LOGFILE%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.home.dir=%HADOOP_YARN_HOME%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.id.str=%YARN_IDENT_STRING%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.home.dir=%HADOOP_YARN_HOME%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.root.logger=%YARN_ROOT_LOGGER%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.root.logger=%YARN_ROOT_LOGGER%
+if defined JAVA_LIBRARY_PATH (
+ set YARN_OPTS=%YARN_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
+)
+set YARN_OPTS=%YARN_OPTS% -Dyarn.policy.file=%YARN_POLICYFILE%
\ No newline at end of file
diff --git a/src/test/clusters/ubuntu-secure/yarn-env.sh b/src/test/clusters/ubuntu-secure/yarn-env.sh
new file mode 100644
index 0000000..16f1b82
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/yarn-env.sh
@@ -0,0 +1,112 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# User for YARN daemons
+export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+
+# resolve links - $0 may be a softlink
+export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+
+# some Java parameters
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+if [ "$JAVA_HOME" != "" ]; then
+ #echo "run java in $JAVA_HOME"
+ JAVA_HOME=$JAVA_HOME
+fi
+
+if [ "$JAVA_HOME" = "" ]; then
+ echo "Error: JAVA_HOME is not set."
+ exit 1
+fi
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx256m
+
+# For setting YARN specific HEAP sizes please use this
+# Parameter and set appropriately
+YARN_HEAPSIZE=256
+
+# check envvars which might override default args
+if [ "$YARN_HEAPSIZE" != "" ]; then
+ JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+fi
+
+# Resource Manager specific parameters
+
+# Specify the max Heapsize for the ResourceManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_RESOURCEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#export YARN_RESOURCEMANAGER_HEAPSIZE=1000
+
+# Specify the JVM options to be used when starting the ResourceManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_RESOURCEMANAGER_OPTS=
+
+# Node Manager specific parameters
+
+# Specify the max Heapsize for the NodeManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_NODEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#export YARN_NODEMANAGER_HEAPSIZE=1000
+
+# Specify the JVM options to be used when starting the NodeManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_NODEMANAGER_OPTS=
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+
+# default log directory & file
+if [ "$YARN_LOG_DIR" = "" ]; then
+ YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
+fi
+if [ "$YARN_LOGFILE" = "" ]; then
+ YARN_LOGFILE='yarn.log'
+fi
+
+# default policy file for service-level authorization
+if [ "$YARN_POLICYFILE" = "" ]; then
+ YARN_POLICYFILE="hadoop-policy.xml"
+fi
+
+# restore ordinary behaviour
+unset IFS
+
+
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
+YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
+YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+ YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi
+YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+
+
diff --git a/src/test/clusters/ubuntu-secure/yarn-site.xml b/src/test/clusters/ubuntu-secure/yarn-site.xml
new file mode 100644
index 0000000..8cac9ef
--- /dev/null
+++ b/src/test/clusters/ubuntu-secure/yarn-site.xml
@@ -0,0 +1,104 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+
+<!-- Site specific YARN configuration properties -->
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>ubuntu:8032</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.webapp.address</name>
+ <value>ubuntu:9081</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>ubuntu:8031</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>1</value>
+ </property>
+ <property>
+ <description>Whether physical memory limits will be enforced for
+ containers.
+ </description>
+ <name>yarn.nodemanager.pmem-check-enabled</name>
+ <value>false</value>
+ </property>
+ <!-- we really don't want checking here-->
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ </property>
+
+ <!-- how long after a failure to see what is left in the directory-->
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>60000</value>
+ </property>
+
+ <!--ten seconds before the process gets a -9 -->
+ <property>
+ <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
+ <value>30000</value>
+ </property>
+
+
+
+ <!-- security -->
+ <property>
+ <name>yarn.resourcemanager.keytab</name>
+ <value>/home/stevel/conf/yarn.keytab</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>yarn/ubuntu@COTHAM</value>
+ </property>
+
+<!--
+ <property>
+ <name>yarn.resourcemanager.webapp.spnego-principal</name>
+ <value>yarn/ubuntu@COTHAM</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.webapp.spnego-keytab-file</name>
+ <value>${yarn.resourcemanager.keytab}</value>
+ </property>
+-->
+ <property>
+ <name>yarn.nodemanager.keytab</name>
+ <value>/home/stevel/conf/yarn.keytab</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.principal</name>
+ <value>yarn/ubuntu@COTHAM</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.webapp.spnego-principal</name>
+ <value>yarn/ubuntu@COTHAM</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.webapp.spnego-keytab-file</name>
+ <value>${yarn.resourcemanager.keytab}</value>
+ </property>
+
+
+ <property>
+ <name>yarn.namenode.kerberos.internal.spnego.principal</name>
+ <value>HTTP/ubuntu@COTHAM</value>
+ </property>
+
+</configuration>
